https://weaviate.io/assets/js/23e07d5f.cc6bf6f2.js

1"use strict";(globalThis.webpackChunkmy_website=globalThis.webpackChunkmy_website||[]).push([[6029],{88943(e,t,i){i.r(t),i.d(t,{assets:()=>c,contentTitle:()=>d,default:()=>u,frontMatter:()=>s,metadata:()=>n,toc:()=>l});var n=i(84409),a=i(74848),o=i(28453),r=i(73284);const s={title:"Embed and Search Text at Scale with Modal and Weaviate",slug:"modal-and-weaviate",description:"Learn how to vectorize ~50 million objects and ingest into Weaviate using Modal!",authors:["charles","erika"],date:new Date("2024-05-08T00:00:00.000Z"),tags:["how-to","integrations"],image:"./img/hero.png"},d=void 0,c={image:i(77527).A,authorsImageUrls:[void 0,void 0]},l=[{value:"Modal: Serverless Infrastructure for GPUs and More",id:"modal-serverless-infrastructure-for-gpus-and-more",level:2},{value:"Using Weaviate and Modal for performant data ingestion",id:"using-weaviate-and-modal-for-performant-data-ingestion",level:2},{value:"Weaviate Features to Improve Indexing Speed",id:"weaviate-features-to-improve-indexing-speed",level:2},{value:"Async Indexing",id:"async-indexing",level:3},{value:"Product Quantization",id:"product-quantization",level:3},{value:"Vector Index Configuration",id:"vector-index-configuration",level:3},{value:"Text Search Configuration",id:"text-search-configuration",level:3},{value:"Batch Imports",id:"batch-imports",level:3},{value:"Take-Aways",id:"take-aways",level:2},...r.RM];function h(e){const t={a:"a",admonition:"admonition",code:"code",h2:"h2",h3:"h3",img:"img",li:"li",ol:"ol",p:"p",pre:"pre",...(0,o.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(t.p,{children:(0,a.jsx)(t.img,{alt:"Modal and Weaviate",src:i(52756).A+"",width:"1200",height:"630"})}),"\n",(0,a.jsx)(t.p,{children:"A full application of AI or embeddings is much more than just a prompt or a model: it\u2019s an integrated system that combines data, models, compute, and user interfaces."}),"\n",(0,a.jsxs)(t.p,{children:["In this blog post, we\u2019ll talk about how we built a full application that discovers analogies between Wikipedia articles by combining serverless infrastructure from Modal with the search and storage capabilities of Weaviate. Try it ",(0,a.jsx)(t.a,{href:"https://vector-analogies-wikipedia.vercel.app/",children:"here"}),"!"]}),"\n",(0,a.jsxs)("figure",{style:{maxWidth:"70%",margin:"0 auto"},children:[(0,a.jsx)("img",{src:i(66419).A,alt:"alt",style:{width:"100%"}}),(0,a.jsx)("figcaption",{style:{marginBottom:"1em"},children:(0,a.jsxs)(t.p,{children:["An analogy from our system: Kobe Bryant is the Albert Einstein of Basketball. ",(0,a.jsx)("br",{})]})})]}),"\n",(0,a.jsxs)(t.p,{children:["The goal of this blog post is to provide you with a quick overview of Modal and Weaviate and how we used them together to build out the search backend for our application. A tutorial on how to run and deploy the demo yourself can be found in ",(0,a.jsx)(t.a,{href:"https://modal.com/docs/examples/vector-analogies-wikipedia",children:"Modal's documentation"}),"."]}),"\n",(0,a.jsx)(t.h2,{id:"modal-serverless-infrastructure-for-gpus-and-more",children:"Modal: Serverless Infrastructure for GPUs and More"}),"\n",(0,a.jsx)(t.p,{children:"We\u2019ll be searching Wikipedia for articles that complete analogies using a combination of text search and vector-based semantic search. Naturally, that makes Weaviate a great fit!"}),"\n",(0,a.jsxs)(t.p,{children:["To provide the infrastructure for computing the vector embeddings, we used ",(0,a.jsx)(t.a,{href:"https://modal.com/",children:"Modal"}),"."]}),"\n",(0,a.jsx)(t.p,{children:"Modal makes it easy for Python developers to deploy serverless applications. That means they can deploy applications that dynamically scale to match their workload \u2013 no PhD in Kubernetes required! Your application can scale up to tens or hundreds of GPUs during peak times and then back down to zero when requests do \u2013 and you only pay for what you use."}),"\n",(0,a.jsx)(t.p,{children:"Unlike other serverless cloud offerings, Modal uses a custom container runtime written in Rust and designed to run data/ML/AI workloads at low latency and high throughput. Most importantly for us, it has GPUs, which make inference faster for embedding and generative models. And we can run them only when we need to, spinning them up in seconds!"}),"\n",(0,a.jsx)(t.p,{children:"And Modal does more than just inference \u2013 it\u2019s a whole serverless infrastru

1cture platform. So we can also host the client that talks to our Weaviate database on Modal, giving us the flexibility to further customize our search using Python\u2019s powerful ML libraries."}),"\n",(0,a.jsx)(t.h2,{id:"using-weaviate-and-modal-for-performant-data-ingestion",children:"Using Weaviate and Modal for performant data ingestion"}),"\n",(0,a.jsx)(t.p,{children:"In designing a complex pipeline, like data ingestion for a search engine, it\u2019s important to first look for a bottleneck, a step whose speed determines the speed of all the others."}),"\n",(0,a.jsxs)(t.p,{children:["Because embedding vectors are so hot and cutting edge these days, you might expect that creating the embeddings is the bottleneck. But it\u2019s not: we can easily scale up to the point where ",(0,a.jsx)(t.a,{href:"https://modal.com/blog/embedding-wikipedia",children:"all of English Wikipedia is converted into vectors in under fifteen minutes"}),"."]}),"\n",(0,a.jsx)(t.p,{children:"In fact, creating the text and vector search indexes is the bottleneck. This shouldn\u2019t be too surprising: databases are frequently the bottleneck. They have a treacherously difficult job to do, providing a storage abstraction that has the durability of disk but the flexibility and speed of memory."}),"\n",(0,a.jsxs)(t.p,{children:["Once you\u2019ve identified the bottleneck, the ",(0,a.jsx)(t.a,{href:"https://www.leanproduction.com/theory-of-constraints/",children:"first step in managing it"})," is to maximize the speed of the bottleneck. In addition to having the combination text and vector search capabilities our application required, Weaviate provided both a scalable, managed deployment with ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/wcs",children:"Weaviate Cloud"})," and all the knobs we needed to configure our indexing to maximize speed while providing the needed features."]}),"\n",(0,a.jsx)(t.p,{children:"With careful tuning, described below, we were able to increase the throughput from the low hundreds of rows up to ~7500 rows/second, ingesting and indexing ~50,000,000 snippets from Wikipedia articles in under two hours."}),"\n",(0,a.jsxs)(t.p,{children:["The ",(0,a.jsx)(t.a,{href:"https://www.leanproduction.com/theory-of-constraints/#:~:text=Integrating%20with%20Lean-,What%20Is%20the%20Theory%20of%20Constraints%3F,no%20longer%20the%20limiting%20factor.",children:"second step in managing a bottleneck"})," is to subordinate the rest of the system to that bottleneck. With serverless infrastructure defined as code, this was as easy as changing a few configuration values in our ingestion scripts. To find out what those numbers should be, all we needed to do was calculate the throughput per GPU worker and divide the indexing throughput by that number to get the maximum number of workers."]}),"\n",(0,a.jsx)(t.h2,{id:"weaviate-features-to-improve-indexing-speed",children:"Weaviate Features to Improve Indexing Speed"}),"\n",(0,a.jsx)(t.p,{children:"In this section, we walk through some of the key Weaviate features we used to improve the indexing performance while preserving search quality."}),"\n",(0,a.jsxs)(t.ol,{children:["\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"#async-indexing",children:"Async Indexing"})}),"\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"#product-quantization",children:"Product Quantization"})}),"\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"#vector-index-configuration",children:"Vector Index Configuration"})}),"\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"#text-search-configuration",children:"Text Search Configuration"})}),"\n",(0,a.jsx)(t.li,{children:(0,a.jsx)(t.a,{href:"#batch-imports",children:"Batch Imports"})}),"\n"]}),"\n",(0,a.jsx)(t.h3,{id:"async-indexing",children:"Async Indexing"}),"\n",(0,a.jsxs)(t.p,{children:["Importing data into a vector database is a two-step process: 1. Object creation, and 2. Vector index construction. Typically, the database won\u2019t receive another batch of objects until it has finished indexing the prior batch. Weaviate ",(0,a.jsx)(t.code,{children:"1.22"})," introduced ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/release-notes/older-releases/release_1_22#asynchronous-vector-indexing-experimental",children:"async indexing"})," to speed up data ingestion."]}),"\n",(0,a.jsxs)(t.p,{children:["Async indexing speeds up object ingestion by separating it from the ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/concepts/vector-index",children:"vector index"})," construction process. Rather than waiting for the new objects to be indexed, the objects are put in an in-memory queue before getting indexed. Since we imported millions of objects, it was important to enable this feature to reduce the import time."]}),"\n",(0,a.jsxs)(t.p,{children:["For open-source users, you can enable it by adding ",(0,a.jsx)(t.code,{children:"ASYNC_INDEXING=true"})," in your environment variable, or you can enable it through Weaviate Cloud."]}),"\n",(0,a.jsx)(t.h3,{id:"product-quantization",children:"Product Quantization"}),"\n",(0,a.jsxs)(t.p,{children:["While async indexing helped with importing, ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/concepts/vector-quantization#product-quantization",children:"product quantization"})," (PQ) was important for alleviating the memory footprint. PQ is a technique used to compress the size of vector embeddings by representing the vectors as centroid ids. The centroid ids are calculated by clustering each segment. We enabled PQ to reduce the memory requirements without slowing down the ingestion."]}),"\n",(0,a.jsxs)(t.p,{children:["You can either ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/configuration/compression/pq-compression#manually-configure-pq",children:"manually"})," enable PQ or have it ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/configuration/compression/pq-compression#configure-autopq",children:"automatically triggered"})," based on the number of vectors. You can manually enable PQ by setting it at the collection level. You\u2019ll first need to import at least 10,000 to 100,000 vectors in order to form the centroids. AutoPQ can be used for new collections and the training step to form the centroids is set at 100,000 vectors by default."]}),"\n",(0,a.jsx)(t.p,{children:"It is important to keep in mind that PQ will have a tradeoff between recall, performance, and memory usage. You can tune HNSW parameters to complement the PQ configuration you selected."}),"\n",(0,a.jsx)(t.h3,{id:"vector-index-configuration",children:"Vector Index Configuration"}),"\n",(0,a.jsxs)(t.p,{children:["Since our application didn\u2019t require high recall, we opted in to save costs by enabling PQ. There are a few ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/config-refs/schema/vector-index#pq-configuration-parameters",children:"PQ parameters"})," that you can configure like ",(0,a.jsx)(t.code,{children:"trainingLimit"}),", ",(0,a.jsx)(t.code,{children:"segments"}),", and ",(0,a.jsx)(t.code,{children:"distribution"}),". Additionally, you can configure the ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/config-refs/schema/vector-index#hnsw-index-parameters",children:"HNSW parameters"})," like the ",(0,a.jsx)(t.code,{children:"distance"}),", ",(0,a.jsx)(t.code,{children:"ef"}),", ",(0,a.jsx)(t.code,{children:"efConstruction"}),", and more. It is important to note that some parameters cannot be changed once the collection is created."]}),"\n",(0,a.jsx)(t.p,{children:"When configuring HNSW, you should consider a few questions:"}),"\n",(0,a.jsxs)(t.ol,{children:["\n",(0,a.jsx)(t.li,{children:"How many queries do you expect per second?"}),"\n",(0,a.jsx)(t.li,{children:"Do you expect a lot of imports or updates?"}),"\n",(0,a.jsx)(t.li,{children:"How high should the recall be?"}),"\n"]}),"\n",(0,a.jsxs)(t.p,{children:["Consider the answers in this ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/config-refs/schema/vector-index#hnsw-configuration-tips",children:"table"})," to help with your use case!"]}),"\n",(0,a.jsx)(t.h3,{id:"text-search-configuration",children:"Text Search Configuration"}),"\n",(0,a.jsx)(t.p,{children:"Weaviate lets you tune how each property is indexed, or if they\u2019re indexed at all! By skipping the indexes, you speed up the insert time and reduce the memory for indexing."}),"\n",(0,a.jsxs)(t.p,{children:["In this demo, we determined that search performance could be improved by exactly matching a particular ",(0,a.jsx)(t.code,{children:"title"})," value. This requires creating an inverted index to find text matches on the ",(0,a.jsx)(t.code,{children:"title"})," property. This is denoted in your schema by setting ",(0,a.jsx)(t.code,{children:"indexFilterable"})," to ",(0,a.jsx)(t.code,{children:"True"}),". We further want to use BM25 scoring on the title, so we create an inverted index for keyword scoring as well. This is achieved by setting ",(0,a.jsx)(t.code,{children:"indexSearchable"})," to ",(0,a.jsx)(t.code,{children:"True"}),". However, for the ",(0,a.jsx)(t.code,{children:"url"})," and ",(0,a.jsx)(t.code,{children:"content"})," properties, although we want the data stored, we do not need an index, so we turn it off by setting both ",(0,a.jsx)(t.code,{children:"indexFilterable"})," and ",(0,a.jsx)(t.code,{children:"indexSearchable"})," to ",(0,a.jsx)(t.code,{children:"False"}),"."]}),"\n",(0,a.jsx)(t.h3,{id:"batch-imports",children:"Batch Imports"}),"\n",(0,a.jsx)(t.p,{children:"When importing data into Weaviate, we suggest using batch imports. This will send multiple objects in a single request, rather than 

1importing them individually."}),"\n",(0,a.jsx)(t.admonition,{title:"Update since publication",type:"note",children:(0,a.jsxs)(t.p,{children:["Weaviate has since added ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/manage-objects/import#server-side-batching",children:"server-side batching"}),", and it is now the recommended starting point. The server tells the client how fast to send data based on its own load, so there is no batch size or concurrency for you to tune. Open the ",(0,a.jsx)(t.code,{children:"stream()"})," context manager, or use the one-shot ",(0,a.jsx)(t.code,{children:"collection.data.ingest(objects)"})," for a straight bulk load. Both need Weaviate ",(0,a.jsx)(t.code,{children:"1.36"})," or newer and Python client ",(0,a.jsx)(t.code,{children:"4.20"})," or newer. The three client-side methods below still work, and they are what we used for this demo in May 2024."]})}),"\n",(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-python",children:"with collection.batch.stream() as batch:\n"})}),"\n",(0,a.jsxs)(t.p,{children:["There are ",(0,a.jsx)(t.a,{href:"https://docs.weaviate.io/weaviate/client-libraries/python#batch-processing",children:"three client-side methods"})," to choose from:"]}),"\n",(0,a.jsxs)(t.ol,{children:["\n",(0,a.jsxs)(t.li,{children:[(0,a.jsx)(t.code,{children:"dynamic"}),": The batch size and number of requests are dynamically adjusted as you import data. The size and number of each are dependent on the server load."]}),"\n"]}),"\n",(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-python",children:"with client.batch.dynamic() as batch:\n"})}),"\n",(0,a.jsxs)(t.ol,{start:"2",children:["\n",(0,a.jsxs)(t.li,{children:[(0,a.jsx)(t.code,{children:"fixed_size"}),": The user specifies the batch size and number of requests."]}),"\n"]}),"\n",(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-python",children:"with client.batch.fixed_size(batch_size=100, concurrent_requests=4) as batch:\n"})}),"\n",(0,a.jsxs)(t.ol,{start:"3",children:["\n",(0,a.jsxs)(t.li,{children:[(0,a.jsx)(t.code,{children:"rate_limit"}),": If there is a third-party vectorization rate limit you want to avoid, you can specify the number of objects to send per minute."]}),"\n"]}),"\n",(0,a.jsx)(t.pre,{children:(0,a.jsx)(t.code,{className:"language-python",children:"with client.batch.rate_limit(requests_per_minute=600) as batch:\n"})}),"\n",(0,a.jsx)(t.p,{children:"For our application, we used the fixed size batch import set to 1,024 objects."}

1),"\n",(0,a.jsx)(t.h2,{id:"take-aways",children:"Take-Aways"}),"\n",(0,a.jsx)(t.p,{children:"We built an end-to-end application that searches Wikipedia for an article that completes an analogy, like \u201cthe peso is to Mexico as the dollar is to the United States\u201d."}),"\n",(0,a.jsxs)(t.p,{children:["We powered our text and vector search with Weaviate and our embeddings, ingestion, and database queries with ",(0,a.jsx)(t.a,{href:"https://modal.com",children:"Modal"}),"."]}),"\n",(0,a.jsxs)(t.p,{children:["We think the two tools make a great pair for data- and compute-intensive applications of generative models and artificial intelligence. Take them for a spin on your next project, whether it\u2019s a ",(0,a.jsx)(t.a,{href:"https://github.com/weaviate-tutorials/Hurricane",children:"blog post generator"})," or a ",(0,a.jsx)(t.a,{href:"https://modal.com/docs/examples/dreambooth_app",children:"pet art generator"}),", and let us know what you build!"]}),"\n","\n",(0,a.jsx)(r.Ay,{})]})}function u(e={}){const{wrapper:t}={...(0,o.R)(),...e.components};return t?(0,a.jsx)(t,{...e,children:(0,a.jsx)(h,{...e})}):h(e)}},77527(e,t,i){i.d(t,{A:()=>n});const n=i.p+"assets/images/hero-698cd72752ad6775139fa3ed43dfc170.png"},66419(e,t,i){i.d(t,{A:()=>n});const n=i.p+"assets/images/example-96908e9c5fda659a5648d89821bc4772.png"},52756(e,t,i){i.d(t,{A:()=>n});const n=i.p+"assets/images/hero-698cd72752ad6775139fa3ed43dfc170.png"},84409(e){e.exports=JSON.parse('{"permalink":"/blog/modal-and-weaviate","editUrl":"https://github.com/weaviate/weaviate-io/tree/main/blog/2024-05-08-Modal-and-Weaviate/index.mdx","source":"@site/blog/2024-05-08-Modal-and-Weaviate/index.mdx","title":"Embed and Search Text at Scale with Modal and Weaviate","description":"Learn how to vectorize ~50 million objects and ingest into Weaviate using Modal!","date":"2024-05-08T00:00:00.000Z","tags":[{"inline":true,"label":"how-to","permalink":"/blog/tags/how-to"},{"inline":true,"label":"integrations","permalink":"/blog/tags/integrations"}],"readingTime":8.77,"hasTruncateMarker":false,"authors":[{"name":"Charles Frye","title":"AI Engineer","url":"https://twitter.com/charles_irl","imageURL":"/img/people/icon/charles-modal.jpg","key":"charles","page":null},{"name":"Erika Shorten","title":"Technology Partner Manager","url":"https://github.com/erika-shorten","imageURL":"/img/people/icon/erika.jpg","key":"erika","page":null}],"frontMatter":{"title":"Embed and Search Text at Scale with Modal and Weaviate","slug":"modal-and-weaviate","description":"Learn how to vectorize ~50 million objects and ingest into Weaviate using Modal!","authors":["charles","erika"],"date":"2024-05-08T00:00:00.000Z","tags":["how-to","integrations"],"image":"./img/hero.png"},"unlisted":false,"prevItem":{"title":"Shipping Trustworthy AI Products with Instabase and Weaviate","permalink":"/blog/instabase-and-weaviate"},"nextItem":{"title":"Using Weaviate to Find Waldo","permalink":"/blog/find-waldo-with-weaviate"}}')}}]);