1"use strict";(globalThis.webpackChunkmy_website=globalThis.webpackChunkmy_website||[]).push([[259],{27749(e,a,t){t.r(a),t.d(a,{assets:()=>d,contentTitle:()=>s,default:()=>m,frontMatter:()=>r,metadata:()=>n,toc:()=>c});var n=t(93891),l=t(74848),i=t(28453),o=t(73284);const r={title:"Building a Local RAG System for Privacy Preservation with\xa0Ollama and Weaviate",slug:"local-rag-with-ollama-and-weaviate",description:"How to implement a local Retrieval-Augmented Generation pipeline with Ollama language models and a self-hosted Weaviate vector database via Docker in\xa0Python.",authors:["leonie"],date:new Date("2024-05-21T00:00:00.000Z"),tags:["how-to"],image:"./img/hero.png"},s=void 0,d={image:t(83719).A,authorsImageUrls:[void 0]},c=[{value:"How to Setup Local Language Models with\xa0Ollama",id:"how-to-setup-local-language-models-withollama",level:2},{value:"Step 1: Download and install\xa0Ollama",id:"step-1-download-and-installollama",level:3},{value:"Step 2: Pull\xa0models",id:"step-2-pullmodels",level:3},{value:"Step 3: Install Ollama Python\xa0library",id:"step-3-install-ollama-pythonlibrary",level:3},{value:"How to Setup a Local Vector Database Instance with\xa0Docker",id:"how-to-setup-a-local-vector-database-instance-withdocker",level:2},{value:"Step 1: Download and install\xa0Docker",id:"step-1-download-and-installdocker",level:3},{value:"Step 2: Start up the Docker container with a Weaviate\xa0instance",id:"step-2-start-up-the-docker-container-with-a-weaviateinstance",level:3},{value:"Step 3: Install the Weaviate Python\xa0client",id:"step-3-install-the-weaviate-pythonclient",level:3},{value:"How to Build a Local RAG\xa0Pipeline",id:"how-to-build-a-local-ragpipeline",level:2},{value:"Preparation: Ingest data into vector\xa0database",id:"preparation-ingest-data-into-vectordatabase",level:3},{value:"Step 1: Retrieve\xa0context",id:"step-1-retrievecontext",level:3},{value:"Step 2: Augment the\xa0prompt",id:"step-2-augment-theprompt",level:3},{value:"Step 3: Generate a\xa0response",id:"step-3-generate-aresponse",level:3},{value:"Summary",id:"summary",level:2},...o.RM];function h(e){const a={a:"a",code:"code",h2:"h2",h3:"h3",img:"img",li:"li",p:"p",pre:"pre",ul:"ul",...(0,i.R)(),...e.components};return(0,l.jsxs)(l.Fragment,{children:[(0,l.jsx)(a.p,{children:(0,l.jsx)(a.img,{alt:"On-premises Retrieval-Augmented Generation system with Ollama and Weaviate",src:t(66032).A+"",width:"1200",height:"630"})}),"\n",(0,l.jsxs)(a.p,{children:["Prototyping an LLM-powered application is fun and easy but when you want to make it production-ready to use in your company, you will quickly face a variety of challenges, such as mitigating hallucinations or protecting data privacy. While ",(0,l.jsx)(a.a,{href:"https://towardsdatascience.com/retrieval-augmented-generation-rag-from-theory-to-langchain-implementation-4e9bd5f6a4f2",children:"Retrieval-Augmented Generation (RAG)"})," has been shown to be effective against hallucinations, local deployments are a popular approach for privacy preservation."]}),"\n",(0,l.jsx)(a.p,{children:"This article showcases how you can implement a local RAG-based chatbot in Python in an on-premises environment without any dependencies on the outside world using the following local components:"}),"\n",(0,l.jsxs)(a.ul,{children:["\n",(0,l.jsxs)(a.li,{children:["Local LLM and embedding models via ",(0,l.jsx)(a.a,{href:"https://ollama.com/",children:"Ollama"})]}),"\n",(0,l.jsxs)(a.li,{children:["Local ",(0,l.jsx)(a.a,{href:"https://weaviate.io/",children:"Weaviate"})," ",(0,l.jsx)(a.a,{href:"https://weaviate.io/blog/what-is-a-vector-database",children:"vector database"})," instance via ",(0,l.jsx)(a.a,{href:"https://www.docker.com/",children:"Docker"})]}),"\n"]}),"\n",(0,l.jsx)(a.p,{children:"Everything is local, open source, and doesn\u2019t require any API keys!"}),"\n",(0,l.jsx)(a.h2,{id:"how-to-setup-local-language-models-withollama",children:"How to Setup Local Language Models with\xa0Ollama"}),"\n",(0,l.jsx)(a.p,{children:"Had I known that getting set up with Ollama takes less than 5 minutes, I wouldn\u2019t have put it off for so long."}),"\n",(0,l.jsx)(a.h3,{id:"step-1-download-and-installollama",children:"Step 1: Download and install\xa0Ollama"}),"\n",(0,l.jsxs)(a.p,{children:["Download the Ollama version for your operating system from the official ",(0,l.jsx)(a.a,{href:"https://ollama.com/download",children:"download page"})," and follow the few installation steps."]}),"\n",(0,l.jsx)(a.h3,{id:"step-2-pullmodels",children:"Step 2: Pull\xa0models"}),"\n",(0,l.jsxs)(a.p,{children:["Open a terminal and pull the relevant LLMs and ",(0,l.jsx)(a.a,{href:"https://ollama.com/blog/embedding-models",children:"embedding models"})," of your choice. For this tutorial, we will be using ",(0,l.jsx)(a.a,{href:"https://ollama.com/library/llama2",children:(0,l.jsx)(a.code,{children:"llama2"})})," by Meta for the LLM and ",(0,l.jsx)(a.a,{href:"https://ollama.com/library/all-minilm",children:(0,l.jsx)(a.code,{children:"all-minilm"})})," for the embedding model."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{children:"ollama pull llama2\nollama pull all-minilm\n"})}),"\n",(0,l.jsxs)(a.p,{children:["Other available embedding models are ",(0,l.jsx)(a.a,{href:"https://ollama.com/library/mxbai-embed-large",children:(0,l.jsx)(a.code,{children:"mxbai-embed-large"})})," (334 M parameters) and ",(0,l.jsx)(a.a,{href:"https://ollama.com/library/nomic-embed-text",children:(0,l.jsx)(a.code,{children:"nomic-embed-text"})})," (137 M parameters)."]}),"\n",(0,l.jsx)(a.h3,{id:"step-3-install-ollama-pythonlibrary",children:"Step 3: Install Ollama Python\xa0library"}),"\n",(0,l.jsxs)(a.p,{children:["Since we\u2019re implementing the RAG pipeline in Python, you will need to ",(0,l.jsx)(a.code,{children:"pip"})," install the Python library. In this tutorial, we\u2019re using version 0.1.8."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{children:"pip install ollama\n"})}),"\n",(0,l.jsxs)(a.p,{children:["Alternatively, a ",(0,l.jsx)(a.a,{href:"https://github.com/ollama/ollama/blob/main/docs/api.md#generate-embeddings",children:"REST API"})," or a ",(0,l.jsx)(a.a,{href:"https://github.com/ollama/ollama-js",children:"JavaScript"})," library are also available."]}),"\n",(0,l.jsx)(a.h2,{id:"how-to-setup-a-local-vector-database-instance-withdocker",children:"How to Setup a Local Vector Database Instance with\xa0Docker"}),"\n",(0,l.jsxs)(a.p,{children:["In a local RAG pipeline, you want to host your vector database locally as well. This tutorial shows you how you can ",(0,l.jsx)(a.a,{href:"https://docs.weaviate.io/weaviate/installation/docker-compose",children:"host an instance of the open source Weaviate vector database locally with Docker"}),"."]}),"\n",(0,l.jsx)(a.h3,{id:"step-1-download-and-installdocker",children:"Step 1: Download and install\xa0Docker"}),"\n",(0,l.jsxs)(a.p,{children:["Install both the ",(0,l.jsx)(a.a,{href:"https://docs.docker.com/desktop/",children:(0,l.jsx)(a.code,{children:"docker"})})," (Docker 17.09.0 or higher) and the ",(0,l.jsx)(a.a,{href:"https://docs.docker.com/compose/install/",children:(0,l.jsx)(a.code,{children:"docker-compose"})})," (",(0,l.jsx)(a.a,{href:"https://docs.docker.com/compose/migrate/",children:"Docker Compose V2"}),") CLI tools."]}),"\n",(0,l.jsx)(a.h3,{id:"step-2-start-up-the-docker-container-with-a-weaviateinstance",children:"Step 2: Start up the Docker
1container with a Weaviate\xa0instance"}),"\n",(0,l.jsx)(a.p,{children:"Now, you can run the following command from a terminal to start up a Weaviate instance from the default Docker image."}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{children:"docker run -p 8080:8080 -p 50051:50051 cr.weaviate.io/semitechnologies/weaviate:1.38.8\n"})}),"\n",(0,l.jsx)(a.h3,{id:"step-3-install-the-weaviate-pythonclient",children:"Step 3: Install the Weaviate Python\xa0client"}),"\n",(0,l.jsxs)(a.p,{children:["Since we\u2019re implementing the RAG pipeline in Python, you will need to ",(0,l.jsx)(a.code,{children:"pip"})," install the Python library. This tutorial uses the ",(0,l.jsx)(a.code,{children:"batch.stream()"})," server-side batching API, which needs client version ",(0,l.jsx)(a.code,{children:"4.20"})," or newer against Weaviate ",(0,l.jsx)(a.code,{children:"1.36"})," or newer."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{children:"pip install -U weaviate-client\n"})}),"\n",(0,l.jsx)(a.h2,{id:"how-to-build-a-local-ragpipeline",children:"How to Build a Local RAG\xa0Pipeline"}),"\n",(0,l.jsx)(a.p,{children:"Once you have the relevant models pulled locally and ready to be served with Ollama and your vector database self-hosted via Docker, you can start implementing the RAG pipeline."}),"\n",(0,l.jsxs)(a.p,{children:["The following example is based on a post in the Ollama blog titled \u201c",(0,l.jsx)(a.a,{href:"https://ollama.com/blog/embedding-models",children:"Embedding models"}),"\u201d."]}),"\n",(0,l.jsx)(a.p,{children:(0,l.jsx)(a.img,{alt:"Technical diagram of local retrieval augmented generation system with Ollama and Weaviate",src:t(35361).A+"",width:"1200",height:"600"})}),"\n",(0,l.jsx)(a.p,{children:"Local Retrieval-Augmented Generation System with language models via\xa0Ollama"}),"\n",(0,l.jsx)(a.h3,{id:"preparation-ingest-data-into-vectordatabase",children:"Preparation: Ingest data into vector\xa0database"}),"\n",(0,l.jsx)(a.p,{children:"The first step in building a RAG pipeline is to ingest your data into a vector database. For this, you will need to generate the data's embeddings."}),"\n",(0,l.jsxs)(a.p,{children:["Below, you can see a few sample documents from an ",(0,l.jsx)(a.a,{href:"https://ollama.com/blog/embedding-models",children:"example used in an Ollama blog post"}),"."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{className:"language-python",children:'documents = [\n "Llamas are members of the camelid family meaning they\'re pretty closely related to vicu\xf1as and camels",\n "Llamas were first domesticated and used as pack animals 4,000 to 5,000 years ago in the Peruvian highlands",\n "Llamas can grow as much as 6 feet tall though the average llama between 5 feet 6 inches and 5 feet 9 inches tall",\n "Llamas weigh between 280 and 450 pounds and can carry 25 to 30 percent of their body weight",\n "Llamas are vegetarians and have very efficient digestive systems",\n "Llamas live to be about 20 years old, though some only live for 15 years and others live to be 30 years old",\n]\n'})}),"\n",(0,l.jsx)(a.p,{children:"Next, you will connect to your locally running vector database instance."}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{className:"language-python",children:"import weaviate\n\nclient = weaviate.connect_to_local()\n"})}),"\n",(0,l.jsxs)(a.p,{children:["At startup, this vector database is empty. To populate it with your data, you first need to define the structure (a collection, called ",(0,l.jsx)(a.code,{children:"docs"})," in this example) in which to store your data. Since the example data is just a simple list of strings, you can define a single property with the name ",(0,l.jsx)(a.code,{children:"text"})," and the datatype ",(0,l.jsx)(a.code,{children:"DataType.TEXT"}),"."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{className:"language-python",children:'import weaviate.classes as wvc\nfrom weaviate.classes.config import Property, DataType\n\n# Create a new data collection\ncollection = client.collections.create(\n name = "docs", # Name of the data collection\n properties=[\n Property(name="text", data_type=DataType.TEXT), # Name and data type of the property\n ],\n)\n'})}),"\n",(0,l.jsxs)(a.p,{children:["Now, you can load the data into the predefined structure. For this, you can iterate over your ",(0,l.jsx)(a.code,{children:"documents"})," and generate the embeddings for each data object with Ollama\u2019s ",(0,l.jsx)(a.code,{children:"embeddings()"})," method. Then, the text and its embedding are stored together in the vector database. The ",(0,l.jsx)(a.code,{children:"batch.stream()"})," context manager uses ",(0,l.jsx)(a.a,{href:"https://docs.weaviate.io/weaviate/manage-objects/import#server-side-batching",children:"server-side batching"}),", so the server sets the pace and you add objects one at a time as Ollama returns each embedding."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{className:"language-python",children:'import ollama\n\n# Store each document in a vector embedding database\nwith collection.batch.stream() as batch:\n for i, d in enumerate(documents):\n # Generate embeddings\n response = ollama.embeddings(model = "all-minilm",\n prompt = d)\n\n # Add data object with text and embedding\n batch.add_object(\n properties = {"text" : d},\n vector = response["embedding"],\n )\n'})}),"\n",(0,l.jsx)(a.h3,{id:"step-1-retrievecontext",children:"Step 1: Retrieve\xa0context"}),"\n",(0,l.jsx)(a.p,{children:"Now, at inference, you will want to retrieve additional context for your prompt. For this, you will c
1onduct a simple similarity search for your prompt (e.g., \u201cWhat animals are llamas related to?\u201d)."}),"\n",(0,l.jsxs)(a.p,{children:["For the similarity search, you will have to first generate the vector embeddings of your search query (here, the prompt) similar to the ingestion stage with the ",(0,l.jsx)(a.code,{children:"embeddings()"})," method. Then, you can pass the resulting embeddings to Weaviate\u2019s ",(0,l.jsx)(a.code,{children:"near_vector()"})," method and specify to retrieve only the closest result (",(0,l.jsx)(a.code,{children:"limit = 1"}),")."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{className:"language-python",children:'# An example prompt\nprompt = "What animals are llamas related to?"\n\n# Generate an embedding for the prompt and retrieve the most relevant doc\nresponse = ollama.embeddings(\n model = "all-minilm",\n prompt = prompt,\n)\n\nresults = collection.query.near_vector(near_vector = response["embedding"],\n limit = 1)\n\ndata = results.objects[0].properties[\'text\']\n'})}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{children:"Llamas are members of the camelid family meaning they're pretty closely related to vicu\xf1as and camels\n"})}),"\n",(0,l.jsx)(a.h3,{id:"step-2-augment-theprompt",children:"Step 2: Augment the\xa0prompt"}),"\n",(0,l.jsx)(a.p,{children:"Next, you can augment a prompt template with the original prompt and the retrieved context:"}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{className:"language-python",children:'prompt_template = f"Using this data: {data}. Respond to this prompt: {prompt}"\n'})}),"\n",(0,l.jsx)(a.h3,{id:"step-3-generate-aresponse",children:"Step 3: Generate a\xa0response"}),"\n",(0,l.jsxs)(a.p,{children:["Finally, you can use Ollama\u2019s ",(0,l.jsx)(a.code,{children:"generate()"})," method to generate a response from the augmented prompt template."]}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{className:"language-python",children:"# Generate a response combining the prompt and data we retrieved in step 2\noutput = ollama.generate(\n model = \"llama2\",\n prompt = prompt_template,\n)\n\nprint(output['response'])\n"})}),"\n",(0,l.jsx)(a.pre,{children:(0,l.jsx)(a.code,{children:'Llamas are members of the camelid family, which means they are closely related to other animals in the same family, including:\n1. Vicu\xf1as: Vicu\xf1as are small, wild relatives of llamas and alpacas. They are found in the Andean region and are known for their soft, woolly coats.\n2. Camels: Camels are large, even-toed ungulates that are closely related to llamas and vicu\xf1as. They are found in hot, dry climates around the world and are known for their ability to go without water for long periods of time.\n3. Guanacos: Guanacos are large, wild animals that are related to llamas and vicu\xf1as. They are found in the Andean region and are known for their distinctive long necks and legs.\n4. Llama-like creatures: There are also other animals that are sometimes referred to as "llamas," such as the lama-like creatures found in China, which are actually a different species altogether. These creatures are not closely related to vicu\xf1as or camels, but are sometimes referred to as "llamas" due to their physical similarities.\nIn summary, llamas are related to vicu\xf1as, camels, guanacos, and other animals that are sometimes referred to as "llamas."\n'})}),"\n",(0,l.jsx)(a.h2,{id:"summary",children:"Summary"}),"\n",(0,l.jsx)(a.p,{children:"This article guided you through a very simple example of a RAG pipeline to highlight how you can build a local RAG system for privacy preservation using local components (language models via Ollama, Weaviate vector database self-hosted via Docker)."}),"\n",(0,l.jsxs)(a.p,{children:["Note, that this tutorial does not use any orchestration frameworks, such as ",(0,l.jsx)(a.a,{href:"https://www.langchain.com/",children:"LangChain"})," or ",(0,l.jsx)(a.a,{href:"https://www.llamaindex.ai/",children:"LlamaIndex"}),". However, both LangChain and LlamaIndex have integrations with both Ollama and Weaviate, so that you can easily modify the above tutorial to the orchestration framework of your choice."]}),"\n",(0,l.jsxs)(a.p,{children:["You can find the Notebook related to this article in ",(0,l.jsx)(a.a,{href:"https://github.com/weaviate/recipes/blob/main/weaviate-features/generative-search/generative_search_ollama/ollama_local_rag.ipynb",children:"this GitHub repository"}),"."]}),"\n","\n",(0,l.jsx)(o.Ay,{})]})}function m(e={}){const{wrapper:a}={...(0,i.R)(),...e.components};return a?(0,l.jsx)(a,{...e,children:(0,l.jsx)(h,{...e})}):h(e)}},83719(e,a,t){t.d(a,{A:()=>n});const n=t.p+"assets/images/hero-f0235cb6218bc8b95596de10a47334c2.png"},66032(e,a,t){t.d(a,{A:()=>n});const n=t.p+"assets/images/hero-f0235cb6218bc8b95596de10a47334c2.png"},35361(e,a,t){t.d(a,{A:()=>n});const n=t.p+"assets/images/rag-ollama-diagram-c5e713fbc8bc1592f586a3107587519b.png"},93891(e){e.exports=JSON.parse('{"permalink":"/blog/local-rag-with-ollama-and-weaviate","editUrl":"https://github.com/weaviate/weaviate-io/tree/main/blog/2024-05-21-local-rag-ollama/index.mdx","source":"@site/blog/2024-05-21-local-rag-ollama/index.mdx","title":"Building a Local RAG System for Privacy Preservation with\xa0Ollama and Weaviate","description":"How to implement a local Retrieval-Augmented Generation pipeline with Ollama language models and a self-hosted Weaviate vector database via Docker in\xa0Python.","date":"2024-05-21T00:00:00.000Z","tags":[{"inline":true,"label":"how-to","permalink":"/blog/tags/how-to"}],"readingTime":7.78,"hasTruncateMarker":false,"authors":[{"name":"Leonie Monigatti","title":"Machine Learning Engineer","url":"https://www.linkedin.com/in/804250ab/","imageURL":"/img/people/icon/leonie.jpg","key":"leonie","page":null}],"frontMatter":{"title":"Building a Local RAG System for Privacy Preservation with\xa0Ollama and Weaviate","slug":"local-rag-with-ollama-and-weaviate","description":"How to implement a local Retrieval-Augmented Generation pipeline with Ollama language models and a self-hosted Weaviate vector database via Docker in\xa0Python.","authors":["leonie"],"date":"2024-05-21T00:00:00.000Z","tags":["how-to"],"image":"./img/hero.png"},"unlisted":false,"prevItem":{"title":"Evaluation Metrics for Search and Recommendation Systems","permalink":"/blog/retrieval-evaluation-metrics"},"nextItem":{"title":"Weaviate 1.25 Release","permalink":"/blog/weaviate-1-25-release"}}')}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.