1"use strict";(self.webpackChunktlab_docs=self.webpackChunktlab_docs||[]).push([[4897],{26914:(e,n,t)=>{t.d(n,{A:()=>r});const r=t.p+"assets/images/vllm-service-configuration-a0bb9d32450f724dce7698ec6cbb97bf.png"},28453:(e,n,t)=>{t.d(n,{R:()=>a,x:()=>c});var r=t(96540);const i={},s=r.createContext(i);function a(e){const n=r.useContext(s);return r.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function c(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(i):e.components||i:a(e.components),r.createElement(s.Provider,{value:n},e.children)}},30739:(e,n,t)=>{t.d(n,{A:()=>r});const r=t.p+"assets/images/select-service-type-2b4c460d2828cc57068d3fca05e9cc52.png"},33717:(e,n,t)=>{t.d(n,{A:()=>r});const r=t.p+"assets/images/screenshot-providershealth-525b60f39d0699f907c4f16d19473b04.png"},47167:(e,n,t)=>{t.d(n,{A:()=>r});const r=t.p+"assets/images/interact-page-new-c2ba411ed9a345899d100e607d2fb167.png"},59207:(e,n,t)=>{t.d(n,{A:()=>r});const r=t.p+"assets/images/vllm-service-card-interactive-setup-d75058a8b46ee38e3d01b9bab076947f.png"},73344:(e,n,t)=>{t.d(n,{A:()=>r});const r=t.p+"assets/images/vllm-access-service-d5b519eb2506ef74813e17eeb0b2f4cb.png"},76643:(e,n,t)=>{t.d(n,{A:()=>r});const r=t.p+"assets/images/screenshot-addprovider-e1b599d63a798a35b27bab4da3847567.png"},78274:(e,n,t)=>{t.r(n),t.d(n,{assets:()=>l,contentTitle:()=>c,default:()=>h,frontMatter:()=>a,metadata:()=>r,toc:()=>o});const r=JSON.parse('{"id":"interact-examples/vllm","title":"Interact with a Model using vLLM","de
1scription":"Running a vLLM Server Service","source":"@site/for-teams/interact-examples/vllm.md","sourceDirName":"interact-examples","slug":"/interact-examples/vllm","permalink":"/for-teams/interact-examples/vllm","draft":false,"unlisted":false,"tags":[],"version":"current","sidebarPosition":8,"frontMatter":{"title":"Interact with a Model using vLLM","sidebar_position":8},"sidebar":"documentationSidebar","previous":{"title":"Get Direct SSH Access to a Node","permalink":"/for-teams/interact-examples/ssh"},"next":{"title":"Run an Ollama Server","permalink":"/for-teams/interact-examples/ollama"}}');var i=t(74848),s=t(28453);const a={title:"Interact with a Model using vLLM",sidebar_position:8},c=void 0,l={},o=[{value:"Running a vLLM Server Service",id:"running-a-vllm-server-service",level:2},{value:"Prerequisites",id:"prerequisites",level:2},{value:"Steps to Run a vLLM Server Service",id:"steps-to-run-a-vllm-server-service",level:2}];function d(e){const n={h2:"h2",img:"img",li:"li",ol:"ol",p:"p",ul:"ul",...(0,s.R)(),...e.components};return(0,i.jsxs)(i.Fragment,{children:[(0,i.jsx)(n.h2,{id:"running-a-vllm-server-service",children:"Running a vLLM Server Service"}),"\n",(0,i.jsx)(n.p,{children:"Interactive services in Transformer Lab allow you to launch environments like vLLM Server to interact with models directly from the interface. This guide walks you through running a vLLM Server service."}),"\n",(0,i.jsx)(n.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,i.jsx)(n.p,{children:"Before running a vLLM Server service, ensure you have a Compute Provider set up and active."}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Navigate to Team Settings and set up a Compute Provider.\n",(0,i.jsx)(n.img,{alt:"Compute Provider Setup",src:t(76643).A+"",width:"851",height:"640"})]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Make sure the provider is active by clicking on the health button.\n",(0,i.jsx)(n.img,{alt:"Provider Health Check",src:t(33717).A+"",width:"2317",height:"323"})]}),"\n"]}),"\n"]}),"\n",(0,i.jsx)(n.h2,{id:"steps-to-run-a-vllm-server-service",children:"Steps to Run a vLLM Server Service"}),"\n",(0,i.jsxs)(n.ol,{children:["\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"Go to the Interact page in Transformer Lab."}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:['Click on the "New" button to create a new interactive service.\n',(0,i.jsx)(n.img,{alt:"Interact Page New Button",src:t(47167).A+"",width:"2366",height:"679"})]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:['Select "vLLM Server" as the type of interactive service to launch.\n',(0,i.jsx)(n.img,{alt:"Select vLLM Server",src:t(30739).A+"",width:"2073",height:"648"})]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:"Configure the service:"}),"\n",(0,i.jsxs)(n.ul,{children:["\n",(0,i.jsx)(n.li,{children:"Enter a name for the service."}),"\n",(0,i.jsx)(n.li,{children:"Select the Compute Provider to use."}),"\n",(0,i.jsx)(n.li,{children:"Specify the resources: CPU, memory, and GPUs."}),"\n",(0,i.jsx)(n.li,{children:"Provide the Hugging Face ID of the model to use."}),"\n",(0,i.jsx)(n.li,{children:"Set the Tensor parallel size for the vLLM engine."}),"\n",(0,i.jsxs)(n.li,{children:["Provide any additional inputs if required (e.g., ngrok auth token for tunneling).\n",(0,i.jsx)(n.img,{alt:"vLLM Configuration",src:t(26914).A+"",width:"1851",height:"1099"})]}),"\n"]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsx)(n.p,{children:'Click "Launch" to start the vLLM Server service.'}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:['Once launched, a card will appear for the service. Click the "Interactive Setup" button on the card.\n',(0,i.jsx)(n.img,{alt:"Service Card Interactive Setup",src:t(59207).A+"",width:"1020",height:"542"})]}),"\n"]}),"\n",(0,i.jsxs)(n.li,{children:["\n",(0,i.jsxs)(n.p,{children:["Follow the provided URL or steps to access and interact with the model via the vLLM Server.\n",(0,i.jsx)(n.img,{alt:"Access vLLM Server",src:t(73344).A+"",width:"980",height:"711"})]}),"\n"]}),"\n"]})]})}function h(e={}){const{wrapper:n}={...(0,s.R)(),...e.components};return n?(0,i.jsx)(n,{...e,children:(0,i.jsx)(d,{...e})}):d(e)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.