1"use strict";(globalThis.webpackChunkllm_d_website=globalThis.webpackChunkllm_d_website||[]).push([[27150],{9042(e,n,t){t.r(n),t.d(n,{assets:()=>o,contentTitle:()=>r,default:()=>h,frontMatter:()=>l,metadata:()=>s,toc:()=>d});const s=JSON.parse('{"id":"infrastructure/gateway/istio","title":"Istio","description":"This guide shows how to deploy llm-d with Istio as your inference gateway. By the end, inference requests will flow from an Istio-managed Gateway to your model servers via the llm-d EPP.","source":"@site/docs/infrastructure/gateway/istio.md","sourceDirName":"infrastructure/gateway","slug":"/infrastructure/gateway/istio","permalink":"/docs/dev/infrastructure/gateway/istio","draft":false,"unlisted":false,"editUrl":"https://github.com/llm-d/llm-d/edit/main/docs/infrastructure/gateway/istio.md","tags":[],"version":"current","frontMatter":{},"sidebar":"docsSidebar","previous":{"title":"Gateway Guides","permalink":"/docs/dev/infrastructure/gateway/"},"next":{"title":"GKE","permalink":"/docs/dev/infrastructure/gateway/gke"}}');var a=t(74848),i=t(28453);const l={},r="Istio",o={},d=[{value:"Prerequisites",id:"prerequisites",level:2},{value:"Step 1: Install Gateway API and Gateway API Inference Extension CRDs",id:"step-1-install-gateway-api-and-gateway-api-inference-extension-crds",level:2},{value:"Step 2: Install Istio",id:"step-2-install-istio",level:2},{value:"Step 3: Deploy the Gateway",id:"step-3-deploy-the-gateway",level:2},{value:"Step 4: Send a Request",id:"step-4-send-a-request",level:2},{value:"Cleanup",id:"cleanup",level:2},{value:"Troubleshooting",id:"troubleshooting",level:2},{value:"Gateway not showing <code>PROGRAMMED=True</code>",id:"gateway-not-showing-programmedtrue",level:3},{value:"HTTPRoute not accepted",id:"httproute-not-accepted",level:3},{value:"No response from Gateway IP",id:"no-response-from-gateway-ip",level:3}];function c(e){const n={a:"a",admonition:"admonition",code:"code",h1:"h1",h2:"h2",h3:"h3",header:"header",li:"li",ol:"ol",p:"p",pre:"pre",...(0,i.R)(),...e.components};return(0,a.jsxs)(a.Fragment,{children:[(0,a.jsx)(n.header,{children:(0,a.jsx)(n.h1,{id:"istio",children:"Istio"})}),"\n",(0,a.jsxs)(n.p,{children:["This guide shows how to deploy llm-d with ",(0,a.jsx)(n.a,{href:"https://istio.io/",children:"Istio"})," as your inference gateway. By the end, inference requests will flow from an Istio-managed ",(0,a.jsx)(n.code,{children:"Gateway"})," to your model servers via the llm-d EPP."]}),"\n",(0,a.jsx)(n.admonition,{type:"note",children:(0,a.jsxs)(n.p,{children:["This guide assumes familiarity with ",(0,a.jsx)(n.a,{href:"https://gateway-api.sigs.k8s.io/",children:"Gateway API"})," and llm-d."]})}),"\n",(0,a.jsx)(n.h2,{id:"prerequisites",children:"Prerequisites"}),"\n",(0,a.jsxs)(n.ol,{children:["\n",(0,a.jsxs)(n.li,{children:["The environment variables ",(0,a.jsx)(n.code,{children:"${GUIDE_NAME}"}),", ",(0,a.jsx)(n.code,{children:"${MODEL_NAME}"})," and ",(0,a.jsx)(n.code,{children:"${NAMESPACE}"})," should be set as part of deploying one of the well-lit path guides."]}),"\n",(0,a.jsxs)(n.li,{children:["A Kubernetes cluster running one of the three most recent ",(0,a.jsx)(n.a,{href:"https://kubernetes.io/releases/",children:"Kubernetes releases"})]}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"https://helm.sh/docs/intro/install/",children:"Helm"})}),"\n",(0,a.jsx)(n.li,{children:(0,a.jsx)(n.a,{href:"https://jqlang.org/download/",children:"jq"})}),"\n"]}),"\n",(0,a.jsx)(n.h2,{id:"step-1-install-gateway-api-and-gateway-api-inference-extension-crds",children:"Step 1: Install Gateway API and Gateway API Inference Extension CRDs"}),"\n",(0,a.jsxs)(n.p,{children:["Install the required CRDs by following the ",(0,a.jsx)(n.a,{href:"/docs/dev/infrastructure/gateway/install-crds",children:"CRD installation guide"}),"."]}),"\n",(0,a.jsx)(n.h2,{id:"step-2-install-istio",children:"Step 2: Install Istio"}),"\n",(0,a.jsx)(n.p,{children:"Install Istio with inference extension support enabled:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'ISTIO_VERSION=1.29.2\ncurl -L https://istio.io/downloadIstio | ISTIO_VERSION=${ISTIO_VERSION}
1 sh -\nexport PATH="$PWD/istio-${ISTIO_VERSION}/bin:$PATH"\nistioctl install -y \\\n --set values.pilot.env.ENABLE_GATEWAY_API_INFERENCE_EXTENSION=true\n'})}),"\n",(0,a.jsx)(n.p,{children:"Verify the installation:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"kubectl get pods -n istio-system\n"})}),"\n",(0,a.jsx)(n.p,{children:"Expected output:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-text",children:"NAME READY STATUS RESTARTS AGE\nistiod-xxxxxxxxxx-xxxxx 1/1 Running 0 30s\n"})}),"\n",(0,a.jsx)(n.h2,{id:"step-3-deploy-the-gateway",children:"Step 3: Deploy the Gateway"}),"\n",(0,a.jsx)(n.p,{children:"Set the llm-d version to match your deployment:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"LLM_D_VERSION=main # Use 'main' for latest, or a release tag like 'v0.7.0'\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Create a ",(0,a.jsx)(n.code,{children:"Gateway"})," resource. Istio watches this resource and creates an Envoy-based proxy that accepts incoming traffic."]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'kubectl apply -k "https://github.com/llm-d/llm-d/guides/recipes/gateway/istio?ref=${LLM_D_VERSION}" -n ${NAMESPACE}\n'})}),"\n",(0,a.jsx)(n.p,{children:"Verify the Gateway is programmed:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"kubectl get gateway llm-d-inference-gateway -n ${NAMESPACE}\n"})}),"\n",(0,a.jsx)(n.p,{children:"Expected output:"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-text",children:"NAME CLASS ADDRESS PROGRAMMED AGE\nllm-d-inference-gateway istio 10.xx.xx.xx True 30s\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Wait until ",(0,a.jsx)(n.code,{children:"PROGRAMMED"})," shows ",(0,a.jsx)(n.code,{children:"True"})," before proceeding."]}),"\n",(0,a.jsx)(n.h2,{id:"step-4-send-a-request",children:"Step 4: Send a Request"}),"\n",(0,a.jsx)(n.admonition,{type:"info",children:(0,a.jsxs)(n.p,{children:["Before sending requests, you must deploy a well-lit path guide. This sets up a model server deployment, an ",(0,a.jsx)(n.code,{children:"InferencePool"}),", and an ",(0,a.jsx)(n.code,{children:"HTTPRoute"})," to connect the Gateway to the pool."]})}),"\n",(0,a.jsxs)(n.p,{children:["Get the ",(0,a.jsx)(n.code,{children:"Gateway"})," external address:"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"export IP=$(kubectl get gateway llm-d-inference-gateway -n ${NAMESPACE} -o jsonpath='{.status.addresses[0].value}')\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Send an inference request via the managed ",(0,a.jsx)(n.code,{children:"Gateway"}),":"]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:'curl -X POST http://${IP}/v1/completions \\\n -H \'Content-Type: application/json\' \\\n -d \'{\n "model": \'\\"${MODEL_NAME}\\"\',\n "prompt": "How are you today?"\n }\' | jq\n'})}),"\n",(0,a.jsx)(n.h2,{id:"cleanup",children:"Cleanup"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"kubectl delete gateway llm-d-inference-gateway -n ${NAMESPACE}\nistioctl uninstall --purge -y\nkubectl delete namespace istio-system\nkubectl delete gatewayclass istio istio-remote\n"})}),"\n",(0,a.jsxs)(n.p,{children:["To uninstall the Gateway API and Gateway API Inference Extension CRDs, see the ",(0,a.jsx)(n.a,{href:"/docs/dev/infrastructure/gateway/install-crds#uninstalling-gateway-api-crds",children:"CRD installation guide"}),"."]}),"\n",(0,a.jsx)(n.h2,{id:"troubleshooting",children:"Troubleshooting"}),"\n",(0,a.jsxs)(n.h3,{id:"gateway-not-showing-programmedtrue",children:["Gateway not showing ",(0,a.jsx)(n.code,{children:"PROGRAMMED=True"})]}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"kubectl describe gateway llm-d-inference-gateway -n ${NAMESPACE}\nkubectl get pods -n istio-system\nkubectl logs -n istio-system deployment/istiod --tail=20\n"})}),"\n",(0,a.jsx)(n.p,{children:"Verify Istio was installed with the inference extension flag enabled."}),"\n",(0,a.jsx)(n.h3,{id:"httproute-not-accepted",children:"HTTPRoute not accepted"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"kubectl describe httproute ${GUIDE_NAME} -n ${NAMESPACE}\n"})}),"\n",(0,a.jsxs)(n.p,{children:["Verify that ",(0,a.jsx)(n.code,{children:"parentRefs"})," matches the Gateway name and ",(0,a.jsx)(n.code,{children:"backendRefs"})," matches the InferencePool name."]}),"\n",(0,a.jsx)(n.h3,{id:"no-response-from-gateway-ip",children:"No response from Gateway IP"}),"\n",(0,a.jsx)(n.pre,{children:(0,a.jsx)(n.code,{className:"language-bash",children:"kubectl get gateway llm-d-inference-gateway -o jsonpath='{.status.addresses[0].value}' -n ${NAMESPACE}\n"})}),"\n",(0,a.jsx)(n.p,{children:"If the address is empty, your Gateway may still be waiting for a LoadB
1alancer service. Check that your cluster supports external load balancers."})]})}function h(e={}){const{wrapper:n}={...(0,i.R)(),...e.components};return n?(0,a.jsx)(n,{...e,children:(0,a.jsx)(c,{...e})}):c(e)}},28453(e,n,t){t.d(n,{R:()=>l,x:()=>r});var s=t(96540);const a={},i=s.createContext(a);function l(e){const n=s.useContext(i);return s.useMemo(function(){return"function"==typeof e?e(n):{...n,...e}},[n,e])}function r(e){let n;return n=e.disableParentContext?"function"==typeof e.components?e.components(a):e.components||a:l(e.components),s.createElement(i.Provider,{value:n},e.children)}}}]);
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.