1import{H as Ss}from"../chunks/dmAq0LiT.js";import"../chunks/DsnmJJEf.js";import"../chunks/CU1QhgsP.js";import{f as u,s,a as b,an as f,b as h,p as z,t as R,c as U,d as g,r as m,e as A,O as w,ai as I,ao as Ms,al as q,am as Ps,ah as Is}from"../chunks/BN1SZ1vA.js";import{h as e}from"../chunks/BIlqOjG3.js";import{i as N}from"../chunks/InhFHllE.js";import{e as $}from"../chunks/BmyB_2jS.js";import{c as Ls}from"../chunks/D0kmE35K.js";import{h as Ds}from"../chunks/pBEQZU1u.js";import{s as S}from"../chunks/qsbTZ_ho.js";import{N as Rs,F as Os}from"../chunks/kkwqtSwP.js";import{A as Ns}from"../chunks/COCSf-wL.js";function Gs(o,a){throw new Ss(o,a)}const W={title:"Add Text-to-Speech to Your Flutter App in 15 Minutes",description:"A step-by-step guide to adding high-quality, on-device TTS to a Flutter app using Xybrid and the Kokoro model. No cloud APIs, no API keys, no per-request costs.",date:"2026-04-06",author:"Glenn Sonna",tags:["flutter","tts","tutorial","on-device-ai"],category:"tutorial",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:8},{title:je,description:Ve,date:Qe,author:Ye,tags:Je,category:Ze,image:sa,imageAlt:ea,draft:aa,featured:na,readingTime:oa}=W;var zs=u('<p>Cloud TTS APIs are convenient â until you see the bill. Or deal with the latency. Or explain to your users why their voice assistant needs an internet connection.</p> <p>Letâs add <strong>on-device text-to-speech</strong> to a Flutter app. The model runs locally, works offline, and costs nothing per request.</p> <p>Weâll use <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a> with the Kokoro TTS model (82M parameters, multiple voices, high quality output).</p> <h2>What Youâll Build</h2> <p>A Flutter app that:</p> <ul><li>Converts any text to natural-sounding speech</li> <li>Runs entirely on-device (no network after initial model download)</li> <li>Supports multiple voices</li> <li>Works on iOS, Android, and macOS</li></ul> <h2>Prerequisites</h2> <ul><li>Flutter 3.x installed</li> <li>Rust toolchain (<code>rustup</code> installed)</li> <li>~10 minutes</li></ul> <h2>Step 1: Add the Dependency</h2> <p>Add <code>xybrid_flutter</code> to your <code>pubspec.yaml</code>:</p> <!> <!> <h2>Step 2: Initialize the SDK</h2> <p>In your appâs startup (e.g., <code>main.dart</code>):</p> <!> <blockquote><p><strong>Android only</strong>: Call <code>initSdkCacheDir()</code> with a path from <code>path_provider</code> before loading models. This tells Xybrid where to cache downloads.</p></blockquote> <!> <h2>Step 3: Load the Model</h2> <!> <p>The first call downloads the model (~80MB) and caches it locally. Subsequent calls load from cache instantly.</p> <h2>Step 4: Run TTS</h2> <!> <p>Thatâs it. <code>result</code> contains the audio as WAV bytes.</p> <h2>Step 5: Play the Audio</h2> <p>Use any audio player package. Hereâs a simple approach with <code>audioplayers</code>:</p> <!> <!> <p>Call it with the result:</p> <!> <h2>Putting It Together</h2> <p>Hereâs a complete, minimal TTS screen:</p> <!> <h2>Bonus: Choose a Voice</h2> <p>Kokoro supports multiple voices. List them and let the user pick:</p> <!> <p>Build a dropdown:</p> <!> <h2>Bonus: Warmup for Instant Response</h2> <p>The first inference is slower (model weights load into memory). Use <code>warmup()</code> to preload:</p> <!> <p>Call this during your loading screen or splash screen.</p> <h2>Performance</h2> <p>Measured on real devices:</p> <table><thead><tr><th>Device</th><th>First Inference</th><th>Subsequent</th><th>Notes</th></tr></thead><tbody><tr><td>iPhone 15 Pro</td><td>~800ms</td><td>~200ms</td><td>CoreML acceleration</td></tr><tr><td>Pixel 8</td><td>~1.2s</td><td>~400ms</td><td>CPU inference</td></tr><tr><td>MacBook Pro M2</td><td>~300ms</td><td>~100ms</td><td>Metal acceleration</td></tr></tbody></table> <p>These are for a short sentence (~10 words). Longer text scales linearly.</p> <h2>Why Not Just Use a Cloud API?</h2> <table><thead><tr><th></th><th>Cloud TTS</th><th>On-Device (Xybrid)</th></tr></thead><tbody><tr><td><strong>Latency</strong></td><td>200-500ms network + processing</td><td>100-400ms total</td></tr><tr><td><strong>Privacy</strong></td><td>Text sent to third party</td><td>Never leaves device</td></tr><tr><td><strong>Cost</strong></td><td>$4-16 per 1M characters</td><td>Free forever</td></tr><tr><td><strong>Offline</strong></td><td>No</td><td>Yes</td></tr><tr><td><strong>API key</strong></td><td>Required</td><td>Not needed</td></tr></tbody></table> <p>For apps where privacy matters (journaling, health, accessibility) or where you need offline support â on-device wins.</p> <h2>Next Steps</h2> <ul><li><strong>Add ASR</strong>: Use <code>whisper-tiny</code> for speech-to-text with <code>XybridStreamer</code> for real-time transcription</li> <li><strong>Chain models</strong>: Build a pipeline that listens, processes, and speaks back</li> <li><strong>Explore voices</strong>: Kokoro has dozens of voice options across languages</li></ul> <p>Check out the <a href="https://github.com/xybrid-ai/xybrid/tree/main/examples/flutter" rel="nofollow">Xybrid Flutter example app</a> for a full-featured demo with 8 screens.</p> <hr/> <p><strong>Xybrid is open-source</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p>',1);function Us(o){var a=zs(),n=s(b(a),20);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#85E89D">dependencies</span><span style="color:#E1E4E8">:</span></span> 2<span class="line"><span style="color:#85E89D"> xybrid_flutter</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">^0.1.0</span></span></code></pre>`);var l=s(n,2);e(l,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">flutter</span><span style="color:#9ECBFF"> pub</span><span style="color:#9ECBFF"> get</span></span></code></pre>');var t=s(l,6);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">
2import</span><span style="color:#9ECBFF"> 'package:xybrid_flutter/xybrid_flutter.dart'</span><span style="color:#E1E4E8">;</span></span> 3<span class="line"></span> 4<span class="line"><span style="color:#F97583">void</span><span style="color:#B392F0"> main</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 5<span class="line"><span style="color:#79B8FF"> WidgetsFlutterBinding</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">ensureInitialized</span><span style="color:#E1E4E8">();</span></span> 6<span class="line"><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">init</span><span style="color:#E1E4E8">();</span></span> 7<span class="line"><span style="color:#B392F0"> runApp</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">const</span><span style="color:#79B8FF"> MyApp</span><span style="color:#E1E4E8">());</span></span> 8<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var p=s(t,4);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Android setup (add to your init flow)</span></span> 9<span class="line"><span style="color:#F97583">import</span><span style="color:#9ECBFF"> 'package:path_provider/path_provider.dart'</span><span style="color:#E1E4E8">;</span></span> 10<span class="line"></span> 11<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> dir </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#B392F0"> getApplicationSupportDirectory</span><span style="color:#E1E4E8">();</span></span> 12<span class="line"><span style="color:#F97583">await</span><span style="color:#B392F0"> initSdkCacheDir</span><span style="color:#E1E4E8">(cacheDir</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> dir.path);</span></span></code></pre>`);var r=s(p,4);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'kokoro-82m'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span></code></pre>`);var c=s(r,6);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 13<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> "Hello! This is running entirely on your device."</span><span style="color:#E1E4E8">),</span></span> 14<span class="line"><span style="color:#E1E4E8">);</span></span></code></pre>`);var d=s(c,8);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># pubspec.yaml</span></span> 15<span class="line"><span style="color:#85E89D">dependencies</span><span style="color:#E1E4E8">:</span></span> 16<span class="line"><span style="color:#85E89D"> audioplayers</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">^6.0.0</span></span></code></pre>`);var i=s(d,2);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">
16import</span><span style="color:#9ECBFF"> 'package:audioplayers/audioplayers.dart'</span><span style="color:#E1E4E8">;</span></span> 17<span class="line"></span> 18<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> player </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> AudioPlayer</span><span style="color:#E1E4E8">();</span></span> 19<span class="line"></span> 20<span class="line"><span style="color:#79B8FF">Future</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">void</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">speak</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">Uint8List</span><span style="color:#E1E4E8"> audioBytes) </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 21<span class="line"><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> player.</span><span style="color:#B392F0">play</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">BytesSource</span><span style="color:#E1E4E8">(audioBytes));</span></span> 22<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var y=s(i,4);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> audioBytes </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> result.</span><span style="color:#B392F0">audioBytes</span><span style="color:#E1E4E8">();</span></span> 23<span class="line"><span style="color:#F97583">if</span><span style="color:#E1E4E8"> (audioBytes </span><span style="color:#F97583">!=</span><span style="color:#79B8FF"> null</span><span style="color:#E1E4E8">) {</span></span> 24<span class="line"><span style="color:#F97583"> await</span><span style="color:#B392F0"> speak</span><span style="color:#E1E4E8">(audioBytes);</span></span> 25<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var E=s(y,6);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">import</span><span style="color:#9ECBFF"> 'package:flutter/material.dart'</span><span style="color:#E1E4E8">;</span></span> 26<span class="line"><span style="color:#F97583">import</span><span style="color:#9ECBFF"> 'package:xybrid_flutter/xybrid_flutter.dart'</span><span style="color:#E1E4E8">;</span></span> 27<span class="line"><span style="color:#F97583">import</span><span style="color:#9ECBFF"> 'package:audioplayers/audioplayers.dart'</span><span style="color:#E1E4E8">;</span></span> 28<span class="line"></span> 29<span class="line"><span style="color:#F97583">class</span><span style="color:#79B8FF"> TTSScreen</span><span style="color:#F97583"> extends</span><span style="color:#79B8FF"> StatefulWidget</span><span style="color:#E1E4E8"> {</span></span> 30<span class="line"><span style="color:#F97583"> const</span><span style="color:#79B8FF"> TTSScreen</span><span style="color:#E1E4E8">({</span><span style="color:#79B8FF">super</span><span style="color:#E1E4E8">.key});</span></span> 31<span class="line"></span> 32<span class="line"><span style="color:#F97583">
32 @override</span></span> 33<span class="line"><span style="color:#79B8FF"> State</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">TTSScreen</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">createState</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">=></span><span style="color:#79B8FF"> _TTSScreenState</span><span style="color:#E1E4E8">();</span></span> 34<span class="line"><span style="color:#E1E4E8">}</span></span> 35<span class="line"></span> 36<span class="line"><span style="color:#F97583">class</span><span style="color:#79B8FF"> _TTSScreenState</span><span style="color:#F97583"> extends</span><span style="color:#79B8FF"> State</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">TTSScreen</span><span style="color:#E1E4E8">> {</span></span> 37<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> _controller </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> TextEditingController</span><span style="color:#E1E4E8">();</span></span> 38<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> _player </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> AudioPlayer</span><span style="color:#E1E4E8">();</span></span> 39<span class="line"></span> 40<span class="line"><span style="color:#79B8FF"> XybridModel</span><span style="color:#F97583">?</span><span style="color:#E1E4E8"> _model;</span></span> 41<span class="line"><span style="color:#79B8FF"> bool</span><span style="color:#E1E4E8"> _loading </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> true</span><span style="color:#E1E4E8">;</span></span> 42<span class="line"><span style="color:#79B8FF"> bool</span><span style="color:#E1E4E8"> _speaking </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">;</span></span> 43<span class="line"></span> 44<span class="line"><span style="color:#F97583">
44 @override</span></span> 45<span class="line"><span style="color:#F97583"> void</span><span style="color:#B392F0"> initState</span><span style="color:#E1E4E8">() {</span></span> 46<span class="line"><span style="color:#79B8FF"> super</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">initState</span><span style="color:#E1E4E8">();</span></span> 47<span class="line"><span style="color:#B392F0"> _loadModel</span><span style="color:#E1E4E8">();</span></span> 48<span class="line"><span style="color:#E1E4E8"> }</span></span> 49<span class="line"></span> 50<span class="line"><span style="color:#79B8FF"> Future</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">void</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">_loadModel</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 51<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'kokoro-82m'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 52<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() {</span></span> 53<span class="line"><span style="color:#E1E4E8"> _model </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model;</span></span> 54<span class="line"><span style="color:#E1E4E8"> _loading </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">;</span></span> 55<span class="line"><span style="color:#E1E4E8"> });</span></span> 56<span class="line"><span style="color:#E1E4E8"> }</span></span> 57<span class="line"></span> 58<span class="line"><span style="color:#79B8FF"> Future</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">void</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">_speak</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 59<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> text </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> _controller.text.</span><span style="color:#B392F0">trim</span><span style="color:#E1E4E8">();</span></span> 60<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> (text.isEmpty </span><span style="color:#F97583">||</span><span style="color:#E1E4E8"> _model </span><span style="color:#F97583">==</span><span style="color:#79B8FF"> null</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">return</span><span style="color:#E1E4E8">;</span></span> 61<span class="line"></span> 62<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _speaking </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> true</span><span style="color:#E1E4E8">);</span></span> 63<span class="line"></span> 64<span class="line"><span style="color:#F97583"> try</span><span style="color:#E1E4E8"> {</span></span> 65<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> _model</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 66<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> text),</span></span> 67<span class="line"><span style="color:#E1E4E8"> );</span></span> 68<span class="line"></span> 69<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> audioBytes </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> result.</span><span style="color:#B392F0">audioBytes</span><span style="color:#E1E4E8">();</span></span> 70<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> (audioBytes </span><span style="color:#F97583">!=</span><span style="color:#79B8FF"> null</span><span style="color:#E1E4E8">) {</span></span> 71<span class="line"><span style="color:#F97583"> await</span><span style="color:#E1E4E8">
71 _player.</span><span style="color:#B392F0">play</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">BytesSource</span><span style="color:#E1E4E8">(audioBytes));</span></span> 72<span class="line"><span style="color:#E1E4E8"> }</span></span> 73<span class="line"><span style="color:#E1E4E8"> } </span><span style="color:#F97583">finally</span><span style="color:#E1E4E8"> {</span></span> 74<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _speaking </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">);</span></span> 75<span class="line"><span style="color:#E1E4E8"> }</span></span> 76<span class="line"><span style="color:#E1E4E8"> }</span></span> 77<span class="line"></span> 78<span class="line"><span style="color:#F97583"> @override</span></span> 79<span class="line"><span style="color:#79B8FF"> Widget</span><span style="color:#B392F0"> build</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">BuildContext</span><span style="color:#E1E4E8"> context) {</span></span> 80<span class="line"><span style="color:#F97583"> return</span><span style="color:#79B8FF"> Scaffold</span><span style="color:#E1E4E8">(</span></span> 81<span class="line"><span style="color:#E1E4E8"> appBar</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> AppBar</span><span style="color:#E1E4E8">(title</span><span style="color:#F97583">:</span><span style="color:#F97583"> const</span><span style="color:#79B8FF"> Text</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">'On-Device TTS'</span><span style="color:#E1E4E8">)),</span></span> 82<span class="line"><span style="color:#E1E4E8"> body</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Padding</span><span style="color:#E1E4E8">(</span></span> 83<span class="line"><span style="color:#E1E4E8"> padding</span><span style="color:#F97583">:</span><span style="color:#F97583"> const</span><span style="color:#79B8FF"> EdgeInsets</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">all</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">16</span><span style="color:#E1E4E8">),</span></span> 84<span class="line"><span style="color:#E1E4E8"> child</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Column</span><span style="color:#E1E4E8">(</span></span> 85<span class="line"><span style="color:#E1E4E8"> children</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> [</span></span> 86<span class="line"><span style="color:#79B8FF"> TextField</span><span style="color:#E1E4E8">(</span></span> 87<span class="line"><span style="color:#E1E4E8"> controller</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> _controller,</span></span> 88<span class="line"><span style="color:#E1E4E8"> maxLines</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 4</span><span style="color:#E1E4E8">,</span></span> 89<span class="line"><span style="color:#E1E4E8"> decoration</span><span style="color:#F97583">:</span><span style="color:#F97583"> const</span><span style="color:#79B8FF"> InputDecoration</span><span style="color:#E1E4E8">(</span></span> 90<span class="line"><span style="color:#E1E4E8"> hintText</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'Enter text to speak...'</span><span style="color:#E1E4E8">,</span></span> 91<span class="line"><span style="color:#E1E4E8"> border</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> OutlineInputBorder</span><span style="color:#E1E4E8">(),</span></span> 92<span class="line"><span style="color:#E1E4E8"> ),</span></span> 93<span class="line"><span style="color:#E1E4E8"> ),</span></span> 94<span class="line"><span style="color:#F97583"> const</span><span style="color:#79B8FF"> SizedBox</span><span style="color:#E1E4E8">(height</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 16</span><span style="color:#E1E4E8">),</span></span> 95<span class="line"><span style="color:#79B8FF"> SizedBox</span><span style="color:#E1E4E8">(</span></span> 96<span class="line"><span style="color:#E1E4E8"> width</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> double</span><span style="color:#E1E4E8">.infinity,</span></span> 97<span class="line"><span style="color:#E1E4E8"> child</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> ElevatedButton</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">icon</span><span style="color:#E1E4E8">(</span></span> 98<span class="line"><span style="color:#E1E4E8"> onPressed</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> _loading </span><span style="color:#F97583">||</span><span style="color:#E1E4E8"> _speaking </span><span style="color:#F97583">?</span><span style="color:#79B8FF"> null</span><span style="color:#F97583"> :</span><span style="color:#E1E4E8"> _speak,</span></span> 99<span class="line"><span style="color:#E1E4E8"> icon</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> _speaking</span></span> 100<span class="line"><span style="color:#F97583"> ?</span><span style="color:#F97583"> const</span><span style="color:#79B8FF"> SizedBox</span><span style="color:#E1E4E8">(</span></span> 101<span class="line"><span style="color:#E1E4E8"> width</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 16</span><span style="color:#E1E4E8">,</span></span> 102<span class="line"><span style="color:#E1E4E8"> height</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 16</span><span style="color:#E1E4E8">,</span></span> 103<span class="line"><span style="color:#E1E4E8"> child</span><span style="color:#F97583">:</span><span style="color:#79B8FF">
103 CircularProgressIndicator</span><span style="color:#E1E4E8">(strokeWidth</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 2</span><span style="color:#E1E4E8">),</span></span> 104<span class="line"><span style="color:#E1E4E8"> )</span></span> 105<span class="line"><span style="color:#F97583"> :</span><span style="color:#F97583"> const</span><span style="color:#79B8FF"> Icon</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">Icons</span><span style="color:#E1E4E8">.volume_up),</span></span> 106<span class="line"><span style="color:#E1E4E8"> label</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Text</span><span style="color:#E1E4E8">(</span></span> 107<span class="line"><span style="color:#E1E4E8"> _loading</span></span> 108<span class="line"><span style="color:#F97583"> ?</span><span style="color:#9ECBFF"> 'Loading model...'</span></span> 109<span class="line"><span style="color:#F97583"> :</span><span style="color:#E1E4E8"> _speaking</span></span> 110<span class="line"><span style="color:#F97583"> ?</span><span style="color:#9ECBFF"> 'Speaking...'</span></span> 111<span class="line"><span style="color:#F97583"> :</span><span style="color:#9ECBFF"> 'Speak'</span><span style="color:#E1E4E8">,</span></span> 112<span class="line"><span style="color:#E1E4E8"> ),</span></span> 113<span class="line"><span style="color:#E1E4E8"> ),</span></span> 114<span class="line"><span style="color:#E1E4E8"> ),</span></span> 115<span class="line"><span style="color:#E1E4E8"> ],</span></span> 116<span class="line"><span style="color:#E1E4E8"> ),</span></span> 117<span class="line"><span style="color:#E1E4E8"> ),</span></span> 118<span class="line"><span style="color:#E1E4E8"> );</span></span> 119<span class="line"><span style="color:#E1E4E8"> }</span></span> 120<span class="line"></span> 121<span class="line"><span style="color:#F97583">
121 @override</span></span> 122<span class="line"><span style="color:#F97583"> void</span><span style="color:#B392F0"> dispose</span><span style="color:#E1E4E8">() {</span></span> 123<span class="line"><span style="color:#E1E4E8"> _controller.</span><span style="color:#B392F0">dispose</span><span style="color:#E1E4E8">();</span></span> 124<span class="line"><span style="color:#E1E4E8"> _player.</span><span style="color:#B392F0">dispose</span><span style="color:#E1E4E8">();</span></span> 125<span class="line"><span style="color:#79B8FF"> super</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">dispose</span><span style="color:#E1E4E8">();</span></span> 126<span class="line"><span style="color:#E1E4E8"> }</span></span> 127<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var F=s(E,6);e(F,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Get available voices</span></span> 128<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> voices </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> _model</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">voices</span><span style="color:#E1E4E8">();</span></span> 129<span class="line"></span> 130<span class="line"><span style="color:#6A737D">// Run with a specific voice</span></span> 131<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> _model</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 132<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> text),</span></span> 133<span class="line"><span style="color:#E1E4E8"> voiceId</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> voices[selectedIndex].id,</span></span> 134<span class="line"><span style="color:#E1E4E8">);</span></span></code></pre>`);var v=s(F,4);e(v,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#79B8FF">DropdownButton</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">int</span><span style="color:#E1E4E8">>(</span></span> 135<span class="line"><span style="color:#E1E4E8"> value</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> _selectedVoice,</span></span> 136<span class="line"><span style="color:#E1E4E8"> items</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> voices.</span><span style="color:#B392F0">asMap</span><span style="color:#E1E4E8">().entries.</span><span style="color:#B392F0">map</span><span style="color:#E1E4E8">((entry) {</span></span> 137<span class="line"><span style="color:#F97583"> return</span><span style="color:#79B8FF"> DropdownMenuItem</span><span style="color:#E1E4E8">(</span></span> 138<span class="line"><span style="color:#E1E4E8"> value</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> entry.key,</span></span> 139<span class="line"><span style="color:#E1E4E8"> child</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Text</span><span style="color:#E1E4E8">(entry.value.name),</span></span> 140<span class="line"><span style="color:#E1E4E8"> );</span></span> 141<span class="line"><span style="color:#E1E4E8"> }).</span><span style="color:#B392F0">toList</span><span style="color:#E1E4E8">(),</span></span> 142<span class="line"><span style="color:#E1E4E8"> onChanged</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> (index) </span><span style="color:#F97583">=></span><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _selectedVoice </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> index</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">),</span></span> 143<span class="line"><span style="color:#E1E4E8">),</span></span></code></pre>`);var _=s(v,6);e(_,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'kokoro-82m'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 144<span class="line"><span style="color:#F97583">await</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">warmup</span><span style="color:#E1E4E8">(); </span><span style="color:#6A737D">// Pre-loads weights, compiles shaders</span></span> 145<span class="line"></span> 146<span class="line"><span style="color:#6A737D">// Now the first speak() call is as fast as subsequent ones</span></span></code></pre>`),f(26),h(o,a)}const K=Object.freeze(Object.defineProperty({__proto__:null,default:Us,metadata:W},Symbol.toStringTag,{value:"Module"})),H={title:"Building a Cross-Platform ML Inference SDK in Rust",description:"How we built a single Rust core that powers ML inference across CLI, Flutter, Swift, Kotlin, and Unity â and the architectural decisions that made it possible.",date:"2026-06-12",author:"Glenn Sonna",tags:["rust","ai","architecture","opensource"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:7},{title:la,description:ta,date:pa,author:ra,tags:ca,category:ia,image:da,imageAlt:ya,draft:Ea,featured:ha,readingTime:ua}=H;var qs=u('<p>We set ourselves an ambitious goal: build <strong>one inference engine</strong> that runs on every major platform â desktops, phones, game engines, and the terminal. Not five separate implementations. One.</p> <p>18 months later, <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a> ships ML inference to iOS, Android, macOS, Linux, Windows, Flutter, Swift, Kotlin, Unity, and the CLI â all from the same Rust core.</p> <p>Hereâs how we designed it.</p> <hr/> <h2>The Architecture</h2> <!> <p>Every platform SDK is a thin binding over the same Rust code. The business logic â model loading, preprocessing, execution, postprocessing â lives in <code>xybrid-core</code> and <code>xybrid-sdk</code>. Itâs written once, tested once, and deployed everywhere.</p> <h2>Decision 1: The Envelope Pattern</h2> <p>The first problem: how do you pass data between pipeline stages when the data type varies? ASR takes audio bytes, TTS takes text, embeddings produce float vectors.</p> <p>We created the <code>Envelope</code> â a tagged union that carries any payload through the system:</p> <!> <p>Every pipeline stage accepts an <code>Envelope</code> and returns an <code>Envelope</code>. This makes stages composable without knowing about each other:</p> <!> <p>The <code>metadata</code> map carries side-channel info (sample rate, voice ID, message role) without polluting the type. Itâs intentionally stringly-typed â pipeline stages are heterogeneous and we donât want a combinatorial explosion of types.</p> <h2>Decision 2: model_metadata.json as the Contract</h2> <p>The hardest part of ML inference isnât running the model. Itâs everything around it: preprocessing inputs, configuring the session, and postprocessing outputs.</p> <p>We solved this with a declarative metadata file that ships with every model:</p> <!> <p>The <code>TemplateExecutor</code> reads this file and handles the full execution flow. No platform-specific inference code. The same metadata drives inference on iOS, Android, and your laptop.</p> <p>This is the key architectural insight: <strong>the model knows how to run itself</strong>
146. The runtime just follows instructions.</p> <!> <p>Three lines. Works for TTS, ASR, classification, embeddings â any model type.</p> <h2>Decision 3: Three FFI Strategies</h2> <p>Rust is great for writing the core. But getting it into Dart, Swift, Kotlin, and C# requires FFI. We use three different approaches:</p> <table><thead><tr><th>Platform</th><th>FFI Tool</th><th>Why</th></tr></thead><tbody><tr><td><strong>Flutter</strong></td><td>flutter_rust_bridge (FRB)</td><td>Auto-generates Dart bindings, handles async, supports streaming callbacks</td></tr><tr><td><strong>Swift & Kotlin</strong></td><td>UniFFI</td><td>Mozillaâs tool, generates idiomatic bindings from a single UDL definition</td></tr><tr><td><strong>Unity (C#)</strong></td><td>C FFI + cbindgen</td><td>Unity needs raw C headers; cbindgen generates them from Rust</td></tr></tbody></table> <p>Why three? Because each ecosystem has different expectations:</p> <ul><li><strong>Flutter</strong> needs async Futures and Streams. FRB handles this natively.</li> <li><strong>Swift</strong> expects <code>async/await</code> and value types. UniFFI maps Rust types to Swift idioms.</li> <li><strong>Kotlin</strong> expects suspending functions and data classes. UniFFI handles this too.</li> <li><strong>Unity</strong> needs <code>DllImport</code> with C calling conventions. Only C FFI works here.</li></ul> <p>The binding layer is thin by design. Hereâs the entire Kotlin API for running a model:</p> <!> <p>All the complexity is in Rust. The bindings are projections, not reimplementations.</p> <h2>Decision 4: Feature Flags for Platform Presets</h2> <p>Not every platform supports every backend. CoreML only works on Apple. Metal only on Apple GPUs. Dynamic ORT loading is needed on Android where we canât statically link.</p> <p>We use Cargo feature flags composed into platform presets:</p> <!> <p>Invalid combinations are caught at compile time:</p> <!> <p>The build system (<code>cargo xtask</code>) auto-detects the target triple and applies the correct preset. Developers donât think about feature flags â they just build for their target.</p> <h2>Decision 5: Always-Available Types</h2> <p>A subtle but important pattern: types that <em>describe</em> capabilities are available even when the capability is disabled.</p> <!> <p>Why? Because downstream code (Flutter bindings, SDK) needs to reference these types in function signatures even on platforms where LLM inference isnât available. Without this, youâd need <code>#[cfg]</code> conditionals everywhere in the binding layer.</p> <p>The types always exist. The <em>implementations</em> are gated behind features. Clean separation.</p> <h2>The Hard Parts</h2> <h3>ONNX Runtime on iOS</h3> <p>ORT doesnât ship a nice iOS package. We vendor a pre-built <code>xcframework</code> in <code>vendor/ort-ios/</code> and share it across every build path (xtask, Flutter, SPM) via a single <code>resolve_ort_lib_location()</code> function.</p> <p>A symlink at <code>bindings/flutter/ios/Frameworks/onnxruntime.xcframework</code> keeps CocoaPods happy. Itâs ugly, but it works on every CI machine without manual setup.</p> <h3>Candle on Android</h3> <p>Candle uses the <code>gemm-f16</code> crate which requires ARM <code>+fp16</code> instructions. We had to add target-specific rustflags:</p> <!> <p>Runtime dispatch still works on older devices without FP16. The flag only affects compilation.</p> <h3>Mutable Whisper Decoding</h3> <p>ONNX models use <code>session.run()</code> with <code>&self</code>. But Whisperâs autoregressive decoding via Candle needs <code>&mut self</code> â each token generation modifies internal state.</p> <p>Our solution: <code>TemplateExecutor</code> detects Candle models and handles mutability internally, while the public API stays immutable. Callers donât know or care.</p> <h2>What Weâd Do Differently</h2> <ol><li><p><strong>Start with UniFFI for all native bindings.</strong> We built the C FFI layer first, then realized UniFFI gives us Swift + Kotlin for free. The C layer is only still needed for Unity.</p></li> <li><p><strong>Define the API contract earlier.</strong> We now have an <code>api-surface.yaml</code> that defines the public API across all SDKs. Adding it from day one would have prevented drift between platforms.</p></li> <li><p><strong>Vendor fewer things.</strong>
146 ORT iOS vendoring was necessary, but every vendored dependency is a maintenance burden. Weâd push harder for upstream packages.</p></li></ol> <h2>Results</h2> <p>From a single Rust codebase:</p> <ul><li><strong>6 platforms</strong> shipping (CLI, Flutter, Swift, Kotlin, Unity, native Rust)</li> <li><strong>3 inference backends</strong> (ONNX Runtime, Candle, llama.cpp)</li> <li><strong>58 unit tests + 7 doctests</strong> in core, all passing</li> <li><strong>One model_metadata.json</strong> per model, works everywhere</li></ul> <p>The Rust core is ~15K lines. Each binding layer is under 1K lines. Thatâs the power of putting the logic in one place.</p> <hr/> <p><strong>Xybrid is open-source</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <p>If youâre building cross-platform AI features and tired of maintaining separate inference code per platform â check it out. Weâd love your feedback.</p> <hr/> <p><em>Have questions about the architecture? Drop them in the comments â happy to go deeper on any of these decisions.</em></p>',1);function $s(o){var a=qs(),n=s(b(a),10);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>âââââââââââââââââââââââââââââââââââââââââââââââ</span></span> 147<span class="line"><span>â Platform SDKs â</span></span> 148<span class="line"><span>â Flutter â Swift â Kotlin â Unity â CLI â</span></span> 149<span class="line"><span>âââââââââââââââââââââââââââââââââââââââââââââââ¤</span></span> 150<span class="line"><span>â xybrid-sdk (Rust) â</span></span> 151<span class="line"><span>â Registry â Cache â Pipeline â Telemetry â</span></span> 152<span class="line"><span>âââââââââââââââââââââââââââââââââââââââââââââââ¤</span></span> 153<span class="line"><span>â xybrid-core (Rust) â</span></span> 154<span class="line"><span>â TemplateExecutor â Envelope â Preprocessing â</span></span> 155<span class="line"><span>âââââââââââââââââââââââââââââââââââââââââââââââ¤</span></span> 156<span class="line"><span>â Runtime Backends â</span></span> 157<span class="line"><span>â ONNX Runtime â Candle â llama.cpp â</span></span> 158<span class="line"><span>âââââââââââââââââââââââââââââââââââââââââââââââ</span></span></code></pre>`);var l=s(n,10);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> Envelope</span><span style="color:#E1E4E8"> {</span></span> 159<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> kind</span><span style="color:#F97583">:</span><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#E1E4E8">,</span></span> 160<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> HashMap</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">>,</span></span> 161<span class="line"><span style="color:#E1E4E8">}</span></span> 162<span class="line"></span> 163<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> enum</span><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#E1E4E8"> {</span></span> 164<span class="line"><span style="color:#B392F0"> Audio</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">u8</span><span style="color:#E1E4E8">>),</span></span> 165<span class="line"><span style="color:#B392F0"> Text</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">),</span></span> 166<span class="line"><span style="color:#B392F0"> Embedding</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">f32</span><span style="color:#E1E4E8">>),</span></span> 167<span class="line"><span style="color:#B392F0"> Tokens</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">i64</span><span style="color:#E1E4E8">>),</span></span> 168<span class="line"><span style="color:#B392F0"> Tensor</span><span style="color:#E1E4E8"> { data</span><span style="color:#F97583">:</span><span style="color:#B392F0"> Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">f32</span><span style="color:#E1E4E8">>, shape</span><span style="color:#F97583">:</span><span style="color:#B392F0"> Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">usize</span><span style="color:#E1E4E8">> },</span></span> 169<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var t=s(l,4);e(t,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>Audio(wav_bytes) â [AudioDecode] â Tensor(samples) â [Model] â Tensor(logits) â [CTCDecode] â Text(transcript)</span></span></code></pre>');var p=s(t,10);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">
169{</span></span> 170<span class="line"><span style="color:#79B8FF"> "model_id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span><span style="color:#E1E4E8">,</span></span> 171<span class="line"><span style="color:#79B8FF"> "execution_template"</span><span style="color:#E1E4E8">: {</span></span> 172<span class="line"><span style="color:#79B8FF"> "type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"SimpleMode"</span><span style="color:#E1E4E8">,</span></span> 173<span class="line"><span style="color:#79B8FF"> "model_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model.onnx"</span></span> 174<span class="line"><span style="color:#E1E4E8"> },</span></span> 175<span class="line"><span style="color:#79B8FF"> "preprocessing"</span><span style="color:#E1E4E8">: [</span></span> 176<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Phonemize"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"backend"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"MisakiDictionary"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"tokens_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"tokens.txt"</span><span style="color:#E1E4E8"> }</span></span> 177<span class="line"><span style="color:#E1E4E8"> ],</span></span> 178<span class="line"><span style="color:#79B8FF"> "postprocessing"</span><span style="color:#E1E4E8">: [</span></span> 179<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"TTSAudioEncode"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"sample_rate"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">24000</span><span style="color:#E1E4E8"> }</span></span> 180<span class="line"><span style="color:#E1E4E8"> ]</span></span> 181<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var r=s(p,6);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ModelMetadata</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> serde_json</span><span style="color:#F97583">::</span><span style="color:#B392F0">from_str</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#B392F0">std</span><span style="color:#F97583">::</span><span style="color:#B392F0">fs</span><span style="color:#F97583">::</span><span style="color:#B392F0">read_to_string</span><span style="color:#E1E4E8">(path)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 182<span class="line"><span style="color:#F97583">let</span><span style="color:#F97583"> mut</span><span style="color:#E1E4E8"> executor </span><span style="color:#F97583">=</span><span style="color:#B392F0"> TemplateExecutor</span><span style="color:#F97583">::</span><span style="color:#B392F0">with_base_path</span><span style="color:#E1E4E8">(model_dir);</span></span> 183<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> executor</span><span style="color:#F97583">.</span><span style="color:#B392F0">execute</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">metadata, </span><span style="color:#F97583">&</span><span style="color:#E1E4E8">input)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span></code></pre>`);var c=s(r,16);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> XybridModelLoader.</span><span style="color:#B392F0">fromRegistry</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"kokoro-82m"</span><span style="color:#E1E4E8">)</span></span> 184<span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">
184(Envelope.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Hello"</span><span style="color:#E1E4E8">))</span></span> 185<span class="line"><span style="color:#6A737D">// result.audioBytes() â play it</span></span></code></pre>`);var d=s(c,10);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">[</span><span style="color:#B392F0">features</span><span style="color:#E1E4E8">]</span></span> 186<span class="line"><span style="color:#E1E4E8">platform-macos = [</span><span style="color:#9ECBFF">"ort-download"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"ort-coreml"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"candle-metal"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"llm-llamacpp"</span><span style="color:#E1E4E8">]</span></span> 187<span class="line"><span style="color:#E1E4E8">platform-ios = [</span><span style="color:#9ECBFF">"ort-download"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"ort-coreml"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"candle-metal"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"llm-llamacpp"</span><span style="color:#E1E4E8">]</span></span> 188<span class="line"><span style="color:#E1E4E8">platform-android = [</span><span style="color:#9ECBFF">"ort-dynamic"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"candle"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"llm-llamacpp"</span><span style="color:#E1E4E8">]</span></span> 189<span class="line"><span style="color:#E1E4E8">platform-desktop = [</span><span style="color:#9ECBFF">"ort-download"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"llm-llamacpp"</span><span style="color:#E1E4E8">]</span></span></code></pre>`);var i=s(d,4);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">#[cfg(all(feature </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "ort-download"</span><span style="color:#E1E4E8">, feature </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "ort-dynamic"</span><span style="color:#E1E4E8">))]</span></span> 190<span class="line"><span style="color:#B392F0">compile_error!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"ort-download and ort-dynamic are mutually exclusive"</span><span style="color:#E1E4E8">);</span></span></code></pre>`);var y=s(i,8);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// These compile without any feature flags:</span></span> 191<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> GenerationConfig</span><span style="color:#E1E4E8"> {</span></span> 192<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> max_tokens</span><span style="color:#F97583">:</span><span style="color:#B392F0"> usize</span><span style="color:#E1E4E8">,</span></span> 193<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> temperature</span><span style="color:#F97583">:</span><span style="color:#B392F0"> f32</span><span style="color:#E1E4E8">,</span></span> 194<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> top_p</span><span style="color:#F97583">:</span><span style="color:#B392F0"> f32</span><span style="color:#E1E4E8">,</span></span> 195<span class="line"><span style="color:#E1E4E8">}</span></span> 196<span class="line"></span> 197<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> ChatMessage</span><span style="color:#E1E4E8"> {</span></span> 198<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> role</span><span style="color:#F97583">:</span><span style="color:#B392F0"> MessageRole</span><span style="color:#E1E4E8">,</span></span> 199<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> content</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">,</span></span> 200<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var E=s(y,18);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># .cargo/config.toml</span></span> 201<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#B392F0">target</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">aarch64-linux-android</span><span style="color:#E1E4E8">]</span></span> 202<span class="line"><span style="color:#E1E4E8">rustflags = [</span><span style="color:#9ECBFF">"-C"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"target-feature=+fp16"</span><span style="color:#E1E4E8">]</span></span></code></pre>`),f(30),h(o,a)}const X=Object.freeze(Object.defineProperty({__proto__:null,default:$s,metadata:H},Symbol.toStringTag,{value:"Module"})),j={title:"Building a Voice Agent That Runs Entirely On-Device",description:"A step-by-step tutorial for building an on-device voice agent using Whisper, a local LLM, and Kokoro TTS â no cloud APIs, no internet required.",date:"2026-03-17",author:"Glenn Sonna",tags:["tutorial","voice-agent","tts","asr","on-device-ai","flutter"],category:"tutorial",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:11},{title:ma,description:Fa,date:ga,author:fa,tags:ba,category:Ba,image:va,imageAlt:ka,draft:wa,featured:xa,readingTime:Ca}=j;var Ws=u('<p>Most voice assistants send your audio to a server, process it remotely, and stream an answer back. That roun
202d trip adds latency, costs money per request, and requires a network connection. It also means your voice data leaves your device.</p> <p>We are going to build the opposite: a voice agent where everything runs locally. The user taps a button, speaks a question, and the app transcribes it, generates a response, and reads it back out loud â all without touching the internet.</p> <p>The full loop looks like this:</p> <!> <p>By the end of this tutorial you will have a working Flutter app that performs this entire pipeline on an iPhone or Android device.</p> <h2>Architecture Overview</h2> <p>The voice agent is a three-stage pipeline. Each stage is a separate ML model, and Xybrid handles the data flow between them automatically.</p> <p><strong>Stage 1 â Speech Recognition (ASR).</strong> Whisper Tiny takes raw audio and produces a text transcript. It runs well on mobile hardware and supports multiple languages, though we will stick with English here.</p> <p><strong>Stage 2 â Reasoning (LLM).</strong> Llama 3.2 1B takes the transcript and generates a conversational response. The 1B parameter variant is small enough to run on-device while still producing coherent, useful answers.</p> <p><strong>Stage 3 â Voice Synthesis (TTS).</strong> Kokoro 82M takes the generated text and produces natural-sounding speech audio. At only 82 million parameters, it is fast enough for real-time synthesis on modern phones.</p> <h2>Prerequisites</h2> <p>Before you start, make sure you have:</p> <ul><li><strong>Flutter SDK</strong> 3.22 or later installed and configured</li> <li><strong>A physical device</strong> for testing (iPhone 12+ or Pixel 6+). The simulator works for development but performance benchmarks require real hardware.</li> <li><strong>About 2 GB of free storage</strong> on the test device for the three models</li> <li>Basic familiarity with Flutter and async Dart</li></ul> <h2>Step 1: Set Up the Flutter Project</h2> <p>Create a new Flutter project or open an existing one. Add the Xybrid Flutter package to your <code>pubspec.yaml</code>:</p> <!> <p>Run <code>flutter pub get</code>, then initialize Xybrid in your appâs entry point:</p> <!> <h2>Step 2: Load the Models</h2> <p>Load all three models and verify they are ready:</p> <!> <p>The first call downloads each model to the device cache. Subsequent calls load from cache instantly.</p> <h2>Step 3: Implement Speech Recognition</h2> <p>Record audio from the device microphone. The recorded audio should be 16kHz mono WAV â the format Whisper expects.</p> <!> <h2>Step 4: Add the Reasoning Layer</h2> <p>Pass the transcript into the local LLM:</p> <!> <p>The LLM generates a conversational response. Keep <code>max_tokens</code> moderate (100â256) â voice responses that run longer than 20â30 seconds feel unnatural.</p> <h2>Step 5: Add Voice Synthesis</h2> <p>Convert the LLMâs text response to speech:</p> <!> <p>Kokoro produces high-quality, natural-sounding audio at 24kHz sample rate.</p> <h2>Step 6: Wire It All Together</h2> <p>Here is the complete interaction flow as a single widget:</p> <!> <p>The interaction flow:</p> <ol><li>User taps the microphone button to start recording</li> <li>Taps again to stop â recorded bytes go to <code>_processAudio</code></li> <li>Whisper transcribes the audio</li> <li>Llama generates a response</li> <li>Kokoro synthesizes speech and plays it back</li></ol> <h2>Performance Results</h2> <p>Benchmarked on real devices with models loaded in memory:</p> <h3>iPhone 15 Pro (A17 Pro, 8 GB RAM)</h3> <table><thead><tr><th>Stage</th><th>Model</th><th>Latency</th></tr></thead><tbody><tr><td>ASR</td><td>Whisper Tiny</td><td>~200ms</td></tr><tr><td>LLM</td><td>Llama 3.2 1B (Q4)</td><td>~500ms to first token</td></tr><tr><td>TTS</td><td>Kokoro 82M</td><td>~150ms</td></tr><tr><td><strong>Total</strong></td><td></td><td><strong>~850ms to first audio</strong></td></tr></tbody></table> <h3>Pixel 8 Pro (Tensor G3, 12 GB RAM)</h3> <table><thead><tr><th>Stage</th><th>Model</th><th>Latency</th></tr></thead><tbody><tr><td>ASR</td><td>Whisper Tiny</td><td>~280ms</td></tr><tr><td>LLM</td><td>Llama 3.2 1B (Q4)</td><td>~650ms to first token</td></tr><tr><td>TTS</td><td>Kokoro 82M</td><td>~180ms</td></tr><tr><td><strong>Total</strong></td><td></td><td><strong>~1.1s to first audio</strong></td></tr></tbody></table> <p>Under one second to first audio on iPhone makes the interaction feel responsive. The LLM is the bottleneck â streaming LLM output to TTS can reduce perceived latency significantly.</p> <p>Memory usage peaks around 1.8 GB with all three models loaded. On devices with 6 GB or more RAM, this leaves plenty of room for the rest of the app.</p> <h2>Next Steps</h2> <h3>Conversation Memory</h3> <p>Right now, each interaction is stateless. Xybrid supports conversation context that carries history across turns:</p> <!> <p>The <code>ConversationContext</code> manages a rolling window of past messages (default: 50 turns).</p> <h3>Wake Word Detection</h3> <p>A hands-free experience requires wake word detection. You can add a lightweight wake word model that listens continuously and triggers the main flow only when it hears a specific phrase.</p> <h3>Streaming Responses</h3> <p>The current implementation waits for the LLM to finish before starting TTS. With streaming, you can begin synthesizing audio as soon as the first sentence is complete, reducing perceived latency.</p> <hr/> <p>The complete source code for this tutorial is available in the <a href="https://github.com/xybrid-ai/xybrid/tree/main/examples/flutter" rel="nofollow">Xybrid examples repository</a>
202. If you run into issues, open a discussion on the GitHub repo.</p>',1);function Ks(o){var a=Ws(),n=s(b(a),6);e(n,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>Audio In â Whisper (ASR) â Text â Llama 3.2 (LLM) â Text â Kokoro (TTS) â Audio Out</span></span></code></pre>');var l=s(n,24);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#85E89D">dependencies</span><span style="color:#E1E4E8">:</span></span> 203<span class="line"><span style="color:#85E89D"> flutter</span><span style="color:#E1E4E8">:</span></span> 204<span class="line"><span style="color:#85E89D"> sdk</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">flutter</span></span> 205<span class="line"><span style="color:#85E89D"> xybrid_flutter</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">^0.1.0</span></span></code></pre>`);var t=s(l,4);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">import</span><span style="color:#9ECBFF"> 'package:xybrid_flutter/xybrid_flutter.dart'</span><span style="color:#E1E4E8">;</span></span> 206<span class="line"></span> 207<span class="line"><span style="color:#F97583">void</span><span style="color:#B392F0"> main</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 208<span class="line"><span style="color:#79B8FF"> WidgetsFlutterBinding</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">ensureInitialized</span><span style="color:#E1E4E8">();</span></span> 209<span class="line"><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">init</span><span style="color:#E1E4E8">();</span></span> 210<span class="line"><span style="color:#B392F0"> runApp</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">const</span><span style="color:#79B8FF"> VoiceAgentApp</span><span style="color:#E1E4E8">());</span></span> 211<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var p=s(t,6);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> whisper </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'whisper-tiny'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 212<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> llm </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'llama-3.2-1b'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 213<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> tts </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'kokoro-82m'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span></code></pre>`);var r=s(p,8);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#79B8FF"> Uint8List</span><span style="color:#E1E4E8"> audioBytes </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8">
213 recorder.</span><span style="color:#B392F0">stopAndGetBytes</span><span style="color:#E1E4E8">();</span></span> 214<span class="line"></span> 215<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> transcript </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> whisper.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 216<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">audio</span><span style="color:#E1E4E8">(bytes</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> audioBytes),</span></span> 217<span class="line"><span style="color:#E1E4E8">);</span></span> 218<span class="line"></span> 219<span class="line"><span style="color:#B392F0">print</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">'User said: </span><span style="color:#9ECBFF">\${</span><span style="color:#79B8FF">transcript</span><span style="color:#9ECBFF">.</span><span style="color:#79B8FF">text</span><span style="color:#9ECBFF">}</span><span style="color:#9ECBFF">'</span><span style="color:#E1E4E8">);</span></span></code></pre>`);var c=s(r,6);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> response </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> llm.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 220<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> transcript.text</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">),</span></span> 221<span class="line"><span style="color:#E1E4E8">);</span></span> 222<span class="line"></span> 223<span class="line"><span style="color:#B392F0">print</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">'Assistant: </span><span style="color:#9ECBFF">\${</span><span style="color:#79B8FF">response</span><span style="color:#9ECBFF">.</span><span style="color:#79B8FF">text</span><span style="color:#9ECBFF">}</span><span style="color:#9ECBFF">'</span><span style="color:#E1E4E8">);</span></span></code></pre>`);var d=s(c,8);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> audio </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> tts.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 224<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> response.text</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">),</span></span> 225<span class="line"><span style="color:#E1E4E8">);</span></span> 226<span class="line"></span> 227<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> audioPlayer </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> AudioPlayer</span><span style="color:#E1E4E8">();</span></span> 228<span class="line"><span style="color:#F97583">await</span><span style="color:#E1E4E8"> audioPlayer.</span><span style="color:#B392F0">playBytes</span><span style="color:#E1E4E8">(audio.audioBytes</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">);</span></span></code></pre>`);var i=s(d,8);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">class</span><span style="color:#79B8FF"> VoiceAgentScreen</span><span style="color:#F97583"> extends</span><span style="color:#79B8FF"> StatefulWidget</span><span style="color:#E1E4E8"> {</span></span> 229<span class="line"><span style="color:#F97583"> const</span><span style="color:#79B8FF"> VoiceAgentScreen</span><span style="color:#E1E4E8">
229({</span><span style="color:#79B8FF">super</span><span style="color:#E1E4E8">.key});</span></span> 230<span class="line"></span> 231<span class="line"><span style="color:#F97583"> @override</span></span> 232<span class="line"><span style="color:#79B8FF"> State</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">VoiceAgentScreen</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">createState</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">=></span><span style="color:#79B8FF"> _VoiceAgentScreenState</span><span style="color:#E1E4E8">();</span></span> 233<span class="line"><span style="color:#E1E4E8">}</span></span> 234<span class="line"></span> 235<span class="line"><span style="color:#F97583">class</span><span style="color:#79B8FF"> _VoiceAgentScreenState</span><span style="color:#F97583"> extends</span><span style="color:#79B8FF"> State</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">VoiceAgentScreen</span><span style="color:#E1E4E8">> {</span></span> 236<span class="line"><span style="color:#79B8FF"> XybridModel</span><span style="color:#F97583">?</span><span style="color:#E1E4E8"> _whisper;</span></span> 237<span class="line"><span style="color:#79B8FF"> XybridModel</span><span style="color:#F97583">?</span><span style="color:#E1E4E8"> _llm;</span></span> 238<span class="line"><span style="color:#79B8FF"> XybridModel</span><span style="color:#F97583">?</span><span style="color:#E1E4E8"> _tts;</span></span> 239<span class="line"><span style="color:#79B8FF"> bool</span><span style="color:#E1E4E8"> _isProcessing </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">;</span></span> 240<span class="line"><span style="color:#79B8FF"> String</span><span style="color:#E1E4E8"> _transcript </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> ''</span><span style="color:#E1E4E8">;</span></span> 241<span class="line"><span style="color:#79B8FF"> String</span><span style="color:#E1E4E8"> _response </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> ''</span><span style="color:#E1E4E8">;</span></span> 242<span class="line"></span> 243<span class="line"><span style="color:#F97583">
243 @override</span></span> 244<span class="line"><span style="color:#F97583"> void</span><span style="color:#B392F0"> initState</span><span style="color:#E1E4E8">() {</span></span> 245<span class="line"><span style="color:#79B8FF"> super</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">initState</span><span style="color:#E1E4E8">();</span></span> 246<span class="line"><span style="color:#B392F0"> _loadModels</span><span style="color:#E1E4E8">();</span></span> 247<span class="line"><span style="color:#E1E4E8"> }</span></span> 248<span class="line"></span> 249<span class="line"><span style="color:#79B8FF"> Future</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">void</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">_loadModels</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 250<span class="line"><span style="color:#E1E4E8"> _whisper </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'whisper-tiny'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 251<span class="line"><span style="color:#E1E4E8"> _llm </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'llama-3.2-1b'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 252<span class="line"><span style="color:#E1E4E8"> _tts </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'kokoro-82m'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 253<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() {});</span></span> 254<span class="line"><span style="color:#E1E4E8"> }</span></span> 255<span class="line"></span> 256<span class="line"><span style="color:#79B8FF"> Future</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">void</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">_processAudio</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">Uint8List</span><span style="color:#E1E4E8"> audioBytes) </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 257<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _isProcessing </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> true</span><span style="color:#E1E4E8">);</span></span> 258<span class="line"></span> 259<span class="line"><span style="color:#F97583"> try</span><span style="color:#E1E4E8"> {</span></span> 260<span class="line"><span style="color:#6A737D"> // 1. Transcribe</span></span> 261<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> transcript </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> _whisper</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 262<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">audio</span><span style="color:#E1E4E8">(bytes</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> audioBytes),</span></span> 263<span class="line"><span style="color:#E1E4E8"> );</span></span> 264<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _transcript </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> transcript.text </span><span style="color:#F97583">??</span><span style="color:#9ECBFF"> ''</span><span style="color:#E1E4E8">);</span></span> 265<span class="line"></span> 266<span class="line"><span style="color:#6A737D"> // 2. Generate response</span></span> 267<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> response </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> _llm</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 268<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> transcript.text</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">),</span></span> 269<span class="line"><span style="color:#E1E4E8"> );</span></span> 270<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _response </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> response.text </span><span style="color:#F97583">??</span><span style="color:#9ECBFF"> ''</span><span style="color:#E1E4E8">);</span></span> 271<span class="line"></span> 272<span class="line"><span style="color:#6A737D"> // 3. Speak</span></span> 273<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> audio </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> _tts</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 274<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> response.text</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">),</span></span> 275<span class="line"><span style="color:#E1E4E8"> );</span></span> 276<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> player </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> AudioPlayer</span><span style="color:#E1E4E8">();</span></span> 277<span class="line"><span style="color:#F97583"> await</span><span style="color:#E1E4E8">
277 player.</span><span style="color:#B392F0">playBytes</span><span style="color:#E1E4E8">(audio.audioBytes</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">);</span></span> 278<span class="line"><span style="color:#E1E4E8"> } </span><span style="color:#F97583">finally</span><span style="color:#E1E4E8"> {</span></span> 279<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _isProcessing </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">);</span></span> 280<span class="line"><span style="color:#E1E4E8"> }</span></span> 281<span class="line"><span style="color:#E1E4E8"> }</span></span> 282<span class="line"></span> 283<span class="line"><span style="color:#F97583"> @override</span></span> 284<span class="line"><span style="color:#79B8FF"> Widget</span><span style="color:#B392F0"> build</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">BuildContext</span><span style="color:#E1E4E8"> context) {</span></span> 285<span class="line"><span style="color:#F97583"> return</span><span style="color:#79B8FF"> Scaffold</span><span style="color:#E1E4E8">(</span></span> 286<span class="line"><span style="color:#E1E4E8"> body</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Center</span><span style="color:#E1E4E8">(</span></span> 287<span class="line"><span style="color:#E1E4E8"> child</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Column</span><span style="color:#E1E4E8">(</span></span> 288<span class="line"><span style="color:#E1E4E8"> mainAxisAlignment</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> MainAxisAlignment</span><span style="color:#E1E4E8">.center,</span></span> 289<span class="line"><span style="color:#E1E4E8"> children</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> [</span></span> 290<span class="line"><span style="color:#79B8FF"> Text</span><span style="color:#E1E4E8">(_transcript, style</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Theme</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">of</span><span style="color:#E1E4E8">(context).textTheme.bodyLarge),</span></span> 291<span class="line"><span style="color:#F97583"> const</span><span style="color:#79B8FF"> SizedBox</span><span style="color:#E1E4E8">(height</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 16</span><span style="color:#E1E4E8">),</span></span> 292<span class="line"><span style="color:#79B8FF"> Text</span><span style="color:#E1E4E8">(_response, style</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Theme</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">of</span><span style="color:#E1E4E8">(context).textTheme.headlineSmall),</span></span> 293<span class="line"><span style="color:#E1E4E8"> ],</span></span> 294<span class="line"><span style="color:#E1E4E8"> ),</span></span> 295<span class="line"><span style="color:#E1E4E8"> ),</span></span> 296<span class="line"><span style="color:#E1E4E8"> floatingActionButton</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> FloatingActionButton</span><span style="color:#E1E4E8">(</span></span> 297<span class="line"><span style="color:#E1E4E8"> onPressed</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> () {</span></span> 298<span class="line"><span style="color:#6A737D"> // Record audio, then call _processAudio(audioBytes)</span></span> 299<span class="line"><span style="color:#E1E4E8"> },</span></span> 300<span class="line"><span style="color:#E1E4E8"> child</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Icon</span><span style="color:#E1E4E8">(_isProcessing </span><span style="color:#F97583">?</span><span style="color:#79B8FF"> Icons</span><span style="color:#E1E4E8">.hourglass_empty </span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Icons</span><span style="color:#E1E4E8">.mic),</span></span> 301<span class="line"><span style="color:#E1E4E8"> ),</span></span> 302<span class="line"><span style="color:#E1E4E8"> );</span></span> 303<span class="line"><span style="color:#E1E4E8"> }</span></span> 304<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var y=s(i,28);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> context </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> ConversationContext</span><span style="color:#E1E4E8">();</span></span> 305<span class="line"></span> 306<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> response </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> llm.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 307<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> transcript.text</span><span style="color:#F97583">!</span><span style="color:#E1E4E8">),</span></span> 308<span class="line"><span style="color:#E1E4E8"> context</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> context,</span></span> 309<span class="line"><span style="color:#E1E4E8">);</span></span></code></pre>`),f(14),h(o,a)}const V=Object.freeze(Object.defineProperty({__proto__:null,default:Ks,metadata:j},Symbol.toStringTag,{value:"Module"})),Q={title:"Edge AI: Why Edge-First Should Be Your Default Architecture",description:"On-device AI should be the starting position, not the optimization. The industry defaults to cloud out of habit, not necessity.",date:"2026-06-19",author:"Glenn Sonna",tags:["edge-ai","on-device-ai","edge-first","private-ai","ai-architecture"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:7},{title:_a,description:Aa,date:Ta,author:Sa,tags:Ma,category:Pa,image:Ia,imageAlt:La,draft:Da,featured:Ra,readingTime:Oa}=Q;var Hs=u('<p>We already published <a href="/blog/edge-vs-cloud-inference">a decision framework for edge vs cloud AI</a>. Five factors, a comparison table, âit depends.â</p> <p>This isnât that. This is the argument that <strong>on-device should be your starting position</strong>, and cloud should be what you escalate to â not the other way around.</p> <p>The industry has it backwards. We default to cloud and treat on-device as an optimization. It should be the reverse.</p> <hr/> <h2>The Default Is Wrong</h2> <p>When a team adds AI to a product in 2026, the conversation goes:</p> <ol><li>âLetâs call the OpenAI APIâ</li> <li>Ship it</li> <li>
309Get the bill</li> <li>Panic</li> <li><em>Maybe</em> investigate on-device as a cost optimization</li></ol> <p>This is backwards because it starts with the most expensive, most latent, least private option and works down. Itâs the equivalent of renting a GPU cluster to resize images because âweâll optimize later.â</p> <p>The right default: <strong>run it on the device. Escalate to cloud only when the model literally doesnât fit.</strong></p> <h2>The Numbers Are Already There</h2> <p>Hereâs what an iPhone 15 can do right now:</p> <table><thead><tr><th>Task</th><th>Model</th><th>On-Device Latency</th><th>Cloud Latency</th></tr></thead><tbody><tr><td>Speech-to-text</td><td>Whisper Tiny (39M)</td><td>~200ms</td><td>300-800ms</td></tr><tr><td>Text-to-speech</td><td>Kokoro (82M)</td><td>~150ms</td><td>400-900ms</td></tr><tr><td>Text generation</td><td>Qwen 3.5 0.8B (Q4)</td><td>~350ms first token</td><td>200-500ms first token</td></tr><tr><td>Multimodal (text + image + audio)</td><td>Gemma 4 E2B (2.3B effective)</td><td>~500ms first token</td><td>300-800ms first token</td></tr><tr><td>Image classification</td><td>MobileNet (4M)</td><td>~2ms</td><td>150-400ms</td></tr></tbody></table> <p>For four out of five tasks, the phone is faster than the cloud. The fifth â a multimodal model that processes text, images, and audio â is competitive. And itâs free, private, and works offline.</p> <h2>Cloud-First Is a Tax on Your Users</h2> <p>Every cloud API call has a floor you canât engineer away:</p> <ul><li>DNS: 10-50ms</li> <li>TLS handshake: 50-150ms</li> <li>Serialization: 10-30ms</li> <li>Network transit: 20-100ms</li> <li>Queue time: 0-500ms</li></ul> <p><strong>100-300ms before the model even starts.</strong> Thatâs a tax on every request. Your users pay it in latency. You pay it in dollars.</p> <p>On-device inference starts at zero. The model is already loaded. The data is already local. There is no floor.</p> <h2>The Cost Inversion</h2> <p>Cloud AI pricing per month at moderate scale:</p> <table><thead><tr><th>Feature</th><th>Usage</th><th>Cloud Cost</th><th>On-Device Cost</th></tr></thead><tbody><tr><td>TTS</td><td>10K requests/day</td><td>~$1,200</td><td>$0</td></tr><tr><td>ASR</td><td>100K minutes/day</td><td>~$18,000</td><td>$0</td></tr><tr><td>Classification</td><td>1M images/day</td><td>~$30,000</td><td>$0</td></tr></tbody></table> <p>On-device cost is zero because the userâs hardware does the work. You pay once for model development. They run inference for free, forever.</p> <p>This isnât an optimization. Itâs a different economic model. Cloud AI has marginal cost per request. On-device has fixed cost and zero marginal cost. At any meaningful scale, on-device wins by orders of magnitude.</p> <p>For consumer apps with millions of users and low ARPU â the kind where you canât absorb $0.006 per API call â on-device isnât an optimization. Itâs the only viable architecture.</p> <h2>Privacy Is Architecture, Not Policy</h2> <p>When inference runs locally, sensitive data physically cannot leave the device. This isnât a policy you enforce. Itâs a property of the system.</p> <p>No data processing agreements. No SOC 2 for inference logs. No âwe encrypt your audio in transit.â The audio never transits.</p> <p>For health apps, voice journals, childrenâs products, legal tools â this is the difference between âwe have a privacy policyâ and âthereâs nothing to protect because we never had it.â</p> <p>GDPR, HIPAA, CCPA all get simpler when the data stays on the userâs device. Not easy â simpler. Fewer categories of risk to manage.</p> <h2>âBut My Model Is Too Bigâ</h2> <p>This is the one legitimate reason to use cloud inference: the model doesnât fit.</p> <p>GPT-4, Claude, Gemini Ultra â these are cloud-only. A 70B parameter model needs dedicated GPU memory that phones donât have.</p> <p>But look at what <em>does</em> fit:</p> <ul><li><strong>Whisper Tiny</strong> (39M) â real-time ASR on any phone</li> <li><strong>Kokoro</strong> (82M) â high-quality multi-voice TTS</li> <li><strong>SmolLM2</strong> (360M) â capable text generation</li> <li><strong>Qwen 3.5</strong> (0.8B) â 201 languages, Q4 quantized fits in 500MB</li> <li><strong>Gemma 4 E2B</strong> (5.1B total, 2.3B effective) â multimodal (text, image, audio, video), Q4 fits in 3.1GB, <strong>runs on iPhone and Android</strong></li> <li><strong>Gemma 4 E4B</strong> (8B total, 4.5B effective) â same multimodal capabilities, Q4 fits in 5GB, runs on flagship phones and tablets</li></ul> <p>Read that last one again. A multimodal model that understands text, images, audio, and video â running on a phone. Not a server. Not a GPU cluster. A phone in your pocket, with 3GB of storage.</p> <p>Googleâs Per-Layer Embeddings architecture is key here: Gemma 4 E2B has 5.1B total parameters but only 2.3B effective parameters per forward pass. Thatâs how a model this capable fits on mobile hardware.</p> <p>The gap between âruns on a phoneâ and ârequires a data centerâ is narrowing faster than most teams realize. The set of tasks that genuinely require cloud is shrinking. The set of tasks youâre sending to cloud out of habit is large.</p> <h2>The Hybrid Pattern (But Edge-First)</h2> <p>Iâm not arguing for pure on-device everything. The right architecture is hybrid â but with edge as the default:</p> <!> <p>The key word is <em>first</em>. Try on-device. If itâs good enough (and for most tasks with modern models, it is), youâre done. If not, escalate.</p> <p>This gives you:</p> <ul><li><strong>95% of requests</strong> handled on-device (fast, free, private)</li> <li><strong>5% of requests</strong> escalated to cloud (complex reasoning, frontier tasks)</li> <li><strong>100% of requests</strong> work offline (edge always available as fallback)</li></ul> <h2>The Industry Is Moving This Way</h2> <p>Apple shipped a 3B parameter model on-device in iOS 18. Googleâs Gemma 4 E2B â a multimodal model that understands text, images, audio, and video â runs on a phone with 3GB of storage. Qualcommâs latest Snapdragon has 45 TOPS of NPU compute.</p> <p>The silicon vendors are betting on local inference. The model researchers are optimizing for efficiency (Per-Layer Embeddings, distillation, quantization). The regulatory environment is pushing toward data minimization.</p> <p>
309Cloud-first AI is the legacy architecture. Edge-first is where the industry is going. The question is whether you arrive there by design or by cost pressure.</p> <h2>Start Edge-First</h2> <!> <p>Default to on-device. Escalate to cloud when you must. Not the other way around.</p> <hr/> <p><strong>Build edge-first</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <hr/> <p><em>Where do you draw the line? Whatâs still cloud-only in your stack that could move to the edge?</em></p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/edge-vs-cloud-inference">Edge AI vs Cloud AI: When to Run Models On-Device</a> â the detailed decision framework with cost analysis.</li> <li><a href="/blog/private-ai-run-models-without-cloud">Private AI: How to Run AI Models Without Sending Data to the Cloud</a> â the privacy and compliance angle.</li> <li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide</a> â hardware, models, and getting started.</li> <li><a href="/blog/run-llm-locally-complete-guide">How to Run LLMs Locally</a> â practical guide to running language models without cloud APIs.</li></ul>',1);function Xs(o){var a=Hs(),n=s(b(a),78);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>User Input</span></span> 310<span class="line"><span> â</span></span> 311<span class="line"><span> â¼</span></span> 312<span class="line"><span>ââââââââââââââââ ââââââââââââââââ</span></span> 313<span class="line"><span>â Edge Model ââââââ¶â Good enough? â</span></span> 314<span class="line"><span>â (fast, free)â â (confidence â</span></span> 315<span class="line"><span>ââââââââââââââââ â threshold) â</span></span> 316<span class="line"><span> ââââââââ¬ââââââââ</span></span> 317<span class="line"><span> Yes â No</span></span> 318<span class="line"><span> â â</span></span> 319<span class="line"><span> â¼ â¼</span></span> 320<span class="line"><span> Done ââââââââââââââââ</span></span> 321<span class="line"><span> â Cloud Model â</span></span> 322<span class="line"><span> â (powerful, â</span></span> 323<span class="line"><span> â costs $$$) â</span></span> 324<span class="line"><span> ââââââââââââââââ</span></span></code></pre>`);var l=s(n,18);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> install</span><span style="color:#9ECBFF"> xybrid-cli</span></span> 325<span class="line"></span> 326<span class="line"><span style="color:#6A737D"># TTS â no API key, no cloud</span></span> 327<span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> kokoro-82m</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> "Edge first, cloud when needed."</span></span> 328<span class="line"></span> 329<span class="line"><span style="color:#6A737D"># ASR â no data leaves the device</span></span> 330<span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> whisper-tiny</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> recording.wav</span></span> 331<span class="line"></span> 332<span class="line"><span style="color:#6A737D"># LLM â runs on your laptop</span></span> 333<span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> qwen3.5-0.8b</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> "Summarize this document."</span></span></code></pre>`),f(16),h(o,a)}const Y=Object.freeze(Object.defineProperty({__proto__:null,default:Xs,metadata:Q},Symbol.toStringTag,{value:"Module"})),J={title:"Edge AI vs Cloud AI: When to Run Models On-Device",description:"A practical decision framework for choosing between on-device and cloud-based AI inference, with cost analysis, c
333omparison tables, and real-world use cases.",date:"2026-03-20",author:"Glenn Sonna",tags:["edge-ai","cloud-ai","inference","architecture","cost-optimization"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:10},{title:Na,description:Ga,date:za,author:Ua,tags:qa,category:$a,image:Wa,imageAlt:Ka,draft:Ha,featured:Xa,readingTime:ja}=J;var js=u("<h2>It Is a Spectrum, Not a Binary Choice</h2> <p>The âedge vs cloudâ framing is misleading. In practice, most AI-powered applications exist on a continuum between three deployment modes:</p> <ul><li><strong>Fully on-device</strong>: The model runs entirely on the userâs hardware. No network calls, no server costs, no data leaving the device.</li> <li><strong>Hybrid</strong>: Some models run locally while others route to the cloud. The system makes routing decisions based on model size, device capabilities, and network conditions.</li> <li><strong>Fully cloud</strong>: All inference happens on remote servers. The client sends data, waits for a response, and renders the result.</li></ul> <p>Each mode has legitimate use cases. The goal is not to pick a side but to understand where each model in your stack belongs. A single application might run speech recognition on-device, route complex reasoning to the cloud, and synthesize audio locally â all in the same request pipeline.</p> <h2>A Decision Framework for On-Device vs Cloud Inference</h2> <p>Choosing where to run a model comes down to five factors. Evaluate each one for every model in your pipeline, not once for the entire application.</p> <h3>Latency Requirements</h3> <p>This is often the deciding factor. On-device inference eliminates network round-trips entirely.</p> <table><thead><tr><th>Scenario</th><th>Acceptable Latency</th><th>Recommendation</th></tr></thead><tbody><tr><td>Real-time voice interaction</td><td>Under 50ms</td><td>On-device</td></tr><tr><td>Live camera/video processing</td><td>Under 100ms</td><td>On-device</td></tr><tr><td>Interactive chat responses</td><td>Under 500ms</td><td>Either (depends on model size)</td></tr><tr><td>Document processing</td><td>Under 5s</td><td>Cloud is fine</td></tr><tr><td>Batch analytics</td><td>Minutes</td><td>Cloud is fine</td></tr></tbody></table> <p>For voice-first applications, cloud latency is often a dealbreaker. A round-trip to a cloud API typically adds 100-300ms of network overhead before the model even begins processing. For text-to-speech, that delay breaks the conversational flow. For wake word detection, it makes the feature unusable.</p> <h3>Privacy Constraints</h3> <p>Some data should never leave the device. This is not just a preference â it is increasingly a legal requirement.</p> <ul><li><strong>Regulated industries</strong> (healthcare, finance, legal): Patient records, financial data, and attorney-client communications often cannot be transmitted to third-party servers, even encrypted.</li> <li><strong>User expectations</strong>: Voice recordings, camera feeds, and biometric data create friction when users learn it is sent to the cloud.</li> <li><strong>Compliance overhead</strong>: Using cloud inference with sensitive data means managing data processing agreements, SOC 2 compliance, and regional data residency requirements. On-device inference sidesteps all of this.</li></ul> <p>If your data is sensitive and your model is small enough to run locally, on-device inference removes an entire category of compliance risk.</p> <h3>Model Complexity</h3> <p>Model size is the hard constraint. A device can only run what fits in its available memory and compute budget.</p> <table><thead><tr><th>Model Size</th><th>Parameter Count</th><th>On-Device Feasibility</th></tr></thead><tbody><tr><td>Under 50 MB</td><td>Under 10M params</td><td>Runs on any modern phone</td></tr><tr><td>50-200 MB</td><td>10-80M params</td><td>Runs on mid-range phones, all desktops</td></tr><tr><td>200 MB - 1 GB</td><td>80M-500M params</td><td>Runs on flagship phones, all desktops</td></tr><tr><td>1-4 GB</td><td>500M-3B params</td><td>Runs on desktops and high-end tablets</td></tr><tr><td>4+ GB</td><td>3B+ params</td><td>Desktop only, or cloud</td></tr><tr><td>20+ GB</td><td>13B+ params</td><td>Cloud (or high-end workstations with quantization)</td></tr></tbody></table> <p>The practical ceiling for mobile on-device inference today is roughly 1-3B parameters with 4-bit quantization. Models like Whisper Tiny (39M params), MobileNet (4M params), and Kokoro TTS (82M params) run comfortably on any modern device. A 7B parameter LLM can run on a desktop with quantization. Anything beyond 13B parameters generally requires cloud infrastru
333cture.</p> <h3>Connectivity Reliability</h3> <p>If your users might be offline, on-device is not optional â it is required.</p> <p>This applies more broadly than you might expect: aircraft cabins, rural areas, subway commutes, factory floors, field service operations, and disaster response scenarios all involve unreliable or nonexistent connectivity. If your application must work in these environments, the core inference pipeline needs to run locally.</p> <p>Even in always-connected environments, on-device inference provides resilience against cloud outages. An application that falls back gracefully to local models when the API is down delivers a better experience than one that shows an error screen.</p> <h3>Cost at Scale</h3> <p>This is where the math gets interesting. Cloud inference has a per-call cost. On-device inference has a fixed cost (model download) and zero marginal cost per inference.</p> <h2>Cost Analysis: The Break-Even Point</h2> <p>Cloud AI pricing is straightforward: you pay per inference. On-device AI has a one-time cost (downloading the model to the device) and then runs for free.</p> <h3>Cloud Inference Costs</h3> <p>Typical cloud API pricing for common model types:</p> <table><thead><tr><th>Model Type</th><th>Cost Per Inference</th><th>Cost Per 1M Inferences</th></tr></thead><tbody><tr><td>Text classification</td><td>~$0.001</td><td>$1,000</td></tr><tr><td>Speech recognition (per minute)</td><td>~$0.006</td><td>$6,000</td></tr><tr><td>Text-to-speech (per 1K chars)</td><td>~$0.015</td><td>$15,000</td></tr><tr><td>Image classification</td><td>~$0.002</td><td>$2,000</td></tr><tr><td>LLM chat (per 1K tokens)</td><td>~$0.002-0.06</td><td>$2,000-60,000</td></tr><tr><td>Embedding generation</td><td>~$0.0001</td><td>$100</td></tr></tbody></table> <p>These costs compound quickly. An application that makes 10 inference calls per user session, with 100,000 daily active users, generates 1 million inferences per day. At $0.005 per inference, that is $5,000 per day or $150,000 per month.</p> <h3>On-Device Inference Costs</h3> <p>On-device costs are fundamentally different:</p> <table><thead><tr><th>Cost Component</th><th>One-Time</th><th>Recurring</th></tr></thead><tbody><tr><td>Model download bandwidth</td><td>~$0.01-0.10 per user (CDN cost for 50-500MB model)</td><td>$0</td></tr><tr><td>Model storage on device</td><td>0 (userâs storage)</td><td>$0</td></tr><tr><td>Compute per inference</td><td>0 (userâs CPU/GPU)</td><td>$0</td></tr><tr><td>API server infrastructure</td><td>$0</td><td>$0</td></tr></tbody></table> <p>The model download is typically 50-500MB, served from a CDN. At standard CDN rates ($0.08-0.12 per GB), downloading a 200MB model costs roughly $0.02 per user. After that, every inference is free.</p> <h3>Break-Even Analysis</h3> <p>The break-even point depends on how many inferences each user makes:</p> <table><thead><tr><th>Cloud Cost Per Inference</th><th>Model Download Cost</th><th>Break-Even Point</th></tr></thead><tbody><tr><td>$0.001</td><td>$0.02</td><td>20 inferences per user</td></tr><tr><td>$0.005</td><td>$0.02</td><td>4 inferences per user</td></tr><tr><td>$0.01</td><td>$0.05</td><td>5 inferences per user</td></tr><tr><td>$0.05</td><td>$0.10</td><td>2 inferences per user</td></tr></tbody></table> <p>For any model where users make more than a handful of inferences, on-device wins on cost â often by orders of magnitude. A user who runs text-to-speech 100 times costs $1.50 in cloud API fees or $0.02 in CDN bandwidth for the on-device model download. At scale, this difference becomes the dominant line item in your infrastructure budget.</p> <h2>Side-by-Side Comparison</h2> <table><thead><tr><th>Factor</th><th>On-Device</th><th>Cloud</th><th>Hybrid</th></tr></thead><tbody><tr><td><strong>Latency</strong></td><td>5-50ms (no network)</td><td>100-500ms (network + inference)</td><td>Varies by route</td></tr><tr><td><strong>Privacy</strong></td><td>Data never leaves device</td><td>Data sent to server</td><td>Sensitive data stays local</td></tr><tr><td><strong>Cost per inference</strong></td><td>$0 (after model download)</td><td>$0.001-0.06</td><td>Reduced cloud spend</td></tr><tr><td><strong>Model size limit</strong></td><td>~1-3B params (mobile), ~13B (desktop)</td><td>Unlimited</td><td>Best of both</td></tr><tr><td><strong>Accuracy</strong></td><td>Limited by model size</td><td>Can use largest models</td><td>Largest where needed</td></tr><tr><td><strong>Offline support</strong></td><td>Full</td><td>None</td><td>Partial (on-device models work)</td></tr><tr><td><strong>Scalability</strong></td><td>Scales with user devices (free)</td><td>Scales with server spend</td><td>Balanced</td></tr><tr><td><strong>Setup complexity</strong></td><td>Model packaging, device testing</td><td>API key, HTTP calls</td><td>Routing logic required</td></tr><tr><td><strong>Update cycle</strong></td><td>Requires model re-download</td><td>Instant (server-side)</td><td>Mixed</td></tr></tbody></table> <h2>Use Cases That Favor On-Device Inference</h2> <p>These workloads share common traits: they are latency-sensitive, use relatively small models, run frequently, or handle sensitive data.</p> <p><strong>Real-time speech recognition (ASR).</strong> Whisper Tiny (39M params, ~75MB) transcribes audio with sub-100ms latency on modern phones. Streaming transcription for voice interfaces, dictation, and accessibility features should almost always run on-device. The latency improvement alone justifies it, and the cost savings at scale are substantial.</p> <p><strong>Text-to-speech (TTS).</strong> Models like Kokoro (82M params, ~180MB) generate natural-sounding speech on-device. For voice assistants, audiobook readers, and accessibility tools, on-device TTS eliminates the noticeable delay that cloud TTS introduces in conversational flows.</p> <p><strong>Image classification and object detection.</strong> MobileNet, EfficientNet, and YOLO variants are designed for edge deployment. Camera-based features (barcode scanning, plant identification, accessibility descriptions) benefit from the instant feedback loop of on-device inference.</p> <p><strong>Text embeddings.</strong>
333 Small embedding models (MiniLM, all-MiniLM-L6-v2 at ~80MB) generate vector representations locally. This enables on-device semantic search, document similarity, and retrieval-augmented generation without sending user documents to a server.</p> <p><strong>Wake word and keyword detection.</strong> Always-on listening for trigger phrases must run on-device. Streaming audio to the cloud continuously is both a privacy concern and a bandwidth problem.</p> <h2>Use Cases That Favor Cloud Inference</h2> <p>These workloads require model sizes or compute budgets that exceed what consumer devices can provide.</p> <p><strong>Large language models (70B+ parameters).</strong> Frontier LLMs like GPT-4, Claude, and Llama 3 70B require tens of gigabytes of memory and significant GPU compute. These models will remain cloud-only for consumer devices for the foreseeable future.</p> <p><strong>Training and fine-tuning.</strong> Model training is fundamentally a cloud workload. Even fine-tuning requires GPU clusters with large memory pools that consumer hardware cannot provide.</p> <p><strong>Multi-modal reasoning.</strong> Models that process images, video, and text together (vision-language models) are typically large and compute-intensive. The accuracy gains from using a 13B+ multi-modal model in the cloud outweigh the latency cost for most use cases.</p> <p><strong>Low-volume or sporadic usage.</strong> If a feature is used rarely (a few times per month per user), the cost of downloading and storing a model on-device is harder to justify. Cloud inference makes more sense when the per-user inference count is low.</p> <p><strong>Rapidly evolving models.</strong> When you need to swap models frequently (A/B testing, weekly model updates), cloud deployment is simpler. On-device models require the user to download updates.</p> <h2>The Hybrid Pattern: Intelligent Routing</h2> <p>The most practical architecture for production applications is hybrid: run what you can on-device, route what you must to the cloud, and make the decision automatically based on runtime conditions.</p> <p>A well-designed orchestrator evaluates device capabilities, model availability, and network conditions at runtime, then routes each inference request to the appropriate backend. For example, a voice assistant pipeline might always run speech recognition and speech synthesis on-device (both are latency-sensitive and use small models), while routing the LLM stage to the cloud when the query exceeds what the local model can handle â or when the device does not have enough resources to run it.</p> <p>The application code remains the same regardless of where execution happens. It submits an inference request and receives a result. Whether that result came from the local NPU or a cloud GPU is an infrastructure decision, not an application concern.</p> <h3>Routing Strategies</h3> <p>Several routing patterns work well in practice:</p> <ul><li><strong>Prefer-device with cloud fallback</strong>: Try on-device first. If the model is not downloaded, the device is under load, or the task requires a larger model, route to the cloud. This is the most common pattern.</li> <li><strong>Latency-aware routing</strong>: Measure on-device inference time. If it exceeds a threshold (device is too slow for the model), route to the cloud for subsequent requests.</li> <li><strong>Complexity-based routing</strong>: Use a small on-device classifier to estimate query complexity. Simple queries go to the local model; complex ones go to a larger cloud model.</li> <li><strong>Offline-first</strong>: Always attempt on-device execution. Only route to the cloud when the local model cannot handle the task and connectivity is available.</li></ul> <h2>Decision Checklist</h2> <p>Before deploying a model, run through these questions. If you answer âyesâ to three or more, on-device inference is likely the right default.</p> <ol><li><strong>Is latency critical?</strong> Does the user experience degrade noticeably with 200ms+ of added latency?</li> <li><strong>Is the data sensitive?</strong> Would sending this data to a third-party server create privacy, compliance, or trust concerns?</li> <li><strong>Is the model small enough?</strong> Is the model under 1GB (mobile) or under 4GB (desktop)?</li> <li><strong>Will users run it frequently?</strong> Will each user trigger this model more than 10 times over the appâs lifetime?</li> <li><strong>Do users need offline access?</strong> Must this feature work without an internet connection?</li> <li><strong>Is the model stable?</strong> Will you keep the same model for weeks or months (not swapping daily)?</li> <li><strong>Are you scaling to many users?</strong> Will cloud inference costs become a significant budget line at your projected user count?</li></ol> <p>For models where you answer ânoâ to most of these â large models, infrequent use, non-sensitive data, always-connected users â cloud inference is simpler to deploy and maintain.</p> <h2>Conclusion</h2> <p>The edge AI vs cloud AI decision is ultimately about matching each model to the deployment mode where it performs best across latency, privacy, cost, and feasibility. Small, fast, frequently-used models belong on the device. Large, complex, infrequently-used models belong in the cloud. And for everything in between, a hybrid approach with intelligent routing gives you the flexibility to optimize for the constraints that matter most to your application.</p> <p>The key insight is to make this decision per-model, not per-application. A single product can â and usually should â use both on-device and cloud inference for different parts of its AI pipeline. The architecture that enables this flexibility is what separates production-grade AI applications from prototypes.</p>",1);function Vs(o){var a=js();f(134),h(o,a)}const Z=Object.freeze(Object.defineProperty({__proto__:null,default:Vs,metadata:J},Symbol.toStringTag,{value:"Module"})),ss={title:"model_metadata.json: A Declarative Schema for ML Model Execution",description:"How a single JSON file replaces custom inference scripts and makes any ML model self-describing. No more 'works on my machine.'",date:"2026-07-10",author:"Glenn Sonna",tags:["machinelearning","architecture","opensource","ai"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:6},{title:Va,description:Qa,date:Ya,author:Ja,tags:Za,category:sn,image:en,imageAlt:an,draft:nn,featured:on,readingTime:ln}=ss;var Qs=u('<p>Every ML model youâve ever deployed had the same hidden problem: <strong>the inference script</strong>.</p> <p>That Python file with hardcoded tensor shapes, tokenizer paths, magic normalization constants, and platform-specific workarounds. The one that âjust worksâ on the original developerâs machine and breaks everywhere else.</p> <p>We replaced it with a JSON file.</p> <hr/> <h2>The Problem</h2> <p>Hereâs what typical model deployment looks like:</p> <!> <p>This script contains critical deployment knowledge:</p> <ul><li>Input tensor name is <code>input_ids</code></li> <li>Input must be <code>int64</code></li> <li>Padding to 128 is required</li> <li>Output needs argmax â skip first â decode</li></ul> <p>None of this is discoverable from the ONNX file alone. Lose the script, lose the ability to run the model.</p> <h2>The Solution: model_metadata.json</h2> <p>Every model in Xybrid ships with a <code>model_metadata.json</code> that <strong>declares everything needed to run it</strong>:</p> <!> <p>The runtime reads this file, builds the execution pipeline, and runs
333it. No inference script. No custom code. No implicit knowledge.</p> <h2>How It Works</h2> <p>The <code>TemplateExecutor</code> (Xybridâs core execution engine) processes the metadata in three phases:</p> <!> <p>In Rust:</p> <!> <p>Three lines. Works for ASR, TTS, classification, embeddings â any model type. The metadata file is the only thing that changes.</p> <h2>The Schema</h2> <h3>Execution Templates</h3> <p>The <code>execution_template</code> field defines how the model file is loaded and run:</p> <!> <p>The template type tells the executor which backend to use (ONNX Runtime, llama.cpp, Candle) and what files are needed.</p> <h3>Preprocessing Steps</h3> <p>Available preprocessing steps and what they do:</p> <table><thead><tr><th>Step</th><th>Input</th><th>Output</th><th>Config</th></tr></thead><tbody><tr><td><code>AudioDecode</code></td><td>WAV bytes</td><td>f32 samples</td><td><code>sample_rate</code>, <code>channels</code></td></tr><tr><td><code>Phonemize</code></td><td>Text</td><td>Token IDs</td><td><code>backend</code>, <code>tokens_file</code>, <code>dict_file</code></td></tr><tr><td><code>Tokenize</code></td><td>Text</td><td>Token IDs</td><td><code>vocab_file</code>, <code>max_length</code></td></tr></tbody></table> <p>Example â TTS preprocessing:</p> <!> <p>This converts <code>"Hello world"</code> â <code>[phoneme token IDs]</code> using a built-in phonemizer. No external dependencies (no espeak-ng installation needed).</p> <h3>Postprocessing Steps</h3> <table><thead><tr><th>Step</th><th>Input</th><th>Output</th><th>Config</th></tr></thead><tbody><tr><td><code>CTCDecode</code></td><td>Logits tensor</td><td>Text</td><td><code>vocab_file</code>, <code>blank_index</code></td></tr><tr><td><code>TTSAudioEncode</code></td><td>f32 waveform</td><td>WAV bytes</td><td><code>sample_rate</code>, <code>apply_postprocessing</code></td></tr><tr><td><code>ArgMax</code></td><td>Logits tensor</td><td>Class index</td><td>â</td></tr></tbody></table> <h3>Voice Configuration (TTS-Specific)</h3> <p>TTS models can declare available voices:</p> <!> <p>The runtime loads the voice embedding from <code>voices.bin</code> and passes it as an additional model input. Consumers just pick a voice by name.</p> <h2>Why Not ONNX Model Zoo Format? Or MLflow?</h2> <p><strong>ONNX Model Zoo</strong> has model cards, but theyâre documentation, not execution configuration. They donât tell you how to preprocess inputs or postprocess outputs.</p> <p><strong>MLflow</strong> has model signatures, but they describe tensor shapes, not the pipeline around the model. You still need custom code for tokenization, audio decoding, etc.</p> <p><strong>model_metadata.json</strong> is execution-complete: everything from âraw user inputâ to âusable outputâ is declared. The runtime needs zero custom code per model.</p> <table><thead><tr><th></th><th>ONNX Zoo</th><th>MLflow</th><th>model_metadata.json</th></tr></thead><tbody><tr><td>Tensor shapes</td><td>Yes</td><td>Yes</td><td>Implicit</td></tr><tr><td>Preprocessing</td><td>No</td><td>No</td><td>Yes</td></tr><tr><td>Postprocessing</td><td>No</td><td>No</td><td>Yes</td></tr><tr><td>Multi-file models</td><td>No</td><td>Partial</td><td>Yes (<code>files</code> array)</td></tr><tr><td>Voice/variant config</td><td>No</td><td>No</td><td>Yes</td></tr><tr><td>Runtime-executable</td><td>No</td><td>Partial</td><td>Yes</td></tr></tbody></table> <h2>Real Examples</h2> <h3>ASR (wav2vec2)</h3> <!> <p>Input: WAV audio bytes â Output: transcribed text.</p> <h3>TTS (Kokoro)</h3> <!> <p>Input: text string â Output: WAV audio bytes.</p> <h3>LLM (Qwen 3.5)</h3> <!> <p>Input: text prompt â Output: generated text. No pre/postprocessing needed â the GGUF backend handles tokenization internally.</p> <h2>Extending the Schema</h2> <p>Need a new preprocessing step? Add a variant to the enum and implement the handler:</p> <!> <p>Implement the handler, and every model that uses <code>MelSpectrogram</code> in its metadata will automatically work across all platforms.</p> <h2>The Payoff</h2> <p>With <code>model_metadata.json</code>:</p> <ul><li><strong>Model authors</strong> declare how their model runs, once</li> <li><strong>Runtime developers</strong> implement preprocessing/postprocessing steps, once</li> <li><strong>App developers</strong> write <code>executor.execute(&metadata, &input)</code> â no model-specific code</li> <li><strong>New platforms</strong> (Flutter, Swift, Kotlin) get every model for free â they use the same metadata</li></ul> <p>One JSON file. Zero inference scripts. Every model is self-describing and portable.</p> <hr/> <p><strong>See it in action</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a> â browse <code>fixtures/models/</code> for real model_metadata.json examples.</p> <hr/> <p><em>Thoughts on the schema design? Things we should add? Let us know in the comments.</em></p>',1);function Ys(o){var a=Qs(),n=s(b(a),12);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># inference.py â "just copy this file with the model"</span></span> 334<span class="line"><span style="color:#F97583">
334import</span><span style="color:#E1E4E8"> onnxruntime </span><span style="color:#F97583">as</span><span style="color:#E1E4E8"> ort</span></span> 335<span class="line"><span style="color:#F97583">import</span><span style="color:#E1E4E8"> numpy </span><span style="color:#F97583">as</span><span style="color:#E1E4E8"> np</span></span> 336<span class="line"><span style="color:#F97583">from</span><span style="color:#E1E4E8"> tokenizer </span><span style="color:#F97583">import</span><span style="color:#E1E4E8"> load_tokenizer </span><span style="color:#6A737D"># custom module, good luck finding it</span></span> 337<span class="line"></span> 338<span class="line"><span style="color:#E1E4E8">session </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> ort.InferenceSession(</span><span style="color:#9ECBFF">"model.onnx"</span><span style="color:#E1E4E8">)</span></span> 339<span class="line"><span style="color:#E1E4E8">tokenizer </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> load_tokenizer(</span><span style="color:#9ECBFF">"vocab.json"</span><span style="color:#E1E4E8">)</span></span> 340<span class="line"></span> 341<span class="line"><span style="color:#F97583">def</span><span style="color:#B392F0"> run</span><span style="color:#E1E4E8">(text):</span></span> 342<span class="line"><span style="color:#E1E4E8"> tokens </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> tokenizer.encode(text)</span></span> 343<span class="line"><span style="color:#E1E4E8"> tokens </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> np.array([tokens], </span><span style="color:#FFAB70">dtype</span><span style="color:#F97583">=</span><span style="color:#E1E4E8">np.int64)</span></span> 344<span class="line"><span style="color:#6A737D"> # Magic: pad to 128 because the model was exported that way</span></span> 345<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> tokens.shape[</span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8">] </span><span style="color:#F97583"><</span><span style="color:#79B8FF"> 128</span><span style="color:#E1E4E8">:</span></span> 346<span class="line"><span style="color:#E1E4E8"> tokens </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> np.pad(tokens, ((</span><span style="color:#79B8FF">0</span><span style="color:#E1E4E8">,</span><span style="color:#79B8FF">0</span><span style="color:#E1E4E8">), (</span><span style="color:#79B8FF">0</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">128</span><span style="color:#F97583"> -</span><span style="color:#E1E4E8"> tokens.shape[</span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8">])))</span></span> 347<span class="line"><span style="color:#E1E4E8"> outputs </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> session.run(</span><span style="color:#79B8FF">None</span><span style="color:#E1E4E8">, {</span><span style="color:#9ECBFF">"input_ids"</span><span style="color:#E1E4E8">: tokens})</span></span> 348<span class="line"><span style="color:#6A737D"> # Magic: take argmax of last dim, skip first token, decode</span></span> 349<span class="line"><span style="color:#E1E4E8"> predicted </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> np.argmax(outputs[</span><span style="color:#79B8FF">0</span><span style="color:#E1E4E8">], </span><span style="color:#FFAB70">axis</span><span style="color:#F97583">=-</span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8">)[</span><span style="color:#79B8FF">0</span><span style="color:#E1E4E8">][</span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8">:]</span></span> 350<span class="line"><span style="color:#F97583"> return</span><span style="color:#E1E4E8"> tokenizer.decode(predicted)</span></span></code></pre>`);var l=s(n,12);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">{</span></span> 351<span class="line"><span style="color:#79B8FF"> "model_id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"wav2vec2-base-960h"</span><span style="color:#E1E4E8">,</span></span> 352<span class="line"><span style="color:#79B8FF"> "version"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"1.0"</span><span style="color:#E1E4E8">,</span></span> 353<span class="line"><span style="color:#79B8FF"> "description"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Speech recognition using CTC decoding"</span><span style="color:#E1E4E8">,</span></span> 354<span class="line"></span> 355<span class="line"><span style="color:#79B8FF"> "execution_template"</span><span style="color:#E1E4E8">: {</span></span> 356<span class="line"><span style="color:#79B8FF"> "type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"SimpleMode"</span><span style="color:#E1E4E8">,</span></span> 357<span class="line"><span style="color:#79B8FF"> "model_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model.onnx"</span></span> 358<span class="line"><span style="color:#E1E4E8"> },</span></span> 359<span class="line"></span> 360<span class="line"><span style="color:#79B8FF"> "preprocessing"</span><span style="color:#E1E4E8">: [</span></span> 361<span class="line"><span style="color:#E1E4E8"> {</span></span> 362<span class="line"><span style="color:#79B8FF"> "type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"AudioDecode"</span><span style="color:#E1E4E8">,</span></span> 363<span class="line"><span style="color:#79B8FF"> "sample_rate"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">16000</span><span style="color:#E1E4E8">,</span></span> 364<span class="line"><span style="color:#79B8FF"> "channels"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">1</span></span> 365<span class="line"><span style="color:#E1E4E8"> }</span></span> 366<span class="line"><span style="color:#E1E4E8"> ],</span></span> 367<span class="line"></span> 368<span class="line"><span style="color:#79B8FF"> "postprocessing"</span><span style="color:#E1E4E8">: [</span></span> 369<span class="line"><span style="color:#E1E4E8"> {</span></span> 370<span class="line"><span style="color:#79B8FF"> "type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"CTCDecode"</span><span style="color:#E1E4E8">,</span></span> 371<span class="line"><span style="color:#79B8FF"> "vocab_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"vocab.json"</span><span style="color:#E1E4E8">,</span></span> 372<span class="line"><span style="color:#79B8FF"> "blank_index"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">0</span></span> 373<span class="line"><span style="color:#E1E4E8"> }</span></span> 374<span class="line"><span style="color:#E1E4E8"> ],</span></span> 375<span class="line"></span> 376<span class="line"><span style="color:#79B8FF"> "files"</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">"model.onnx"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"vocab.json"</span><span style="color:#E1E4E8">],</span></span> 377<span class="line"></span> 378<span class="line"><span style="color:#79B8FF"> "metadata"</span><span style="color:#E1E4E8">: {</span></span> 379<span class="line"><span style="color:#79B8FF"> "task"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"speech-recognition"</span><span style="color:#E1E4E8">,</span></span> 380<span class="line"><span style="color:#79B8FF">
380 "architecture"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"wav2vec2"</span><span style="color:#E1E4E8">,</span></span> 381<span class="line"><span style="color:#79B8FF"> "language"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"en"</span></span> 382<span class="line"><span style="color:#E1E4E8"> }</span></span> 383<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var t=s(l,8);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>Input Envelope</span></span> 384<span class="line"><span> â</span></span> 385<span class="line"><span> â¼</span></span> 386<span class="line"><span>âââââââââââââââ</span></span> 387<span class="line"><span>â Preprocess â AudioDecode: WAV bytes â f32 samples at 16kHz</span></span> 388<span class="line"><span>ââââââââ¬âââââââ</span></span> 389<span class="line"><span> â</span></span> 390<span class="line"><span> â¼</span></span> 391<span class="line"><span>âââââââââââââââ</span></span> 392<span class="line"><span>â Model â ONNX inference: samples â logits</span></span> 393<span class="line"><span>ââââââââ¬âââââââ</span></span> 394<span class="line"><span> â</span></span> 395<span class="line"><span> â¼</span></span> 396<span class="line"><span>âââââââââââââââ</span></span> 397<span class="line"><span>â Postprocess â CTCDecode: logits â text via vocab.json</span></span> 398<span class="line"><span>ââââââââ¬âââââââ</span></span> 399<span class="line"><span> â</span></span> 400<span class="line"><span> â¼</span></span> 401<span class="line"><span>Output Envelope</span></span></code></pre>`);var p=s(t,4);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ModelMetadata</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> serde_json</span><span style="color:#F97583">::</span><span style="color:#B392F0">from_str</span><span style="color:#E1E4E8">(</span></span> 402<span class="line"><span style="color:#F97583"> &</span><span style="color:#B392F0">std</span><span style="color:#F97583">::</span><span style="color:#B392F0">fs</span><span style="color:#F97583">::</span><span style="color:#B392F0">read_to_string</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"model_metadata.json"</span><span style="color:#E1E4E8">)</span><span style="color:#F97583">?</span></span> 403<span class="line"><span style="color:#E1E4E8">)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 404<span class="line"></span> 405<span class="line"><span style="color:#F97583">let</span><span style="color:#F97583"> mut</span><span style="color:#E1E4E8"> executor </span><span style="color:#F97583">=</span><span style="color:#B392F0"> TemplateExecutor</span><span style="color:#F97583">::</span><span style="color:#B392F0">with_base_path</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"./model-dir"</span><span style="color:#E1E4E8">);</span></span> 406<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> executor</span><span style="color:#F97583">.</span><span style="color:#B392F0">execute</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">metadata, </span><span style="color:#F97583">&</span><span style="color:#E1E4E8">input)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span></code></pre>`);var r=s(p,10);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>// Standard ONNX model</span></span> 407<span class="line"><span>{ "type": "SimpleMode", "model_file": "model.onnx" }</span></span> 408<span class="line"><span></span></span> 409<span class="line"><span>// GGUF model (llama.cpp)</span></span> 410<span class="line"><span>{</span></span> 411<span class="line"><span>
411 "type": "Gguf",</span></span> 412<span class="line"><span> "model_file": "model.gguf",</span></span> 413<span class="line"><span> "context_length": 4096</span></span> 414<span class="line"><span>}</span></span> 415<span class="line"><span></span></span> 416<span class="line"><span>// Whisper (Candle, autoregressive)</span></span> 417<span class="line"><span>{</span></span> 418<span class="line"><span> "type": "WhisperMode",</span></span> 419<span class="line"><span> "model_file": "model.safetensors",</span></span> 420<span class="line"><span> "tokenizer_file": "tokenizer.json",</span></span> 421<span class="line"><span> "config_file": "config.json"</span></span> 422<span class="line"><span>}</span></span></code></pre>`);var c=s(r,12);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">{</span></span> 423<span class="line"><span style="color:#79B8FF"> "preprocessing"</span><span style="color:#E1E4E8">: [</span></span> 424<span class="line"><span style="color:#E1E4E8"> {</span></span> 425<span class="line"><span style="color:#79B8FF"> "type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Phonemize"</span><span style="color:#E1E4E8">,</span></span> 426<span class="line"><span style="color:#79B8FF"> "backend"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"MisakiDictionary"</span><span style="color:#E1E4E8">,</span></span> 427<span class="line"><span style="color:#79B8FF"> "tokens_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"tokens.txt"</span></span> 428<span class="line"><span style="color:#E1E4E8"> }</span></span> 429<span class="line"><span style="color:#E1E4E8"> ]</span></span> 430<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var d=s(c,12);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">{</span></span> 431<span class="line"><span style="color:#79B8FF"> "metadata"</span><span style="color:#E1E4E8">: {</span></span> 432<span class="line"><span style="color:#79B8FF"> "voices"</span><span style="color:#E1E4E8">: {</span></span> 433<span class="line"><span style="color:#79B8FF"> "voices_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"voices.bin"</span><span style="color:#E1E4E8">,</span></span> 434<span class="line"><span style="color:#79B8FF"> "format"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Float32Array"</span><span style="color:#E1E4E8">,</span></span> 435<span class="line"><span style="color:#79B8FF"> "embedding_dim"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">256</span><span style="color:#E1E4E8">,</span></span> 436<span class="line"><span style="color:#79B8FF"> "voices"</span><span style="color:#E1E4E8">: [</span></span> 437<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"af_heart"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"name"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Heart"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"language"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"en-us"</span><span style="color:#E1E4E8"> },</span></span> 438<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"af_sky"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"name"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Sky"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"language"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"en-us"</span><span style="color:#E1E4E8"> }</span></span> 439<span class="line"><span style="color:#E1E4E8"> ],</span></span> 440<span class="line"><span style="color:#79B8FF"> "default_voice"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"af_heart"</span></span> 441<span class="line"><span style="color:#E1E4E8"> }</span></span> 442<span class="line"><span style="color:#E1E4E8"> }</span></span> 443<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var i=s(d,18);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">
443{</span></span> 444<span class="line"><span style="color:#79B8FF"> "model_id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"wav2vec2-base-960h"</span><span style="color:#E1E4E8">,</span></span> 445<span class="line"><span style="color:#79B8FF"> "execution_template"</span><span style="color:#E1E4E8">: { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"SimpleMode"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"model_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model.onnx"</span><span style="color:#E1E4E8"> },</span></span> 446<span class="line"><span style="color:#79B8FF"> "preprocessing"</span><span style="color:#E1E4E8">: [</span></span> 447<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"AudioDecode"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"sample_rate"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">16000</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"channels"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8"> }</span></span> 448<span class="line"><span style="color:#E1E4E8"> ],</span></span> 449<span class="line"><span style="color:#79B8FF"> "postprocessing"</span><span style="color:#E1E4E8">: [</span></span> 450<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"CTCDecode"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"vocab_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"vocab.json"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"blank_index"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">0</span><span style="color:#E1E4E8"> }</span></span> 451<span class="line"><span style="color:#E1E4E8"> ]</span></span> 452<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var y=s(i,6);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">{</span></span> 453<span class="line"><span style="color:#79B8FF"> "model_id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span><span style="color:#E1E4E8">,</span></span> 454<span class="line"><span style="color:#79B8FF"> "execution_template"</span><span style="color:#E1E4E8">: { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"SimpleMode"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"model_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model.onnx"</span><span style="color:#E1E4E8"> },</span></span> 455<span class="line"><span style="color:#79B8FF"> "preprocessing"</span><span style="color:#E1E4E8">: [</span></span> 456<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Phonemize"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"backend"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"MisakiDictionary"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"tokens_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"tokens.txt"</span><span style="color:#E1E4E8"> }</span></span> 457<span class="line"><span style="color:#E1E4E8"> ],</span></span> 458<span class="line"><span style="color:#79B8FF"> "postprocessing"</span><span style="color:#E1E4E8">: [</span></span> 459<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"TTSAudioEncode"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"sample_rate"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">24000</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"apply_postprocessing"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">true</span><span style="color:#E1E4E8"> }</span></span> 460<span class="line"><span style="color:#E1E4E8"> ]</span></span> 461<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var E=s(y,6);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">
461{</span></span> 462<span class="line"><span style="color:#79B8FF"> "model_id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"qwen3.5-0.8b"</span><span style="color:#E1E4E8">,</span></span> 463<span class="line"><span style="color:#79B8FF"> "execution_template"</span><span style="color:#E1E4E8">: {</span></span> 464<span class="line"><span style="color:#79B8FF"> "type"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Gguf"</span><span style="color:#E1E4E8">,</span></span> 465<span class="line"><span style="color:#79B8FF"> "model_file"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Qwen3.5-0.8B-Q4_K_M.gguf"</span><span style="color:#E1E4E8">,</span></span> 466<span class="line"><span style="color:#79B8FF"> "context_length"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">4096</span></span> 467<span class="line"><span style="color:#E1E4E8"> },</span></span> 468<span class="line"><span style="color:#79B8FF"> "preprocessing"</span><span style="color:#E1E4E8">: [],</span></span> 469<span class="line"><span style="color:#79B8FF"> "postprocessing"</span><span style="color:#E1E4E8">: [],</span></span> 470<span class="line"><span style="color:#79B8FF"> "metadata"</span><span style="color:#E1E4E8">: {</span></span> 471<span class="line"><span style="color:#79B8FF"> "task"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"text-generation"</span><span style="color:#E1E4E8">,</span></span> 472<span class="line"><span style="color:#79B8FF"> "architecture"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"qwen35"</span><span style="color:#E1E4E8">,</span></span> 473<span class="line"><span style="color:#79B8FF"> "backend"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"llamacpp"</span></span> 474<span class="line"><span style="color:#E1E4E8"> }</span></span> 475<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var F=s(E,8);e(F,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// In steps.rs</span></span> 476<span class="line"><span style="color:#E1E4E8">#[derive(</span><span style="color:#B392F0">Debug</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">Clone</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">Serialize</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">Deserialize</span><span style="color:#E1E4E8">)]</span></span> 477<span class="line"><span style="color:#E1E4E8">#[serde(tag </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "type"</span><span style="color:#E1E4E8">)]</span></span> 478<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> enum</span><span style="color:#B392F0"> PreprocessingStep</span><span style="color:#E1E4E8"> {</span></span> 479<span class="line"><span style="color:#B392F0"> AudioDecode</span><span style="color:#E1E4E8"> { sample_rate</span><span style="color:#F97583">:</span><span style="color:#B392F0"> u32</span><span style="color:#E1E4E8">, channels</span><span style="color:#F97583">:</span><span style="color:#B392F0"> u16</span><span style="color:#E1E4E8"> },</span></span> 480<span class="line"><span style="color:#B392F0"> Phonemize</span><span style="color:#E1E4E8"> { backend</span><span style="color:#F97583">:</span><span style="color:#B392F0"> PhonemizerBackend</span><span style="color:#E1E4E8">, tokens_file</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8"> },</span></span> 481<span class="line"><span style="color:#B392F0"> Tokenize</span><span style="color:#E1E4E8"> { vocab_file</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8"> },</span></span> 482<span class="line"><span style="color:#6A737D"> // Add your new step:</span></span> 483<span class="line"><span style="color:#B392F0"> MelSpectrogram</span><span style="color:#E1E4E8"> { n_fft</span><span style="color:#F97583">:</span><span style="color:#B392F0"> usize</span><span style="color:#E1E4E8">, hop_length</span><span style="color:#F97583">:</span><span style="color:#B392F0"> usize</span><span style="color:#E1E4E8">, n_mels</span><span style="color:#F97583">:</span><span style="color:#B392F0"> usize</span><span style="color:#E1E4E8"> },</span></span> 484<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`),f(18),h(o,a)}const es=Object.freeze(Object.defineProperty({__proto__:null,default:Ys,metadata:ss},Symbol.toStringTag,{value:"Module"})),as={title:"On-Device AI: The Complete Guide to Running ML Models Locally",description:"Everything you need to know about running machine learning models directly on mobile and desktop devices â privacy, latency, cost benefits, and how to get started.",date:"2026-03-23",author:"Glenn Sonna",tags:["on-device-ai","edge-inference","mobile-ml","privacy"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!0,readingTime:12},{title:tn,description:pn,date:rn,author:cn,tags:dn,category:yn,image:En,imageAlt:hn,draft:un,featured:mn,readingTime:Fn}=as;var Js=u('<h2>What Is On-Device AI?</h2> <p>On-device AI â also called edge AI inference or local machine learning â refers to executing ML models directly on the end userâs hardware. Instead of sending input data to a remote server, processing it, and returning results, the entire inference pipeline runs on the device itself: the phone in your pocket, the laptop on your desk, or an embedded system in a vehicle.</p> <p>This stands in contrast to the traditional cloud inference model, where every prediction requires a network round-trip to a GPU cluster. Cloud inference has its merits â effectively unlimited compute, easy model updates, centralized logging â but it comes with fundamental tradeoffs in latency, privacy, cost, and availability that on-device inference sidesteps entirely.</p> <p>The distinction matters because it changes the architecture of your application. With cloud inference, your app is a thin client that sends requests and renders responses. With on-device inference, your app contains the model itself. The model ships with your binary (or is downloaded on first launch), loads into device memory, and runs against hardware accelerators that are already sitting idle in most consumer devices.</p> <h2>Why On-Device AI Matters Now</h2> <p>The shift toward on-device AI is not a theoretical trend. It is driven by concrete hardware improvements that have made local inference practical for production workloads.</p> <h3>Neural Processing Units Are Everywhere</h3> <p>Modern mobile and desktop processors ship with dedicated ML accelerators:</p> <ul><li><strong>Apple Neural Engine (ANE):</strong> Present in every iPhone since the A11 (2017) and every Mac with Apple Silicon. The M-series ANE can sustain 15+ TOPS (trillion operations per second), enough to run most sub-1B parameter models in real time.</li> <li><strong>Qualcomm Hexagon NPU:</strong> Found in Snapdragon 8-series and 7-series chipsets powering the majority of Android flagships. The Snapdragon 8 Gen 3 delivers up to 45 TOPS.</li> <li><strong>Intel and AMD NPUs:</strong> Recent laptop processors from both vendors include dedicated AI accelerators, with Intelâs Meteor Lake shipping a dedicated NPU tile.</li></ul> <p>These accelerators are purpose-built for the matrix multiplications and convolutions that dominate ML inference. They are more power-efficient than running the same workload on the CPU or GPU, which means better battery life and lower thermal impact.</p> <h3>Model Compression Has Matured</h3> <p>Running a model on-device requires it to fit in device memory and execute within acceptable latency. Quantization techniques have made this feasible for a wide range of models:</p> <ul><li><strong>INT8 quantization</strong> reduces model size by 4x compared to FP32 with minimal accuracy loss for most tasks.</li> <li><strong>INT4 quantization (Q4_K_M, Q4_0)</strong> pushes this further, enabling 7B-parameter language models to run on phones with 6-8 GB of RAM.</li> <li><strong>Knowledge distillation</strong> produces smaller âstudentâ models that approximate the behavior of larger âteacherâ models at a fraction of the size.</li> <li><strong>Structured pruning</strong> removes entire neurons or attention heads that contribute little to output quality.</li></ul> <p>A model like Whisper Tiny (39M parameters) occupies roughly 75 MB in FP32 and under 40 MB quantized. That is small enough to bundle directly in a mobile app without meaningfully impacting download size.</p> <h2>Privacy and Data Sovereignty</h2> <p>Privacy is not just a feature â for many applications, it is a regulatory requirement. On-device AI provides privacy guarantees that cloud inference fundamentally cannot.</p> <h3>Data Never Leaves the Device</h3> <p>When inference runs locally, the input data â whether it is a voice recording, a medical image, or a financial document â never traverses a network. There is no server to breach, no API logs to subpoena, and no third-party data processing agreement to negotiate.</p> <p>This is not a policy decision that can be reversed. It is an architectural property. If the model runs on-device, the data physically cannot leave unless the application explicitly transmits it.</p> <h3>Regulatory Compliance</h3> <p>Regulations like GDPR, HIPAA, and CCPA impose strict requirements on how personal data is collected, processed, and stored. On-device inference simplifies compliance:</p> <ul><li><strong>GDPR (EU):</strong> Processing personal data on-device can avoid triggering cross-border data transfer rules entirely. There is no âdata controllerâ relationship with a cloud provider for inference data.</li> <li><strong>HIPAA (US Healthcare):</strong> Protected Health Information (PHI) processed locally on a patientâs own device may fall outside the scope of HIPAAâs data handling requirements for the app developer, depending on implementation.</li> <li><strong>Financial Services:</strong> Voice-based banking assistants that transcribe speech on-device avoid the compliance burden of transmitting audio containing account numbers and PINs to external servers.</li></ul> <h3>User Trust</h3> <p>Beyond compliance, on-device processing is a trust signal. Users are increasingly aware of how their data is handled. An app that can truthfully state âyour voice never leaves your deviceâ has a meaningful advantage over one that qualifies with âwe encrypt your data in transit and delete it after processing.â</p> <h2>Latency and Offline Capability</h2> <h3>Real-Time Inference Without Network Dependencies</h3> <p>Cloud inference latency has a hard floor: the network round-trip. Even on a fast connection, sending audio to a server, running inference, and returning results adds 100-500 ms of latency that no amount of model optimization can eliminate. On congested networks or in regi
484ons with limited infrastructure, this balloons to seconds.</p> <p>On-device inference eliminates this entirely. A well-optimized model running on an NPU can return results in 10-50 ms â fast enough for real-time applications like:</p> <ul><li><strong>Live speech transcription</strong> with per-word latency under 100 ms</li> <li><strong>Camera-based object detection</strong> at 30+ FPS</li> <li><strong>Text prediction and autocomplete</strong> that feels instantaneous</li> <li><strong>Voice assistants</strong> that respond before the user finishes their sentence</li></ul> <h3>Offline-First Architecture</h3> <p>On-device AI works without any network connection. This enables use cases that cloud inference cannot serve:</p> <ul><li>Field workers using translation or transcription tools in areas without cellular coverage</li> <li>Aircraft passengers using voice-to-text during flights</li> <li>Medical devices that must function regardless of hospital network status</li> <li>Military and emergency response applications where network availability is not guaranteed</li></ul> <p>Offline capability is not an edge case. It is a reliability guarantee. An app that degrades to âno internet connectionâ when the network drops is fragile. An app that continues to function identically is robust.</p> <h2>Cost at Scale</h2> <p>Cloud inference pricing follows a per-request model. At small scale, this is convenient. At production scale, it becomes a significant line item.</p> <h3>The Math on API Costs</h3> <p>Consider a speech-to-text feature handling 1 million inference requests per month:</p> <table><thead><tr><th>Approach</th><th>Cost per inference</th><th>Monthly cost (1M requests)</th><th>Annual cost</th></tr></thead><tbody><tr><td>Cloud API (typical)</td><td>$0.006 / 15s audio</td><td>$6,000</td><td>$72,000</td></tr><tr><td>Self-hosted GPU</td><td>~$0.001</td><td>$1,000</td><td>$12,000</td></tr><tr><td>On-device</td><td>$0.00</td><td>$0</td><td>$0</td></tr></tbody></table> <p>On-device inference has zero marginal cost per inference. The cost is fixed and upfront: engineering time to integrate the model and the slight increase in app binary size. Once deployed, whether your app serves 1,000 users or 10 million, the inference cost does not change.</p> <p>This cost structure is particularly compelling for consumer apps where per-user revenue is low. A free-tier app cannot absorb $0.006 per API call across millions of daily active users. On-device inference makes AI features viable in products where cloud inference would be economically impossible.</p> <h3>Bandwidth Savings</h3> <p>Sending audio or image data to a server consumes user bandwidth. A 15-second audio clip at 16 kHz mono is approximately 480 KB. At 1 million requests per month, that is 480 GB of upstream data transfer â a real cost for users on metered connections and a real cost for you if you are paying for ingress.</p> <h2>Technical Challenges</h2> <p>On-device AI is not without trade-offs. Understanding these challenges is essential for making informed architecture decisions.</p> <h3>Model Size Constraints</h3> <p>Device memory is limited. A flagship phone in 2026 ships with 8-12 GB of RAM, shared between the operating system, foreground apps, and background processes. Your model needs to fit within the memory budget the OS allocates to your app, which is typically 1-4 GB depending on the platform and device tier.</p> <p>This constrains model selection. A 7B-parameter model quantized to Q4 occupies roughly 4 GB â feasible on flagships, but it will cause out-of-memory kills on mid-range devices. Careful profiling and fallback strategies are required.</p> <h3>Device Fragmentation</h3> <p>The Android ecosystem spans thousands of device configurations with different chipsets, driver versions, and accelerator capabilities. A model that runs correctly on a Snapdragon 8 Gen 3 may behave differently (or fail entirely) on a MediaTek Dimensity or an older Snapdragon 600-series chip.</p> <p>Testing across this matrix is expensive but necessary. Automated device farms help, but they cannot cover every combination. Defensive coding â graceful fallbacks when an accelerator is unavailable, runtime capability detection, and conservative default configurations â is essential.</p> <h3>Thermal Management</h3> <p>Sustained inference generates heat. Mobile devices thermal-throttle aggressively to protect battery health and prevent user discomfort. A model that runs at 50 ms per inference when cold may slow to 200 ms after 30 seconds of continuous use as the device throttles.</p> <p>Batch processing strategies, inference scheduling, and respecting thermal state APIs help manage this, but it remains a constraint that cloud inference does not face.</p> <h3>Model Updates</h3> <p>Cloud models can be updated instantly â deploy a new version to the server, and every subsequent request uses it. On-device models require a download-and-replace cycle. This means:</p> <ul><li>Users may run different model versions simultaneously</li> <li>Model updates compete with app update fatigue</li> <li>Rollback requires another download cycle</li> <li>A/B testing requires shipping multiple model variants</li></ul> <h2>The Hybrid Approach: Intelligent Routing</h2> <p>The most practical architecture for production applications is neither pure cloud nor pure on-device. It is a hybrid approach that routes inference requests to the optimal backend based on real-time conditions.</p> <h3>
484When to Run On-Device</h3> <ul><li>The model fits comfortably in device memory</li> <li>Latency requirements are strict (under 100 ms)</li> <li>Privacy constraints prohibit data transmission</li> <li>The device has a capable accelerator for the model type</li> <li>Network connectivity is unreliable or unavailable</li></ul> <h3>When to Route to Cloud</h3> <ul><li>The model is too large for the target device (e.g., 70B+ parameter LLMs)</li> <li>The task requires capabilities not available on-device (e.g., image generation at high resolution)</li> <li>The device is thermal-throttled and inference quality would degrade</li> <li>A newer, more accurate model is available server-side</li></ul> <h3>Intelligent Routing in Practice</h3> <p>A well-designed orchestrator evaluates these factors at runtime and routes each request to the appropriate backend. The application code remains the same â it submits an inference request and receives a result. Whether that result came from the local NPU or a cloud GPU is an infrastructure decision, not an application concern.</p> <h2>Getting Started with Xybrid</h2> <p><a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a> is an open-source ML inference orchestrator that runs models on-device across iOS, Android, macOS, Linux, and Windows. It handles model downloading, caching, and hardware acceleration selection so you can focus on your application logic.</p> <h3>Flutter</h3> <p>The Flutter SDK is the fastest path to on-device inference in a cross-platform app:</p> <!> <p>First run downloads the model from the registry and caches it locally. Subsequent runs load from cache with no network dependency.</p> <h3>Swift and Kotlin</h3> <p>Native mobile SDKs follow the same pattern:</p> <!> <!> <h3>CLI</h3> <p>For quick evaluation without writing code:</p> <!> <p>Models are hosted on HuggingFace and cached locally after first download.</p> <h2>Where On-Device AI Is Headed</h2> <p>The trajectory is clear. Hardware accelerators are getting faster and more power-efficient with each generation. Models are getting smaller through better training techniques, architecture innovations, and quantization methods. The gap between what you can run on-device and what requires a data center is narrowing rapidly.</p> <p>Within the next two years, expect sub-3B parameter models to match the quality of todayâs 7-13B models for domain-specific tasks. Expect NPU performance to double again. Expect on-device inference to become the default for latency-sensitive, privacy-sensitive, and cost-sensitive applications.</p> <p>The question is no longer whether on-device AI is viable. It is whether your application can afford not to use it. Network latency, API costs, and privacy concerns are not going away. The hardware to solve them is already in your usersâ hands.</p> <p>Start building for it.</p>',1);function Zs(o){var a=Js(),n=s(b(a),136);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">import</span><span style="color:#9ECBFF"> 'package:xybrid_flutter/xybrid_flutter.dart'</span><span style="color:#E1E4E8">;</span></span> 485<span class="line"></span> 486<span class="line"><span style="color:#F97583">await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">init</span><span style="color:#E1E4E8">();</span></span> 487<span class="line"></span> 488<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'whisper-tiny'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 489<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span></span> 490<span class="line"><span style="color:#E1E4E8"> envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">audio</span><span style="color:#E1E4E8">(bytes</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> audioBytes),</span></span> 491<span class="line"><span style="color:#E1E4E8">);</span></span> 492<span class="line"><span style="color:#B392F0">print</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">'Transcription: </span><span style="color:#9ECBFF">\${</span><span style="color:#79B8FF">result</span><span style="color:#9ECBFF">.</span><span style="color:#79B8FF">text</span><span style="color:#9ECBFF">}</span><span style="color:#9ECBFF">'</span><span style="color:#E1E4E8">);</span></span></code></pre>`);var l=s(n,8);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8">
492 xybrid </span><span style="color:#F97583">=</span><span style="color:#F97583"> try</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">()</span></span> 493<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> try</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> xybrid.</span><span style="color:#79B8FF">model</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">id</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span><span style="color:#E1E4E8">).</span><span style="color:#79B8FF">load</span><span style="color:#E1E4E8">()</span></span> 494<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#F97583"> try</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> model.</span><span style="color:#79B8FF">run</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">envelope</span><span style="color:#E1E4E8">: .</span><span style="color:#79B8FF">text</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Hello from Swift"</span><span style="color:#E1E4E8">))</span></span></code></pre>`);var t=s(l,2);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8">
494 xybrid </span><span style="color:#F97583">=</span><span style="color:#B392F0"> Xybrid</span><span style="color:#E1E4E8">()</span></span> 495<span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> xybrid.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(id </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "kokoro-82m"</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">()</span></span> 496<span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(envelope </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> Envelope.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Hello from Kotlin"</span><span style="color:#E1E4E8">))</span></span></code></pre>`);var p=s(t,6);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> install</span><span style="color:#9ECBFF"> xybrid-cli</span></span> 497<span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> kokoro-82m</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> "Hello from the edge"</span><span style="color:#79B8FF"> --output</span><span style="color:#9ECBFF"> hello.wav</span></span></code></pre>`),f(12),h(o,a)}const ns=Object.freeze(Object.defineProperty({__proto__:null,default:Zs,metadata:as},Symbol.toStringTag,{value:"Module"})),os={title:"On-Device AI for Mobile Apps: A Developer's Guide to iOS and Android",description:"How to add on-device AI to iOS and Android apps â comparing CoreML, ONNX Runtime, TensorFlow Lite, and llama.cpp. Practical guide with model sizes, performance benchmarks, and integration patterns.",date:"2026-06-23",author:"Glenn Sonna",tags:["on-device-ai","mobile-ai","ai-on-iphone","ai-on-android","coreml","mobile-ml"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:12},{title:gn,description:fn,date:bn,author:Bn,tags:vn,category:kn,image:wn,imageAlt:xn,draft:Cn,featured:_n,readingTime:An}=os;var se=u('<p>Adding AI features to a mobile app used to mean integrating a cloud API. Send user data to a server, wait for a response, pay per request. That model works, but it is no longer the only option â and for many features, it is no longer the best one.</p> <p>Modern smartphones have hardware specifically designed for ML inference: Appleâs Neural Engine, Qualcommâs Hexagon DSP, Googleâs Tensor TPU. A flagship phone from 2024 can run a speech recognition model in under 100ms, generate natural-sounding speech, classify images in real-time, and even run a 1B parameter language model. All without an internet connection.</p> <p>This guide covers the practical side of adding on-device AI to iOS and Android apps: which frameworks to use, which models fit on mobile hardware, what performance to expect, and how to handle the engineering challenges of bundling and running models on constrained devices.</p> <h2>The Mobile AI Hardware Landscape</h2> <p>Before choosing a framework, understand what hardware is available to you.</p> <h3>iOS (Apple Silicon)</h3> <p>Every iPhone since the iPhone 8 (2017) includes a Neural Engine â dedicated hardware for ML inference. The capability has scaled dramatically:</p> <table><thead><tr><th>Chip</th><th>Neural Engine</th><th>TOPS</th><th>Available Memory</th></tr></thead><tbody><tr><td>A11 (iPhone 8/X)</td><td>2-core</td><td>0.6</td><td>2-3 GB</td></tr><tr><td>A14 (iPhone 12)</td><td>16-core</td><td>11</td><td>4 GB</td></tr><tr><td>A15 (iPhone 13/14)</td><td>16-core</td><td>15.8</td><td>4-6 GB</td></tr><tr><td>A16 (iPhone 14 Pro/15)</td><td>16-core</td><td>17</td><td>6 GB</td></tr><tr><td>A17 Pro (iPhone 15 Pro)</td><td>16-core</td><td>35</td><td>8 GB</td></tr><tr><td>A18 Pro (iPhone 16 Pro)</td><td>16-core</td><td>35+</td><td>8 GB</td></tr></tbody></table> <p>The Neural Engine is accessed through CoreML, Appleâs ML framework. Models that CoreML can optimize for the Neural Engine run dramatically faster than CPU-only inference â we have measured 6.8x speedups on vision models like MobileNet.</p> <p>The GPU (Metal) is also available for ML workloads that do not map well to the Neural Engine, such as models with dynamic shapes or certain attention patterns.</p> <h3>Android</h3> <p>Android hardware is more varied. The key accelerators:</p> <p><strong>Qualcomm (Snapdragon):</strong> The Hexagon DSP and AI Engine provide ML acceleration. Accessed via ONNX Runtimeâs QNN provider or Qualcommâs AI Hub SDK. Performance varies significantly by chipset generation.</p> <p><strong>Google Tensor:</strong> Custom chips in Pixel phones with dedicated TPU cores. Accessed via TensorFlow Lite with the Google Edge TPU delegate.</p> <p><strong>MediaTek:</strong> APU (AI Processing Unit) available in Dimensity chips. Accessed via the MediaTek NeuroPilot SDK or ONNX Runtime.</p> <p><strong>Samsung (Exynos):</strong> NPU available in some Galaxy devices. Accessed via Samsungâs Neural SDK.</p> <p>The fragmentation is real. An AI feature that runs great on a Pixel 8 (Tensor G3) might be slow on a mid-range Snapdragon. Practical approach: target CPU inference as the baseline, use hardware acceleration as an optimization for specific chipsets.</p> <table><thead><tr><th>
497Chipset</th><th>Category</th><th>AI Performance</th><th>Available Memory</th></tr></thead><tbody><tr><td>Snapdragon 8 Gen 3</td><td>Flagship</td><td>45 TOPS</td><td>8-16 GB</td></tr><tr><td>Snapdragon 8 Gen 2</td><td>Flagship</td><td>33 TOPS</td><td>8-12 GB</td></tr><tr><td>Snapdragon 7 Gen 1</td><td>Mid-range</td><td>13 TOPS</td><td>6-8 GB</td></tr><tr><td>Tensor G3 (Pixel 8)</td><td>Flagship</td><td>~20 TOPS</td><td>12 GB</td></tr><tr><td>Dimensity 9300</td><td>Flagship</td><td>41 TOPS</td><td>8-16 GB</td></tr></tbody></table> <h2>Inference Frameworks Compared</h2> <p>Five frameworks dominate mobile ML inference. Each has different strengths.</p> <h3>CoreML (iOS only)</h3> <p>Appleâs native ML framework. First-class access to the Neural Engine, GPU (Metal), and CPU. Tightly integrated with Xcode and the Apple ecosystem.</p> <p><strong>Strengths:</strong></p> <ul><li>Best Neural Engine utilization on iOS</li> <li>Optimized for Apple Silicon</li> <li>Supports model conversion from ONNX, TensorFlow, and PyTorch</li> <li>Xcode integration with model preview and profiling</li></ul> <p><strong>Tradeoffs:</strong></p> <ul><li>iOS and macOS only</li> <li>Limited model format support (must convert to .mlmodel/.mlpackage)</li> <li>Less control over execution providers than ONNX Runtime</li> <li>Dynamic shapes can force CPU fallback (losing Neural Engine benefits)</li></ul> <p><strong>Best for:</strong> iOS-exclusive apps where maximum performance on Apple hardware matters.</p> <h3>ONNX Runtime</h3> <p>Microsoftâs cross-platform inference engine. Runs on iOS (CoreML backend), Android (NNAPI, QNN), and desktop. Supports the standardized ONNX model format.</p> <p><strong>Strengths:</strong></p> <ul><li>Cross-platform: same model runs on iOS, Android, Windows, macOS, Linux</li> <li>Multiple execution providers (CoreML, CUDA, DirectML, NNAPI, QNN)</li> <li>Broad model format support</li> <li>Active development and optimization</li></ul> <p><strong>Tradeoffs:</strong></p> <ul><li>Binary size is significant (~15-30 MB depending on providers)</li> <li>CoreML provider on iOS does not always match native CoreML performance</li> <li>NNAPI on Android is deprecated in favor of vendor-specific APIs</li></ul> <p><strong>Best for:</strong> Cross-platform apps that need to run the same model on iOS and Android. Vision, audio, and embedding models.</p> <h3>TensorFlow Lite</h3> <p>Googleâs mobile inference framework. Mature ecosystem with good model conversion tools and Android-first optimizations.</p> <p><strong>Strengths:</strong></p> <ul><li>Mature model conversion from TensorFlow/Keras</li> <li>Good GPU delegate for Android</li> <li>Google Edge TPU support on Pixel devices</li> <li>Extensive model zoo and documentation</li></ul> <p><strong>Tradeoffs:</strong></p> <ul><li>TensorFlow ecosystem lock-in</li> <li>iOS support exists but is not the primary focus</li> <li>Less active development than ONNX Runtime</li> <li>Cannot run ONNX models directly</li></ul> <p><strong>Best for:</strong> Android-first apps already in the TensorFlow ecosystem.</p> <h3>llama.cpp</h3> <p>C/C++ library for LLM inference. The de facto standard for running language models on consumer hardware, including mobile.</p> <p><strong>Strengths:</strong></p> <ul><li>Best local LLM inference performance</li> <li>Metal acceleration on iOS, Vulkan on Android</li> <li>Minimal dependencies, small binary</li> <li>GGUF format is the standard for quantized LLMs</li></ul> <p><strong>Tradeoffs:</strong></p> <ul><li>LLMs only (not for vision, audio, or embeddings)</li> <li>Requires C/C++ FFI from Swift, Kotlin, Dart, etc.</li> <li>Memory management on mobile requires careful handling</li></ul> <p><strong>Best for:</strong> Any app that needs to run an LLM on-device.</p> <h3>Platform Summary</h3> <table><thead><tr><th>Feature</th><th>CoreML</th><th>ONNX Runtime</th><th>TF Lite</th><th>llama.cpp</th></tr></thead><tbody><tr><td>iOS</td><td>Native</td><td>Via CoreML EP</td><td>Supported</td><td>Metal</td></tr><tr><td>Android</td><td>No</td><td>NNAPI/QNN</td><td>Native</td><td>Vulkan</td></tr><tr><td>Model types</td><td>Any</td><td>Any</td><td>Any</td><td>LLMs only</td></tr><tr><td>LLM support</td><td>Limited</td><td>Limited</td><td>Limited</td><td>Excellent</td></tr><tr><td>Neural Engine</td><td>Yes</td><td>Via CoreML</td><td>No</td><td>No</td></tr><tr><td>Binary size</td><td>0 (system)</td><td>15-30 MB</td><td>5-10 MB</td><td>2-5 MB</td></tr></tbody></table> <p><strong>Practical recommendation:</strong> Use ONNX Runtime or CoreML for non-LLM models (ASR, TTS, embeddings, vision). Use llama.cpp for LLMs. This gives you the best performance for each model type.</p> <h2>What Runs on a Phone Today</h2> <p>Here are models that run well on current mobile hardware, tested on real devices:</p> <h3>Speech Recognition (ASR)</h3> <table><thead><tr><th>
497Model</th><th>Size</th><th>iPhone 15 Pro</th><th>Pixel 8 Pro</th><th>Quality</th></tr></thead><tbody><tr><td>Whisper Tiny</td><td>75 MB</td><td>~60ms/sec audio</td><td>~90ms/sec audio</td><td>Good for English</td></tr><tr><td>Whisper Base</td><td>140 MB</td><td>~120ms/sec audio</td><td>~180ms/sec audio</td><td>Better multilingual</td></tr><tr><td>Whisper Small</td><td>460 MB</td><td>~400ms/sec audio</td><td>~600ms/sec audio</td><td>Best mobile quality</td></tr></tbody></table> <p>Whisper Tiny processes audio faster than real-time on any modern phone. It handles English accurately and is the best choice for real-time transcription on mobile.</p> <h3>Text-to-Speech (TTS)</h3> <table><thead><tr><th>Model</th><th>Size</th><th>iPhone 15 Pro</th><th>Pixel 8 Pro</th><th>Quality</th></tr></thead><tbody><tr><td>Kokoro 82M</td><td>180 MB</td><td>~0.8s per sentence</td><td>~1.2s per sentence</td><td>Natural, expressive</td></tr><tr><td>Piper (onnx)</td><td>60-100 MB</td><td>~0.3s per sentence</td><td>~0.5s per sentence</td><td>Fast, slightly robotic</td></tr></tbody></table> <p>Kokoro produces remarkably natural speech for its size. The slight synthesis delay is acceptable for most use cases â users do not notice a sub-second pause before speech begins.</p> <h3>Language Models (LLM)</h3> <table><thead><tr><th>Model</th><th>Size (Q4)</th><th>iPhone 15 Pro</th><th>Pixel 8 Pro</th><th>Quality</th></tr></thead><tbody><tr><td>Qwen 3.5 0.8B</td><td>~500 MB</td><td>~20 tok/s</td><td>~12 tok/s</td><td>Good for simple tasks</td></tr><tr><td>Llama 3.2 1B</td><td>~700 MB</td><td>~18 tok/s</td><td>~10 tok/s</td><td>Better reasoning</td></tr><tr><td>Llama 3.2 3B</td><td>~1.8 GB</td><td>~8 tok/s</td><td>~5 tok/s</td><td>Desktop-like quality</td></tr></tbody></table> <p>The 1B models are the sweet spot for mobile: fast enough for conversational use, small enough to download over cellular, and capable enough for summarization, Q&A, and simple chat.</p> <h3>Embeddings</h3> <table><thead><tr><th>Model</th><th>Size</th><th>iPhone 15 Pro</th><th>Pixel 8 Pro</th><th>Use Case</th></tr></thead><tbody><tr><td>MiniLM-L6</td><td>80 MB</td><td>~5ms per query</td><td>~8ms per query</td><td>Semantic search</td></tr><tr><td>all-MiniLM-L6</td><td>80 MB</td><td>~5ms per query</td><td>~8ms per query</td><td>Similarity matching</td></tr></tbody></table> <p>Embedding models are small, fast, and enable powerful features: on-device semantic search, document clustering, and RAG without sending documents to the cloud.</p> <h2>Integration Patterns</h2> <h3>Pattern 1: Download on First Launch</h3> <p>The most common approach. The app ships without models. On first launch, it downloads the required model(s) from a CDN.</p> <p><strong>Pros:</strong> Small initial app size. Models can be updated independently of the app. <strong>Cons:</strong> First-launch experience requires download and wait time. Needs network on first use.</p> <p><strong>Implementation notes:</strong></p> <ul><li>Show a progress indicator with estimated time</li> <li>Download in the background if possible</li> <li>Cache the model in the appâs documents directory (persists across updates)</li> <li>Verify integrity with a SHA-256 checksum after download</li></ul> <h3>Pattern 2: Bundle in App</h3> <p>Ship the model file inside the app binary. The app works immediately with no download step.</p> <p><strong>Pros:</strong> Works offline from first launch. No download management code. <strong>Cons:</strong> Increases app binary size significantly. App Store size limits apply (iOS: 4 GB, but >200 MB triggers a warning. Google Play: 150 MB AAB + 2 GB expansion files).</p> <p><strong>Best for:</strong> Apps where the model is small (under 50 MB) or where offline-first is a hard requirement.</p> <h3>Pattern 3: Lazy Loading With Fallback</h3> <p>Load models on demand when the user first accesses the feature. If the model is not available, show a download prompt or fall back to a cloud API.</p> <p><strong>Pros:</strong> Minimizes initial footprint. Only downloads what the user actually uses. <strong>Cons:</strong> Feature is not available until model is downloaded. Requires cloud fallback or graceful degradation.</p> <h2>Memory Management on Mobile</h2> <p>Mobile devices have strict memory limits. iOS will kill your app if it uses too much memory. Android has similar constraints.</p> <p><strong>Rules of thumb:</strong></p> <ul><li>Keep total model memory under 50% of available RAM</li> <li>Unload models when not actively in use</li> <li>Use lazy loading â do not load all models at app startup</li> <li>Monitor memory warnings (iOS <code>didReceiveMemoryWarning</code>, Android <code>onTrimMemory</code>)</li> <li>Prefer quantized models (Q4) to reduce memory footprint</li></ul> <p>For LLMs, the context window length directly impacts memory usage. A 1B model with a 2K context uses significantly less memory than the same model with an 8K context. Cap the context length to what the use case actually requires.</p> <h2>Performance Optimization</h2> <h3>Use Hardware Acceleration</h3> <ul><li><strong>iOS:</strong> CoreML with Neural Engine for vision/audio models. Metal for LLMs via llama.cpp.</li> <li><strong>Android:</strong> NNAPI or vendor-specific delegates for vision/audio. Vulkan for LLMs via llama.cpp.</li></ul> <p>Hardware acceleration can provide 2-10x speedups over CPU-only inference, depending on the model architecture.</p> <h3>Profile Before Optimizing</h3> <p>Use platform profiling tools to identify bottlenecks:</p> <ul><li><strong>iOS:</strong> Xcode Instruments (Neural Engine, GPU, CPU profiling)</li> <li><strong>Android:</strong> Android Studio Profiler, Perfetto for system-level tracing</li></ul> <p>Common bottleneck: preprocessing (decoding audio, resizing images) can take longer than inference itself. Optimize the full pipeline, not just the model execution.</p> <h3>Quantize Aggressively</h3> <p>Q4_K_M quantization reduces model size by 4x compared to FP16 with minimal quality loss. For mobile, always use quantized models. The quality difference is negligible for most tasks, and the memory and speed improvements are substantial.</p> <h2>Getting Started</h2> <ol><li><p><strong>Pick your model.</strong> Start with one model for one feature. Speech recognition (Whisper Tiny), TTS (Kokoro), embeddings (MiniLM), or chat (Llama 3.2 1B) are all proven on mobile.</p></li> <li><p><strong>Pick your runtime.</strong> ONNX Runtime for non-LLM models, llama.cpp for LLMs. Both work on iOS and Android.</p></li> <li><p><strong>Prototype on-device.</strong> Get the model running on a physical device (not a simulator). Measure actual latency and memory usage.</p></li> <li><p><strong>Handle the packaging.</strong> Decide between download-on-first-launch and bundled. For models over 50 MB, download-on-first-launch is usually the better UX.</p></li> <li><p><strong>Test on lower-end devices.</strong> Your model runs fast on an iPhone 15 Pro. Does it run acceptably on an iPhone 12? On a Pixel 6a? Define your minimum supported hardware and benchmark there.</p></li></ol> <p>The hardware is ready. The models are small enough. The frameworks are mature enough. On-device AI on mobile is not a future possibility â it is a present capability waiting for developers to use it.</p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide to Running ML Models Locally</a> â the broader guide covering desktop and mobile inference.</li> <li><a href="/blog/run-llm-locally-complete-guide">How to Run LLMs Locally</a> â deep dive into local LLM runtimes, quantization, and model selection.</li> <li><a href="/blog/add-text-to-speech-to-flutter-app-in-15-minutes">Add Text-to-Speech to Your Flutter App in 15 Minutes</a> â hands-on tutorial for on-device TTS in Flutter.</li> <li><a href="/blog/building-voice-agent-on-device">Building a Voice Agent That Runs Entirely On-Device</a> â full voice pipeline tutorial: ASR + LLM + TTS on mobile.</li></ul>',1);function ee(o){var a=se();f(196),h(o,a)}const ls=Object.freeze(Object.defineProperty({__proto__:null,default:ee,metadata:os},Symbol.toStringTag,{value:"Module"})),ts={title:"One Codebase, Five Platforms: Shipping ML to Flutter, Swift, Kotlin, Unity & CLI",description:"How UniFFI, flutter_rust_bridge, and C FFI let us write ML inference once in Rust and bind it to five platform SDKs. The cross-platform FFI playbook.",date:"2026-06-05",author:"Glenn Sonna",tags:["rust","flutter","kotlin","gamedev"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:8}
497,{title:Tn,description:Sn,date:Mn,author:Pn,tags:In,category:Ln,image:Dn,imageAlt:Rn,draft:On,featured:Nn,readingTime:Gn}=ts;var ae=u('<p>We had a rule: <strong>every platform gets the same inference behavior</strong>. Not âsimilar.â Not âequivalent.â The same.</p> <p>Same model. Same preprocessing. Same output bytes. Whether youâre on an iPhone, an Android tablet, a Unity game, or a terminal.</p> <p>The only way to guarantee that? One implementation, many bindings.</p> <p>Hereâs how we ship <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a> to five platform SDKs from a single Rust core â and the FFI playbook we developed along the way.</p> <hr/> <h2>The Problem</h2> <p>You want to add TTS to your app. You find a great model. Now:</p> <ul><li><strong>iOS team</strong> writes inference code in Swift</li> <li><strong>Android team</strong> writes it in Kotlin</li> <li><strong>Flutter team</strong> writes it in Dart</li> <li><strong>Game team</strong> writes it in C#</li> <li><strong>Backend team</strong> writes it in⦠Python? Rust? Node?</li></ul> <p>Five implementations. Five sets of bugs. Five preprocessing behaviors that <em>should</em> be identical but inevitably drift. Someone hard-codes a sample rate. Someone normalizes differently. Users get different audio quality on different platforms.</p> <p>This is the N-platform problem. And FFI is the solution.</p> <h2>The Architecture</h2> <!> <p>One Rust core. Three FFI strategies. Five platform SDKs.</p> <h2>Strategy 1: flutter_rust_bridge (Flutter)</h2> <p><strong>flutter_rust_bridge (FRB)</strong> auto-generates Dart bindings from Rust code. Itâs the most sophisticated of our three FFI tools.</p> <h3>What It Handles</h3> <ul><li>Async functions â Dart <code>Future</code>s</li> <li>Streaming callbacks â Dart <code>Stream</code>s</li> <li>Complex types â Dart classes</li> <li>Error types â Dart exceptions</li> <li>Memory management â automatic</li></ul> <h3>The Rust API</h3> <!> <h3>The Generated Dart API</h3> <!> <p>FRB handles the entire async dance: spawning the Rust code on an isolate, marshalling data across FFI, and surfacing results on the Dart side.</p> <h3>The High-Level Dart Wrapper</h3> <p>We add a thin, idiomatic Dart layer on top:</p> <!> <p><strong>Total Dart code</strong>: ~800 lines (wrapper + types). All inference logic is in Rust.</p> <h2>Strategy 2: UniFFI (Swift + Kotlin)</h2> <p><strong>UniFFI</strong> (Mozillaâs tool) generates Swift and Kotlin bindings from a single Rust definition. Itâs less magical than FRB but covers two platforms at once.</p> <h3>The UDL Definition</h3> <!> <h3>Generated Swift</h3> <!> <h3>Generated Kotlin</h3> <!> <p>One UDL file â two platform SDKs. The generated code is idiomatic â Swift gets optionals and throwing functions, Kotlin gets nullable types and exceptions.</p> <h3>Adding a Higher-Level Wrapper</h3> <p>Like Flutter, we wrap the raw bindings:</p> <!> <p><strong>Total wrapper code per platform</strong>: ~150 lines.</p> <h2>Strategy 3: C FFI + cbindgen (Unity)</h2> <p>Unityâs C# interop requires raw C headers. No fancy codegen â just <code>extern "C"</code> functions and <code>DllImport</code>.</p> <h3>The Rust C API</h3> <!> <p><strong>cbindgen</strong> generates the C header:</p> <!> <h3>The C# Wrapper (Unity)</h3> <!> <p>More boilerplate than UniFFI, but itâs the only option for Unity. The total C# wrapper is ~300 lines.</p> <h2>The FFI Decision Matrix</h2> <table><thead><tr><th></th><th>FRB</th><th>UniFFI</th><th>C FFI</th></tr></thead><tbody><tr><td><strong>Languages</strong></td><td>Dart</td><td>Swift + Kotlin</td><td>Any (C ABI)</td></tr><tr><td><strong>Async support</strong></td><td>Native (Futures, Streams)</td><td>Manual</td><td>Manual</td></tr><tr><td><strong>Type richness</strong></td><td>High (enums, generics)</td><td>Medium (UDL types)</td><td>Low (C types)</td></tr><tr><td><strong>Code generation</strong></td><td>Automatic from Rust</td><td>From UDL definition</td><td>From Rust (cbindgen)</td></tr><tr><td><strong>Memory management</strong></td><td>Automatic</td><td>Automatic</td><td>Manual (alloc/free)</td></tr><tr><td><strong>Best for</strong></td><td>Flutter</td><td>Mobile native SDKs</td><td>Game engines, C interop</td></tr></tbody></table> <p>If we were starting over and only needed one tool, UniFFI covers the most ground. But FRBâs async/streaming support is essential for Flutter, and C FFI is the only option for Unity.</p> <h2>Lessons Learned</h2> <h3>1. Keep the FFI Surface Small</h3> <p>Every function in the FFI layer is a maintenance point across all platforms. We minimize it:</p> <!> <p>~7 functions serve the entire SDK. All complexity is inside Rust.</p> <h3>2. Use Wrapper Types, Not Domain Types</h3> <p>
497Donât expose internal types across FFI. Use flat, serializable wrapper types:</p> <!> <p>Conversion happens at the boundary. Internal types can evolve without breaking FFI.</p> <h3>3. Test Across Platforms in CI</h3> <p>We run platform-specific CI for each binding:</p> <!> <p>A Rust change that breaks any platform is caught before merge.</p> <h3>4. API Contract as Source of Truth</h3> <p>We maintain <code>api-surface.yaml</code> that defines the public API across all SDKs:</p> <!> <p>SDK implementations are validated against this contract. No platform can drift.</p> <hr/> <h2>The Payoff</h2> <p>From one Rust codebase:</p> <table><thead><tr><th>Platform</th><th>SDK</th><th>Binding Lines</th><th>Full Feature Parity</th></tr></thead><tbody><tr><td>Flutter</td><td>xybrid_flutter</td><td>~800</td><td>Yes</td></tr><tr><td>Swift</td><td>Xybrid (SPM)</td><td>~400</td><td>Yes</td></tr><tr><td>Kotlin</td><td>xybrid-kotlin (Maven)</td><td>~400</td><td>Yes</td></tr><tr><td>Unity</td><td>Xybrid.cs</td><td>~300</td><td>Yes</td></tr><tr><td>CLI</td><td>xybrid-cli</td><td>~500</td><td>Yes</td></tr></tbody></table> <p>~2,400 lines of binding code for five platforms. The Rust core is ~15,000 lines. Thatâs a 6:1 ratio of shared-to-platform-specific code.</p> <p>One model, one metadata file, one preprocessing pipeline, one set of tests â five platforms that behave identically.</p> <hr/> <p><strong>Explore the SDKs</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <hr/> <p><em>Building cross-platform AI features? Whatâs your FFI strategy? Share in the comments.</em></p>',1);function ne(o){var a=ae(),n=s(b(a),22);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>ââââââââââââââââââââââââââââââââââââââââââââ</span></span> 498<span class="line"><span>â Platform SDKs â</span></span> 499<span class="line"><span>â â</span></span> 500<span class="line"><span>â âââââââââââ âââââââââ ââââââââââ â</span></span> 501<span class="line"><span>â â Flutter â â Swift â â Kotlin â â</span></span> 502<span class="line"><span>â â (Dart) â â â â â â</span></span> 503<span class="line"><span>â ââââââ¬ââââââ âââââ¬ââââ âââââ¬âââââ â</span></span> 504<span class="line"><span>â â FRB â UniFFI â UniFFI â</span></span> 505<span class="line"><span>â ââââââ´ââââ ââââââ´âââââââââ´âââââ â</span></span> 506<span class="line"><span>â â Unity â â â â</span></span> 507<span class="line"><span>â â (C#) â â â â</span></span> 508<span class="line"><span>â ââââââ¬âââââ â xybrid-sdk â â</span></span> 509<span class="line"><span>â â C FFI â xybrid-core â â</span></span> 510<span class="line"><span>â â â (Rust) â â</span></span> 511<span class="line"><span>â ââââââââ⤠â â</span></span> 512<span class="line"><span>â âââââââââââââââââââ â</span></span> 513<span class="line"><span>ââââââââââââââââââââââââââââââââââââââââââââ</span></span></code></pre>`);var l=s(n,14);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// This is the FRB-visible API</span></span> 514<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> run_model</span><span style="color:#E1E4E8">(model_id</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">, input</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ApiEnvelope</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> Result</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">ApiEnvelope</span><span style="color:#E1E4E8">> {</span></span> 515<span class="line"><span style="color:#F97583">
515 let</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#B392F0"> load_model</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">model_id)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 516<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model</span><span style="color:#F97583">.</span><span style="color:#B392F0">execute</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">input</span><span style="color:#F97583">.</span><span style="color:#B392F0">into</span><span style="color:#E1E4E8">())</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 517<span class="line"><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(output</span><span style="color:#F97583">.</span><span style="color:#B392F0">into</span><span style="color:#E1E4E8">())</span></span> 518<span class="line"><span style="color:#E1E4E8">}</span></span> 519<span class="line"></span> 520<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> run_streaming</span><span style="color:#E1E4E8">(</span></span> 521<span class="line"><span style="color:#E1E4E8"> model_id</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">,</span></span> 522<span class="line"><span style="color:#E1E4E8"> input</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ApiEnvelope</span><span style="color:#E1E4E8">,</span></span> 523<span class="line"><span style="color:#E1E4E8"> sink</span><span style="color:#F97583">:</span><span style="color:#B392F0"> StreamSink</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">ApiPartialToken</span><span style="color:#E1E4E8">>,</span></span> 524<span class="line"><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> Result</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">ApiEnvelope</span><span style="color:#E1E4E8">> {</span></span> 525<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> callback </span><span style="color:#F97583">=</span><span style="color:#F97583"> move</span><span style="color:#F97583"> |</span><span style="color:#E1E4E8">token</span><span style="color:#F97583">:</span><span style="color:#B392F0"> PartialToken</span><span style="color:#F97583">|</span><span style="color:#E1E4E8"> {</span></span> 526<span class="line"><span style="color:#E1E4E8"> sink</span><span style="color:#F97583">.</span><span style="color:#B392F0">add</span><span style="color:#E1E4E8">(token</span><span style="color:#F97583">.</span><span style="color:#B392F0">into</span><span style="color:#E1E4E8">())</span><span style="color:#F97583">.</span><span style="color:#B392F0">is_ok</span><span style="color:#E1E4E8">()</span></span> 527<span class="line"><span style="color:#E1E4E8"> };</span></span> 528<span class="line"><span style="color:#6A737D"> // ...</span></span> 529<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var t=s(l,4);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Auto-generated â we don't write this</span></span> 530<span class="line"><span style="color:#79B8FF">Future</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">ApiEnvelope</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">runModel</span><span style="color:#E1E4E8">({</span></span> 531<span class="line"><span style="color:#F97583"> required</span><span style="color:#79B8FF"> String</span><span style="color:#E1E4E8"> modelId,</span></span> 532<span class="line"><span style="color:#F97583"> required</span><span style="color:#79B8FF"> ApiEnvelope</span><span style="color:#E1E4E8"> input,</span></span> 533<span class="line"><span style="color:#E1E4E8">}
533) </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> { ... }</span></span> 534<span class="line"></span> 535<span class="line"><span style="color:#79B8FF">Stream</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">ApiPartialToken</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">runStreaming</span><span style="color:#E1E4E8">({</span></span> 536<span class="line"><span style="color:#F97583"> required</span><span style="color:#79B8FF"> String</span><span style="color:#E1E4E8"> modelId,</span></span> 537<span class="line"><span style="color:#F97583"> required</span><span style="color:#79B8FF"> ApiEnvelope</span><span style="color:#E1E4E8"> input,</span></span> 538<span class="line"><span style="color:#E1E4E8">}) { ... }</span></span></code></pre>`);var p=s(t,8);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">class</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8"> {</span></span> 539<span class="line"><span style="color:#F97583"> static</span><span style="color:#79B8FF"> Future</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">void</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">init</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> { ... }</span></span> 540<span class="line"><span style="color:#F97583"> static</span><span style="color:#79B8FF"> XybridModelBuilder</span><span style="color:#B392F0"> model</span><span style="color:#E1E4E8">({</span><span style="color:#F97583">required</span><span style="color:#79B8FF"> String</span><span style="color:#E1E4E8"> modelId}) </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> ...;</span></span> 541<span class="line"><span style="color:#F97583"> static</span><span style="color:#79B8FF"> XybridPipelineBuilder</span><span style="color:#B392F0"> pipeline</span><span style="color:#E1E4E8">({</span><span style="color:#F97583">required</span><span style="color:#79B8FF"> String</span><span style="color:#E1E4E8"> yamlContent}) </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> ...;</span></span> 542<span class="line"><span style="color:#E1E4E8">}</span></span> 543<span class="line"></span> 544<span class="line"><span style="color:#6A737D">// Usage</span></span> 545<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'kokoro-82m'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 546<span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(envelope</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'Hello'</span><span style="color:#E1E4E8">));</span></span></code></pre>`);var r=s(p,10);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>// xybrid.udl â defines the FFI surface</span></span> 547<span class="line"><span>namespace xybrid {</span></span> 548<span class="line"><span> [Throws=XybridError]</span></span> 549<span class="line"><span> XybridModelHandle load_model(string model_id);</span></span> 550<span class="line"><span>};</span></span> 551<span class="line"><span></span></span> 552<span class="line"><span>interface XybridModelHandle {</span></span> 553<span class="line"><span> [Throws=XybridError]</span></span> 554<span class="line"><span> XybridEnvelope run(XybridEnvelope input);</span></span> 555<span class="line"><span></span></span> 556<span class="line"><span> sequence<XybridVoiceInfo> voices();</span></span> 557<span class="line"><span>};</span></span> 558<span class="line"><span></span></span> 559<span class="line"><span>dictionary XybridEnvelope {</span></span> 560<span class="line"><span> string kind;</span></span> 561<span class="line"><span> bytes data;</span></span> 562<span class="line"><span>};</span></span> 563<span class="line"><span></span></span> 564<span class="line"><span>[Error]</span></span> 565<span class="line"><span>
565enum XybridError {</span></span> 566<span class="line"><span> "ModelNotFound",</span></span> 567<span class="line"><span> "InferenceFailed",</span></span> 568<span class="line"><span> "InvalidInput",</span></span> 569<span class="line"><span>};</span></span></code></pre>`);var c=s(r,4);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Auto-generated</span></span> 570<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> try</span><span style="color:#79B8FF"> loadModel</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">modelId</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span><span style="color:#E1E4E8">)</span></span> 571<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#F97583"> try</span><span style="color:#E1E4E8"> model.</span><span style="color:#79B8FF">run</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">input</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">XybridEnvelope</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">kind</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"text"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">data</span><span style="color:#E1E4E8">: textData))</span></span> 572<span class="line"></span> 573<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> voices </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model.</span><span style="color:#79B8FF">voices</span><span style="color:#E1E4E8">() </span><span style="color:#6A737D">// [XybridVoiceInfo]</span></span></code></pre>`);var d=s(c,4);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Auto-generated</span></span> 574<span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#B392F0"> loadModel</span><span style="color:#E1E4E8">(modelId </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "kokoro-82m"</span><span style="color:#E1E4E8">)</span></span> 575<span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(input </span><span style="color:#F97583">=</span><span style="color:#B392F0"> XybridEnvelope</span><span style="color:#E1E4E8">(kind </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "text"</span><span style="color:#E1E4E8">, </span><span style="color:#F97583">data</span><span style="color:#F97583"> =</span><span style="color:#E1E4E8"> textData))</span></span> 576<span class="line"></span> 577<span class="line"><span style="color:#F97583">val</span><span style="color:#E1E4E8"> voices </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">voices</span><span style="color:#E1E4E8">() </span><span style="color:#6A737D">// List<XybridVoiceInfo></span></span></code></pre>`);var i=s(d,8);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Kotlin SDK wrapper</span></span> 578<span class="line"><span style="color:#F97583">object</span><span style="color:#B392F0"> XybridModelLoader</span><span style="color:#E1E4E8"> {</span></span> 579<span class="line"><span style="color:#F97583"> fun</span><span style="color:#B392F0"> fromRegistry</span><span style="color:#E1E4E8">(modelId: </span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">): </span><span style="color:#B392F0">XybridModel</span><span style="color:#E1E4E8"> {</span></span> 580<span class="line"><span style="color:#F97583"> val</span><span style="color:#E1E4E8"> handle </span><span style="color:#F97583">=</span><span style="color:#B392F0"> loadModel</span><span style="color:#E1E4E8">(modelId </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> modelId)</span></span> 581<span class="line"><span style="color:#F97583"> return</span><span style="color:#B392F0"> XybridModel</span><span style="color:#E1E4E8">(handle)</span></span> 582<span class="line"><span style="color:#E1E4E8"> }</span></span> 583<span class="line"><span style="color:#E1E4E8">}</span></span> 584<span class="line"></span> 585<span class="line"><span style="color:#F97583">class</span><span style="color:#B392F0"> XybridModel</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">private</span><span style="color:#F97583"> val</span><span style="color:#E1E4E8"> handle: </span><span style="color:#B392F0">XybridModelHandle</span><span style="color:#E1E4E8">) {</span></span> 586<span class="line"><span style="color:#F97583"> fun</span><span style="color:#B392F0"> run</span><span style="color:#E1E4E8">
586(input: </span><span style="color:#B392F0">Envelope</span><span style="color:#E1E4E8">): </span><span style="color:#B392F0">Envelope</span><span style="color:#F97583"> =</span><span style="color:#E1E4E8"> handle.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(input.</span><span style="color:#B392F0">toFFI</span><span style="color:#E1E4E8">()).</span><span style="color:#B392F0">fromFFI</span><span style="color:#E1E4E8">()</span></span> 587<span class="line"><span style="color:#F97583"> fun</span><span style="color:#B392F0"> voices</span><span style="color:#E1E4E8">(): </span><span style="color:#B392F0">List</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">VoiceInfo</span><span style="color:#E1E4E8">> </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> handle.</span><span style="color:#B392F0">voices</span><span style="color:#E1E4E8">().</span><span style="color:#B392F0">map</span><span style="color:#E1E4E8"> { it.</span><span style="color:#B392F0">fromFFI</span><span style="color:#E1E4E8">() }</span></span> 588<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var y=s(i,10);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// xybrid-ffi/src/lib.rs</span></span> 589<span class="line"></span> 590<span class="line"><span style="color:#E1E4E8">#[no_mangle]</span></span> 591<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> extern</span><span style="color:#9ECBFF"> "C"</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> xybrid_load_model</span><span style="color:#E1E4E8">(</span></span> 592<span class="line"><span style="color:#E1E4E8"> model_id</span><span style="color:#F97583">:</span><span style="color:#F97583"> *const</span><span style="color:#E1E4E8"> c_char,</span></span> 593<span class="line"><span style="color:#E1E4E8"> out_handle</span><span style="color:#F97583">:</span><span style="color:#F97583"> *mut</span><span style="color:#F97583"> *mut</span><span style="color:#B392F0"> XybridHandle</span><span style="color:#E1E4E8">,</span></span> 594<span class="line"><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> i32</span><span style="color:#E1E4E8"> {</span></span> 595<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> model_id </span><span style="color:#F97583">=</span><span style="color:#F97583"> unsafe</span><span style="color:#E1E4E8"> { </span><span style="color:#B392F0">CStr</span><span style="color:#F97583">::</span><span style="color:#B392F0">from_ptr</span><span style="color:#E1E4E8">(model_id) }</span><span style="color:#F97583">.</span><span style="color:#B392F0">to_str</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">.</span><span style="color:#B392F0">unwrap</span><span style="color:#E1E4E8">();</span></span> 596<span class="line"><span style="color:#F97583"> match</span><span style="color:#B392F0"> load_model</span><span style="color:#E1E4E8">(model_id) {</span></span> 597<span class="line"><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(handle) </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> {</span></span> 598<span class="line"><span style="color:#F97583"> unsafe</span><span style="color:#E1E4E8"> { </span><span style="color:#F97583">*</span><span style="color:#E1E4E8">out_handle </span><span style="color:#F97583">=</span><span style="color:#B392F0"> Box</span><span style="color:#F97583">::</span><span style="color:#B392F0">into_raw</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Box</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">(handle)) };</span></span> 599<span class="line"><span style="color:#79B8FF"> 0</span><span style="color:#6A737D"> // success</span></span> 600<span class="line"><span style="color:#E1E4E8"> }</span></span> 601<span class="line"><span style="color:#B392F0"> Err</span><span style="color:#E1E4E8">(_) </span><span style="color:#F97583">=></span><span style="color:#F97583"> -</span><span style="color:#79B8FF">1</span><span style="color:#6A737D"> // error</span></span> 602<span class="line"><span style="color:#E1E4E8"> }</span></span> 603<span class="line"><span style="color:#E1E4E8">}</span></span> 604<span class="line"></span> 605<span class="line"><span style="color:#E1E4E8">#[no_mangle]</span></span> 606<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> extern</span><span style="color:#9ECBFF"> "C"</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> xybrid_free_model</span><span style="color:#E1E4E8">(handle</span><span style="color:#F97583">:</span><span style="color:#F97583"> *mut</span><span style="color:#B392F0"> XybridHandle</span><span style="color:#E1E4E8">) {</span></span> 607<span class="line"><span style="color:#F97583"> if</span><span style="color:#F97583"> !</span><span style="color:#E1E4E8">handle</span><span style="color:#F97583">.</span><span style="color:#B392F0">is_null</span><span style="color:#E1E4E8">() {</span></span> 608<span class="line"><span style="color:#F97583"> unsafe</span><span style="color:#E1E4E8"> { </span><span style="color:#B392F0">drop</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Box</span><span style="color:#F97583">::</span><span style="color:#B392F0">from_raw</span><span style="color:#E1E4E8">(handle)) };</span></span> 609<span class="line"><span style="color:#E1E4E8"> }</span></span> 610<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var E=s(y,4);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>// xybrid.h (auto-generated)</span></span> 611<span class="line"><span>int32_t xybrid_load_model(const char *model_id, XybridHandle **out_handle);</span></span> 612<span class="line"><span>void xybrid_free_model(XybridHandle *handle);</span></span></code></pre>`);var F=s(E,4);e(F,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">public</span><span style="color:#F97583"> class</span><span style="color:#B392F0"> Xybrid</span><span style="color:#E1E4E8"> {</span></span> 613<span class="line"><span style="color:#E1E4E8"> [</span><span style="color:#B392F0">
613DllImport</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"xybrid_ffi"</span><span style="color:#E1E4E8">)]</span></span> 614<span class="line"><span style="color:#F97583"> private</span><span style="color:#F97583"> static</span><span style="color:#F97583"> extern</span><span style="color:#F97583"> int</span><span style="color:#B392F0"> xybrid_load_model</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">string</span><span style="color:#B392F0"> modelId</span><span style="color:#E1E4E8">, </span><span style="color:#F97583">out</span><span style="color:#B392F0"> IntPtr</span><span style="color:#B392F0"> handle</span><span style="color:#E1E4E8">);</span></span> 615<span class="line"></span> 616<span class="line"><span style="color:#E1E4E8"> [</span><span style="color:#B392F0">DllImport</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"xybrid_ffi"</span><span style="color:#E1E4E8">)]</span></span> 617<span class="line"><span style="color:#F97583"> private</span><span style="color:#F97583"> static</span><span style="color:#F97583"> extern</span><span style="color:#F97583"> void</span><span style="color:#B392F0"> xybrid_free_model</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">IntPtr</span><span style="color:#B392F0"> handle</span><span style="color:#E1E4E8">);</span></span> 618<span class="line"></span> 619<span class="line"><span style="color:#F97583"> public</span><span style="color:#F97583"> static</span><span style="color:#B392F0"> XybridModel</span><span style="color:#B392F0"> LoadModel</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">string</span><span style="color:#B392F0"> modelId</span><span style="color:#E1E4E8">) {</span></span> 620<span class="line"><span style="color:#B392F0"> IntPtr</span><span style="color:#B392F0"> handle</span><span style="color:#E1E4E8">;</span></span> 621<span class="line"><span style="color:#F97583"> int</span><span style="color:#B392F0"> result</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> xybrid_load_model</span><span style="color:#E1E4E8">(modelId, </span><span style="color:#F97583">out</span><span style="color:#E1E4E8"> handle);</span></span> 622<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> (result </span><span style="color:#F97583">!=</span><span style="color:#79B8FF"> 0</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">throw</span><span style="color:#F97583"> new</span><span style="color:#B392F0"> Exception</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Failed to load model"</span><span style="color:#E1E4E8">);</span></span> 623<span class="line"><span style="color:#F97583"> return</span><span style="color:#F97583"> new</span><span style="color:#B392F0"> XybridModel</span><span style="color:#E1E4E8">(handle);</span></span> 624<span class="line"><span style="color:#E1E4E8"> }</span></span> 625<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var v=s(F,16);e(v,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>FFI surface:</span></span> 626<span class="line"><span> - init()</span></span> 627<span class="line"><span> - load_model(id) â handle</span></span> 628<span class="line"><span> - run(handle, input) â output</span></span> 629<span class="line"><span> - run_streaming(handle, input) â stream</span></span> 630<span class="line"><span> - voices(handle) â list</span></span> 631<span class="line"><span> - warmup(handle)</span></span> 632<span class="line"><span> - free(handle)</span></span></code></pre>`);var _=s(v,8);e(_,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Internal (rich, nested)</span></span> 633<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> Envelope</span><span style="color:#E1E4E8"> {</span></span> 634<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> kind</span><span style="color:#F97583">:</span><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#E1E4E8">,</span></span> 635<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> HashMap</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">>,</span></span> 636<span class="line"><span style="color:#E1E4E8">}</span></span> 637<span class="line"></span> 638<span class="line"><span style="color:#6A737D">// FFI (flat, simple)</span></span> 639<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> ApiEnvelope</span><span style="color:#E1E4E8"> {</span></span> 640<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> kind</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">, </span><span style="color:#6A737D">// "text", "audio", "embedding"</span></span> 641<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> data</span><span style="color:#F97583">:</span><span style="color:#B392F0"> Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">u8</span><span style="color:#E1E4E8">>, </span><span style="color:#6A737D">// serialized payload</span></span> 642<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var T=s(_,8);e(T,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># .github/workflows/</span></span> 643<span class="line"><span style="color:#9ECBFF">build-flutter.yml</span><span style="color:#6A737D"> # Flutter build + Dart analysis</span></span> 644<span class="line"><span style="color:#9ECBFF">build-apple.yml</span><span style="color:#6A737D"> # XCFramework + Swift package</span></span> 645<span class="line"><span style="color:#9ECBFF">build-android.yml</span><span style="color:#6A737D"> # AAR + Kotlin compilation + ORT verification</span></span> 646<span class="line"><span style="color:#9ECBFF">build-unity.yml</span><span style="color:#6A737D"> # Native lib + C header generation</span></span></code></pre>`);var L=s(T,8);e(L,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#85E89D">methods</span><span style="color:#E1E4E8">:</span></span> 647<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">name</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">init</span></span> 648<span class="line"><span style="color:#85E89D"> platforms</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">flutter</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">
648swift</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">kotlin</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">unity</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">cli</span><span style="color:#E1E4E8">]</span></span> 649<span class="line"><span style="color:#85E89D"> status</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">stable</span></span> 650<span class="line"></span> 651<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">name</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">model.run</span></span> 652<span class="line"><span style="color:#85E89D"> platforms</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">flutter</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">swift</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">kotlin</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">unity</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">cli</span><span style="color:#E1E4E8">]</span></span> 653<span class="line"><span style="color:#85E89D"> args</span><span style="color:#E1E4E8">:</span></span> 654<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">name</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">envelope</span></span> 655<span class="line"><span style="color:#85E89D"> type</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">Envelope</span></span> 656<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">name</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">voiceId</span></span> 657<span class="line"><span style="color:#85E89D"> type</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">string?</span></span> 658<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">name</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">generationConfig</span></span> 659<span class="line"><span style="color:#85E89D"> type</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">GenerationConfig?</span></span> 660<span class="line"><span style="color:#85E89D"> status</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">stable</span></span></code></pre>`),f(22),h(o,a)}const ps=Object.freeze(Object.defineProperty({__proto__:null,default:ne,metadata:ts},Symbol.toStringTag,{value:"Module"})),rs={title:"How We Made ONNX Runtime 6.8x Faster on Apple Silicon with CoreML",description:"Real benchmarks showing when Apple's Neural Engine helps (and when it hurts). Lessons from optimizing ML inference across execution providers.",date:"2026-07-03",author:"Glenn Sonna",tags:["onnx-runtime","coreml","apple-silicon-ml","neural-engine","on-device-ai"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:6},{title:zn,description:Un,date:qn,author:$n,tags:Wn,category:Kn,image:Hn,imageAlt:Xn,draft:jn,featured:Vn,readingTime:Qn}=rs;var oe=u('<p>âJust enable CoreML, itâll be faster.â</p> <p>We heard this advice constantly. So we benchmarked it properly. The results surprised us: <strong>6.8x faster</strong> for some models, <strong>actually slower</strong> for others.</p> <p>Hereâs what we learned optimizing ONNX Runtime inference on Apple Silicon, and the rules of thumb that came out of it.</p> <hr/> <h2>The Setup</h2> <p>We built <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a>, an on-device ML inference SDK in Rust. It uses ONNX Runtime as one of its backends, which supports multiple <strong>execution providers</strong> (EPs):</p> <ul><li><strong>CPU</strong> â Default, works everywhere</li> <li><strong>CoreML</strong> â Appleâs ML framework, can target the <strong>Neural Engine (ANE)</strong></li> <li><strong>Metal</strong> â GPU compute via Appleâs Metal API</li></ul> <p>We wanted to know: for which models does CoreML actually help?</p> <h2>The Benchmark</h2> <p>We used Criterion.rs with three representative models:</p> <table><thead><tr><th>
660Model</th><th>Type</th><th>Input Shape</th><th>Size</th></tr></thead><tbody><tr><td>MNIST</td><td>Vision (tiny)</td><td>[1, 1, 28, 28]</td><td>26 KB</td></tr><tr><td>MobileNet v2</td><td>Vision (CNN)</td><td>[1, 3, 224, 224]</td><td>14 MB</td></tr><tr><td>Kokoro 82M</td><td>TTS (transformer)</td><td>dynamic</td><td>330 MB</td></tr></tbody></table> <!> <h2>The Results</h2> <p>Tested on M2 MacBook Pro:</p> <table><thead><tr><th>Model</th><th>CPU</th><th>CoreML (ANE)</th><th>Speedup</th></tr></thead><tbody><tr><td>MNIST</td><td>0.15ms</td><td>0.8ms</td><td><strong>0.2x (slower)</strong></td></tr><tr><td>MobileNet v2</td><td>12.4ms</td><td>1.8ms</td><td><strong>6.8x faster</strong></td></tr><tr><td>Kokoro 82M</td><td>280ms</td><td>290ms</td><td><strong>~1x (no benefit)</strong></td></tr></tbody></table> <p>Wait â MNIST got <em>slower</em> with CoreML? And Kokoro saw no improvement?</p> <h2>Why CoreML Isnât Always Faster</h2> <h3>Rule 1: Tiny Models Lose to Dispatch Overhead</h3> <p>The ANE is a dedicated hardware accelerator. Sending work to it has fixed overhead: the CoreML framework compiles the model graph, schedules it on the ANE, and marshals data between CPU and ANE memory.</p> <p>For MNIST (26 KB, runs in 0.15ms on CPU), that dispatch overhead <strong>costs more than the inference itself</strong>. The model is too small to benefit.</p> <p><strong>Guideline</strong>: Models under ~1MB with inference times under 1ms wonât benefit from CoreML. Stick with CPU.</p> <h3>Rule 2: CNNs with Fixed Shapes Are the Sweet Spot</h3> <p>MobileNet is the perfect ANE workload:</p> <ul><li><strong>Fixed input shape</strong> [1, 3, 224, 224] â the CoreML compiler can optimize the graph once</li> <li><strong>Regular operations</strong> â convolutions, batch norms, ReLUs map directly to ANE instructions</li> <li><strong>Sufficient compute</strong> â enough work to amortize dispatch overhead</li></ul> <p>The 6.8x speedup is real and consistent. Vision models (image classification, object detection, segmentation) with static shapes will almost always benefit.</p> <p><strong>Guideline</strong>: Fixed-shape CNN models â enable CoreML. Expect 5-10x speedup.</p> <h3>Rule 3: Dynamic Shapes Kill ANE Performance</h3> <p>Kokoro TTS is a transformer with dynamic input shapes â the token sequence length varies per input. Each new shape triggers a <strong>recompilation</strong> of the CoreML graph.</p> <p>Even when the graph is cached, dynamic shapes prevent the ANE from using its most aggressive optimizations. The result: inference time roughly matches CPU.</p> <p><strong>Guideline</strong>: Models with dynamic input dimensions (most transformers, seq2seq, attention-based models) â CoreML wonât help unless you pad to fixed sizes.</p> <h3>Rule 4: First Inference Has a Cold-Start Penalty</h3> <p>CoreML compiles the model to ANE instructions on first run. This adds 1-5 seconds of startup time depending on model complexity.</p> <p>For benchmarks, we use warm iterations. But in production, this cold start matters. Xybridâs <code>warmup()</code> API exists specifically for this:</p> <!> <p><strong>Guideline</strong>: Always warmup CoreML models before latency-sensitive inference.</p> <h2>How We Integrated This in Xybrid</h2> <p>Rather than forcing users to choose execution providers, Xybrid selects them based on the model type and platform:</p> <!> <p>When CoreML is available, ONNX Runtime automatically evaluates which ops can run on ANE and falls back to CPU for the rest. This <strong>per-op routing</strong> means even partially supported models get some benefit.</p> <h2>Summary: When to Use What</h2> <table><thead><tr><th>Model Type</th><th>Best EP</th><th>Expected Speedup</th><th>Why</th></tr></thead><tbody><tr><td>Vision (CNN, fixed shape)</td><td>CoreML ANE</td><td>5-10x</td><td>Regular ops, static shapes</td></tr><tr><td>Embeddings (fixed shape)</td><td>CoreML ANE</td><td>2-4x</td><td>Matrix multiply benefits</td></tr><tr><td>TTS (dynamic shapes)</td><td>CPU</td><td>1x</td><td>Recompilation overhead</td></tr><tr><td>Tiny models (<1MB)</td><td>CPU</td><td>n/a</td><td>Dispatch overhead dominates</td></tr><tr><td>LLMs (GGUF)</td><td>Metal GPU</td><td>2-5x</td><td>llama.cpp uses Metal directly</td></tr></tbody></table> <h2>Run the Benchmarks Yourself</h2> <!> <p>
660The Criterion reports include statistical analysis, confidence intervals, and comparison between runs. No hand-wavy âfelt fasterâ â real numbers.</p> <hr/> <h2>Key Takeaways</h2> <ol><li><strong>Benchmark, donât assume.</strong> âApple Silicon is fastâ is not the same as âCoreML is always faster.â</li> <li><strong>Model shape matters more than model size.</strong> A 14MB CNN (MobileNet) benefits enormously; a 330MB transformer (Kokoro) doesnât.</li> <li><strong>Warmup is mandatory.</strong> CoreML cold-start can dominate perceived latency.</li> <li><strong>Per-op routing is free.</strong> ONNX Runtimeâs CoreML EP automatically splits graphs â you donât have to manually decide.</li></ol> <p>The 6.8x on MobileNet is real. But it only comes when you know which models to point at the ANE.</p> <hr/> <p><strong>Run your own benchmarks</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <hr/> <p><em>Got benchmark results on different hardware? Share them in the comments â weâre collecting data across device generations.</em></p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/on-device-ai-mobile-ios-android">On-Device AI for Mobile Apps: iOS and Android Guide</a> â framework comparison across CoreML, ONNX Runtime, TF Lite, and llama.cpp.</li> <li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide</a> â the full landscape of running ML models locally.</li> <li><a href="/blog/edge-vs-cloud-inference">Edge AI vs Cloud AI</a> â when on-device beats cloud, with cost analysis.</li></ul>',1);function le(o){var a=oe(),n=s(b(a),22);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># CPU baseline</span></span> 661<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> bench</span><span style="color:#79B8FF"> -p</span><span style="color:#9ECBFF"> xybrid-core</span></span> 662<span class="line"></span> 663<span class="line"><span style="color:#6A737D"># CoreML (routes eligible ops to ANE)</span></span> 664<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> bench</span><span style="color:#79B8FF"> -p</span><span style="color:#9ECBFF"> xybrid-core</span><span style="color:#79B8FF"> --features</span><span style="color:#9ECBFF"> coreml-ep</span></span></code></pre>`);var l=s(n,44);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Pre-compile for ANE during loading screen</span></span> 665<span class="line"><span style="color:#E1E4E8">executor</span><span style="color:#F97583">.</span><span style="color:#B392F0">warmup</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">metadata)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span></code></pre>`);var t=s(l,8);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># Platform presets handle this automatically</span></span> 666<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#B392F0">features</span><span style="color:#E1E4E8">]</span></span> 667<span class="line"><span style="color:#E1E4E8">platform-macos = [</span><span style="color:#9ECBFF">"ort-download"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"ort-coreml"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"candle-metal"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"llm-llamacpp"</span><span style="color:#E1E4E8">]</span></span> 668<span class="line"><span style="color:#E1E4E8">platform-ios = [</span><span style="color:#9ECBFF">"ort-download"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"ort-coreml"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"candle-metal"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"llm-llamacpp"</span><span style="color:#E1E4E8">]</span></span> 669<span class="line"><span style="color:#E1E4E8">platform-android = [</span><span style="color:#9ECBFF">"ort-dynamic"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"llm-llamacpp"</span><span style="color:#E1E4E8">]</span></span></code></pre>`);var p=s(t,10);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">git</span><span style="color:#9ECBFF"> clone</span><span style="color:#9ECBFF"> https://github.com/xybrid-ai/xybrid.git</span></span> 670<span class="line"><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> xybrid</span></span> 671<span class="line"></span> 672<span class="line"><span style="color:#6A737D"># Download benchmark models</span></span> 673<span class="line"><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> integration-tests</span><span style="color:#E1E4E8"> && </span><span style="color:#B392F0">./download.sh</span><span style="color:#9ECBFF"> mnist</span><span style="color:#9ECBFF"> mobilenet</span><span style="color:#9ECBFF"> kokoro-82m</span><span style="color:#E1E4E8"> && </span><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> ..</span></span> 674<span class="line"></span> 675<span class="line"><span style="color:#6A737D"># CPU baseline</span></span> 676<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> bench</span><span style="color:#79B8FF"> -p</span><span style="color:#9ECBFF"> xybrid-core</span></span> 677<span class="line"></span> 678<span class="line"><span style="color:#6A737D"># CoreML</span></span> 679<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> bench</span><span style="color:#79B8FF"> -p</span><span style="color:#9ECBFF"> xybrid-core</span><span style="color:#79B8FF"> --features</span><span style="color:#9ECBFF"> coreml-ep</span></span> 680<span class="line"></span> 681<span class="line"><span style="color:#6A737D">
681# View HTML report</span></span> 682<span class="line"><span style="color:#B392F0">open</span><span style="color:#9ECBFF"> target/criterion/report/index.html</span></span></code></pre>`),f(24),h(o,a)}const cs=Object.freeze(Object.defineProperty({__proto__:null,default:le,metadata:rs},Symbol.toStringTag,{value:"Module"})),is={title:"Private AI: How to Run AI Models Without Sending Data to the Cloud",description:"How to run AI models privately on-device â no cloud APIs, no data leaving the device. Covers GDPR, HIPAA, and SOC 2 compliance, private LLMs, and practical implementation patterns.",date:"2026-06-16",author:"Glenn Sonna",tags:["private-ai","private-llm","on-device-ai","ai-privacy","gdpr-ai","hipaa-ai"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:10},{title:Yn,description:Jn,date:Zn,author:so,tags:eo,category:ao,image:no,imageAlt:oo,draft:lo,featured:to,readingTime:po}=is;var te=u('<p>Every time your app calls a cloud AI API, your usersâ data travels to someone elseâs server. The prompt, the audio, the image â it leaves the device, crosses the network, and lands on infrastructure you do not control. For many applications, this is fine. For many others, it is a liability.</p> <p>Private AI means running inference locally â on the userâs phone, laptop, or edge server â so that data never leaves the device. No API calls, no third-party data processing agreements, no trust assumptions about what happens to user data after the request completes.</p> <p>This is not a philosophical position. It is an architecture decision with concrete implications for compliance, cost, and user trust.</p> <h2>Why Private AI Matters Now</h2> <p>Three forces are converging to make private AI practical and, in some cases, mandatory.</p> <h3>Regulation Is Tightening</h3> <p>The regulatory landscape for AI and data processing has shifted significantly:</p> <ul><li><strong>GDPR (EU)</strong> requires a lawful basis for processing personal data. Sending voice recordings or text prompts to a cloud AI provider means that provider becomes a data processor, requiring a Data Processing Agreement, impact assessments, and documented lawful basis. Running inference on-device eliminates the data transfer entirely.</li> <li><strong>HIPAA (US healthcare)</strong> restricts how Protected Health Information (PHI) is transmitted and stored. A medical app that sends patient notes to a cloud LLM for summarization must ensure the provider signs a Business Associate Agreement, implements encryption in transit and at rest, and maintains audit logs. On-device inference sidesteps all of this â the data never enters the network.</li> <li><strong>SOC 2</strong> compliance for B2B SaaS products increasingly requires documenting every third-party service that touches customer data. Each cloud AI API adds a vendor to your supply chain, a risk to your audit, and a dependency to your architecture. On-device inference removes the vendor from the equation.</li> <li><strong>EU AI Act</strong> introduces obligations around transparency and data governance for AI systems. Keeping inference local simplifies compliance by reducing the number of parties involved in data processing.</li> <li><strong>State-level US privacy laws</strong> (CCPA/CPRA in California, VCDPA in Virginia, CPA in Colorado) are expanding data subject rights. On-device processing reduces the surface area of personal data collection.</li></ul> <p>The trend is clear: every year, sending user data to third parties gets harder to justify and more expensive to do correctly.</p> <h3>Models Are Small Enough</h3> <p>Two years ago, on-device AI meant running a MobileNet classifier. Today, you can run capable models across the full AI stack on consumer hardware:</p> <table><thead><tr><th>Task</th><th>Model</th><th>Size</th><th>Runs On</th></tr></thead><tbody><tr><td>Speech recognition</td><td>Whisper Tiny</td><td>75 MB</td><td>Any modern phone</td></tr><tr><td>Text-to-speech</td><td>Kokoro 82M</td><td>180 MB</td><td>Mid-range phones and up</td></tr><tr><td>Text generation</td><td>Llama 3.2 1B (Q4)</td><td>~700 MB</td><td>Flagship phones, all desktops</td></tr><tr><td>Text generation</td><td>Qwen 3.5 0.8B (Q4)</td><td>~500 MB</td><td>Flagship phones, all desktops</td></tr><tr><td>Embeddings</td><td>MiniLM-L6-v2</td><td>80 MB</td><td>Any modern phone</td></tr><tr><td>Image classification</td><td>MobileNet v2</td><td>14 MB</td><td>Any device</td></tr></tbody></table> <p>A complete voice assistant pipeline â speech recognition, language model, and text-to-speech â fits in under 1 GB and runs on a 2022 smartphone. That was not possible three years ago.</p> <h3>Users Are Paying Attention</h3> <p>User awareness of data privacy has moved from niche concern to mainstream expectation. When an app records audio and âprocesses it in the cloud,â users increasingly w
682ant to know: whose cloud? What happens to the recording after? How long is it stored?</p> <p>Apps that can truthfully say âyour data never leaves your deviceâ have a trust advantage that no privacy policy can replicate. It is the difference between promising privacy and architecting it.</p> <h2>Who Needs Private AI</h2> <p>Not every application needs private inference. Here are the cases where it is either required or strongly advantageous.</p> <h3>Regulated Industries</h3> <p><strong>Healthcare.</strong> Patient-facing AI features â transcribing doctor-patient conversations, summarizing medical records, triaging symptoms â handle PHI. Cloud processing triggers HIPAA obligations. On-device processing does not create a data transmission event, dramatically simplifying compliance.</p> <p><strong>Finance.</strong> AI features that process transaction data, account information, or financial documents face regulatory scrutiny under SOX, PCI-DSS, and regional banking regulations. Private inference keeps sensitive financial data off third-party servers.</p> <p><strong>Legal.</strong> Attorney-client privilege applies to AI-assisted legal work. Sending privileged documents to a cloud LLM for summarization could waive privilege depending on jurisdiction. On-device inference eliminates the risk.</p> <p><strong>Education.</strong> Student data is protected under FERPA (US) and similar regulations globally. AI tutoring or assessment tools that process student responses locally avoid the compliance overhead of cloud data processing.</p> <h3>Consumer Applications With Sensitive Data</h3> <p>Voice assistants, journaling apps, health trackers, messaging apps with AI features, photo organization tools â any application where the AI input is inherently personal. Users do not expect their private journal entries or health data to be sent to a server for processing. On-device inference matches the expectation.</p> <h3>Enterprise and B2B</h3> <p>Corporate data classification, document summarization, code completion, and internal search â enterprises are increasingly reluctant to send proprietary data to cloud AI providers, even with enterprise agreements in place. On-device or on-premise inference keeps intellectual property inside the organizationâs boundary.</p> <h3>Offline and Air-Gapped Environments</h3> <p>Military, government classified systems, industrial facilities, and field operations in remote areas. These environments require AI that works without any network connectivity. Private AI is not a preference here â it is a hard requirement.</p> <h2>Practical Architecture for Private AI</h2> <p>Moving from cloud inference to private on-device inference is not just a deployment change. It requires different architectural decisions at the model selection, packaging, and runtime layers.</p> <h3>Model Selection</h3> <p>Choose models that run well on target hardware. The goal is not to run the largest possible model â it is to run the most capable model that fits within the deviceâs compute and memory budget.</p> <p><strong>Rules of thumb:</strong></p> <ul><li>Mobile phones (2024+): Up to 1-3B parameters with 4-bit quantization</li> <li>Laptops and desktops: Up to 7-13B parameters with quantization</li> <li>Edge servers with GPU: Up to 30B+ parameters</li></ul> <p>Quantization is critical. A 1B parameter model in full precision (FP32) requires ~4 GB of memory. The same model quantized to Q4_K_M requires ~700 MB. Quantization reduces both memory usage and inference latency, typically with minimal accuracy loss for conversational and generative tasks.</p> <h3>Model Packaging</h3> <p>On-device models need to be packaged for distribution. Users should not be downloading raw model files from GitHub. A good packaging strategy includes:</p> <ol><li><strong>Versioned bundles</strong> with checksums for integrity verification</li> <li><strong>Progressive download</strong> so users only download what they need</li> <li><strong>Background model updates</strong> that do not interrupt the user experience</li> <li><strong>Platform-specific optimization</strong> (CoreML on Apple, NNAPI on Android) for maximum performance</li></ol> <h3>Runtime Execution</h3> <p>The inference runtime must handle:</p> <ul><li><strong>Model loading</strong> from local storage with lazy initialization</li> <li><strong>Memory management</strong> to avoid OOM on constrained devices</li> <li><strong>Hardware acceleration</strong> using the deviceâs NPU, GPU, or Neural Engine when available</li> <li><strong>Preprocessing and postprocessing</strong> so the application code works with natural data types (text in, audio out) rather than tensors</li></ul> <h3>Hybrid Fallback</h3> <p>Private AI does not mean cloud-never. A well-designed system runs what it can locally and routes to the cloud only when necessary â and only with explicit user consent. For example:</p> <ul><li>
682Speech recognition and TTS always run on-device (small models, latency-sensitive)</li> <li>A 1B LLM handles simple queries locally</li> <li>Complex queries optionally route to a cloud LLM if the user enables it</li></ul> <p>The key word is âoptionally.â Private AI means the default is local. Cloud is opt-in, not opt-out.</p> <h2>Common Objections</h2> <h3>âOn-device models are not good enoughâ</h3> <p>This was true in 2023. It is increasingly false. Whisper Tiny transcribes English accurately. Llama 3.2 1B handles conversational queries, summarization, and simple reasoning. Kokoro produces natural-sounding speech. These are not toy models â they are production-quality for the tasks they target.</p> <p>For tasks that genuinely require frontier models (complex multi-step reasoning, advanced code generation, nuanced creative writing), cloud inference is still necessary. But most AI features in most applications do not need GPT-4-class capabilities.</p> <h3>âIt is too hard to deploy models to devicesâ</h3> <p>Model packaging and distribution tooling has matured significantly. Frameworks like ONNX Runtime, llama.cpp, and CoreML provide cross-platform inference. The challenge is integration â getting models bundled, downloaded, and running correctly across iOS, Android, macOS, Windows, and Linux. This is an engineering problem, not a research problem, and it is solvable with the right SDK layer.</p> <h3>âUsers do not care about privacyâ</h3> <p>Users say they do not care about privacy until they do. A voice journaling app that âsends recordings to the cloud for processingâ will face more support tickets, App Store review pushback, and churn than one that processes locally. Privacy-by-architecture is a product quality, not just a compliance checkbox.</p> <h3>âWe need the latest modelsâ</h3> <p>You need the latest models for research and benchmarks. You need adequate models for production features. A text-to-speech model that sounds natural is adequate. A speech recognition model that transcribes accurately is adequate. Chasing SOTA on a leaderboard is a different problem from shipping a feature that works.</p> <h2>The Privacy-Performance Sweet Spot</h2> <p>The best candidates for private AI are models that are:</p> <ul><li><strong>Small</strong> (under 1 GB quantized)</li> <li><strong>Focused</strong> (single task, not general-purpose)</li> <li><strong>Frequently used</strong> (high inference volume per user)</li> <li><strong>Processing sensitive input</strong> (voice, text, images, health data)</li></ul> <p>This describes a large percentage of AI features in production applications. Speech recognition, text-to-speech, embeddings, classification, summarization of short documents, simple chat â these are the bread and butter of applied AI, and they all run well on modern consumer devices.</p> <p>The models that do not fit this profile â frontier LLMs, large multi-modal models, training workloads â belong in the cloud. Private AI is not about eliminating cloud inference entirely. It is about ensuring that the default for sensitive, frequently-used AI features is local, and that cloud processing is a conscious, opt-in choice.</p> <h2>Getting Started</h2> <p>If you are building an application with AI features and want to evaluate on-device inference:</p> <ol><li><strong>Audit your current AI API calls.</strong> List every cloud AI endpoint your app hits. For each one, note the data type (text, audio, image), the model task, and the inference volume.</li> <li><strong>Identify candidates for local inference.</strong> Any call that processes sensitive data, runs frequently, or uses a model under 1B parameters is a candidate.</li> <li><strong>Prototype with local inference tooling.</strong> Run models locally using ONNX Runtime, llama.cpp, or a framework that abstracts the runtime layer. Measure latency and accuracy on target devices.</li> <li><strong>Evaluate the compliance impact.</strong> For each model you move on-device, document the data processing agreements and compliance obligations you can eliminate.</li> <li><strong>Ship with hybrid architecture.</strong> Start with the easiest wins (embeddings, classification, TTS) and move toward more complex tasks (AS
682R, LLM) as you validate on-device performance.</li></ol> <p>The goal is not to rewrite your AI stack overnight. It is to establish on-device as the default and cloud as the exception â starting with the features where privacy matters most.</p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide to Running ML Models Locally</a> â the full guide to on-device inference: hardware, models, and getting started.</li> <li><a href="/blog/edge-vs-cloud-inference">Edge AI vs Cloud AI: When to Run Models On-Device</a> â a decision framework with cost analysis for choosing where to run each model.</li> <li><a href="/blog/run-llm-locally-complete-guide">How to Run LLMs Locally</a> â run language models on your laptop or phone with no cloud API.</li> <li><a href="/blog/building-voice-agent-on-device">Building a Voice Agent That Runs Entirely On-Device</a> â tutorial: Whisper + LLM + Kokoro TTS in a Flutter app, no internet required.</li></ul>',1);function pe(o){var a=te();f(134),h(o,a)}const ds=Object.freeze(Object.defineProperty({__proto__:null,default:pe,metadata:is},Symbol.toStringTag,{value:"Module"})),ys={title:"Publishing ML Models as Bundles with xybrid-pack",description:"A packaging pipeline that turns raw model files into versioned, checksummed, cross-platform bundles hosted on HuggingFace. No more 'download the model and put it here.'",date:"2026-07-17",author:"Glenn Sonna",tags:["machinelearning","devops","opensource","ai"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:5},{title:ro,description:co,date:io,author:yo,tags:Eo,category:ho,image:uo,imageAlt:mo,draft:Fo,featured:go,readingTime:fo}=ys;var re=u('<p>Youâve trained a model. It works on your machine. Now you need to get it to users.</p> <p>The typical approach: upload the ONNX file somewhere, write a README explaining which files to download, what preprocessing is needed, and hope people follow the instructions.</p> <p>We built something better: a packaging pipeline that produces <strong>self-contained, versioned, checksummed bundles</strong> that any Xybrid client can download and run automatically.</p> <hr/> <h2>The Problem with Raw Model Distribution</h2> <p>A typical model âdistributionâ looks like this:</p> <!> <p>Problems:</p> <ol><li><strong>No single download</strong> â users must fetch 5 files individually</li> <li><strong>No integrity checks</strong> â how do you know the files arenât corrupted?</li> <li><strong>No metadata</strong> â which files are required vs. optional?</li> <li><strong>No versioning</strong> â updating one file can break compatibility</li> <li><strong>No platform variants</strong> â Android might need a different quantization than desktop</li></ol> <h2>The .xyb Bundle Format</h2> <p>Xybrid packages models as <code>.xyb</code> bundles: <strong>tar + zstd compression</strong> with a <code>manifest.json</code>:</p> <!> <p>The manifest declares contents, checksums, and compatibility:</p> <!> <p>
682One download, verified integrity, self-describing contents.</p> <h2>The xybrid-pack Pipeline</h2> <p><code>xybrid-pack</code> is our model packaging tool. It takes a <strong>model spec</strong> (YAML) and produces a published, registered bundle.</p> <h3>Step 1: Write a Model Spec</h3> <!> <p>The spec is the single source of truth for how this model is fetched, packaged, and published.</p> <h3>Step 2: Dry Run</h3> <!> <p>This runs the full pipeline <strong>without publishing</strong>:</p> <!> <h3>Step 3: Publish</h3> <!> <p>The full pipeline:</p> <ol><li><strong>Validate</strong> â checks the spec against the schema</li> <li><strong>Fetch</strong> â downloads source files from HuggingFace (or GitHub, or URL)</li> <li><strong>Generate</strong> â creates <code>model_metadata.json</code> from the specâs execution config</li> <li><strong>Build</strong> â creates the <code>.xyb</code> bundle (tar + zstd)</li> <li><strong>Publish</strong> â uploads to HuggingFace under <code>xybrid-ai/kokoro-82m</code></li> <li><strong>Verify</strong> â re-downloads the bundle and checks SHA256 matches</li> <li><strong>Register</strong> â adds the model to <code>registry.json</code> with download URL and checksum</li></ol> <h3>Step 4: Verify</h3> <!> <p>Users can now <code>xybrid fetch kokoro-82m</code> and the SDK handles everything.</p> <h2>The Registry</h2> <p><code>registry.json</code> is a flat index of all published models:</p> <!> <p><strong>Critical rule</strong>: <code>registry.json</code> is <strong>never edited manually</strong>. Only <code>xybrid-pack register</code> or <code>xybrid-pack pipeline</code> writes to it. This ensures the SHA256 always matches what HuggingFace actually serves.</p> <p>The pipeline order is intentional: <strong>publish â verify â register</strong>. We upload first, then re-download and hash, then record. This catches upload corruption.</p> <h2>Platform Variants</h2> <p>Some models need different formats per platform. GGUF for LLMs, ONNX for everything else. Spec supports multiple variants:</p> <!> <p>The SDK resolves the best variant for the current platform automatically.</p> <h2>Why HuggingFace?</h2> <p>We host bundles on HuggingFace because:</p> <ol><li><strong>Free hosting</strong> for open-source models</li> <li><strong>CDN</strong> â fast downloads globally</li> <li><strong>Versioning</strong> via git LFS</li> <li><strong>Community</strong> â users can browse models on the HF website</li></ol> <p>The <code>.xyb</code> format is HF-agnostic though. The registry just needs a URL and SHA256. You could host on S3, GCS, or your own CDN.</p> <h2>Model Spec as the Source of Truth</h2> <p>The model spec YAML drives everything:</p> <ul><li><strong>What to download</strong> (source files, source repo)</li> <li><strong>How to package</strong> (which files, compression)</li> <li><strong>How to run</strong> (execution template, pre/post processing)</li> <li><strong>Where to publish</strong> (HF org, repo name)</li> <li><strong>What platforms</strong> (variant â platform mapping)</li></ul> <p>Change the spec, re-run the pipeline, and the entire distribution updates consistently.</p> <hr/> <h2>Quick Reference</h2> <!> <hr/> <p><strong>Explore the specs</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <p>If youâre distributing ML models and tired of âdownload these 5 files and put them in this directoryâ â this pattern might help.</p> <hr/> <p><em>How do you distribute your models? Let us know in the comments.</em></p>',1);function ce(o){var a=re(),n=s(b(a),12);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>huggingface.co/org/my-model/</span></span> 683<span class="line"><span>âââ model.onnx (330MB)</span></span> 684<span class="line"><span>âââ vocab.json (1.2MB)</span></span> 685<span class="line"><span>âââ tokens.txt (45KB)</span></span> 686<span class="line"><span>âââ voices.bin (12MB)</span></span> 687<span class="line"><span>âââ config.json (2KB)</span></span> 688<span class="line"><span>âââ README.md ("download all files to the same directory")</span></span></code></pre>`);var l=s(n,10);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>kokoro-82m-universal.xyb (compressed)</span></span> 689<span class="line"><span>âââ manifest.json</span></span> 690<span class="line"><span>âââ model_metadata.json</span></span> 691<span class="line"><span>âââ model.onnx</span></span> 692<span class="line"><span>âââ tokens.txt</span></span> 693<span class="line"><span>âââ voices.bin</span></span> 694<span class="line"><span>âââ README.md</span></span></code></pre>`);var t=s(l,4);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">
694{</span></span> 695<span class="line"><span style="color:#79B8FF"> "model_id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span><span style="color:#E1E4E8">,</span></span> 696<span class="line"><span style="color:#79B8FF"> "variant"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"universal"</span><span style="color:#E1E4E8">,</span></span> 697<span class="line"><span style="color:#79B8FF"> "version"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"1.0"</span><span style="color:#E1E4E8">,</span></span> 698<span class="line"><span style="color:#79B8FF"> "files"</span><span style="color:#E1E4E8">: [</span></span> 699<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"path"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model.onnx"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"sha256"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"a1b2c3..."</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"size"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">345678901</span><span style="color:#E1E4E8"> },</span></span> 700<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"path"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"tokens.txt"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"sha256"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"d4e5f6..."</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"size"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">45123</span><span style="color:#E1E4E8"> },</span></span> 701<span class="line"><span style="color:#E1E4E8"> { </span><span style="color:#79B8FF">"path"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"voices.bin"</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"sha256"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"g7h8i9..."</span><span style="color:#E1E4E8">, </span><span style="color:#79B8FF">"size"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">12345678</span><span style="color:#E1E4E8"> }</span></span> 702<span class="line"><span style="color:#E1E4E8"> ],</span></span> 703<span class="line"><span style="color:#79B8FF"> "platforms"</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">"android"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"ios"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"macos"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"linux"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"windows"</span><span style="color:#E1E4E8">],</span></span> 704<span class="line"><span style="color:#79B8FF"> "format"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"onnx"</span><span style="color:#E1E4E8">,</span></span> 705<span class="line"><span style="color:#79B8FF"> "created_at"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"2026-03-15T10:30:00Z"</span></span> 706<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var p=s(t,10);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># model-specs/tts/kokoro-82m.yaml</span></span> 707<span class="line"><span style="color:#85E89D">version</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"1.0"</span></span> 708<span class="line"><span style="color:#85E89D">status</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">ready</span></span> 709<span class="line"></span> 710<span class="line"><span style="color:#85E89D">model</span><span style="color:#E1E4E8">:</span></span> 711<span class="line"><span style="color:#85E89D"> id</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span></span> 712<span class="line"><span style="color:#85E89D"> family</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro"</span></span> 713<span class="line"><span style="color:#85E89D"> task</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"text-to-speech"</span></span> 714<span class="line"><span style="color:#85E89D"> description</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"High-quality multi-voice TTS, 82M parameters"</span></span> 715<span class="line"><span style="color:#85E89D"> license</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"Apache-2.0"</span></span> 716<span class="line"><span style="color:#85E89D"> parameters</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">82000000</span></span> 717<span class="line"></span> 718<span class="line"><span style="color:#85E89D">source</span><span style="color:#E1E4E8">:</span></span> 719<span class="line"><span style="color:#85E89D"> type</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">huggingface</span></span> 720<span class="line"><span style="color:#85E89D"> repo</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"hexgrad/Kokoro-82M"</span></span> 721<span class="line"></span> 722<span class="line"><span style="color:#85E89D">files</span><span style="color:#E1E4E8">:</span></span> 723<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">source</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-v0_19.onnx"</span></span> 724<span class="line"><span style="color:#85E89D"> target</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model.onnx"</span></span> 725<span class="line"><span style="color:#85E89D"> required</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">true</span></span> 726<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">source</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"voices.bin"</span></span> 727<span class="line"><span style="color:#85E89D"> required</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">true</span></span> 728<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">source</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"tokens.txt"</span></span> 729<span class="line"><span style="color:#85E89D"> required</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">true</span></span> 730<span class="line"></span> 731<span class="line"><span style="color:#85E89D">execution</span><span style="color:#E1E4E8">:</span></span> 732<span class="line"><span style="color:#85E89D"> template</span><span style="color:#E1E4E8">:</span></span> 733<span class="line"><span style="color:#85E89D"> type</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">SimpleMode</span></span> 734<span class="line"><span style="color:#85E89D"> model_file</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model.onnx"</span></span> 735<span class="line"><span style="color:#85E89D"> preprocessing</span><span style="color:#E1E4E8">:</span></span> 736<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">type</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">Phonemize</span></span> 737<span class="line"><span style="color:#85E89D"> backend</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">MisakiDictionary</span></span> 738<span class="line"><span style="color:#85E89D"> tokens_file</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"tokens.txt"</span></span> 739<span class="line"><span style="color:#85E89D"> postprocessing</span><span style="color:#E1E4E8">:</span></span> 740<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">type</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">TTSAudioEncode</span></span> 741<span class="line"><span style="color:#85E89D"> sample_rate</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">24000</span></span> 742<span class="line"><span style="color:#85E89D"> apply_postprocessing</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">true</span></span> 743<span class="line"></span> 744<span class="line"><span style="color:#85E89D">variants</span><span style="color:#E1E4E8">:</span></span> 745<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">id</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"universal"</span></span> 746<span class="line"><span style="color:#85E89D"> default</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">true</span></span> 747<span class="line"><span style="color:#85E89D"> format</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">onnx</span></span> 748<span class="line"><span style="color:#85E89D"> platforms</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">android</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">ios</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">macos</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">linux</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">windows</span><span style="color:#E1E4E8">]</span></span> 749<span class="line"></span> 750<span class="line"><span style="color:#85E89D">publish</span><span style="color:#E1E4E8">:</span></span> 751<span class="line"><span style="color:#85E89D"> hf_org</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"xybrid-ai"</span></span> 752<span class="line"><span style="color:#85E89D"> hf_repo</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span></span></code></pre>`);var r=s(p,6);e(r,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">just</span><span style="color:#9ECBFF"> pack-pipeline-dry</span><span style="color:#9ECBFF"> model-specs/tts/kokoro-82m.yaml</span></span></code></pre>');var c=s(r,4);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>[1/7] Validate â Spec is valid</span></span> 753<span class="line"><span>[2/7] Fetch â Downloaded 3 files from hexgrad/Kokoro-82M</span></span> 754<span class="line"><span>[3/7] Generate â Created model_metadata.json</span></span> 755<span class="line"><span>[4/7] Build â Built kokoro-82m-universal.xyb (156MB â 142MB compressed)</span></span> 756<span class="line"><span>[5/7] Publish âï¸ Skipped (dry run)</span></span> 757<span class="line"><span>[6/7] Verify âï¸ Skipped (dry run)</span></span> 758<span class="line"><span>[7/7] Register âï¸ Skipped (dry run)</span></span></code></pre>`);var d=s(c,4);
758e(d,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">HF_TOKEN</span><span style="color:#F97583">=</span><span style="color:#9ECBFF">hf_xxx</span><span style="color:#B392F0"> just</span><span style="color:#9ECBFF"> pack-pipeline</span><span style="color:#9ECBFF"> model-specs/tts/kokoro-82m.yaml</span></span></code></pre>');var i=s(d,8);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> models</span><span style="color:#9ECBFF"> list</span></span> 759<span class="line"><span style="color:#6A737D"># kokoro-82m text-to-speech 82M Apache-2.0</span></span> 760<span class="line"></span> 761<span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> models</span><span style="color:#9ECBFF"> info</span><span style="color:#9ECBFF"> kokoro-82m</span></span> 762<span class="line"><span style="color:#6A737D"># ID: kokoro-82m</span></span> 763<span class="line"><span style="color:#6A737D"># Task: text-to-speech</span></span> 764<span class="line"><span style="color:#6A737D"># Variants: universal (onnx, 142MB)</span></span> 765<span class="line"><span style="color:#6A737D"># Platforms: android, ios, macos, linux, windows</span></span></code></pre>`);var y=s(i,8);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">{</span></span> 766<span class="line"><span style="color:#79B8FF"> "kokoro-82m"</span><span style="color:#E1E4E8">: {</span></span> 767<span class="line"><span style="color:#79B8FF"> "model_id"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"kokoro-82m"</span><span style="color:#E1E4E8">,</span></span> 768<span class="line"><span style="color:#79B8FF"> "task"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"text-to-speech"</span><span style="color:#E1E4E8">,</span></span> 769<span class="line"><span style="color:#79B8FF"> "variants"</span><span style="color:#E1E4E8">: {</span></span> 770<span class="line"><span style="color:#79B8FF"> "universal"</span><span style="color:#E1E4E8">: {</span></span> 771<span class="line"><span style="color:#79B8FF">
771 "url"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"https://huggingface.co/xybrid-ai/kokoro-82m/resolve/main/kokoro-82m-universal.xyb"</span><span style="color:#E1E4E8">,</span></span> 772<span class="line"><span style="color:#79B8FF"> "sha256"</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"abc123..."</span><span style="color:#E1E4E8">,</span></span> 773<span class="line"><span style="color:#79B8FF"> "size"</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">148897234</span><span style="color:#E1E4E8">,</span></span> 774<span class="line"><span style="color:#79B8FF"> "platforms"</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">"android"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"ios"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"macos"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"linux"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"windows"</span><span style="color:#E1E4E8">]</span></span> 775<span class="line"><span style="color:#E1E4E8"> }</span></span> 776<span class="line"><span style="color:#E1E4E8"> }</span></span> 777<span class="line"><span style="color:#E1E4E8"> }</span></span> 778<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var E=s(y,10);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#85E89D">variants</span><span style="color:#E1E4E8">:</span></span> 779<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">id</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"universal"</span></span> 780<span class="line"><span style="color:#85E89D"> default</span><span style="color:#E1E4E8">: </span><span style="color:#79B8FF">true</span></span> 781<span class="line"><span style="color:#85E89D"> format</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">onnx</span></span> 782<span class="line"><span style="color:#85E89D"> platforms</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">android</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">ios</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">macos</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">linux</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">windows</span><span style="color:#E1E4E8">]</span></span> 783<span class="line"></span> 784<span class="line"><span style="color:#E1E4E8"> - </span><span style="color:#85E89D">id</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"q4-mobile"</span></span> 785<span class="line"><span style="color:#85E89D"> format</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">onnx</span></span> 786<span class="line"><span style="color:#85E89D"> quantization</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">q4</span></span> 787<span class="line"><span style="color:#85E89D"> source_file</span><span style="color:#E1E4E8">: </span><span style="color:#9ECBFF">"model-q4.onnx"</span></span> 788<span class="line"><span style="color:#85E89D"> platforms</span><span style="color:#E1E4E8">: [</span><span style="color:#9ECBFF">android</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">ios</span><span style="color:#E1E4E8">]</span></span></code></pre>`);var F=s(E,24);e(F,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># List all model specs</span></span> 789<span class="line"><span style="color:#B392F0">just</span><span style="color:#9ECBFF"> pack-list</span></span> 790<span class="line"></span> 791<span class="line"><span style="color:#6A737D"># Validate a spec</span></span> 792<span class="line"><span style="color:#B392F0">just</span><span style="color:#9ECBFF"> pack-validate</span><span style="color:#9ECBFF"> model-specs/tts/kokoro-82m.yaml</span></span> 793<span class="line"></span> 794<span class="line"><span style="color:#6A737D"># Show spec details</span></span> 795<span class="line"><span style="color:#B392F0">just</span><span style="color:#9ECBFF"> pack-info</span><span style="color:#9ECBFF"> model-specs/tts/kokoro-82m.yaml</span></span> 796<span class="line"></span> 797<span class="line"><span style="color:#6A737D"># Dry run (no publish)</span></span> 798<span class="line"><span style="color:#B392F0">just</span><span style="color:#9ECBFF"> pack-pipeline-dry</span><span style="color:#9ECBFF"> model-specs/tts/kokoro-82m.yaml</span></span> 799<span class="line"></span> 800<span class="line"><span style="color:#6A737D"># Full pipeline</span></span> 801<span class="line"><span style="color:#E1E4E8">HF_TOKEN</span><span style="color:#F97583">=</span><span style="color:#9ECBFF">hf_xxx</span><span style="color:#B392F0"> just</span><span style="color:#9ECBFF"> pack-pipeline</span><span style="color:#9ECBFF"> model-specs/tts/kokoro-82m.yaml</span></span></code></pre>`),f(10),h(o,a)}const Es=Object.freeze(Object.defineProperty({__proto__:null,default:ce,metadata:ys},Symbol.toStringTag,{value:"Module"})),hs={title:"Run AI Models On-Device â Zero Config, Five Minutes",description:"CLI, Rust, Flutter, Swift, Kotlin, Unity â run 25+ ML models on-device with one command. No tensor shapes, no preprocessing scripts.",date:"2026-06-02",author:"Glenn Sonna",tags:["on-device-ai","run-ml-locally","rust-ml","edge-inference","open-source-ai"],category:"announcement",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:3},{title:bo,description:Bo,date:vo,author:ko,tags:wo,category:xo,image:Co,imageAlt:_o,draft:Ao,featured:To,readingTime:So}=hs;var ie=u('<p>You already know why on-device AI matters. Privacy, latency, cost. Youâve read the guides.</p> <p>Now you want to actually do it. Hereâs what that looks like with <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a> â no tensor shapes, no preprocessing scripts, no ML expertise.</p> <hr/> <h2>Install</h2> <!> <h2>Text-to-Speech</h2> <!> <p>Thatâs it. Xybrid resolved the model from the registry, downloaded it, ran inference, and saved a WAV file. You configured nothing.</p> <p>Kokoro is an 82M parameter TTS model with 24 voices. First run downloads ~80MB and caches it locally. Subsequent runs are instant.</p> <h2>Speech Recognition</h2> <!> <p>Whisper Tiny transcribes audio in real-time on any modern laptop. Outputs plain text.</p> <h2>Text Generation</h2> <!> <p>Qwen 3.5 0.8B runs locally via llama.cpp. 201 languages, fits in 500MB quantized.</p> <h2>Browse the Registry</h2> <!> <p>25+ models, all hosted on HuggingFace, downloaded on-demand, cached locally:</p> <table><thead><tr><th>
801Model</th><th>Task</th><th>Size</th><th>Notes</th></tr></thead><tbody><tr><td>kokoro-82m</td><td>Text-to-Speech</td><td>82M</td><td>24 voices, high quality</td></tr><tr><td>kitten-tts-nano-0.8</td><td>Text-to-Speech</td><td>15M</td><td>Ultra-lightweight</td></tr><tr><td>qwen3-tts-0.6b</td><td>Text-to-Speech</td><td>600M</td><td>Multilingual</td></tr><tr><td>whisper-tiny</td><td>Speech Recognition</td><td>39M</td><td>Real-time, multilingual</td></tr><tr><td>wav2vec2-base-960h</td><td>Speech Recognition</td><td>95M</td><td>CTC-based</td></tr><tr><td>lfm2.5-350m</td><td>Text Generation</td><td>354M</td><td>9 languages, edge-optimized</td></tr><tr><td>smollm2-360m</td><td>Text Generation</td><td>360M</td><td>Best tiny LLM</td></tr><tr><td>qwen3.5-0.8b</td><td>Text Generation</td><td>800M</td><td>201 languages</td></tr><tr><td>gemma-4-e2b</td><td>Text Generation</td><td>5.1B</td><td>Multimodal</td></tr><tr><td>mistral-7b</td><td>Text Generation</td><td>7B</td><td>Function calling</td></tr></tbody></table> <h2>Beyond the CLI</h2> <p>The CLI is the fastest way to evaluate. When youâre ready to integrate into an app, Xybrid has SDKs for Flutter, Swift, Kotlin, Unity, and Rust â same models, same behavior, every platform.</p> <hr/> <p>Xybrid is in beta (v0.1.0-beta9), open-source under Apache 2.0.</p> <p><strong>GitHub</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide</a> â hardware, privacy, cost, and how to get started.</li> <li><a href="/blog/edge-vs-cloud-inference">Edge AI vs Cloud AI: When to Run Models On-Device</a> â the decision framework.</li> <li><a href="/blog/add-text-to-speech-to-flutter-app-in-15-minutes">Add Text-to-Speech to Your Flutter App in 15 Minutes</a> â hands-on Flutter tutorial.</li></ul>',1);function de(o){var a=ie(),n=s(b(a),8);e(n,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> install</span><span style="color:#9ECBFF"> xybrid-cli</span></span></code></pre>');var l=s(n,4);e(l,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> kokoro-82m</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> "Hello from the edge"</span><span style="color:#79B8FF"> --output</span><span style="color:#9ECBFF"> hello.wav</span></span></code></pre>');var t=s(l,8);e(t,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> whisper-tiny</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> recording.wav</span></span></code></pre>');var p=s(t,6);e(p,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> qwen3.5-0.8b</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> "Explain quantum computing in one sentence"</span></span></code></pre>');var r=s(p,6);e(r,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> models</span><span style="color:#9ECBFF"> list</span></span></code></pre>'),f(20),h(o,a)}const us=Object.freeze(Object.defineProperty({__proto__:null,default:de,metadata:hs},Symbol.toStringTag,{value:"Module"})),ms={title:"How to Run LLMs Locally: A Complete Guide (2026)",description:"How to run large language models locally on your laptop, desktop, or phone â llama.cpp, Ollama, ONNX Runtime, and on-device options compared. No cloud API needed.",date:"2026-06-09",author:"Glenn Sonna",tags:["local-llm","run-llm-locally","offline-ai","private-llm","on-device-ai","local-ai"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:12},{title:Mo,description:Po,date:Io,author:Lo,tags:Do,category:Ro,image:Oo,imageAlt:No,draft:Go,featured:zo,readingTime:Uo}=ms;var ye=u('<p>You do not need a cloud API key to use a large language model. Models like Llama 3.2, Qwen 3.5, Phi-3, and Gemma 2 run on consumer hardware â your laptop, your desktop, even your phone. No internet required, no per-token billing, no data leaving your machine.</p> <p>This guide covers everything you need to start running LLMs locally: which models fit on which hardware, the major runtimes and tools available, how to choose between them, and what performance to expect.</p> <h2>Why Run LLMs Locally</h2> <p>Four reasons drive most people to local LLM inference:</p> <p><strong>Privacy.</strong> Your prompts and conversations stay on your machine. No data is transmitted to OpenAI, Anthropic, Google, or any other provider. For personal use, this means your journal entries, medical questions, and private thoughts never leave your device. For enterprise use, it means proprietary code, internal documents, and customer data stay within your infrastru
801cture.</p> <p><strong>Cost.</strong> Cloud LLM APIs charge per token. At scale, this adds up quickly. Local inference has a one-time cost (downloading the model) and zero marginal cost per token. If you run hundreds of queries per day, local inference pays for itself within the first week.</p> <p><strong>Latency.</strong> Local inference eliminates network round-trips. For applications that need real-time responses â voice assistants, code completion, interactive chat â removing 100-300ms of network overhead makes a noticeable difference.</p> <p><strong>Control.</strong> You choose the model, the quantization level, the system prompt, and the inference parameters. No rate limits, no content filtering you did not ask for, no surprise model deprecations. The model runs when you tell it to, exactly how you configured it.</p> <h2>Hardware Requirements</h2> <p>The constraining factor for local LLMs is memory. A language model must fit entirely in RAM (or VRAM) during inference. Here is what you need:</p> <h3>Desktop and Laptop</h3> <table><thead><tr><th>Model Size</th><th>RAM Required (Q4 quantized)</th><th>Example Models</th><th>Hardware</th></tr></thead><tbody><tr><td>0.5-1B params</td><td>1-2 GB</td><td>Qwen 3.5 0.8B, SmolLM2</td><td>Any modern laptop</td></tr><tr><td>1-3B params</td><td>2-3 GB</td><td>Llama 3.2 1B, Llama 3.2 3B, Phi-3 Mini</td><td>Any laptop with 8 GB RAM</td></tr><tr><td>7-8B params</td><td>5-6 GB</td><td>Llama 3.1 8B, Mistral 7B, Gemma 2 9B</td><td>Laptop with 16 GB RAM</td></tr><tr><td>13B params</td><td>8-10 GB</td><td>Llama 2 13B, CodeLlama 13B</td><td>Laptop with 16+ GB RAM</td></tr><tr><td>30-34B params</td><td>20-24 GB</td><td>CodeLlama 34B, Yi 34B</td><td>Desktop with 32 GB RAM or GPU with 24 GB VRAM</td></tr><tr><td>70B params</td><td>40-48 GB</td><td>Llama 3.1 70B</td><td>High-end workstation or multi-GPU setup</td></tr></tbody></table> <p><strong>Apple Silicon</strong> (M1/M2/M3/M4) is particularly good for local LLMs because the unified memory architecture means CPU and GPU share the same RAM pool. A MacBook with 16 GB of unified memory can run 7-8B models fluently in the GPU.</p> <p><strong>NVIDIA GPUs</strong> with VRAM offload the entire model to the GPU for maximum speed. An RTX 4090 (24 GB VRAM) handles up to 13B models comfortably at full GPU speed, or 30B+ models with partial CPU offload.</p> <p><strong>CPU-only</strong> works for models up to 13B with acceptable speed (5-15 tokens/second). Beyond that, an integrated or discrete GPU is strongly recommended.</p> <h3>Mobile Devices</h3> <table><thead><tr><th>Device Tier</th><th>Available Memory</th><th>Max Model Size</th><th>Example Models</th></tr></thead><tbody><tr><td>Flagship 2024+ (iPhone 15 Pro, Pixel 8 Pro)</td><td>6-8 GB total (~3-4 GB for models)</td><td>1-3B Q4</td><td>Llama 3.2 1B, Qwen 3.5 0.8B</td></tr><tr><td>Mid-range 2024+</td><td>4-6 GB total (~2-3 GB for models)</td><td>0.5-1B Q4</td><td>SmolLM2, Qwen 3.5 0.8B</td></tr><tr><td>Older / budget phones</td><td>Under 4 GB total</td><td>Not recommended</td><td>â</td></tr></tbody></table> <p>Mobile inference is real but constrained. A 1B parameter model quantized to Q4 generates 10-20 tokens per second on a flagship phone â usable for chat, summarization, and simple question-answering. Models larger than 3B are impractical on current mobile hardware.</p> <h2>Model Formats</h2> <p>Local LLM models come in several formats. The format determines which runtimes can load the model and what optimizations are available.</p> <h3>GGUF</h3> <p>The dominant format for local LLM inference. GGUF files are self-contained â the model weights, tokenizer, and metadata are in a single file. llama.cpp, Ollama, LM Studio, and most local LLM tools use GGUF.</p> <p>GGUF supports multiple quantization levels:</p> <table><thead><tr><th>Quantization</th><th>Bits Per Weight</th><th>Quality</th><th>Speed</th><th>Use Case</th></tr></thead><tbody><tr><td>Q2_K</td><td>~2.5</td><td>Low</td><td>Fastest</td><td>Experimentation only</td></tr><tr><td>Q4_K_M</td><td>~4.5</td><td>Good</td><td>Fast</td><td><strong>Recommended default</strong></td></tr><tr><td>Q5_K_M</td><td>~5.5</td><td>Very good</td><td>Moderate</td><td>When quality matters more than speed</td></tr><tr><td>Q6_K</td><td>~6.5</td><td>Near-FP16</td><td>Slower</td><td>Quality-sensitive tasks</td></tr><tr><td>Q8_0</td><td>8</td><td>Excellent</td><td>Slow</td><td>When you have the memory</td></tr><tr><td>FP16</td><td>16</td><td>Full</td><td>Slowest</td><td>Benchmarking only</td></tr></tbody></table> <p><strong>Start with Q4_K_M.</strong> It offers the best balance of quality, speed, and memory usage for most models and most use cases.</p> <h3>ONNX</h3> <p>Used by ONNX Runtime for cross-platform inference. ONNX models can leverage platform-specific accelerators (CoreML on Apple, DirectML on Windows, NNAPI on Android). Less common for LLMs but well-suited for smaller models and non-LLM tasks (AS
801R, TTS, embeddings, classification).</p> <h3>SafeTensors</h3> <p>The native format for Hugging Face models. Used by transformers, Candle (Rust), and other Python-ecosystem tools. Not directly loadable by llama.cpp or Ollama â you need to convert to GGUF first.</p> <h2>Runtime Options Compared</h2> <p>Several tools exist for running LLMs locally. They serve different use cases.</p> <h3>llama.cpp</h3> <p>The foundational project for local LLM inference. A C/C++ library with no dependencies that runs on CPU, CUDA, Metal, Vulkan, and SYCL.</p> <p><strong>Best for:</strong> Developers who need maximum control, custom integrations, embedding in other applications, mobile deployment.</p> <p><strong>Strengths:</strong></p> <ul><li>Fastest CPU inference of any runtime</li> <li>Metal acceleration on Apple Silicon</li> <li>CUDA acceleration on NVIDIA GPUs</li> <li>Used as the backend by Ollama, LM Studio, and most other tools</li> <li>C API for embedding in any language</li></ul> <p><strong>Tradeoffs:</strong></p> <ul><li>No GUI â command-line and API only</li> <li>Compiling from source for optimal performance</li> <li>Configuration requires understanding quantization and inference parameters</li></ul> <h3>Ollama</h3> <p>A user-friendly wrapper around llama.cpp. Install, <code>ollama pull llama3.2</code>, <code>ollama run llama3.2</code>. That is it.</p> <p><strong>Best for:</strong> Getting started quickly, running models as a local API server, replacing cloud API calls with a local endpoint.</p> <p><strong>Strengths:</strong></p> <ul><li>Simplest installation and model management</li> <li>OpenAI-compatible API server (drop-in replacement for many applications)</li> <li>Model library with one-command downloads</li> <li>Good defaults for most parameters</li></ul> <p><strong>Tradeoffs:</strong></p> <ul><li>Less control over inference parameters than raw llama.cpp</li> <li>Abstracts away model format details (harder to use custom GGUF files)</li> <li>Desktop only â no mobile support</li></ul> <h3>LM Studio</h3> <p>A desktop application with a GUI for browsing, downloading, and chatting with local models.</p> <p><strong>Best for:</strong> Non-developers, experimentation, evaluating different models quickly.</p> <p><strong>Strengths:</strong></p> <ul><li>Visual interface for model management</li> <li>Built-in chat UI</li> <li>Local API server (OpenAI-compatible)</li> <li>Model discovery and download from HuggingFace</li></ul> <p><strong>Tradeoffs:</strong></p> <ul><li>Desktop only (macOS, Windows, Linux)</li> <li>Not embeddable in other applications</li> <li>Closed source</li></ul> <h3>Embedding in Mobile and Cross-Platform Apps</h3> <p>For shipping local LLMs inside an application (mobile, desktop, or embedded), you need a runtime that can be linked as a library, not run as a separate process.</p> <p>This is where llama.cppâs C API becomes essential. It can be compiled for iOS, Android, macOS, Windows, and Linux, then called from your application code through FFI bindings. The alternative is ONNX Runtime, which provides similar cross-platform coverage but is more commonly used for non-LLM models.</p> <p>The challenge is packaging: you need to bundle the model with your app or download it on first launch, manage model storage on constrained devices, and handle memory pressure gracefully. This is non-trivial engineering that most developers should not rebuild from scratch.</p> <h2>Choosing a Model</h2> <p>The âbestâ local LLM depends on your hardware and use case. Here are practical recommendations:</p> <h3>For Chat and General Assistance</h3> <table><thead><tr><th>Hardware</th><th>Recommended Model</th><th>Why</th></tr></thead><tbody><tr><td>8 GB laptop</td><td>Llama 3.2 3B Q4_K_M</td><td>Best quality at this memory budget</td></tr><tr><td>16 GB laptop</td><td>Llama 3.1 8B Q4_K_M</td><td>Best general-purpose local model</td></tr><tr><td>32 GB desktop</td><td>Llama 3.1 70B Q4_K_M</td><td>Cloud-competitive quality</td></tr><tr><td>Mobile (flagship)</td><td>Llama 3.2 1B Q4_K_M</td><td>Best mobile chat model</td></tr></tbody></table> <h3>For Code</h3> <table><thead><tr><th>
801Hardware</th><th>Recommended Model</th><th>Why</th></tr></thead><tbody><tr><td>16 GB laptop</td><td>CodeLlama 7B Q4_K_M or DeepSeek Coder 6.7B</td><td>Trained specifically for code generation</td></tr><tr><td>32 GB desktop</td><td>CodeLlama 34B Q4_K_M</td><td>Closest to cloud code assistants</td></tr></tbody></table> <h3>For Summarization and Extraction</h3> <p>Smaller models (1-3B) handle summarization and information extraction well because the task is largely extractive rather than generative. A Llama 3.2 1B can summarize a document nearly as well as a 70B model for straightforward summarization tasks.</p> <h2>Performance Expectations</h2> <p>Realistic token generation speeds on common hardware:</p> <table><thead><tr><th>Model</th><th>Hardware</th><th>Quantization</th><th>Tokens/sec</th><th>Experience</th></tr></thead><tbody><tr><td>Llama 3.2 1B</td><td>iPhone 15 Pro</td><td>Q4_K_M</td><td>15-25</td><td>Usable chat</td></tr><tr><td>Llama 3.2 3B</td><td>M2 MacBook Air</td><td>Q4_K_M</td><td>30-50</td><td>Fluent</td></tr><tr><td>Llama 3.1 8B</td><td>M3 MacBook Pro</td><td>Q4_K_M</td><td>25-40</td><td>Fluent</td></tr><tr><td>Llama 3.1 8B</td><td>RTX 4090</td><td>Q4_K_M</td><td>80-120</td><td>Instant</td></tr><tr><td>Llama 3.1 70B</td><td>M3 Max (48 GB)</td><td>Q4_K_M</td><td>8-15</td><td>Readable</td></tr><tr><td>Llama 3.1 70B</td><td>RTX 4090 (24 GB)</td><td>Q4_K_M</td><td>15-25 (partial offload)</td><td>Usable</td></tr></tbody></table> <p>For conversational use, 10+ tokens/second feels responsive. Anything above 20 tokens/second feels fluent. Below 5 tokens/second, users notice the delay and it starts to feel sluggish.</p> <h2>Getting Started (5 Minutes)</h2> <p>The fastest path from zero to running a local LLM:</p> <h3>Option 1: Ollama (Simplest)</h3> <!> <h3>Option 2: llama.cpp (Most Control)</h3> <!> <h3>Option 3: In a Mobile or Desktop App</h3> <p>For embedding a local LLM in your own application, you need an SDK that handles model management, platform-specific compilation, and runtime abstraction. This is more involved than running a CLI tool, but it gives you full control over the user experience.</p> <p>The general pattern:</p> <ol><li>Bundle or download the GGUF model on first launch</li> <li>Load the model into memory using llama.cpp (or an abstraction over it)</li> <li>Run inference through the libraryâs API</li> <li>Stream tokens to your UI as they are generated</li></ol> <h2>Limitations to Know</h2> <p><strong>Context length is memory-bound.</strong> A longer context window requires more memory. A 7B model with a 4K context fits in 6 GB. The same model with a 32K context needs 10+ GB. If memory is tight, reduce the context length.</p> <p><strong>First-token latency is real.</strong> The first token takes longer than subsequent tokens because the model must process the entire prompt (prefill). For a 2,000-token prompt on a 7B model, expect 1-3 seconds before the first token appears. Subsequent tokens stream at the speeds listed above.</p> <p><strong>Not all tasks benefit.</strong> Tasks that require vast world knowledge, multi-step reasoning chains, or real-time information retrieval still favor larger cloud models. Local LLMs excel at focused tasks: summarization, extraction, code generation, conversational Q&A, and text transformation.</p> <p><strong>Quantization has limits.</strong> Below Q3, most models degrade noticeably. Q4_K_M is the sweet spot. If a model does not perform well at Q4, try the next size up rather than increasing the quantization quality of the same model.</p> <h2>What Is Next for Local LLMs</h2> <p>The trajectory is clear: models are getting smaller and more capable. Each generation of hardware ships with more NPU/Neural Engine compute. The models that required a desktop last year run on a phone this year.</p> <p>Within the next year, expect:</p> <ul><li>3B parameter models that match todayâs 7B quality</li> <li>Dedicated NPU inference paths on iOS and Android for LLMs</li> <li>Better speculative decoding for faster generation on constrained hardware</li> <li>On-device fine-tuning for personalization</li></ul> <p>The gap between cloud and local model quality is closing. For the majority of practical AI features â chat, summarization, extraction, code assistance, voice interaction â local inference is already good enough. It will only get better.</p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide to Running ML Models Locally</a> â broader guide covering all model types, not just LLMs.</li> <li><a href="/blog/edge-vs-cloud-inference">Edge AI vs Cloud AI: When to Run Models On-Device</a> â decision framework for when local inference makes sense vs cloud.</li></ul>',1);function Ee(o){var a=ye(),n=s(b(a),142);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># Install Ollama</span></span> 802<span class="line"><span style="color:#B392F0">curl</span><span style="color:#79B8FF"> -fsSL</span><span style="color:#9ECBFF"> https://ollama.com/install.sh</span><span style="color:#F97583"> |</span><span style="color:#B392F0"> sh</span></span> 803<span class="line"></span> 804<span class="line"><span style="color:#6A737D"># Pull and run a model</span></span> 805<span class="line"><span style="color:#B392F0">ollama</span><span style="color:#9ECBFF"> run</span><span style="color:#9ECBFF"> llama3.2</span></span> 806<span class="line"></span> 807<span class="line"><span style="color:#6A737D"># Or run as an API server</span></span> 808<span class="line"><span style="color:#B392F0">ollama</span><span style="color:#9ECBFF"> serve</span></span> 809<span class="line"><span style="color:#B392F0">curl</span><span style="color:#9ECBFF"> http://localhost:11434/api/generate</span><span style="color:#79B8FF"> -d</span><span style="color:#9ECBFF"> '{"model": "llama3.2", "prompt": "Hello"}'</span></span></code></pre>`);var l=s(n,4);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># Clone and build</span></span> 810<span class="line"><span style="color:#B392F0">git</span><span style="color:#9ECBFF"> clone</span><span style="color:#9ECBFF"> https://github.com/ggerganov/llama.cpp</span></span> 811<span class="line"><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> llama.cpp</span><span style="color:#E1E4E8"> && </span><span style="color:#B392F0">make</span><span style="color:#79B8FF"> -j</span></span> 812<span class="line"></span> 813<span class="line"><span style="color:#6A737D"># Download a model (GGUF format from HuggingFace)</span></span> 814<span class="line"><span style="color:#6A737D"># Then run:</span></span> 815<span class="line"><span style="color:#B392F0">./llama-cli</span><span style="color:#79B8FF"> -m</span><span style="color:#9ECBFF"> path/to/model.gguf</span><span style="color:#79B8FF"> -p</span><span style="color:#9ECBFF"> "Hello, how are you?"</span><span style="color:#79B8FF"> -n</span><span style="color:#79B8FF"> 256</span></span></code></pre>`),f(34),h(o,a)}const Fs=Object.freeze(Object.defineProperty({__proto__:null,default:Ee,metadata:ms},Symbol.toStringTag,{value:"Module"})),gs={title:"Streaming LLM Tokens from Rust to Flutter in Real-Time",description:"How we bridge real-time token streaming from a Rust inference engine to Flutter's reactive UI using flutter_rust_bridge callbacks.",date:"2026-07-07",author:"Glenn Sonna",tags:["flutter","rust","ai","tutorial"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:6},{title:qo,description:$o,date:Wo,author:Ko,tags:Ho,category:Xo,image:jo,imageAlt:Vo,draft:Qo,featured:Yo,readingTime:Jo}=gs;var he=u('<p>Youâre running an LLM locally. Tokens generate one at a time, each in ~20ms. You want the UI to update as each token arrives â the ChatGPT-style typing effect.</p> <p>The problem: your inference engine is in Rust. Your UI is in Flutter. Theyâre in different worlds, connected by FFI.</p> <p>Hereâs how we solved real-time token streaming across the Rust-Flutter boundary in <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a>.</p> <hr/> <h2>The Challenge</h2> <p>Token-by-token streaming has strict requirements:</p> <ol><li><strong>Low latency</strong> â each token must reach the UI within milliseconds of generation</li> <li><strong>Non-blocking</strong> â inference runs on a background thread; the UI stays responsive</li> <li><strong>Backpressure-aware</strong> â if the UI canât keep up, we shouldnât drop tokens</li> <li><strong>Clean shutdown</strong> â cancellation must propagate from Dart to Rust cleanly</li></ol> <p>A naive approach (poll for results, batch tokens) adds visible latency. Users notice when text appears in chunks instead of flowing smoothly.</p> <h2>The Architecture</h2> <!> <p>The key insight: <strong>flutter_rust_bridge (FRB) supports Dart <code>Stream</code> return types from Rust functions.</strong> This means Rust can push values to Dart asynchronously â exactly what we need.</p> <h2>Rust Side: The Streaming Callback</h2> <p>In <code>xybrid-core</code>, streaming is built around a simple callback:</p> <!> <p>The callback receives each token as itâs generated. Returning <code>false</code> signals cancellation â the engine stops generating.</p> <p>The executorâs streaming API:</p> <!> <h2>Bridge Layer: FRB Stream</h2> <p>flutter_rust_bridge can turn a Rust function that calls a callback into a Dart
815<code>Stream</code>. Hereâs the bridge function:</p> <!> <p>The <code>StreamSink</code> is FRBâs mechanism for sending values from Rust to Dart. Each <code>sink.add()</code> call pushes a value to the Dart <code>Stream</code>. If the Dart side cancels (drops the stream subscription), <code>sink.add()</code> returns <code>Err</code>, and we return <code>false</code> to stop generation.</p> <h2>Dart Side: Consuming the Stream</h2> <p>On the Flutter side, tokens arrive as a standard Dart <code>Stream</code>:</p> <!> <p>Thatâs the core pattern. Each token triggers a <code>setState</code>, which rebuilds the text widget. The result is smooth, character-by-character text appearance.</p> <h2>Building a Chat UI</h2> <p>Hereâs a more complete example â a chat screen with streaming responses:</p> <!> <p>Key details:</p> <ul><li><strong>Cancel button</strong>: <code>_subscription.cancel()</code> in Dart â <code>sink.add()</code> returns error in Rust â callback returns <code>false</code> â inference stops. Clean cancellation across the FFI boundary.</li> <li><strong>Last message mutation</strong>: We append tokens to the last message in the list, which triggers a rebuild of only that widget.</li> <li><strong>Error propagation</strong>: Rust errors surface as stream errors in Dart.</li></ul> <h2>GenerationConfig: Controlling Output</h2> <p>Control the LLMâs behavior with <code>GenerationConfig</code>:</p> <!> <p>Presets:</p> <!> <p>These are <strong>always-available types</strong> â they compile even on platforms without LLM support, so your code doesnât need platform conditionals.</p> <h2>Performance Notes</h2> <p>Measured with Qwen 3.5 0.8B (Q4_K_M quantization):</p> <table><thead><tr><th>Device</th><th>Tokens/sec</th><th>First Token</th><th>Notes</th></tr></thead><tbody><tr><td>MacBook Pro M2</td><td>~45 tok/s</td><td>~200ms</td><td>Metal GPU acceleration</td></tr><tr><td>iPhone 15 Pro</td><td>~30 tok/s</td><td>~350ms</td><td>Metal GPU</td></tr><tr><td>Pixel 8</td><td>~15 tok/s</td><td>~500ms</td><td>CPU only</td></tr></tbody></table> <p>The streaming overhead (FRB callback per token) adds <1ms per token â negligible compared to generation time.</p> <h2>Common Pitfalls</h2> <h3>1. Donât Rebuild the Entire List</h3> <!> <h3>2. Handle Reasoning Tags</h3> <p>Some models (like Qwen 3.5) emit <code><think>...</think></code> blocks for chain-of-thought reasoning. Xybrid strips these automatically in streaming mode, so you donât need to filter them in Dart.</p> <h3>3. Warmup Before First Chat</h3> <p>The first token has extra latency (model loading, shader compilation). Call <code>warmup()</code> during your loading screen:</p> <!> <hr/> <h2>Summary</h2> <p>The streaming pipeline:</p> <ol><li><strong>Rust</strong> generates tokens via llama.cpp or Candle</li> <li><strong>StreamingCallback</strong> fires per token, pushes to FRB <code>StreamSink</code></li> <li><strong>FRB</strong> marshals the value across the FFI boundary</li> <li><strong>Dart Stream</strong> delivers it to Flutterâs reactive UI</li> <li><strong>setState</strong> triggers a rebuild, token appears on screen</li></ol> <p>Total overhead per token: <1ms. Users see smooth, real-time text generation from a model running entirely on their device.</p> <hr/> <p><strong>Public SDK source</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a> â check the Flutter binding docs for package usage.</p> <hr/> <p><em>Building something with local LLMs in Flutter? Share it in the comments!</em></p>',1);function ue(o){var a=he(),n=s(b(a),18);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>Rust (background thread) Flutter (main thread)</span></span> 816<span class="line"><span>ââââââââââââââââââââââââ ââââââââââââââââââââââââ</span></span> 817<span class="line"><span>â llama.cpp / Candle â â StreamBuilder â</span></span> 818<span class="line"><span>â generates token â â rebuilds UI â</span></span> 819<span class="line"><span>â â â â â² â</span></span> 820<span class="line"><span>â â¼ â â â â</span></span> 821<span class="line"><span>â StreamingCallback âââââ FRB âââââ¶â Dart Stream â</span></span> 822<span class="line"><span>â (Rust closure) â (async) â (token events) â</span></span> 823<span class="line"><span>ââââââââââââââââââââââââ ââââââââââââââââââââââââ</span></span></code></pre>`);var l=s(n,8);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> type</span><span style="color:#B392F0"> StreamingCallback</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> Box</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">dyn</span><span style="color:#B392F0"> FnMut</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">PartialToken</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> bool</span><span style="color:#F97583"> +</span><span style="color:#B392F0"> Send</span><span style="color:#E1E4E8">>;</span></span> 824<span class="line"></span> 825<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> PartialToken</span><span style="color:#E1E4E8"> {</span></span> 826<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> text</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">,</span></span> 827<span class="line"><span style="color:#F97583"> pub</span><span style="color:#E1E4E8"> is_final</span><span style="color:#F97583">:</span><span style="color:#B392F0"> bool</span><span style="color:#E1E4E8">,</span></span> 828<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var t=s(l,6);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> execute_streaming</span><span style="color:#E1E4E8">(</span></span> 829<span class="line"><span style="color:#F97583"> &
829mut</span><span style="color:#79B8FF"> self</span><span style="color:#E1E4E8">,</span></span> 830<span class="line"><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">ModelMetadata</span><span style="color:#E1E4E8">,</span></span> 831<span class="line"><span style="color:#E1E4E8"> input</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">Envelope</span><span style="color:#E1E4E8">,</span></span> 832<span class="line"><span style="color:#E1E4E8"> callback</span><span style="color:#F97583">:</span><span style="color:#B392F0"> StreamingCallback</span><span style="color:#E1E4E8">,</span></span> 833<span class="line"><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> Result</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">Envelope</span><span style="color:#E1E4E8">> {</span></span> 834<span class="line"><span style="color:#6A737D"> // ... sets up inference, calls callback per token</span></span> 835<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var p=s(t,6);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// In the FRB bridge layer</span></span> 836<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> run_pipeline_streaming</span><span style="color:#E1E4E8">(</span></span> 837<span class="line"><span style="color:#E1E4E8"> yaml_content</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">,</span></span> 838<span class="line"><span style="color:#E1E4E8"> input</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ApiEnvelope</span><span style="color:#E1E4E8">,</span></span> 839<span class="line"><span style="color:#E1E4E8"> sink</span><span style="color:#F97583">:</span><span style="color:#B392F0"> StreamSink</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">ApiPartialToken</span><span style="color:#E1E4E8">>,</span></span> 840<span class="line"><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> Result</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">ApiEnvelope</span><span style="color:#E1E4E8">> {</span></span> 841<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> callback</span><span style="color:#F97583">:</span><span style="color:#B392F0"> StreamingCallback</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> Box</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">move</span><span style="color:#F97583"> |</span><span style="color:#E1E4E8">token</span><span style="color:#F97583">|</span><span style="color:#E1E4E8"> {</span></span> 842<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> api_token </span><span style="color:#F97583">=</span><span style="color:#B392F0"> ApiPartialToken</span><span style="color:#E1E4E8"> {</span></span> 843<span class="line"><span style="color:#E1E4E8"> text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> token</span><span style="color:#F97583">.</span><span style="color:#E1E4E8">text</span><span style="color:#F97583">.</span><span style="color:#B392F0">clone</span><span style="color:#E1E4E8">(),</span></span> 844<span class="line"><span style="color:#E1E4E8"> is_final</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> token</span><span style="color:#F97583">.</span><span style="color:#E1E4E8">is_final,</span></span> 845<span class="line"><span style="color:#E1E4E8"> };</span></span> 846<span class="line"><span style="color:#E1E4E8"> sink</span><span style="color:#F97583">.</span><span style="color:#B392F0">add</span><span style="color:#E1E4E8">(api_token)</span><span style="color:#F97583">.</span><span style="color:#B392F0">is_ok</span><span style="color:#E1E4E8">()</span></span> 847<span class="line"><span style="color:#E1E4E8"> });</span></span> 848<span class="line"></span> 849<span class="line"><span style="color:#6A737D"> // Run inference with streaming</span></span> 850<span class="line"><span style="color:#F97583">
850 let</span><span style="color:#E1E4E8"> result </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> executor</span><span style="color:#F97583">.</span><span style="color:#B392F0">execute_streaming</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">metadata, </span><span style="color:#F97583">&</span><span style="color:#E1E4E8">envelope, callback)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 851<span class="line"><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(result</span><span style="color:#F97583">.</span><span style="color:#B392F0">into</span><span style="color:#E1E4E8">())</span></span> 852<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var r=s(p,8);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> stream </span><span style="color:#F97583">=</span><span style="color:#B392F0"> runPipelineStreaming</span><span style="color:#E1E4E8">(</span></span> 853<span class="line"><span style="color:#E1E4E8"> yamlContent</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> pipelineYaml,</span></span> 854<span class="line"><span style="color:#E1E4E8"> input</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> userMessage),</span></span> 855<span class="line"><span style="color:#E1E4E8">);</span></span> 856<span class="line"></span> 857<span class="line"><span style="color:#79B8FF">String</span><span style="color:#E1E4E8"> fullResponse </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> ''</span><span style="color:#E1E4E8">;</span></span> 858<span class="line"></span> 859<span class="line"><span style="color:#F97583">await</span><span style="color:#F97583"> for</span><span style="color:#E1E4E8"> (</span><span style="color:#F97583">final</span><span style="color:#E1E4E8"> token </span><span style="color:#F97583">in</span><span style="color:#E1E4E8"> stream) {</span></span> 860<span class="line"><span style="color:#E1E4E8"> fullResponse </span><span style="color:#F97583">+=</span><span style="color:#E1E4E8"> token.text;</span></span> 861<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() {</span></span> 862<span class="line"><span style="color:#E1E4E8"> _responseText </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> fullResponse;</span></span> 863<span class="line"><span style="color:#E1E4E8"> });</span></span> 864<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var c=s(r,8);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">class</span><span style="color:#79B8FF"> ChatScreen</span><span style="color:#F97583"> extends</span><span style="color:#79B8FF"> StatefulWidget</span><span style="color:#E1E4E8"> {</span></span> 865<span class="line"><span style="color:#F97583"> const</span><span style="color:#79B8FF"> ChatScreen</span><span style="color:#E1E4E8">({</span><span style="color:#79B8FF">super</span><span style="color:#E1E4E8">.key});</span></span> 866<span class="line"></span> 867<span class="line"><span style="color:#F97583">
867 @override</span></span> 868<span class="line"><span style="color:#79B8FF"> State</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">ChatScreen</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">createState</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">=></span><span style="color:#79B8FF"> _ChatScreenState</span><span style="color:#E1E4E8">();</span></span> 869<span class="line"><span style="color:#E1E4E8">}</span></span> 870<span class="line"></span> 871<span class="line"><span style="color:#F97583">class</span><span style="color:#79B8FF"> _ChatScreenState</span><span style="color:#F97583"> extends</span><span style="color:#79B8FF"> State</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">ChatScreen</span><span style="color:#E1E4E8">> {</span></span> 872<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> _controller </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> TextEditingController</span><span style="color:#E1E4E8">();</span></span> 873<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> _messages </span><span style="color:#F97583">=</span><span style="color:#F97583"> <</span><span style="color:#79B8FF">ChatMessage</span><span style="color:#F97583">></span><span style="color:#E1E4E8">[];</span></span> 874<span class="line"><span style="color:#79B8FF"> bool</span><span style="color:#E1E4E8"> _isGenerating </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">;</span></span> 875<span class="line"><span style="color:#79B8FF"> StreamSubscription</span><span style="color:#F97583">?</span><span style="color:#E1E4E8"> _subscription;</span></span> 876<span class="line"></span> 877<span class="line"><span style="color:#79B8FF"> Future</span><span style="color:#E1E4E8"><</span><span style="color:#F97583">void</span><span style="color:#E1E4E8">> </span><span style="color:#B392F0">_send</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">async</span><span style="color:#E1E4E8"> {</span></span> 878<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> text </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> _controller.text.</span><span style="color:#B392F0">trim</span><span style="color:#E1E4E8">();</span></span> 879<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> (text.isEmpty </span><span style="color:#F97583">||</span><span style="color:#E1E4E8"> _isGenerating) </span><span style="color:#F97583">return</span><span style="color:#E1E4E8">;</span></span> 880<span class="line"></span> 881<span class="line"><span style="color:#E1E4E8"> _controller.</span><span style="color:#B392F0">clear</span><span style="color:#E1E4E8">();</span></span> 882<span class="line"></span> 883<span class="line"><span style="color:#6A737D"> // Add user message</span></span> 884<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() {</span></span> 885<span class="line"><span style="color:#E1E4E8"> _messages.</span><span style="color:#B392F0">add</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">ChatMessage</span><span style="color:#E1E4E8">(role</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'user'</span><span style="color:#E1E4E8">, content</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> text));</span></span> 886<span class="line"><span style="color:#E1E4E8"> _messages.</span><span style="color:#B392F0">add</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">ChatMessage</span><span style="color:#E1E4E8">(role</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'assistant'</span><span style="color:#E1E4E8">, content</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> ''</span><span style="color:#E1E4E8">));</span></span> 887<span class="line"><span style="color:#E1E4E8"> _isGenerating </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> true</span><span style="color:#E1E4E8">;</span></span> 888<span class="line"><span style="color:#E1E4E8"> });</span></span> 889<span class="line"></span> 890<span class="line"><span style="color:#6A737D"> // Stream the response</span></span> 891<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> stream </span><span style="color:#F97583">=</span><span style="color:#B392F0"> runPipelineStreaming</span><span style="color:#E1E4E8">(</span></span> 892<span class="line"><span style="color:#E1E4E8"> yamlContent</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> _pipelineYaml,</span></span> 893<span class="line"><span style="color:#E1E4E8"> input</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> text),</span></span> 894<span class="line"><span style="color:#E1E4E8"> );</span></span> 895<span class="line"></span> 896<span class="line"><span style="color:#E1E4E8"> _subscription </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> stream.</span><span style="color:#B392F0">listen</span><span style="color:#E1E4E8">(</span></span> 897<span class="line"><span style="color:#E1E4E8"> (token) {</span></span> 898<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() {</span></span> 899<span class="line"><span style="color:#E1E4E8"> _messages.last.content </span><span style="color:#F97583">+=</span><span style="color:#E1E4E8"> token.text;</span></span> 900<span class="line"><span style="color:#E1E4E8"> });</span></span> 901<span class="line"><span style="color:#E1E4E8"> },</span></span> 902<span class="line"><span style="color:#E1E4E8"> onDone</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> () {</span></span> 903<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _isGenerating </span><span style="color:#F97583">=</span><span style="color:#79B8FF">
903 false</span><span style="color:#E1E4E8">);</span></span> 904<span class="line"><span style="color:#E1E4E8"> },</span></span> 905<span class="line"><span style="color:#E1E4E8"> onError</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> (error) {</span></span> 906<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() {</span></span> 907<span class="line"><span style="color:#E1E4E8"> _messages.last.content </span><span style="color:#F97583">+=</span><span style="color:#9ECBFF"> '</span><span style="color:#79B8FF"> 908</span><span style="color:#9ECBFF">[Error: </span><span style="color:#9ECBFF">$</span><span style="color:#79B8FF">error</span><span style="color:#9ECBFF">]'</span><span style="color:#E1E4E8">;</span></span> 909<span class="line"><span style="color:#E1E4E8"> _isGenerating </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">;</span></span> 910<span class="line"><span style="color:#E1E4E8"> });</span></span> 911<span class="line"><span style="color:#E1E4E8"> },</span></span> 912<span class="line"><span style="color:#E1E4E8"> );</span></span> 913<span class="line"><span style="color:#E1E4E8"> }</span></span> 914<span class="line"></span> 915<span class="line"><span style="color:#F97583"> void</span><span style="color:#B392F0"> _cancel</span><span style="color:#E1E4E8">() {</span></span> 916<span class="line"><span style="color:#E1E4E8"> _subscription</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">cancel</span><span style="color:#E1E4E8">(); </span><span style="color:#6A737D">// Propagates to Rust via sink.add() failure</span></span> 917<span class="line"><span style="color:#B392F0"> setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _isGenerating </span><span style="color:#F97583">=</span><span style="color:#79B8FF"> false</span><span style="color:#E1E4E8">);</span></span> 918<span class="line"><span style="color:#E1E4E8"> }</span></span> 919<span class="line"></span> 920<span class="line"><span style="color:#F97583">
920 @override</span></span> 921<span class="line"><span style="color:#F97583"> void</span><span style="color:#B392F0"> dispose</span><span style="color:#E1E4E8">() {</span></span> 922<span class="line"><span style="color:#E1E4E8"> _subscription</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">cancel</span><span style="color:#E1E4E8">();</span></span> 923<span class="line"><span style="color:#E1E4E8"> _controller.</span><span style="color:#B392F0">dispose</span><span style="color:#E1E4E8">();</span></span> 924<span class="line"><span style="color:#79B8FF"> super</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">dispose</span><span style="color:#E1E4E8">();</span></span> 925<span class="line"><span style="color:#E1E4E8"> }</span></span> 926<span class="line"></span> 927<span class="line"><span style="color:#F97583"> @override</span></span> 928<span class="line"><span style="color:#79B8FF"> Widget</span><span style="color:#B392F0"> build</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">BuildContext</span><span style="color:#E1E4E8"> context) {</span></span> 929<span class="line"><span style="color:#F97583"> return</span><span style="color:#79B8FF"> Scaffold</span><span style="color:#E1E4E8">(</span></span> 930<span class="line"><span style="color:#E1E4E8"> appBar</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> AppBar</span><span style="color:#E1E4E8">(</span></span> 931<span class="line"><span style="color:#E1E4E8"> title</span><span style="color:#F97583">:</span><span style="color:#F97583"> const</span><span style="color:#79B8FF"> Text</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">'Local LLM Chat'</span><span style="color:#E1E4E8">),</span></span> 932<span class="line"><span style="color:#E1E4E8"> actions</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> [</span></span> 933<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> (_isGenerating)</span></span> 934<span class="line"><span style="color:#79B8FF"> IconButton</span><span style="color:#E1E4E8">(</span></span> 935<span class="line"><span style="color:#E1E4E8"> icon</span><span style="color:#F97583">:</span><span style="color:#F97583"> const</span><span style="color:#79B8FF"> Icon</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">Icons</span><span style="color:#E1E4E8">.stop),</span></span> 936<span class="line"><span style="color:#E1E4E8"> onPressed</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> _cancel,</span></span> 937<span class="line"><span style="color:#E1E4E8"> ),</span></span> 938<span class="line"><span style="color:#E1E4E8"> ],</span></span> 939<span class="line"><span style="color:#E1E4E8"> ),</span></span> 940<span class="line"><span style="color:#E1E4E8"> body</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Column</span><span style="color:#E1E4E8">(</span></span> 941<span class="line"><span style="color:#E1E4E8"> children</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> [</span></span> 942<span class="line"><span style="color:#79B8FF"> Expanded</span><span style="color:#E1E4E8">(</span></span> 943<span class="line"><span style="color:#E1E4E8"> child</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> ListView</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">builder</span><span style="color:#E1E4E8">(</span></span> 944<span class="line"><span style="color:#E1E4E8"> itemCount</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> _messages.length,</span></span> 945<span class="line"><span style="color:#E1E4E8"> itemBuilder</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> (context, index) {</span></span> 946<span class="line"><span style="color:#F97583"> final</span><span style="color:#E1E4E8"> msg </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> _messages[index];
946</span></span> 947<span class="line"><span style="color:#F97583"> return</span><span style="color:#79B8FF"> ChatBubble</span><span style="color:#E1E4E8">(</span></span> 948<span class="line"><span style="color:#E1E4E8"> role</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> msg.role,</span></span> 949<span class="line"><span style="color:#E1E4E8"> content</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> msg.content,</span></span> 950<span class="line"><span style="color:#E1E4E8"> isStreaming</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> index </span><span style="color:#F97583">==</span><span style="color:#E1E4E8"> _messages.length </span><span style="color:#F97583">-</span><span style="color:#79B8FF"> 1</span><span style="color:#F97583"> &&</span><span style="color:#E1E4E8"> _isGenerating,</span></span> 951<span class="line"><span style="color:#E1E4E8"> );</span></span> 952<span class="line"><span style="color:#E1E4E8"> },</span></span> 953<span class="line"><span style="color:#E1E4E8"> ),</span></span> 954<span class="line"><span style="color:#E1E4E8"> ),</span></span> 955<span class="line"><span style="color:#B392F0"> _buildInputBar</span><span style="color:#E1E4E8">(),</span></span> 956<span class="line"><span style="color:#E1E4E8"> ],</span></span> 957<span class="line"><span style="color:#E1E4E8"> ),</span></span> 958<span class="line"><span style="color:#E1E4E8"> );</span></span> 959<span class="line"><span style="color:#E1E4E8"> }</span></span> 960<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var d=s(c,10);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> stream </span><span style="color:#F97583">=</span><span style="color:#B392F0"> runPipelineStreaming</span><span style="color:#E1E4E8">(</span></span> 961<span class="line"><span style="color:#E1E4E8"> yamlContent</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> pipelineYaml,</span></span> 962<span class="line"><span style="color:#E1E4E8"> input</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> Envelope</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">text</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#E1E4E8"> userMessage),</span></span> 963<span class="line"><span style="color:#E1E4E8"> generationConfig</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> GenerationConfig</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">creative</span><span style="color:#E1E4E8">(), </span><span style="color:#6A737D">// temperature=0.8, top_p=0.95</span></span> 964<span class="line"><span style="color:#E1E4E8">);</span></span></code></pre>`);var i=s(d,4);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#79B8FF">GenerationConfig</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">greedy</span><span style="color:#E1E4E8">() </span><span style="color:#6A737D">// temperature=0.0, deterministic</span></span> 965<span class="line"><span style="color:#79B8FF">GenerationConfig</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">creative</span><span style="color:#E1E4E8">() </span><span style="color:#6A737D">// temperature=0.8, top_p=0.95</span></span> 966<span class="line"><span style="color:#79B8FF">GenerationConfig</span><span style="color:#E1E4E8">( </span><span style="color:#6A737D">// custom</span></span> 967<span class="line"><span style="color:#E1E4E8"> maxTokens</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 512</span><span style="color:#E1E4E8">,</span></span> 968<span class="line"><span style="color:#E1E4E8"> temperature</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 0.6</span><span style="color:#E1E4E8">,</span></span> 969<span class="line"><span style="color:#E1E4E8"> topP</span><span style="color:#F97583">:</span><span style="color:#79B8FF"> 0.9</span><span style="color:#E1E4E8">,</span></span> 970<span class="line"><span style="color:#E1E4E8">)</span></span></code></pre>`);var y=s(i,16);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Bad: rebuilds every message on each token</span></span> 971<span class="line"><span style="color:#B392F0">setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _messages </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> [..._messages]);</span></span> 972<span class="line"></span> 973<span class="line"><span style="color:#6A737D">// Good: mutate the last message, use keys for targeted rebuilds</span></span> 974<span class="line"><span style="color:#B392F0">setState</span><span style="color:#E1E4E8">(() </span><span style="color:#F97583">=></span><span style="color:#E1E4E8"> _messages.last.content </span><span style="color:#F97583">+=</span><span style="color:#E1E4E8"> token.text);</span></span></code></pre>`);var E=s(y,10);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">final</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> await</span><span style="color:#79B8FF"> Xybrid</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">model</span><span style="color:#E1E4E8">(modelId</span><span style="color:#F97583">:</span><span style="color:#9ECBFF"> 'qwen3.5-0.8b'</span><span style="color:#E1E4E8">).</span><span style="color:#B392F0">load</span><span style="color:#E1E4E8">();</span></span> 975<span class="line"><span style="color:#F97583">await</span><span style="color:#E1E4E8"> model.</span><span style="color:#B392F0">warmup</span><span style="color:#E1E4E8">(); </span><span style="color:#6A737D">// Pre-load weights into memory</span></span></code></pre>`),f(18),h(o,a)}const fs=Object.freeze(Object.defineProperty({__proto__:null,default:ue,metadata:gs},Symbol.toStringTag,{value:"Module"}
975)),bs={title:"Vendoring llama.cpp in a Rust Workspace (Lessons Learned)",description:"The practical pain of embedding a C++ inference engine in a Rust monorepo â cross-compilation, Android fp16, think-tag stripping, and when to give up on upstream.",date:"2026-07-14",author:"Glenn Sonna",tags:["rust","cpp","ai","devops"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:7},{title:Zo,description:sl,date:el,author:al,tags:nl,category:ol,image:ll,imageAlt:tl,draft:pl,featured:rl,readingTime:cl}=bs;var me=u('<p>We needed local LLM inference in our Rust SDK. The obvious choice: <a href="https://github.com/ggerganov/llama.cpp" rel="nofollow">llama.cpp</a> â fast, well-maintained, supports every quantization format.</p> <p>The less obvious part: actually integrating it into a Rust workspace that cross-compiles to iOS, Android, macOS, Linux, and Windows.</p> <p>Hereâs what we learned vendoring llama.cpp into <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a>.</p> <hr/> <h2>Why Vendor?</h2> <p>We considered three approaches:</p> <table><thead><tr><th>Approach</th><th>Pros</th><th>Cons</th></tr></thead><tbody><tr><td>System dependency</td><td>Simple, always latest</td><td>Users must install it, version mismatch hell</td></tr><tr><td>Git submodule</td><td>Pinned version, upstream updates</td><td>Submodule pain, build integration complexity</td></tr><tr><td><strong>Vendor (copy into repo)</strong></td><td>Full control, reproducible builds</td><td>Manual updates, larger repo</td></tr></tbody></table> <p>We chose vendoring because:</p> <ol><li><strong>Reproducible builds</strong>: CI and users get exactly the same llama.cpp version</li> <li><strong>Patching</strong>: We can fix cross-compilation issues without waiting for upstream</li> <li><strong>Feature gating</strong>: We only compile the parts we need via Cargo features</li></ol> <p>The trade-off is manual updates. When we need a new model architecture (like Qwen 3.5), we update the vendored copy.</p> <h2>The Build System: build.rs</h2> <p>llama.cpp is pure C/C++ with no build system requirement (CMake is optional). We compile it directly from <code>build.rs</code>:</p> <!> <p>The <code>cc</code> crate handles cross-compilation toolchains automatically â it reads <code>CC</code>, <code>CXX</code>, and target-specific env vars from Cargo.</p> <h2>Lesson 1: Android Needs +fp16</h2> <p>Our first Android build failed immediately:</p> <!> <p>The culprit: llama.cpp uses ARM FP16 instructions for performance. The Android NDK compiler doesnât enable them by default on <code>aarch64-linux-android</code>.</p> <p>Fix:</p> <!> <p>This tells LLVM to emit FP16 instructions. But wait â what about older devices without FP16 support?</p> <p>Turns out, <strong>every 64-bit ARM Android device supports FP16</strong> (itâs mandatory in ARMv8-A). The NDK is just conservative with defaults. The flag is safe.</p> <p>However, the <code>gemm-f16</code> crate (used by Candle, another dependency) had the same issue. We had to ensure the flag applied globally, not just to llama.cpp.</p> <h2>Lesson 2: Think Tags in Reasoning Models</h2> <p>We added Qwen 3.5 support (a reasoning model) and immediately hit a UX problem. The model outputs:</p> <!> <p>Users donât want to see the <code><think></code> block. We needed to strip it â but in <strong>both</strong> batch and streaming modes.</p> <h3>Batch Mode (Easy)</h3> <!> <h3>Streaming Mode (Hard)</h3> <p>In streaming mode, tokens arrive one at a time. The <code><think></code> block might span hundreds of tokens. We canât wait for the closing <code></think></code> because that defeats the purpose of streaming.</p> <p>Our approach: a state machine that buffers tokens inside think blocks:</p> <!> <p>When we see <code><</code>, we start buffering. If it becomes <code><think></code>, we enter suppression mode. If it becomes anything else, we flush the buffer as normal tokens. Same logic for <code></think></code>.</p> <p>This adds zero latency to normal tokens and correctly handles:</p> <ul><li>Partial tag matches (<code><th</code> followed by <code>inking</code> â not a tag, flush both)</li> <li>Multiple think blocks in one response</li> <li>Think blocks that span the entire response</li></ul> <h2>Lesson 3: Updating Vendored Code</h2> <p>When Qwen 3.5 came out, our vendored llama.cpp didnât support its architecture. The update process:</p> <ol><li><strong>Identify the minimum commit</strong> in llama.cpp that adds support</li> <li><strong>Copy the relevant files</strong> (we donât vendor the entire repo â just core inference files)</li> <li><strong>Re-apply our patches</strong> (we maintain a small patchset for cross-compilation fixes)</li> <li><strong>Test on all platforms</strong></li></ol> <!> <p>We went from a Jan 30, 2026 snapshot to Mar 11, 2026 â specifically to get <code>qwen35</code> architecture support. The diff was manageable because we only vend
975or the inference core, not the entire project (examples, server, tests, etc.).</p> <h2>Lesson 4: Metal Shader Compilation</h2> <p>On macOS and iOS, llama.cpp uses Metal for GPU acceleration. The Metal shaders (<code>ggml-metal.metal</code>) need to be compiled and accessible at runtime.</p> <p>Two approaches:</p> <ol><li><strong>Embed at compile time</strong> â converts the shader to a C string embedded in the binary</li> <li><strong>Load at runtime</strong> â reads from a file path next to the binary</li></ol> <p>We use option 1 (embed) because itâs simpler for distribution â no shader files to bundle:</p> <!> <p>The trade-off: the binary is ~200KB larger. Worth it for zero-configuration deployment.</p> <h2>Lesson 5: Wrapping C++ in Safe Rust</h2> <p>llama.cppâs API is C-style (it provides a C header). Our Rust wrapper:</p> <!> <p>Key patterns:</p> <ul><li><strong>Owned pointers</strong> with <code>Drop</code> for cleanup â no leaks</li> <li><strong>Result return types</strong> for every fallible operation</li> <li><strong>Error messages that help</strong>: we include the file path, architecture hints, and point to <code>XYBRID_LLAMACPP_VERBOSITY</code> for debugging</li></ul> <h2>Lesson 6: Error Diagnostics Matter</h2> <p>The worst llama.cpp error is a silent failure â the model loads but produces garbage. We added diagnostics:</p> <!> <p>The <code>XYBRID_LLAMACPP_VERBOSITY</code> env var controls llama.cppâs internal logging, which is invaluable for debugging quantization mismatches and architecture issues.</p> <h2>The Feature Flag</h2> <p>llama.cpp is entirely behind a feature flag:</p> <!> <p>When disabled, the LLM types (<code>GenerationConfig</code>, <code>ChatMessage</code>) still compile â theyâre in an always-available module. Only the implementation is gated:</p> <!> <p>This prevents downstream crates (Flutter bindings) from needing conditional compilation for type definitions.</p> <h2>Was Vendoring Worth It?</h2> <p><strong>Yes.</strong> The alternatives had worse trade-offs:</p> <ul><li>A system dependency would require users to install llama.cpp (and the right version) on every platform</li> <li>A git submodule would add CI complexity and still require patching</li> <li>Building against a pre-compiled library would lose Metal/ANE optimization on Apple platforms</li></ul> <p>The cost is ~50MB added to the repo and occasional manual updates. For reproducible, cross-platform builds with full optimization â thatâs a fair trade.</p> <hr/> <p><strong>See the implementation</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <hr/> <p><em>Vendoring C/C++ in Rust? Share your war stories in the comments.</em></p>',1);function Fe(o){var a=me(),n=s(b(a),24);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// build.rs (simplified)</span></span> 976<span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> main</span><span style="color:#E1E4E8">() {</span></span> 977<span class="line"><span style="color:#F97583"> let</span><span style="color:#F97583"> mut</span><span style="color:#E1E4E8"> build </span><span style="color:#F97583">=</span><span style="color:#B392F0"> cc</span><span style="color:#F97583">::</span><span style="color:#B392F0">Build</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">();</span></span> 978<span class="line"></span> 979<span class="line"><span style="color:#E1E4E8"> build</span></span> 980<span class="line"><span style="color:#F97583"> .</span><span style="color:#B392F0">cpp</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">true</span><span style="color:#E1E4E8">)</span></span> 981<span class="line"><span style="color:#F97583"> .</span><span style="color:#B392F0">file</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"vendor/llama.cpp/llama.cpp"</span><span style="color:#E1E4E8">)</span></span> 982<span class="line"><span style="color:#F97583"> .</span><span style="color:#B392F0">file</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"vendor/llama.cpp/ggml.c"</span><span style="color:#E1E4E8">)</span></span> 983<span class="line"><span style="color:#F97583"> .</span><span style="color:#B392F0">file</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"vendor/llama.cpp/ggml-alloc.c"</span><span style="color:#E1E4E8">)</span></span> 984<span class="line"><span style="color:#F97583"> .</span><span style="color:#B392F0">file</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"vendor/llama.cpp/ggml-backend.c"</span><span style="color:#E1E4E8">)</span></span> 985<span class="line"><span style="color:#F97583"> .</span><span style="color:#B392F0">include</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"vendor/llama.cpp/"</span><span style="color:#E1E4E8">);</span></span> 986<span class="line"></span> 987<span class="line"><span style="color:#6A737D"> // Platform-specific backends. In a build script \`cfg!()\` reflects the</span></span> 988<span class="line"><span style="color:#6A737D"> // HOST, not the target you're compiling for â read CARGO_CFG_TARGET_OS</span></span> 989<span class="line"><span style="color:#6A737D"> // so cross-compiles (e.g. building for iOS from a Linux CI box) pick the</span></span> 990<span class="line"><span style="color:#6A737D"> // right backend.</span></span> 991<span class="line"><span style="color:#F97583">
991 let</span><span style="color:#E1E4E8"> target_os </span><span style="color:#F97583">=</span><span style="color:#B392F0"> std</span><span style="color:#F97583">::</span><span style="color:#B392F0">env</span><span style="color:#F97583">::</span><span style="color:#B392F0">var</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"CARGO_CFG_TARGET_OS"</span><span style="color:#E1E4E8">)</span><span style="color:#F97583">.</span><span style="color:#B392F0">unwrap</span><span style="color:#E1E4E8">();</span></span> 992<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> target_os </span><span style="color:#F97583">==</span><span style="color:#9ECBFF"> "macos"</span><span style="color:#F97583"> ||</span><span style="color:#E1E4E8"> target_os </span><span style="color:#F97583">==</span><span style="color:#9ECBFF"> "ios"</span><span style="color:#E1E4E8"> {</span></span> 993<span class="line"><span style="color:#E1E4E8"> build</span><span style="color:#F97583">.</span><span style="color:#B392F0">file</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"vendor/llama.cpp/ggml-metal.m"</span><span style="color:#E1E4E8">);</span></span> 994<span class="line"><span style="color:#B392F0"> println!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"cargo:rustc-link-framework=Metal"</span><span style="color:#E1E4E8">);</span></span> 995<span class="line"><span style="color:#B392F0"> println!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"cargo:rustc-link-framework=MetalPerformanceShaders"</span><span style="color:#E1E4E8">);</span></span> 996<span class="line"><span style="color:#E1E4E8"> }</span></span> 997<span class="line"></span> 998<span class="line"><span style="color:#E1E4E8"> build</span><span style="color:#F97583">.</span><span style="color:#B392F0">compile</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"llama"</span><span style="color:#E1E4E8">);</span></span> 999<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var l=s(n,8);e(l,()=>'<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>error: instruction requires: fullfp16</span></span></code></pre>');var t=s(l,6);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># .cargo/config.toml</span></span> 1000<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#B392F0">target</span><span style="color:#E1E4E8">.</span><span style="color:#B392F0">aarch64-linux-android</span><span style="color:#E1E4E8">]</span></span> 1001<span class="line"><span style="color:#E1E4E8">rustflags = [</span><span style="color:#9ECBFF">"-C"</span><span style="color:#E1E4E8">, </span><span style="color:#9ECBFF">"target-feature=+fp16"</span><span style="color:#E1E4E8">]</span></span></code></pre>`);var p=s(t,12);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span><think></span></span> 1002<span class="line"><span>The user is asking about weather. I should provide a helpful response</span></span> 1003<span class="line"><span>about current conditions. Let me think about what information would</span></span> 1004<span class="line"><span>be most useful...</span></span> 1005<span class="line"><span></think></span></span> 1006<span class="line"><span></span></span> 1007<span class="line"><span>The weather today is sunny with a high of 72°F.</span></span></code></pre>`);var r=s(p,6);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> strip_think_tags</span><span style="color:#E1E4E8">(text</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">str</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8"> {</span></span> 1008<span class="line"><span style="color:#6A737D"> // Remove <think>...</think> blocks (including nested)</span></span> 1009<span class="line"><span style="color:#F97583">
1009 let</span><span style="color:#E1E4E8"> re </span><span style="color:#F97583">=</span><span style="color:#B392F0"> Regex</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">r"(?s)<think>.*?</think>s*"</span><span style="color:#E1E4E8">)</span><span style="color:#F97583">.</span><span style="color:#B392F0">unwrap</span><span style="color:#E1E4E8">();</span></span> 1010<span class="line"><span style="color:#E1E4E8"> re</span><span style="color:#F97583">.</span><span style="color:#B392F0">replace_all</span><span style="color:#E1E4E8">(text, </span><span style="color:#9ECBFF">""</span><span style="color:#E1E4E8">)</span><span style="color:#F97583">.</span><span style="color:#B392F0">trim</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">.</span><span style="color:#B392F0">to_string</span><span style="color:#E1E4E8">()</span></span> 1011<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var c=s(r,8);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">enum</span><span style="color:#B392F0"> ThinkState</span><span style="color:#E1E4E8"> {</span></span> 1012<span class="line"><span style="color:#B392F0"> Normal</span><span style="color:#E1E4E8">,</span></span> 1013<span class="line"><span style="color:#B392F0"> MaybeOpening</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">), </span><span style="color:#6A737D">// Buffering potential "<think>"</span></span> 1014<span class="line"><span style="color:#B392F0"> InsideThink</span><span style="color:#E1E4E8">, </span><span style="color:#6A737D">// Suppressing tokens</span></span> 1015<span class="line"><span style="color:#B392F0"> MaybeClosing</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">), </span><span style="color:#6A737D">// Buffering potential "</think>"</span></span> 1016<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var d=s(c,14);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># Our update script (simplified)</span></span> 1017<span class="line"><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> /tmp</span><span style="color:#E1E4E8"> && </span><span style="color:#B392F0">git</span><span style="color:#9ECBFF"> clone</span><span style="color:#9ECBFF"> https://github.com/ggerganov/llama.cpp</span></span> 1018<span class="line"><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> llama.cpp</span><span style="color:#E1E4E8"> && </span><span style="color:#B392F0">git</span><span style="color:#9ECBFF"> checkout</span><span style="color:#F97583"> <</span><span style="color:#9ECBFF">commi</span><span style="color:#E1E4E8">t</span><span style="color:#F97583">></span></span> 1019<span class="line"></span> 1020<span class="line"><span style="color:#6A737D"># Copy core files</span></span> 1021<span class="line"><span style="color:#B392F0">cp</span><span style="color:#9ECBFF"> llama.cpp</span><span style="color:#9ECBFF"> ggml</span><span style="color:#79B8FF">*</span><span style="color:#9ECBFF">.c</span><span style="color:#9ECBFF"> ggml</span><span style="color:#79B8FF">*</span><span style="color:#9ECBFF">.h</span><span style="color:#9ECBFF"> ggml</span><span style="color:#79B8FF">*</span><span style="color:#9ECBFF">.m</span><span style="color:#9ECBFF"> /path/to/vendor/llama.cpp/</span></span> 1022<span class="line"></span> 1023<span class="line"><span style="color:#6A737D"># Re-apply patches</span></span> 1024<span class="line"><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> /path/to/project</span></span> 1025<span class="line"><span style="color:#B392F0">git</span><span style="color:#9ECBFF"> apply</span><span style="color:#9ECBFF"> patches/llama-cpp-</span><span style="color:#79B8FF">*</span><span style="color:#9ECBFF">.patch</span></span></code></pre>`);var i=s(d,14);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>// In ggml-metal.m (our patch)</span></span> 1026<span class="line"><span>#define GGML_METAL_EMBED_LIBRARY 1</span></span> 1027<span class="line"><span>// This makes the shader source a compile-time constant</span></span></code></pre>`);var y=s(i,8);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> LlamaCppModel</span><span style="color:#E1E4E8"> {</span></span> 1028<span class="line"><span style="color:#E1E4E8"> ctx</span><span style="color:#F97583">:</span><span style="color:#F97583"> *mut</span><span style="color:#E1E4E8"> llama_context,</span></span> 1029<span class="line"><span style="color:#E1E4E8"> model</span><span style="color:#F97583">:</span><span style="color:#F97583"> *mut</span><span style="color:#E1E4E8"> llama_model,</span></span> 1030<span class="line"><span style="color:#E1E4E8">}</span></span> 1031<span class="line"></span> 1032<span class="line"><span style="color:#F97583">impl</span><span style="color:#B392F0"> LlamaCppModel</span><span style="color:#E1E4E8"> {</span></span> 1033<span class="line"><span style="color:#F97583"> pub</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> load</span><span style="color:#E1E4E8">(path</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">str</span><span style="color:#E1E4E8">, config</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">GenerationConfig</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> Result</span><span style="color:#E1E4E8"><</span><span style="color:#79B8FF">
1033Self</span><span style="color:#E1E4E8">> {</span></span> 1034<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> c_path </span><span style="color:#F97583">=</span><span style="color:#B392F0"> CString</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">(path)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1035<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> params </span><span style="color:#F97583">=</span><span style="color:#F97583"> unsafe</span><span style="color:#E1E4E8"> { </span><span style="color:#B392F0">llama_model_default_params</span><span style="color:#E1E4E8">() };</span></span> 1036<span class="line"></span> 1037<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> model </span><span style="color:#F97583">=</span><span style="color:#F97583"> unsafe</span><span style="color:#E1E4E8"> { </span><span style="color:#B392F0">llama_load_model_from_file</span><span style="color:#E1E4E8">(c_path</span><span style="color:#F97583">.</span><span style="color:#B392F0">as_ptr</span><span style="color:#E1E4E8">(), params) };</span></span> 1038<span class="line"><span style="color:#F97583"> if</span><span style="color:#E1E4E8"> model</span><span style="color:#F97583">.</span><span style="color:#B392F0">is_null</span><span style="color:#E1E4E8">() {</span></span> 1039<span class="line"><span style="color:#F97583"> return</span><span style="color:#B392F0"> Err</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">anyhow!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Failed to load model: {}"</span><span style="color:#E1E4E8">, path));</span></span> 1040<span class="line"><span style="color:#E1E4E8"> }</span></span> 1041<span class="line"></span> 1042<span class="line"><span style="color:#6A737D"> // ... context creation, error handling</span></span> 1043<span class="line"><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">Self</span><span style="color:#E1E4E8"> { ctx, model })</span></span> 1044<span class="line"><span style="color:#E1E4E8"> }</span></span> 1045<span class="line"></span> 1046<span class="line"><span style="color:#F97583"> pub</span><span style="color:#F97583"> fn</span><span style="color:#B392F0"> generate</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&mut</span><span style="color:#79B8FF"> self</span><span style="color:#E1E4E8">, prompt</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">str</span><span style="color:#E1E4E8">, callback</span><span style="color:#F97583">:</span><span style="color:#B392F0"> StreamingCallback</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> Result</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">> {</span></span> 1047<span class="line"><span style="color:#6A737D"> // Tokenize, sample, decode â calling back per token</span></span> 1048<span class="line"><span style="color:#E1E4E8"> }</span></span> 1049<span class="line"><span style="color:#E1E4E8">}</span></span> 1050<span class="line"></span> 1051<span class="line"><span style="color:#6A737D">// Safety: LlamaCppModel owns the pointers and frees them on drop</span></span> 1052<span class="line"><span style="color:#F97583">impl</span><span style="color:#B392F0"> Drop</span><span style="color:#F97583"> for</span><span style="color:#B392F0"> LlamaCppModel</span><span style="color:#E1E4E8"> {</span></span> 1053<span class="line"><span style="color:#F97583"> fn</span><span style="color:#B392F0"> drop</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&mut</span><span style="color:#79B8FF"> self</span><span style="color:#E1E4E8">) {</span></span> 1054<span class="line"><span style="color:#F97583"> unsafe</span><span style="color:#E1E4E8"> {</span></span> 1055<span class="line"><span style="color:#B392F0"> llama_free</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">
1055self</span><span style="color:#F97583">.</span><span style="color:#E1E4E8">ctx);</span></span> 1056<span class="line"><span style="color:#B392F0"> llama_free_model</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">self</span><span style="color:#F97583">.</span><span style="color:#E1E4E8">model);</span></span> 1057<span class="line"><span style="color:#E1E4E8"> }</span></span> 1058<span class="line"><span style="color:#E1E4E8"> }</span></span> 1059<span class="line"><span style="color:#E1E4E8">}</span></span> 1060<span class="line"></span> 1061<span class="line"><span style="color:#6A737D">// Send + Sync are safe because we don't share the context across threads</span></span> 1062<span class="line"><span style="color:#F97583">unsafe</span><span style="color:#F97583"> impl</span><span style="color:#B392F0"> Send</span><span style="color:#F97583"> for</span><span style="color:#B392F0"> LlamaCppModel</span><span style="color:#E1E4E8"> {}</span></span></code></pre>`);var E=s(y,10);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">Err</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">anyhow!</span><span style="color:#E1E4E8">(</span></span> 1063<span class="line"><span style="color:#9ECBFF"> "Failed to load GGUF model: {}</span><span style="color:#79B8FF"> 1064</span><span style="color:#9ECBFF"></span></span> 1065<span class="line"><span style="color:#9ECBFF"> Architecture: {}</span><span style="color:#79B8FF"> 1066</span><span style="color:#9ECBFF"></span></span> 1067<span class="line"><span style="color:#9ECBFF"> Hint: Set XYBRID_LLAMACPP_VERBOSITY=2 for detailed llama.cpp output</span><span style="color:#79B8FF"> 1068</span><span style="color:#9ECBFF"></span></span> 1069<span class="line"><span style="color:#9ECBFF"> Hint: Ensure the GGUF file matches a supported architecture"</span><span style="color:#E1E4E8">,</span></span> 1070<span class="line"><span style="color:#E1E4E8"> path,</span></span> 1071<span class="line"><span style="color:#B392F0"> detect_architecture</span><span style="color:#E1E4E8">(path)</span><span style="color:#F97583">.</span><span style="color:#B392F0">unwrap_or</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"unknown"</span><span style="color:#F97583">.</span><span style="color:#B392F0">into</span><span style="color:#E1E4E8">()),</span></span> 1072<span class="line"><span style="color:#E1E4E8">))</span></span></code></pre>`);var F=s(E,8);e(F,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">[</span><span style="color:#B392F0">features</span><span style="color:#E1E4E8">]</span></span> 1073<span class="line"><span style="color:#E1E4E8">llm-llamacpp = [</span><span style="color:#9ECBFF">"dep:llama-cpp-bindings"</span><span style="color:#E1E4E8">]</span></span></code></pre>`);var v=s(F,4);e(v,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">#[cfg(feature </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "llm-llamacpp"</span><span style="color:#E1E4E8">)]</span></span> 1074<span class="line"><span style="color:#F97583">mod</span><span style="color:#B392F0"> llamacpp_backend</span><span style="color:#E1E4E8">;</span></span> 1075<span class="line"></span> 1076<span class="line"><span style="color:#6A737D">// These are always available:</span></span> 1077<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> GenerationConfig</span><span style="color:#E1E4E8"> { </span><span style="color:#F97583">...</span><span style="color:#E1E4E8"> }</span></span> 1078<span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> struct</span><span style="color:#B392F0"> ChatMessage</span><span style="color:#E1E4E8"> { </span><span style="color:#F97583">...</span><span style="color:#E1E4E8"> }</span></span></code></pre>`),f(18),h(o,a)}const Bs=Object.freeze(Object.defineProperty({__proto__:null,default:Fe,metadata:bs},Symbol.toStringTag,{value:"Module"})),vs={title:"Whisper Speech Recognition as a Single Rust Binary",description:"Build a Whisper-powered transcription tool as a single Rust binary. Candle for pure-Rust inference. No runtime dependencies, works airgapped.",date:"2026-06-30",author:"Glenn Sonna",tags:["whisper-rust","offline-ai","speech-recognition","local-ai","private-ai"],category:"tutorial",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:5},{title:il,description:dl,date:yl,author:El,tags:hl,category:ul,image:ml,imageAlt:Fl,draft:gl,featured:fl,readingTime:bl}=vs;var ge=u('<p>You want speech-to-text in a compiled binary â a CLI tool, a backend service, an embedded system. No Python runtime. No cloud API. Just Rust, Whisper, and audio bytes in â text out.</p> <p>Weâll use <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a> with Whisper running via Candle â Hugging Faceâs pure-Rust ML framework. The entire inference stack compiles to a single native binary with zero runtime dependencies.</p> <hr/> <h2>Who This Is For</h2> <ul><li><strong>CLI transcription tools</strong> â batch process audio files on a server</li> <li><strong>Embedded systems</strong> â Raspberry Pi, edge devices, kiosks</li> <li><strong>Desktop apps</strong> â native macOS/Linux/Windows</li> <li><strong>
1078Backend services</strong> â transcribe uploads without paying cloud API fees</li> <li><strong>Privacy-critical systems</strong> â medical, legal, financial where data must stay on-premise</li></ul> <p>For these, you want a compiled binary with no runtime. Rust gives you that. Candle gives you pure-Rust ML inference. No shared libraries to manage.</p> <h2>Quick Start: CLI</h2> <p>If you just want transcription without writing code:</p> <!> <p>Thatâs it. But letâs build it from scratch to understand whatâs happening underneath.</p> <h2>Step 1: Set Up the Project</h2> <!> <!> <p>The <code>candle</code> feature enables pure-Rust inference via Candle. No system dependencies to install.</p> <h2>Step 2: Download and Cache the Model</h2> <!> <p>First run downloads to <code>~/.xybrid/cache/whisper-tiny/</code>. Every subsequent run loads from disk. No network.</p> <h2>Step 3: Transcribe</h2> <!> <p>The <code>TemplateExecutor</code> reads <code>model_metadata.json</code> and handles everything: WAV decoding â mel spectrogram â Whisper encoder-decoder â token decoding â text. You donât configure any of it.</p> <h2>Step 4: Wire It Up</h2> <!> <!> <p>A single binary. No Python. No containers. Deploy it anywhere Rust compiles.</p> <h2>Why Candle Instead of ONNX Runtime</h2> <p>Whisper uses autoregressive decoding â each output token depends on previous tokens. This requires mutable state between inference steps.</p> <p>ONNX Runtimeâs <code>session.run()</code> is stateless (<code>&self</code>). Youâd need to manage decoder state externally, which is complex and error-prone.</p> <p>Candle handles this natively. Xybrid detects Whisper models and automatically routes them through Candle:</p> <!> <p>You donât choose the backend. The metadata file determines it.</p> <h2>Batch Processing</h2> <p>For server-side use cases â transcribing uploads, processing audio archives:</p> <!> <p>The executor reuses loaded model weights across calls. First inference is slower (model loads into memory). Subsequent calls reuse everything.</p> <h2>Performance</h2> <p>Measured transcribing a 10-second audio clip:</p> <table><thead><tr><th>Device</th><th>whisper-tiny</th><th>whisper-base</th></tr></thead><tbody><tr><td>MacBook Pro M2</td><td>0.8s</td><td>2.1s</td></tr><tr><td>Desktop (i7-12700)</td><td>1.2s</td><td>3.5s</td></tr><tr><td>Raspberry Pi 4</td><td>8s</td><td>22s</td></tr></tbody></table> <p>Whisper Tiny processes faster than real-time on any modern laptop. Even a Raspberry Pi 4 handles it â slower, but viable for non-interactive use cases.</p> <h2>The Privacy Architecture</h2> <!> <p>What doesnât happen:</p> <ul><li>No audio sent to any server</li> <li>No telemetry about what you transcribe</li> <li>No API keys that could be revoked</li> <li>No model updates that change behavior without your knowledge</li></ul> <p>The model is a file on your disk. Inference is a function call. The binary works airgapped.</p> <h2>Available Whisper Models</h2> <table><thead><tr><th>Model</th><th>Parameters</th><th>Size</th><th>Speed</th><th>Accuracy</th></tr></thead><tbody><tr><td><code>whisper-tiny</code></td><td>39M</td><td>~75MB</td><td>Real-time</td><td>Good for clear speech</td></tr><tr><td><code>whisper-base</code></td><td>74M</td><td>~150MB</td><td>Near real-time</td><td>Better accuracy</td></tr><tr><td><code>whisper-small</code></td><td>244M</td><td>~500MB</td><td>Slower</td><td>Much better accuracy</td></tr></tbody></table> <p>Start with <code>whisper-tiny</code>. Upgrade if accuracy matters more than speed for your use case.</p> <hr/> <p><strong>Get started</strong>:</p> <!> <p><strong>GitHub</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a></p> <hr/> <p><em>Building privacy-first voice features in Rust? Share your use case in the comments.</em></p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/building-voice-agent-on-device">Building a Voice Agent That Runs Entirely On-Device</a> â full pipeline tutorial: ASR + LLM + TTS in Flutter.</li> <li><a href="/blog/private-ai-run-models-without-cloud">Private AI: How to Run AI Models Without Sending Data to the Cloud</a> â the compliance and privacy case for local inference.</li> <li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide</a> â the broader landscape of on-device ML inference.</li></ul>',1);function fe(o){var a=ge(),n=s(b(a),16);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> install</span><span style="color:#9ECBFF"> xybrid-cli</span></span> 1079<span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> whisper-tiny</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF">
1079 recording.wav</span></span> 1080<span class="line"><span style="color:#6A737D"># Output: "Hello, this is a test of on-device transcription."</span></span></code></pre>`);var l=s(n,6);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> new</span><span style="color:#9ECBFF"> speech-to-text</span></span> 1081<span class="line"><span style="color:#79B8FF">cd</span><span style="color:#9ECBFF"> speech-to-text</span></span></code></pre>`);var t=s(l,2);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># Cargo.toml</span></span> 1082<span class="line"><span style="color:#E1E4E8">[</span><span style="color:#B392F0">dependencies</span><span style="color:#E1E4E8">]</span></span> 1083<span class="line"><span style="color:#E1E4E8">xybrid-core = { version = </span><span style="color:#9ECBFF">"0.1.0"</span><span style="color:#E1E4E8">, features = [</span><span style="color:#9ECBFF">"candle"</span><span style="color:#E1E4E8">] }</span></span> 1084<span class="line"><span style="color:#E1E4E8">xybrid-sdk = </span><span style="color:#9ECBFF">"0.1.0"</span></span> 1085<span class="line"><span style="color:#E1E4E8">anyhow = </span><span style="color:#9ECBFF">"1.0"</span></span></code></pre>`);var p=s(t,6);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">use</span><span style="color:#B392F0"> xybrid_sdk</span><span style="color:#F97583">::</span><span style="color:#B392F0">RegistryClient</span><span style="color:#E1E4E8">;</span></span> 1086<span class="line"><span style="color:#F97583">use</span><span style="color:#B392F0"> std</span><span style="color:#F97583">::</span><span style="color:#B392F0">path</span><span style="color:#F97583">::</span><span style="color:#B392F0">PathBuf</span><span style="color:#E1E4E8">;</span></span> 1087<span class="line"></span> 1088<span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> get_model</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">-></span><span style="color:#B392F0"> anyhow</span><span style="color:#F97583">::</span><span style="color:#B392F0">Result</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">PathBuf</span><span style="color:#E1E4E8">> {</span></span> 1089<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> client </span><span style="color:#F97583">=</span><span style="color:#B392F0"> RegistryClient</span><span style="color:#F97583">::</span><span style="color:#B392F0">default_client</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1090<span class="line"></span> 1091<span class="line"><span style="color:#F97583"> if</span><span style="color:#F97583"> let</span><span style="color:#B392F0"> Some</span><span style="color:#E1E4E8">(path) </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> client</span><span style="color:#F97583">.</span><span style="color:#B392F0">cached_path</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"whisper-tiny"</span><span style="color:#E1E4E8">)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8"> {</span></span> 1092<span class="line"><span style="color:#F97583"> return</span><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(path);</span></span> 1093<span class="line"><span style="color:#E1E4E8"> }</span></span> 1094<span class="line"></span> 1095<span class="line"><span style="color:#B392F0"> println!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Downloading Whisper Tiny (~75MB)..."</span><span style="color:#E1E4E8">);</span></span> 1096<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> path </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> client</span><span style="color:#F97583">.</span><span style="color:#B392F0">fetch</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"whisper-tiny"</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">None</span><span style="color:#E1E4E8">, </span><span style="color:#F97583">|</span><span style="color:#E1E4E8">p</span><span style="color:#F97583">|</span><span style="color:#E1E4E8"> {</span></span> 1097<span class="line"><span style="color:#B392F0"> print!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"</span><span style="color:#79B8FF">\r</span><span style="color:#9ECBFF"> {:.0}%"</span><span style="color:#E1E4E8">, p </span><span style="color:#F97583">*</span><span style="color:#79B8FF"> 100.0</span><span style="color:#E1E4E8">);</span></span> 1098<span class="line"><span style="color:#E1E4E8"> })</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1099<span class="line"><span style="color:#B392F0"> println!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"</span><span style="color:#79B8FF"> 1100</span><span style="color:#9ECBFF">Cached at: {}"</span><span style="color:#E1E4E8">, path</span><span style="color:#F97583">.</span><span style="color:#B392F0">display</span><span style="color:#E1E4E8">());</span></span> 1101<span class="line"></span> 1102<span class="line"><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(path)</span></span> 1103<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var r=s(p,6);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">
1103use</span><span style="color:#B392F0"> xybrid_core</span><span style="color:#F97583">::</span><span style="color:#B392F0">execution</span><span style="color:#F97583">::</span><span style="color:#E1E4E8">{</span><span style="color:#B392F0">ModelMetadata</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">TemplateExecutor</span><span style="color:#E1E4E8">};</span></span> 1104<span class="line"><span style="color:#F97583">use</span><span style="color:#B392F0"> xybrid_core</span><span style="color:#F97583">::</span><span style="color:#B392F0">ir</span><span style="color:#F97583">::</span><span style="color:#E1E4E8">{</span><span style="color:#B392F0">Envelope</span><span style="color:#E1E4E8">, </span><span style="color:#B392F0">EnvelopeKind</span><span style="color:#E1E4E8">};</span></span> 1105<span class="line"></span> 1106<span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> transcribe</span><span style="color:#E1E4E8">(model_dir</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">PathBuf</span><span style="color:#E1E4E8">, audio_path</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">str</span><span style="color:#E1E4E8">) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> anyhow</span><span style="color:#F97583">::</span><span style="color:#B392F0">Result</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">> {</span></span> 1107<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ModelMetadata</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> serde_json</span><span style="color:#F97583">::</span><span style="color:#B392F0">from_str</span><span style="color:#E1E4E8">(</span></span> 1108<span class="line"><span style="color:#F97583"> &</span><span style="color:#B392F0">std</span><span style="color:#F97583">::</span><span style="color:#B392F0">fs</span><span style="color:#F97583">::</span><span style="color:#B392F0">read_to_string</span><span style="color:#E1E4E8">(model_dir</span><span style="color:#F97583">.</span><span style="color:#B392F0">join</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"model_metadata.json"</span><span style="color:#E1E4E8">))</span><span style="color:#F97583">?</span></span> 1109<span class="line"><span style="color:#E1E4E8"> )</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1110<span class="line"></span> 1111<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> audio_bytes </span><span style="color:#F97583">=</span><span style="color:#B392F0"> std</span><span style="color:#F97583">::</span><span style="color:#B392F0">fs</span><span style="color:#F97583">::</span><span style="color:#B392F0">read</span><span style="color:#E1E4E8">(audio_path)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1112<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> input </span><span style="color:#F97583">=</span><span style="color:#B392F0"> Envelope</span><span style="color:#E1E4E8"> {</span></span> 1113<span class="line"><span style="color:#E1E4E8"> kind</span><span style="color:#F97583">:</span><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#F97583">::</span><span style="color:#B392F0">Audio</span><span style="color:#E1E4E8">(audio_bytes),</span></span> 1114<span class="line"><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> std</span><span style="color:#F97583">::</span><span style="color:#B392F0">collections</span><span style="color:#F97583">::</span><span style="color:#B392F0">HashMap</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">(),</span></span> 1115<span class="line"><span style="color:#E1E4E8"> };</span></span> 1116<span class="line"></span> 1117<span class="line"><span style="color:#F97583">
1117 let</span><span style="color:#F97583"> mut</span><span style="color:#E1E4E8"> executor </span><span style="color:#F97583">=</span><span style="color:#B392F0"> TemplateExecutor</span><span style="color:#F97583">::</span><span style="color:#B392F0">with_base_path</span><span style="color:#E1E4E8">(model_dir</span><span style="color:#F97583">.</span><span style="color:#B392F0">to_str</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">.</span><span style="color:#B392F0">unwrap</span><span style="color:#E1E4E8">());</span></span> 1118<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> executor</span><span style="color:#F97583">.</span><span style="color:#B392F0">execute</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">metadata, </span><span style="color:#F97583">&</span><span style="color:#E1E4E8">input)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1119<span class="line"></span> 1120<span class="line"><span style="color:#F97583"> match</span><span style="color:#E1E4E8"> output</span><span style="color:#F97583">.</span><span style="color:#E1E4E8">kind {</span></span> 1121<span class="line"><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#F97583">::</span><span style="color:#B392F0">Text</span><span style="color:#E1E4E8">(text) </span><span style="color:#F97583">=></span><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(text),</span></span> 1122<span class="line"><span style="color:#E1E4E8"> _ </span><span style="color:#F97583">=></span><span style="color:#B392F0"> Err</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">anyhow</span><span style="color:#F97583">::</span><span style="color:#B392F0">anyhow!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Expected text output"</span><span style="color:#E1E4E8">)),</span></span> 1123<span class="line"><span style="color:#E1E4E8"> }</span></span> 1124<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var c=s(r,6);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> main</span><span style="color:#E1E4E8">() </span><span style="color:#F97583">-></span><span style="color:#B392F0"> anyhow</span><span style="color:#F97583">::</span><span style="color:#B392F0">Result</span><span style="color:#E1E4E8"><()> {</span></span> 1125<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> args</span><span style="color:#F97583">:</span><span style="color:#B392F0"> Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">> </span><span style="color:#F97583">=</span><span style="color:#B392F0"> std</span><span style="color:#F97583">::</span><span style="color:#B392F0">env</span><span style="color:#F97583">::</span><span style="color:#B392F0">args</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">.</span><span style="color:#B392F0">collect</span><span style="color:#E1E4E8">();</span></span> 1126<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> audio_path </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> args</span><span style="color:#F97583">.</span><span style="color:#B392F0">get</span><span style="color:#E1E4E8">(</span><span style="color:#79B8FF">1</span><span style="color:#E1E4E8">)</span></span> 1127<span class="line"><span style="color:#F97583"> .</span><span style="color:#B392F0">ok_or_else</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">||</span><span style="color:#B392F0"> anyhow</span><span style="color:#F97583">::</span><span style="color:#B392F0">anyhow!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"Usage: speech-to-text <audio.wav>"</span><span style="color:#E1E4E8">))</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1128<span class="line"></span> 1129<span class="line"><span style="color:#F97583">
1129 let</span><span style="color:#E1E4E8"> model_dir </span><span style="color:#F97583">=</span><span style="color:#B392F0"> get_model</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1130<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> text </span><span style="color:#F97583">=</span><span style="color:#B392F0"> transcribe</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">model_dir, audio_path)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1131<span class="line"><span style="color:#B392F0"> println!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"{}"</span><span style="color:#E1E4E8">, text);</span></span> 1132<span class="line"></span> 1133<span class="line"><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(())</span></span> 1134<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var d=s(c,2);e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --</span><span style="color:#9ECBFF"> recording.wav</span></span> 1135<span class="line"><span style="color:#6A737D"># Hello, this is a test of on-device transcription.</span></span></code></pre>`);var i=s(d,12);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>model_metadata.json: "type": "WhisperMode"</span></span> 1136<span class="line"><span> â TemplateExecutor detects Candle backend</span></span> 1137<span class="line"><span> â CandleRuntimeAdapter with &mut self</span></span> 1138<span class="line"><span> â Autoregressive decoding handled internally</span></span></code></pre>`);var y=s(i,8);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> transcribe_batch</span><span style="color:#E1E4E8">(model_dir</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#B392F0">PathBuf</span><span style="color:#E1E4E8">, files</span><span style="color:#F97583">:</span><span style="color:#F97583"> &</span><span style="color:#E1E4E8">[</span><span style="color:#F97583">&</span><span style="color:#B392F0">str</span><span style="color:#E1E4E8">]) </span><span style="color:#F97583">-></span><span style="color:#B392F0"> anyhow</span><span style="color:#F97583">::</span><span style="color:#B392F0">Result</span><span style="color:#E1E4E8"><()> {</span></span> 1139<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ModelMetadata</span><span style="color:#F97583"> =</span><span style="color:#B392F0"> serde_json</span><span style="color:#F97583">::</span><span style="color:#B392F0">from_str</span><span style="color:#E1E4E8">(</span></span> 1140<span class="line"><span style="color:#F97583"> &</span><span style="color:#B392F0">std</span><span style="color:#F97583">::</span><span style="color:#B392F0">fs</span><span style="color:#F97583">::</span><span style="color:#B392F0">read_to_string</span><span style="color:#E1E4E8">(model_dir</span><span style="color:#F97583">.</span><span style="color:#B392F0">join</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"model_metadata.json"</span><span style="color:#E1E4E8">))</span><span style="color:#F97583">?</span></span> 1141<span class="line"><span style="color:#E1E4E8"> )</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1142<span class="line"><span style="color:#F97583"> let</span><span style="color:#F97583"> mut</span><span style="color:#E1E4E8"> executor </span><span style="color:#F97583">=</span><span style="color:#B392F0"> TemplateExecutor</span><span style="color:#F97583">::</span><span style="color:#B392F0">with_base_path</span><span style="color:#E1E4E8">(model_dir</span><span style="color:#F97583">.</span><span style="color:#B392F0">to_str</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">.</span><span style="color:#B392F0">unwrap</span><span style="color:#E1E4E8">());</span></span> 1143<span class="line"></span> 1144<span class="line"><span style="color:#F97583"> for</span><span style="color:#E1E4E8"> file </span><span style="color:#F97583">in</span><span style="color:#E1E4E8"> files {</span></span> 1145<span class="line"><span style="color:#F97583">
1145 let</span><span style="color:#E1E4E8"> audio_bytes </span><span style="color:#F97583">=</span><span style="color:#B392F0"> std</span><span style="color:#F97583">::</span><span style="color:#B392F0">fs</span><span style="color:#F97583">::</span><span style="color:#B392F0">read</span><span style="color:#E1E4E8">(file)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1146<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> input </span><span style="color:#F97583">=</span><span style="color:#B392F0"> Envelope</span><span style="color:#E1E4E8"> {</span></span> 1147<span class="line"><span style="color:#E1E4E8"> kind</span><span style="color:#F97583">:</span><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#F97583">::</span><span style="color:#B392F0">Audio</span><span style="color:#E1E4E8">(audio_bytes),</span></span> 1148<span class="line"><span style="color:#E1E4E8"> metadata</span><span style="color:#F97583">:</span><span style="color:#B392F0"> std</span><span style="color:#F97583">::</span><span style="color:#B392F0">collections</span><span style="color:#F97583">::</span><span style="color:#B392F0">HashMap</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">(),</span></span> 1149<span class="line"><span style="color:#E1E4E8"> };</span></span> 1150<span class="line"></span> 1151<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> executor</span><span style="color:#F97583">.</span><span style="color:#B392F0">execute</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">metadata, </span><span style="color:#F97583">&</span><span style="color:#E1E4E8">input)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1152<span class="line"><span style="color:#F97583"> if</span><span style="color:#F97583"> let</span><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#F97583">::</span><span style="color:#B392F0">Text</span><span style="color:#E1E4E8">(text) </span><span style="color:#F97583">=</span><span style="color:#F97583"> &</span><span style="color:#E1E4E8">output</span><span style="color:#F97583">.</span><span style="color:#E1E4E8">kind {</span></span> 1153<span class="line"><span style="color:#B392F0"> println!</span><span style="color:#E1E4E8">(</span><span style="color:#9ECBFF">"{}: {}"</span><span style="color:#E1E4E8">, file, text);</span></span> 1154<span class="line"><span style="color:#E1E4E8"> }</span></span> 1155<span class="line"><span style="color:#E1E4E8"> }</span></span> 1156<span class="line"></span> 1157<span class="line"><span style="color:#B392F0"> Ok</span><span style="color:#E1E4E8">(())</span></span> 1158<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var E=s(y,14);e(E,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span>Audio file â Rust binary â Text output</span></span> 1159<span class="line"><span> â</span></span> 1160<span class="line"><span> âââ Model loaded from ~/.xybrid/cache/</span></span> 1161<span class="line"><span> (downloaded once, never phones home)</span></span></code></pre>`);var F=s(E,18);e(F,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> install</span><span style="color:#9ECBFF"> xybrid-cli</span></span> 1162<span class="line"><span style="color:#B392F0">xybrid</span><span style="color:#9ECBFF"> run</span><span style="color:#79B8FF"> --model</span><span style="color:#9ECBFF"> whisper-tiny</span><span style="color:#79B8FF"> --input</span><span style="color:#9ECBFF"> your-audio.wav</span></span></code></pre>`),f(12),h(o,a)}const ks=Object.freeze(Object.defineProperty({__proto__:null,default:fe,metadata:vs},Symbol.toStringTag,{value:"Module"})),ws={title:"Why We Chose Rust Over Python for ML Inference",description:"Not training â inference. How Rust's zero-cost abstractions, lack of GIL, and FFI story make it the better choice for shipping ML to production devices.",date:"2026-06-26",author:"Glenn Sonna",tags:["rust-ml","rust-vs-python","ml-inference","edge-ai","on-device-ai"],category:"engineering",image:"",imageAlt:"",draft:!1,featured:!1,readingTime:7},{title:Bl,description:vl,date:kl,author:wl,tags:xl,category:Cl,image:_l,imageAlt:Al,draft:Tl,featured:Sl,readingTime:Ml}=ws;var be=u('<p>âJust use Python.â</p> <p>Thatâs the default advice for anything ML. And for training, itâs right â Pythonâs ecosystem (PyTorch, JAX, HuggingFace) is unmatched.</p> <p>But weâre not training. Weâre running inference on phones, laptops, and edge devices. And for that, <strong>Rust beats Python in every dimension that matters</strong>.</p> <p>Hereâs why we built <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">Xybrid</a> in Rust and havenât looked back.</p> <hr/> <h2>The Distinction: Training vs. Inference</h2> <table><thead><tr><th>
1162</th><th>Training</th><th>Inference</th></tr></thead><tbody><tr><td><strong>Where</strong></td><td>GPU cluster</td><td>Userâs device</td></tr><tr><td><strong>Who</strong></td><td>ML engineers</td><td>Every user of your app</td></tr><tr><td><strong>Priority</strong></td><td>Flexibility, iteration speed</td><td>Latency, binary size, reliability</td></tr><tr><td><strong>Frequency</strong></td><td>Once (or periodically)</td><td>Millions of times</td></tr><tr><td><strong>Environment</strong></td><td>Controlled</td><td>Uncontrolled (any OS, any device)</td></tr></tbody></table> <p>Training and inference have fundamentally different requirements. Optimizing for one often hurts the other. Python excels at trainingâs requirements. Rust excels at inferenceâs.</p> <h2>Reason 1: No Runtime, No GIL, No Problem</h2> <p>Python inference means shipping a Python runtime with your app. On mobile, thatâs a non-starter. On desktop, itâs a 30-50MB overhead plus dependency management nightmares.</p> <p>Rust compiles to a single native binary. No runtime. No interpreter. No <code>pip install</code> on the userâs machine.</p> <!> <p>The GIL (Global Interpreter Lock) is Pythonâs other gift. Even with threading, Python canât truly parallelize CPU-bound work. For inference pipelines with multiple stages (preprocess â model â postprocess), this means sequential execution.</p> <p>Rust has no GIL. Pipeline stages can genuinely run in parallel:</p> <!> <h2>Reason 2: FFI Is a First-Class Citizen</h2> <p>We need to call our inference engine from Flutter (Dart), Swift, Kotlin, and C# (Unity). From Python, your FFI options are:</p> <ul><li><strong>PyO3</strong> â embeds a Python interpreter in Rust. Now youâre shipping Python anyway.</li> <li><strong>ctypes/cffi</strong> â works, but Python is the host. You canât call Python from Swift.</li> <li><strong>Subprocess</strong> â shell out to a Python script. Serialization overhead kills latency.</li></ul> <p>From Rust, FFI is natural:</p> <table><thead><tr><th>Target</th><th>Tool</th><th>Lines of Binding Code</th></tr></thead><tbody><tr><td>Dart (Flutter)</td><td>flutter_rust_bridge</td><td>~200</td></tr><tr><td>Swift</td><td>UniFFI</td><td>~150</td></tr><tr><td>Kotlin</td><td>UniFFI</td><td>~150</td></tr><tr><td>C# (Unity)</td><td>cbindgen (C FFI)</td><td>~100</td></tr></tbody></table> <p>Rust produces <code>.so</code>, <code>.dylib</code>, <code>.dll</code>, and <code>.a</code> files that any language can load. The type system maps cleanly to C, and tools like UniFFI generate idiomatic wrappers automatically.</p> <p>Our entire Flutter SDK is a thin Dart layer over Rust. The inference, preprocessing, caching â all Rust, called via FFI. Zero Python in the dependency tree.</p> <h2>Reason 3: Memory Safety Without Garbage Collection</h2> <p>ML inference processes large tensors. A 82M parameter model produces multi-megabyte intermediate buffers. In Python (via numpy), memory management is opaque â you rely on reference counting and the GC.</p> <p>Rustâs ownership model gives you:</p> <ul><li><strong>Deterministic deallocation</strong> â buffers free exactly when the executor finishes, not âsometime later when the GC runsâ</li> <li><strong>No use-after-free</strong> â the compiler prevents dangling references to deallocated tensors</li> <li><strong>Zero-copy where possible</strong> â pass slices instead of copying data between pipeline stages</li></ul> <!> <p>On memory-constrained devices (phones, embedded), this predictability matters. No GC pauses during inference. No memory spikes from deferred collection.</p> <h2>Reason 4: Cross-Compilation Just Works</h2> <p>We target 6 platforms from one codebase:</p> <!> <p>Rustâs cross-compilation story is mature. The <code>cc</code> crate handles C/C++ dependencies (like ONNX Runtime and llama.cpp) across toolchains. Cargo features let us conditionally compile platform-specific code:</p> <!> <p>Try cross-compiling a Python inference script with numpy, ONNX Runtime, and a custom tokenizer to iOS. Iâll wait.</p> <h2>Reason 5: Error Handling That Doesnât Lie</h2> <p>Pythonâs approach to errors in inference code:</p> <!> <p>Rust forces you to handle every failure mode explicitly:</p> <!> <p>Every function that can fail returns <code>Result<T, E></code>. The compiler ensures you handle (or explicitly propagate) every error. No silent failures, no âit returned None but I donât know why.â</p> <p>For production inference, this is invaluable. When a model fails on a userâs device, the error message tells you exactly what went wrong.</p> <h2>Reason 6: Zero-Cost Abstractions</h2> <p>Our <code>Envelope</code> type (which carries data between pipeline stages) uses an enum:</p> <!> <p>Pattern matching on this enum compiles to a jump table â no heap allocation, no vtable lookup, no dynamic dispatch. The abstraction costs literally nothing at runtime.</p> <p>In Python, the equivalent would be:</p> <!> <p>Every attribute access goes through the dictionary lookup. Every type check is a runtime operation. For inference pipelines that run millions of times, these costs add up.</p> <h2>When Python Still Wins</h2> <p>Letâs be fair. Python is better when:</p> <ul><li><strong>Prototyping</strong> â Jupyter notebooks for testing model behavior</li> <li><strong>Training</strong> â PyTorch/JAX ecosystem is unmatched</li> <li><strong>One-off scripts</strong> â quick data processing, model conversion</li> <li><strong>Team expertise</strong> â if your team knows Python and not Rust</li></ul> <p>We use Python for model conversion (ONNX export), testing (comparing outputs with reference implementations), and prototyping new preprocessing steps. Then we implement the production version in Rust.</p> <p>The boundary is clear: <strong>Python for research, Rust for production</strong>.</p> <h2>The Numbers</h2> <p>Our Rust inference SDK:</p> <table><thead><tr><th>
1162Metric</th><th>Value</th></tr></thead><tbody><tr><td>Core library size</td><td>~2MB stripped binary</td></tr><tr><td>Cold start (model load)</td><td>50-200ms depending on model</td></tr><tr><td>Inference overhead</td><td><1ms above raw ONNX Runtime</td></tr><tr><td>Memory overhead</td><td>~5MB above model weights</td></tr><tr><td>Platforms</td><td>6 (macOS, iOS, Android, Linux, Windows, WASM)</td></tr><tr><td>Languages served</td><td>5 (Rust, Dart, Swift, Kotlin, C#)</td></tr></tbody></table> <p>Try shipping a Python inference runtime to iOS at 2MB.</p> <hr/> <h2>TL;DR</h2> <table><thead><tr><th></th><th>Python</th><th>Rust</th></tr></thead><tbody><tr><td>Training</td><td>Best choice</td><td>Not practical</td></tr><tr><td>Inference</td><td>Works, but heavy</td><td>Ideal</td></tr><tr><td>Binary size</td><td>30-50MB+ runtime</td><td>2MB</td></tr><tr><td>Cross-platform</td><td>Painful</td><td>Native</td></tr><tr><td>FFI to mobile</td><td>Very hard</td><td>Built-in</td></tr><tr><td>Memory control</td><td>GC-dependent</td><td>Deterministic</td></tr><tr><td>Error handling</td><td>Runtime exceptions</td><td>Compile-time guarantees</td></tr><tr><td>Concurrency</td><td>GIL-limited</td><td>True parallelism</td></tr></tbody></table> <p>Python is for building models. Rust is for shipping them.</p> <hr/> <p><strong>See it in action</strong>: <a href="https://github.com/xybrid-ai/xybrid" rel="nofollow">github.com/xybrid-ai/xybrid</a> â one Rust core, five platform SDKs, zero Python in production.</p> <hr/> <p><em>Agree? Disagree? Drop your take in the comments.</em></p> <hr/> <h2>Related</h2> <ul><li><a href="/blog/on-device-ai-guide">On-Device AI: The Complete Guide</a> â why on-device inference is viable and how to get started.</li> <li><a href="/blog/one-codebase-five-platforms-ml-flutter-swift-kotlin-unity">One Codebase, Five Platforms</a> â how the Rust core powers Flutter, Swift, Kotlin, Unity, and CLI.</li> <li><a href="/blog/edge-vs-cloud-inference">Edge AI vs Cloud AI</a> â the decision framework for where to run each model.</li></ul>',1);function Be(o){var a=be(),n=s(b(a),22);e(n,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># Python inference deployment</span></span> 1163<span class="line"><span style="color:#B392F0">pip</span><span style="color:#9ECBFF"> install</span><span style="color:#9ECBFF"> onnxruntime</span><span style="color:#9ECBFF"> numpy</span><span style="color:#9ECBFF"> tokenizers</span></span> 1164<span class="line"><span style="color:#6A737D"># Hope the user has Python 3.9+</span></span> 1165<span class="line"><span style="color:#6A737D"># Hope numpy doesn't conflict with their system version</span></span> 1166<span class="line"><span style="color:#6A737D"># Hope the wheel exists for their platform</span></span> 1167<span class="line"></span> 1168<span class="line"><span style="color:#6A737D"># Rust inference deployment</span></span> 1169<span class="line"><span style="color:#6A737D"># Ship one binary. Done.</span></span></code></pre>`);var l=s(n,6);e(l,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Preprocess next input while current model runs</span></span> 1170<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> preprocess_handle </span><span style="color:#F97583">=</span><span style="color:#B392F0"> thread</span><span style="color:#F97583">::</span><span style="color:#B392F0">spawn</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">move</span><span style="color:#F97583"> ||</span><span style="color:#B392F0"> preprocess</span><span style="color:#E1E4E8">(next_input));</span></span> 1171<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> model</span><span style="color:#F97583">.</span><span style="color:#B392F0">run</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">current_input)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1172<span class="line"><span style="color:#F97583">let</span><span style="color:#E1E4E8"> next_preprocessed </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> preprocess_handle</span><span style="color:#F97583">.</span><span style="color:#B392F0">join</span><span style="color:#E1E4E8">()</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span></code></pre>`);var t=s(l,24);e(t,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D">// Tensor data freed exactly when executor scope ends</span></span> 1173<span class="line"><span style="color:#E1E4E8">
1173{</span></span> 1174<span class="line"><span style="color:#F97583"> let</span><span style="color:#F97583"> mut</span><span style="color:#E1E4E8"> executor </span><span style="color:#F97583">=</span><span style="color:#B392F0"> TemplateExecutor</span><span style="color:#F97583">::</span><span style="color:#B392F0">new</span><span style="color:#E1E4E8">();</span></span> 1175<span class="line"><span style="color:#F97583"> let</span><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> executor</span><span style="color:#F97583">.</span><span style="color:#B392F0">execute</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">&</span><span style="color:#E1E4E8">metadata, </span><span style="color:#F97583">&</span><span style="color:#E1E4E8">input)</span><span style="color:#F97583">?</span><span style="color:#E1E4E8">;</span></span> 1176<span class="line"><span style="color:#6A737D"> // Process output...</span></span> 1177<span class="line"><span style="color:#E1E4E8">} </span><span style="color:#6A737D">// All intermediate buffers freed here. Deterministic.</span></span></code></pre>`);var p=s(t,8);e(p,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#6A737D"># macOS (host)</span></span> 1178<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> build</span></span> 1179<span class="line"></span> 1180<span class="line"><span style="color:#6A737D"># iOS</span></span> 1181<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> build</span><span style="color:#79B8FF"> --target</span><span style="color:#9ECBFF"> aarch64-apple-ios</span></span> 1182<span class="line"></span> 1183<span class="line"><span style="color:#6A737D"># Android</span></span> 1184<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> build</span><span style="color:#79B8FF"> --target</span><span style="color:#9ECBFF"> aarch64-linux-android</span></span> 1185<span class="line"></span> 1186<span class="line"><span style="color:#6A737D"># Linux</span></span> 1187<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> build</span><span style="color:#79B8FF"> --target</span><span style="color:#9ECBFF"> x86_64-unknown-linux-gnu</span></span> 1188<span class="line"></span> 1189<span class="line"><span style="color:#6A737D"># Windows</span></span> 1190<span class="line"><span style="color:#B392F0">cargo</span><span style="color:#9ECBFF"> build</span><span style="color:#79B8FF"> --target</span><span style="color:#9ECBFF"> x86_64-pc-windows-msvc</span></span></code></pre>`);var r=s(p,4);e(r,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#E1E4E8">#[cfg(target_os </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "macos"</span><span style="color:#E1E4E8">)]</span></span> 1191<span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> setup_accelerator</span><span style="color:#E1E4E8">() {</span></span> 1192<span class="line"><span style="color:#6A737D"> // CoreML + Metal</span></span> 1193<span class="line"><span style="color:#E1E4E8">}</span></span> 1194<span class="line"></span> 1195<span class="line"><span style="color:#E1E4E8">#[cfg(target_os </span><span style="color:#F97583">=</span><span style="color:#9ECBFF"> "android"</span><span style="color:#E1E4E8">)]</span></span> 1196<span class="line"><span style="color:#F97583">fn</span><span style="color:#B392F0"> setup_accelerator</span><span style="color:#E1E4E8">() {</span></span> 1197<span class="line"><span style="color:#6A737D"> // CPU + NNAPI</span></span> 1198<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var c=s(r,8);e(c,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">try</span><span style="color:#E1E4E8">:</span></span> 1199<span class="line"><span style="color:#E1E4E8"> output </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> session.run(</span><span style="color:#79B8FF">None</span><span style="color:#E1E4E8">, inputs)</span></span> 1200<span class="line"><span style="color:#F97583">except</span><span style="color:#79B8FF"> Exception</span><span style="color:#F97583"> as</span><span style="color:#E1E4E8"> e:</span></span> 1201<span class="line"><span style="color:#79B8FF"> print</span><span style="color:#E1E4E8">(</span><span style="color:#F97583">f</span><span style="color:#9ECBFF">"Something went wrong: </span><span style="color:#79B8FF">{</span><span style="color:#E1E4E8">e</span><span style="color:#79B8FF">}</span><span style="color:#9ECBFF">"</span><span style="color:#E1E4E8">)</span></span> 1202<span class="line"><span style="color:#6A737D"> # Good luck debugging which of 15 possible failure modes this is</span></span></code></pre>`);var d=s(c,4);
1202e(d,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> enum</span><span style="color:#B392F0"> ExecutionError</span><span style="color:#E1E4E8"> {</span></span> 1203<span class="line"><span style="color:#B392F0"> ModelNotFound</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">PathBuf</span><span style="color:#E1E4E8">),</span></span> 1204<span class="line"><span style="color:#B392F0"> PreprocessingFailed</span><span style="color:#E1E4E8"> { step</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">, reason</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8"> },</span></span> 1205<span class="line"><span style="color:#B392F0"> InferenceFailed</span><span style="color:#E1E4E8"> { model</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">, error</span><span style="color:#F97583">:</span><span style="color:#B392F0"> ort</span><span style="color:#F97583">::</span><span style="color:#B392F0">Error</span><span style="color:#E1E4E8"> },</span></span> 1206<span class="line"><span style="color:#B392F0"> PostprocessingFailed</span><span style="color:#E1E4E8"> { step</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8">, reason</span><span style="color:#F97583">:</span><span style="color:#B392F0"> String</span><span style="color:#E1E4E8"> },</span></span> 1207<span class="line"><span style="color:#B392F0"> UnsupportedFormat</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">),</span></span> 1208<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var i=s(d,10);e(i,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">pub</span><span style="color:#F97583"> enum</span><span style="color:#B392F0"> EnvelopeKind</span><span style="color:#E1E4E8"> {</span></span> 1209<span class="line"><span style="color:#B392F0"> Audio</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">u8</span><span style="color:#E1E4E8">>),</span></span> 1210<span class="line"><span style="color:#B392F0"> Text</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">String</span><span style="color:#E1E4E8">),</span></span> 1211<span class="line"><span style="color:#B392F0"> Embedding</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">f32</span><span style="color:#E1E4E8">>),</span></span> 1212<span class="line"><span style="color:#B392F0"> Tokens</span><span style="color:#E1E4E8">(</span><span style="color:#B392F0">Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">i64</span><span style="color:#E1E4E8">>),</span></span> 1213<span class="line"><span style="color:#B392F0"> Tensor</span><span style="color:#E1E4E8"> { data</span><span style="color:#F97583">:</span><span style="color:#B392F0"> Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">f32</span><span style="color:#E1E4E8">>, shape</span><span style="color:#F97583">:</span><span style="color:#B392F0"> Vec</span><span style="color:#E1E4E8"><</span><span style="color:#B392F0">usize</span><span style="color:#E1E4E8">> },</span></span> 1214<span class="line"><span style="color:#E1E4E8">}</span></span></code></pre>`);var y=s(i,6);e(y,()=>`<pre class="shiki github-dark" style="background-color:#24292e;color:#e1e4e8" tabindex="0"><code><span class="line"><span style="color:#F97583">class</span><span style="color:#B392F0"> Envelope</span><span style="color:#E1E4E8">:</span></span> 1215<span class="line"><span style="color:#F97583"> def</span><span style="color:#79B8FF"> __init__</span><span style="color:#E1E4E8">(self, kind, data):</span></span> 1216<span class="line"><span style="color:#79B8FF"> self</span><span style="color:#E1E4E8">.kind </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> kind </span><span style="color:#6A737D"># heap-allocated string</span></span> 1217<span class="line"><span style="color:#79B8FF"> self</span><span style="color:#E1E4E8">.data </span><span style="color:#F97583">=</span><span style="color:#E1E4E8"> data </span><span style="color:#6A737D"># heap-allocated, dynamically typed</span></span></code></pre>`),f(42),h(o,a)}const xs=Object.freeze(Object.defineProperty({__proto__:null,default:Be,metadata:ws},Symbol.toStringTag,{value:"Module"}));function Cs(o){const a=o.split(/\s+/).length;return Math.max(1,Math.ceil(a/200))}function _s(o){return o.draft?!1:new Date(o.date+"T00:00:00Z").getTime()<=Date.now()}function As(o){return(o.split("/").pop()??"").replace(".md","")}function G(){const o=Object.assign({"/src/content/blog/add-text-to-speech-to-flutter-app-in-15-minutes.md":K,"/src/content/blog/building-cross-platform-ml-inference-sdk-in-rust.md":X,"/src/content/blog/building-voice-agent-on-device.md":V,"/src/content/blog/edge-first-should-be-your-default.md":Y,"/src/content/blog/edge-vs-cloud-inference.md":Z,"/src/content/blog/model-metadata-json-declarative-schema.md":es,"/src/content/blog/on-device-ai-guide.md":ns,"/src/content/blog/on-device-ai-mobile-ios-android.md":ls,"/src/content/blog/one-codebase-five-platforms-ml-flutter-swift-kotlin-unity.md":ps,"/src/content/blog/onnx-runtime-6x-faster-apple-silicon-coreml.md":cs,"/src/content/blog/private-ai-run-models-without-cloud.md":ds,"/src/content/blog/publishing-ml-models-as-bundles-xybrid-pack.md":Es,"/src/content/blog/run-ai-models-on-device-zero-config.md":us,"/src/content/blog/run-llm-locally-complete-guide.md":Fs,"/src/content/blog/streaming-llm-tokens-rust-to-flutter.md":fs,"/src/content/blog/vendoring-llamacpp-in-rust-workspace.md":Bs,"/src/content/blog/whisper-speech-recognition-single-rust-binary.md":ks,"/src/content/blog/why-rust-over-python-for-ml-inference.md":xs}),a=[];for(const[n,l]of Object.entries(o)){const t=l.metadata;_s(t)&&a.push({...t,slug:As(n),readingTime:t.readingTime??Cs(n)})}return a.sort((n,l)=>new Date(l.date).getTime()-new Date(n.date).getTime())}function ve(o){const a=Object.assign({"/src/content/blog/add-text-to-speech-to-flutter-app-in-15-minutes.md":K,"/src/content/blog/building-cross-platform-ml-inference-sdk-in-rust.md":X,"/src/content/blog/building-voice-agent-on-device.md":V,"/src/content/blog/edge-first-should-be-your-default.md":Y,"/src/content/blog/edge-vs-cloud-inference.md":Z,"/src/content/blog/model-metadata-json-declarative-schema.md":es,"/src/content/blog/on-device-ai-guide.md":ns,"/src/content/blog/on-device-ai-mobile-ios-android.md":ls,"/src/content/blog/one-codebase-five-platforms-ml-flutter-swift-kotlin-unity.md":ps,"/src/content/blog/onnx-runtime-6x-faster-apple-silicon-coreml.md":cs,"/src/content/blog/private-ai-run-models-without-cloud.md":ds,"/src/content/blog/publishing-ml-models-as-bundles-xybrid-pack.md":Es,"/src/content/blog/run-ai-models-on-device-zero-config.md":us,"/src/content/blog/run-llm-locally-complete-guide.md":Fs,"/src/content/blog/streaming-llm-tokens-rust-to-flutter.md":fs,"/src/content/blog/vendoring-llamacpp-in-rust-workspace.md":Bs,"/src/content/blog/whisper-speech-recognition-single-rust-binary.md":ks,"/src/content/blog/why-rust-over-python-for-ml-inference.md":xs});for(const[n,l]of Object.entries(a))if(As(n)===o){const t=l.metadata;return _s(t)?{post:{...t,slug:o,readingTime:t.readingTime??Cs(n)},component:l.default}:void 0}}function ke(o,a=3){const n=G().find(p=>p.slug===o);return n?G().filter(p=>p.slug!==o).map(p=>({post:p,score:p.tags.filter(r=>n.tags.includes(r)).length})).sort((p,r)=>r.score-p.score).slice(0,a).map(p=>p.post):[]}const we=!0,xe=()=>G().map(o=>({slug:o.slug})),Ce=async({params:o})=>{const a=ve(o.slug);
1217if(!a)throw Gs(404,`Post not found: ${o.slug}`);const{post:n,component:l}=a,t=ke(o.slug,3);return{post:n,component:l,related:t,seo:{title:`${n.title} - Xybrid Blog`,description:n.description,url:`https://www.xybrid.ai/blog/${n.slug}`,image:n.image?`https://www.xybrid.ai${n.image}`:void 0}}},Pl=Object.freeze(Object.defineProperty({__proto__:null,entries:xe,load:Ce,prerender:we},Symbol.toStringTag,{value:"Module"}));var _e=u('<img class="svelte-gqkn6s"/>'),Ae=u('<span class="x-article-header__tag svelte-gqkn6s"> </span>'),Te=u('<header class="x-article-header svelte-gqkn6s"><a class="x-article-header__back svelte-gqkn6s" href="/blog">â Back to blog</a> <span class="x-article-header__category svelte-gqkn6s"> </span> <h1 class="x-display x-article-header__title svelte-gqkn6s"> </h1> <p class="x-lead x-article-header__lead svelte-gqkn6s"> </p> <div class="x-article-header__meta svelte-gqkn6s"><div class="x-article-header__author svelte-gqkn6s"><!> <span> </span></div> <span class="x-article-header__sep svelte-gqkn6s">·</span> <time> </time> <span class="x-article-header__sep svelte-gqkn6s">·</span> <span> </span></div> <div class="x-article-header__tags svelte-gqkn6s"></div></header>');function Se(o,a){z(a,!0);const n=I(()=>new Date(a.post.date).toLocaleDateString("en-US",{month:"long",day:"numeric",year:"numeric"})),l=I(()=>({engineering:"Engineering",tutorial:"Tutorial",announcement:"Announcement","case-study":"Case study"})[a.post.category]??a.post.category);var t=Te(),p=s(g(t),2),r=g(p,!0);m(p);var c=s(p,2),d=g(c,!0);m(c);var i=s(c,2),y=g(i,!0);m(i);var E=s(i,2),F=g(E),v=g(F);{var _=C=>{var D=_e();R(()=>{S(D,"src",a.post.authorImage),S(D,"alt",a.post.author)}),h(C,D)};N(v,C=>{a.post.authorImage&&C(_)})}var T=s(v,2),L=g(T,!0);m(T),m(F);var M=s(F,4),k=g(M,!0);m(M);var B=s(M,4),x=g(B);m(B),m(E);var P=s(E,2);$(P,20,()=>a.post.tags,C=>C,(C,D)=>{var O=Ae(),Ts=g(O,!0);m(O),R(()=>A(Ts,D)),h(C,O)}),m(P),m(t),R(()=>{A(r,w(l)),A(d,a.post.title),A(y,a.post.description),A(L,a.post.author),S(M,"datetime",a.post.date),A(k,w(n)),A(x,`${a.post.readingTime??""} min read`)}),h(o,t),U()}var Me=u('<div class="x-share svelte-3gbi91"><span class="x-share__label svelte-3gbi91">Share</span> <a class="x-share__btn svelte-3gbi91" target="_blank" rel="noopener noreferrer" aria-label="Share on X (Twitter)">X</a> <a class="x-share__btn svelte-3gbi91" target="_blank" rel="noopener noreferrer" aria-label="Share on LinkedIn">in</a> <button class="x-share__btn svelte-3gbi91" aria-label="Copy link"> </button></div>');function Pe(o,a){z(a,!0);let n=Ps(!1);const l=I(()=>`https://www.xybrid.ai/blog/${a.post.slug}`),t=I(()=>`https://twitter.com/intent/tweet?text=${encodeURIComponent(a.post.title)}&url=${encodeURIComponent(w(l))}`),p=I(()=>`https://www.linkedin.com/sharing/share-offsite/?url=${encodeURIComponent(w(l))}`);async function r(){await navigator.clipboard.writeText(w(l)),q(n,!0),setTimeout(()=>q(n,!1),2e3)}var c=Me(),d=s(g(c),2),i=s(d,2),y=s(i,2);y.__click=r;var E=g(y,!0);m(y),m(c),R(()=>{S(d,"href",w(t)),S(i,"href",w(p)),A(E,w(n)?"Copied":"Copy link")}),h(o,c),U()}Ms(["click"]);var Ie=u('<div class="x-article__hero svelte-1teoznn"><img class="svelte-1teoznn"/></div>'),Le=u('<div class="x-article__related svelte-1teoznn"><h2 class="x-h2 svelte-1teoznn">Related articles</h2> <div class="x-article__related-grid svelte-1teoznn"></div></div>'),De=u('<div class="x-page x-root svelte-1teoznn"><!> <main><article class="x-article svelte-1teoznn"><div class="x-container x-container--narrow svelte-1teoznn"><!> <!> <div class="x-article__body svelte-1teoznn"><!></div> <div class="x-article__share svelte-1teoznn"><!></div> <!></div></article></main> <!></div>');function Il(o,a){z(a,!0);const n=I(()=>JSON.stringify({"@context":"https://schema.org","@type":"Article",headline:a.data.post.title,description:a.data.post.description,image:a.data.post.image?`https://www.xybrid.ai${a.data.post.image}`:void 0,datePublished:a.data.post.date,dateModified:a.data.post.updated??a.data.post.date,author:{"@type":"Person",name:a.data.post.author}
1217,publisher:{"@type":"Organization",name:"Xybrid",logo:{"@type":"ImageObject",url:"https://www.xybrid.ai/logo.png"}},mainEntityOfPage:{"@type":"WebPage","@id":`https://www.xybrid.ai/blog/${a.data.post.slug}`}}));var l=De();Ds("1teoznn",k=>{var B=Is(),x=b(B);e(x,()=>`<script type="application/ld+json">${w(n)}<\/script>`),h(k,B)});var t=g(l);Rs(t,{ctaLabel:"Get API key"});var p=s(t,2),r=g(p),c=g(r),d=g(c);Se(d,{get post(){return a.data.post}});var i=s(d,2);{var y=k=>{var B=Ie(),x=g(B);m(B),R(()=>{S(x,"src",a.data.post.image),S(x,"alt",a.data.post.imageAlt)}),h(k,B)};N(i,k=>{a.data.post.image&&k(y)})}var E=s(i,2),F=g(E);Ls(F,()=>a.data.component,(k,B)=>{B(k,{})}),m(E);var v=s(E,2),_=g(v);Pe(_,{get post(){return a.data.post}}),m(v);var T=s(v,2);{var L=k=>{var B=Le(),x=s(g(B),2);$(x,21,()=>a.data.related,P=>P.slug,(P,C)=>{Ns(P,{get post(){return w(C)}})}),m(x),m(B),h(k,B)};N(T,k=>{a.data.related.length>0&&k(L)})}m(c),m(r),m(p);var M=s(p,2);Os(M,{}),m(l),h(o,l),U()}export{Il as component,Pl as universal};
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.