https://weaviate.io/assets/js/80537b1e.0867063b.js

1"use strict";(globalThis.webpackChunkmy_website=globalThis.webpackChunkmy_website||[]).push([[16030],{24899(e,n,i){i.r(n),i.d(n,{assets:()=>h,contentTitle:()=>c,default:()=>m,frontMatter:()=>a,metadata:()=>s,toc:()=>d});var s=i(19774),r=i(74848),t=i(28453),o=i(49107),l=i(73284);const a={title:"Mixture-of-Agents Enhances Large Language Model Capabilities",slug:"moa",authors:["zain"],date:new Date("2024-06-29T00:00:00.000Z"),tags:[],image:"./img/hero.png",description:"Combining small LLMs to outperform larger monolithic LLMs!"},c=void 0,h={image:i(48463).A,authorsImageUrls:[void 0]},d=[{value:"\ud83e\udd16Can multiple smaller open-source LLMs be combined to outperform larger monolithic LLMs?",id:"can-multiple-smaller-open-source-llms-be-combined-to-outperform-larger-monolithic-llms",level:2},{value:"MoA Architecture",id:"moa-architecture",level:2},{value:"How do you choose which models to include in the MoA??",id:"how-do-you-choose-which-models-to-include-in-the-moa",level:2},{value:"Details:",id:"details",level:2},...l.RM];function p(e){const n={a:"a",code:"code",h2:"h2",img:"img",li:"li",ol:"ol",p:"p",ul:"ul",...(0,t.R)(),...e.components};return(0,r.jsxs)(r.Fragment,{children:[(0,r.jsx)(n.p,{children:(0,r.jsx)(n.img,{alt:"A preview of the paper",src:i(72042).A+"",width:"15472",height:"5061"})}),"\n","\n",(0,r.jsx)(n.h2,{id:"can-multiple-smaller-open-source-llms-be-combined-to-outperform-larger-monolithic-llms",children:"\ud83e\udd16Can multiple smaller open-source LLMs be combined to outperform larger monolithic LLMs?"}),"\n",(0,r.jsx)(n.p,{children:"New paper shows that LLMs tend to generate better responses when presented with outputs from other models, even if less capable."}),"\n",(0,r.jsx)(n.p,{children:"They use this to build a Mixture of Agents(MoA) architecture where multiple LLMs are used to iteratively enhance the generation quality."}),"\n",(0,r.jsx)(n.p,{children:"LLMs in deeper layers are presented responses from LLMs in earlier layers and iteratively improve the response; mitigates individual model deficiencies and enhance overall response."}),"\n",(0,r.jsx)(n.h2,{id:"moa-architecture",children:"MoA Architecture"}),"\n",(0,r.jsx)(n.p,{children:"The complete architecture consists of LLM agents playing one of two roles:"}),"\n",(0,r.jsxs)(n.ol,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Proposers: These models generate initial reference responses."}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Aggregators: These models synthesize the different responses from the proposers into one high-quality response."}),"\n"]}),"\n"]}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:["Models used: ",(0,r.jsx)(n.code,{children:"Qwen1.5-110B-Chat"}),", ",(0,r.jsx)(n.code,{children:"Qwen1.572B-Chat"}),", ",(0,r.jsx)(n.code,{children:"WizardLM-8x22B"}),", ",(0,r.jsx)(n.code,{children:"LLaMA-3-70B-Instruct"}),", ",(0,r.jsx)(n.code,{children:"Mixtral-8x22B-v0.1"}),", ",(0,r.jsx)(n.code,{children:"dbrx-instruct"})]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"3 MoA layers and use the same set of models in each MoA layer"}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsxs)(n.p,{children:[(0,r.jsx)(n.code,{children:"Qwen1.5-110BChat"})," as the aggregator in the last layer"]}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Some models work better as proposers and others as both proposers and aggregators."}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"how-do-you-choose-which-models-to-include-in-the-moa",children:"How do you choose which models to include in the MoA??"}),"\n",(0,r.jsx)(n.p,{children:"Two metrics are used to assess which models are included in the mixture:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Performance: The average win rate of models in layer i decides if they are included in layer i + 1."}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Diversity: The diversity of model outputs is important - using heterogeneous models across layers is better then using the same model"}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.h2,{id:"details",children:"Details:"}),"\n",(0,r.jsxs)(n.ul,{children:["\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"MoA achieves a new SOTA win rate of 65.8% on AlpacaEval 2.0 compared to the previous best of 57.5% achieved by GPT-4 Omni."}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Overall performance comparable to GPT-4 Turbo while being 2\xd7 more cost-effective."}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"No finetuning required only utilizes the interface of prompting and generation of LLMs."}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"Extends the MoE concept to the model level by operating at the model level rather than at the activation level."}),"\n"]}),"\n",(0,r.jsxs)(n.li,{children:["\n",(0,r.jsx)(n.p,{children:"You can swap the final aggregator to any LLM of your choice (Gemini, GPT-4o, Claude3.5) and this improves performance!"}),"\n"]}),"\n"]}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.a,{href:"https://github.com/togethercomputer/MoA#interactive-cli-demo",children:"Demo"})}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.a,{href:"https://github.com/togethercomputer/MoA",children:"Code"})}),"\n",(0,r.jsx)(n.p,{children:(0,r.jsx)(n.a,{href:"https://www.together.ai/blog/together-moa",children:"Blog"})}),"\n",(0,r.jsx)("p",{children:(0,r.jsx)(o.fd,{link:"https://arxiv.org/abs/2406.04692",children:"\ud83d\udd17 arXiv Link"})}),"\n",(0,r.jsx)("p",{children:(0,r.jsx)(o.fd,{link:"https://arxiv.org/pdf/2406.04692",children:"\ud83d\udcdc Download paper"})}),"\n","\n",(0,r.jsx)(l.Ay,{})]})}function m(e={}){const{wrapper:n}={...(0,t.R)(),...e.components};return n?(0,r.jsx)(n,{...e,children:(0,r.jsx)(p,{...e})}):p(e)}},49107(e,n,i){i.d(n,{Ef:()=>u,fd:()=>m,z9:()=>p});var s=i(28774);i(96540);const r="buttons_NqrD",t="left_ej0y",o="right_kFGn",l="btn_VbJ1",a="btnMain_ywTD",c="btnAccent_uqV6",h="btnOutlined_BYrT";var d=i(74848);function p({children:e,link:n,color:i="main",newTab:r=!0,className:t,style:o}){let p=l;switch(i){case"accent":p+=" "+c;break;case"outlined":p+=" "+h;break;default:p+=" "+a}return t&&(p+=" "+t),(0,d.jsx)(s.A,{className:p,style:o,to:n,target:r?"_blank":"_self",children:e})}function m({children:e,link:n,color:i="main",newTab:s=!0}){let r=l;if("accent"===i)r+=" "+c;else r+=" "+a;return(0,d.jsx)("a",{className:r,href:n,download:!0,children:e})}function u({children:e,position:n="center"}){let i=r;switch(n){case"left":i+=" "+t;break;case"right":i+=" "+o}return(0,d.jsx)("div",{className:i,children:e})}},48463(e,n,i){i.d(n,{A:()=>s});const s=i.p+"assets/images/hero-a226000952df08ea3d493335b94d1825.png"},72042(e,n,i){i.d(n,{A:()=>s});const s=i.p+"assets/images/hero-a226000952df08ea3d493335b94d1825.png"},19774(e){e.exports=JSON.parse('{"permalink":"/papers/moa","source":"@site/papers/2024-06-29-moa/index.mdx","title":"Mixture-of-Agents Enhances Large Language Model Capabilities","description":"Combining small LLMs to outperform larger monolithic LLMs!","date":"2024-06-29T00:00:00.000Z","tags":[],"readingTime":1.99,"hasTruncateMarker":true,"authors":[{"name":"Zain Hasan","title":"Developer Advocate","url":"https://github.com/zainhas","imageURL":"/img/people/icon/zain.jpg","key":"zain","page":null}],"frontMatter":{"title":"Mixture-of-Agents Enhances Large Language Model Capabilities","slug":"moa","authors":["zain"],"date":"2024-06-29T00:00:00.000Z","tags":[],"image":"./img/hero.png","description":"Combining small LLMs to outperform larger monolithic LLMs!"},"unlisted":false,"prevItem":{"title":"Token Pooling to Scale Multi-Vector Retrieval Systems","permalink":"/papers/colbertpooling"},"nextItem":{"title":"GLiNER: Generalist Model for Named Entity Recognition using Bidirectional Transformer","permalink":"/papers/gliner"}}')}}]);