import{s as de,n as fe,o as he}from"../chunks/scheduler.eb244325.js";import{S as ce,i as $e,e as s,s as a,c as k,h as ve,a as o,d as n,b as i,f as ge,g as M,j as r,k as ue,l as we,m as l,n as P,t as S,o as C,p as H}from"../chunks/index.661680a1.js";import{C as xe,H as Z,E as be}from"../chunks/MermaidChart.svelte_svelte_type_style_lang.338b442f.js";function Le(ee){let m,U,G,E,p,q,g,F,u,te=`SGLang is a fast serving framework for large language models and vision language models. It’s very similar to TGI and vLLM and comes
with production ready features.`,I,d,ne="The core features include:",R,f,le=`<li><p><strong>Fast Backend Runtime</strong>:</p> <ul><li>efficient serving with RadixAttention for prefix caching</li> <li>zero-overhead CPU scheduler</li> <li>continuous batching, paged attention, tensor parallelism and pipeline parallelism,</li> <li>expert parallelism, structured outputs, chunked prefill, quantization (FP8/INT4/AWQ/GPTQ), and multi-lora batching</li></ul></li> <li><p><strong>Extensive Model Support</strong>: Supports a wide range of generative models (Llama, Gemma, Mistral, Qwen, DeepSeek, LLaVA, etc.),
embedding models (e5-mistral, gte, mcdse) and reward models (Skywork), with easy extensibility for integrating new models.</p></li>`,A,h,Q,c,ae='<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/sglang/sglang.png" alt="sglang"/>',W,$,ie=`<li><strong>Max Running Request</strong>: the max number of concurrent requests</li> <li><strong>Max Prefill Tokens</strong> (per batch): the maximum number of tokens that can be processed in a single prefill operation. This controls the batch size for the prefill phase and helps manage memory usage during prompt processing.</li> <li><strong>Chunked prefill size</strong>: sets how many tokens are processed at once during the prefill phase. If a prompt is longer than this value,
it will be split into smaller chunks and processed sequentially to avoid out-of-memory errors during prefill with long prompts.
For example, setting —chunked-prefill-size 4096 means each chunk will have up to 4096 tokens processed at a time. Setting this to -1
means disabling chunked prefill.</li> <li><strong>Tensor Parallel Size</strong>: the number of GPUs to use for tensor parallelism. This enables model sharding across multiple GPUs
to handle larger models that don’t fit on a single GPU. For example, setting this to 2 will split the model across 2 GPUs.</li> <li><strong>KV Cache DType</strong>: the data type used for storing the key-value cache during generation. Options include “auto”, “fp8_e5m2”,
and “fp8_e4m3”. Using lower precision types can reduce memory usage but may slightly impact generation quality.</li>`,D,v,se=`For more advanced configuration you can pass any of the <a href="https://docs.sglang.ai/backend/server_arguments.html" rel="nofollow">Server Arguments that SGlang supports</a>
as container arguments. For example changing the <code>schedule-policy</code> to <code>lpm</code> would look like this:`,O,w,oe='<img src="https://raw.githubusercontent.com/huggingface/hf-endpoints-documentation/main/assets/sglang/sglang-advanced.png" alt="sglang-advanced"/>',V,x,j,b,re=`SGlang has wide support for large language models, multimodal language models, embedding models and more. We recommend reading the
<a href="https://docs.sglang.ai/supported_models/generative_models.html" rel="nofollow">supported models</a> section in the SGLang documentation for a full list.`,B,L,me=`In the Inference Endpoints UI, by default, any model on the Hugging Face Hub that has a <code>transformers</code> tag, can be deployed with SGLang.
This is because SGLang <a href="https://docs.sglang.ai/supported_models/transformers_fallback.html#transformers-fallback-in-sglang" rel="nofollow">implements a fallback</a> to use transformers
if SGLang doesn’t have their own implementation of a model.`,K,_,N,y,pe='We also recommend reading the <a href="https://docs.sglang.ai/" rel="nofollow">SGLang documentation</a> for more in-depth information.',J,T,X,z,Y;return p=new xe({props:{containerStyle:"float: right; margin-left: 10px; display: inline-flex; position: relative; z-index: 10;"}}),g=new Z({props:{title:"SGLang",local:"sglang",headingTag:"h1"}}),h=new Z({props:{title:"Configuration",local:"configuration",headingTag:"h2"}}),x=new Z({props:{title:"Supported models",local:"supported-models",headingTag:"h2"}}),_=new Z({props:{title:"References",local:"references",headingTag:"h2"}}),T=new be({props:{source:"https://github.com/huggingface/hf-endpoints-documentation/blob/main/docs/source/engines/sglang.md"}}),{c(){m=s("meta"),U=a(),G=s("p"),E=a(),k(p.$$.fragment),q=a(),k(g.$$.fragment),F=a(),u=s("p"),u.textContent=te,I=a(),d=s("p"),d.textContent=ne,R=a(),f=s("ul"),f.innerHTML=le,A=a(),k(h.$$.fragment),Q=a(),c=s("p"),c.innerHTML=ae,W=a(),$=s("ul"),$.innerHTML=ie,D=a(),v=s("p"),v.innerHTML=se,O=a(),w=s("p"),w.innerHTML=oe,V=a(),k(x.$$.fragment),j=a(),b=s("p"),b.innerHTML=re,B=a(),L=s("p"),L.innerHTML=me,K=a(),k(_.$$.fragment),N=a(),y=s("p"),y.innerHTML=pe,J=a(),k(T.$$.fragment),X=a(),z=s("p"),this.h()},l(e){const t=ve("svelte-u9bgzb",document.head);m=o(t,"META",{name:!0,content:!0}),t.forEach(n),U=i(e),G=o(e,"P",{}),ge(G).forEach(n),E=i(e),M(p.$$.fragment,e),q=i(e),M(g.$$.fragment,e),F=i(e),u=o(e,"P",{"data-svelte-h":!0}),r(u)!=="svelte-i4zzvk"&&(u.textContent=te),I=i(e),d=o(e,"P",{"data-svelte-h":!0}),r(d)!=="svelte-1h7t0xv"&&(d.textContent=ne),R=i(e),f=o(e,"UL",{"data-svelte-h":!0}),r(f)!=="svelte-i8wyr8"&&(f.innerHTML=le),A=i(e),M(h.$$.fragment,e),Q=i(e),c=o(e,"P",{"data-svelte-h":!0}),r(c)!=="svelte-mw3pr2"&&(c.innerHTML=ae),W=i(e),$=o(e,"UL",{"data-svelte-h":!0}),r($)!=="svelte-v7hqhg"&&($.innerHTML=ie),D=i(e),v=o(e,"P",{"data-svelte-h":!0}),r(v)!=="svelte-bu8esv"&&(v.innerHTML=se),O=i(e),w=o(e,"P",{"data-svelte-h":!0}),r(w)!=="svelte-zlixhk"&&(w.innerHTML=oe),V=i(e),M(x.$$.fragment,e),j=i(e),b=o(e,"P",{"data-svelte-h":!0}),r(b)!=="svelte-1b1idos"&&(b.innerHTML=re),B=i(e),L=o(e,"P",{"data-svelte-h":!0}),r(L)!=="svelte-76vkhp"&&(L.innerHTML=me),K=i(e),M(_.$$.fragment,e),N=i(e),y=o(e,"P",{"data-svelte-h":!0}),r(y)!=="svelte-1yhkg23"&&(y.innerHTML=pe),J=i(e),M(T.$$.fragment,e),X=i(e),z=o(e,"P",{}),ge(z).forEach(n),this.h()},h(){ue(m,"name","hf:doc:metadata"),ue(m,"content",_e)},m(e,t){we(document.head,m),l(e,U,t),l(e,G,t),l(e,E,t),P(p,e,t),l(e,q,t),P(g,e,t),l(e,F,t),l(e,u,t),l(e,I,t),l(e,d,t),l(e,R,t),l(e,f,t),l(e,A,t),P(h,e,t),l(e,Q,t),l(e,c,t),l(e,W,t),l(e,$,t),l(e,D,t),l(e,v,t),l(e,O,t),l(e,w,t),l(e,V,t),P(x,e,t),l(e,j,t),l(e,b,t),l(e,B,t),l(e,L,t),l(e,K,t),P(_,e,t),l(e,N,t),l(e,y,t),l(e,J,t),P(T,e,t),l(e,X,t),l(e,z,t),Y=!0},p:fe,i(e){Y||(S(p.$$.fragment,e),S(g.$$.fragment,e),S(h.$$.fragment,e),S(x.$$.fragment,e),S(_.$$.fragment,e),S(T.$$.fragment,e),Y=!0)},o(e){C(p.$$.fragment,e),C(g.$$.fragment,e),C(h.$$.fragment,e),C(x.$$.fragment,e),C(_.$$.fragment,e),C(T.$$.fragment,e),Y=!1},d(e){e&&(n(U),n(G),n(E),n(q),n(F),n(u),n(I),n(d),n(R),n(f),n(A),n(Q),n(c),n(W),n($),n(D),n(v),n(O),n(w),n(V),n(j),n(b),n(B),n(L),n(K),n(N),n(y),n(J),n(X),n(z)),n(m),H(p,e),H(g,e),H(h,e),H(x,e),H(_,e),H(T,e)}}}const _e='{"title":"SGLang","local":"sglang","sections":[{"title":"Configuration","local":"configuration","sections":[],"depth":2},{"title":"Supported models","local":"supported-models","sections":[],"depth":2},{"title":"References","local":"references","sections":[],"depth":2}],"depth":1}';function ye(ee){return he(()=>{new URLSearchParams(window.location.search).get("fw")}),[]}class Pe extends ce{constructor(m){super(),$e(this,m,ye,Le,de,{})}}export{Pe as component};
