> ## Documentation Index > Fetch the complete documentation index at: https://docs.baseten.co/llms.txt > Use this file to discover all available pages before exploring further. # Autoscaling > Configure autoscaling to dynamically adjust replicas based on traffic while minimizing idle compute costs. export const AutoscalerSimPanels = () => { const ref = React.useRef(null); const init = React.useRef(false); React.useEffect(() => { if (!ref.current || init.current) return; init.current = true; const root = ref.current; let cleanup = null, destroyed = false, timer = null, retries = 0; const tryMount = () => { if (destroyed || !ref.current) return; if (window._asEngine && window._asState) { cleanup = mount(); } else if (retries++ < 120) { timer = setTimeout(tryMount, 30); } }; function mount() { const E = window._asEngine, W = 1100; const card = document.createElement("div"); const row = document.createElement("div"); row.style.cssText = "display:grid;grid-template-columns:repeat(auto-fit,minmax(320px,1fr));gap:12px;margin:0 0 14px;align-items:start"; const live = document.createElement("div"), mathP = document.createElement("div"); row.appendChild(live); row.appendChild(mathP); card.appendChild(row); const params = document.createElement("div"); card.appendChild(params); function head(parent, text, body) { const h = document.createElement("div"); h.style.cssText = "display:flex;justify-content:space-between;align-items:baseline;margin:0 0 10px" + (body ? ";cursor:pointer;user-select:none" : ""); const left = document.createElement("span"); left.style.cssText = "display:flex;align-items:center;gap:6px;font:500 11px ui-monospace,Menlo,monospace;text-transform:lowercase;color:#869089"; let arrow = null; if (body) { arrow = document.createElement("span"); arrow.textContent = "▸"; arrow.style.cssText = "font-size:9px;display:inline-block;transition:transform 0.2s ease"; left.appendChild(arrow); } const t = document.createElement("span"); t.textContent = text; left.appendChild(t); const s = document.createElement("span"); s.style.cssText = "font:400 10px ui-monospace,Menlo,monospace;color:#869089"; h.appendChild(left); h.appendChild(s); parent.appendChild(h); if (body) { body.style.display = "none"; h.onclick = () => { const open = body.style.display !== "none"; body.style.display = open ? "none" : ""; arrow.style.transform = open ? "" : "rotate(90deg)"; }; } return s; } const mathBody = document.createElement("div"); const lsSub = head(live, "deployment state"); const mSub = head(mathP, "autoscaler math", mathBody); mathP.appendChild(mathBody); const ppGridWrap = document.createElement("div"); const ppSub = head(params, "parameters", ppGridWrap); params.appendChild(ppGridWrap); const lsBar = document.createElement("div"); lsBar.style.cssText = "display:flex;height:14px;border-radius:4px;overflow:hidden;background:rgba(0,0,0,0.04);margin:0 0 8px"; const segP = document.createElement("div"); segP.style.cssText = "background:#0e863f;height:100%"; const segQ = document.createElement("div"); segQ.style.cssText = "background:#9c7400;height:100%"; const segR = document.createElement("div"); segR.style.cssText = "height:100%"; lsBar.appendChild(segP); lsBar.appendChild(segQ); lsBar.appendChild(segR); live.appendChild(lsBar); const lsLeg = document.createElement("div"); lsLeg.style.cssText = "display:flex;gap:14px;flex-wrap:wrap;font:500 11px ui-monospace,Menlo,monospace;color:#869089;margin:0 0 6px"; function legItem(color) { const w = document.createElement("span"); w.style.cssText = "display:inline-flex;align-items:center;gap:5px"; const dot = document.createElement("span"); dot.style.cssText = "width:8px;height:8px;border-radius:2px;background:" + color; const txt = document.createElement("span"); w.appendChild(dot); w.appendChild(txt); lsLeg.appendChild(w); return txt; } const legP = legItem("#0e863f"), legQ = legItem("#9c7400"), legR = legItem("#9CA59E"); live.appendChild(lsLeg); const lsCap = document.createElement("div"); lsCap.style.cssText = "font:400 10px/1.4 ui-monospace,Menlo,monospace;color:#869089;margin:6px 0 0"; live.appendChild(lsCap); const mRows = []; for (let i = 0; i < 4; i++) { const r = document.createElement("div"); r.style.cssText = "display:flex;justify-content:space-between;padding:5px 0;font:400 12px system-ui,-apple-system,sans-serif;align-items:center;gap:8px"; if (i === 3) r.style.fontWeight = "500"; const a = document.createElement("span"); a.style.cssText = "color:#869089"; const b = document.createElement("span"); b.style.cssText = "font-family:ui-monospace,Menlo,monospace;font-size:12px;text-align:right"; r.appendChild(a); r.appendChild(b); mathBody.appendChild(r); mRows.push({ a, b, r }); } const mCount = document.createElement("div"); mCount.style.cssText = "margin:10px 0 0;opacity:0;transition:opacity 0.25s ease;pointer-events:none;min-height:74px"; const cdLabel = document.createElement("div"); cdLabel.style.cssText = "display:flex;justify-content:space-between;font:500 11px ui-monospace,Menlo,monospace;color:#1960d3;margin:0 0 4px"; const cdL = document.createElement("span"), cdR = document.createElement("span"); cdLabel.appendChild(cdL); cdLabel.appendChild(cdR); const cdTrack = document.createElement("div"); cdTrack.style.cssText = "height:4px;border-radius:2px;background:rgba(33,118,255,0.15);overflow:hidden"; const cdFill = document.createElement("div"); cdFill.style.cssText = "height:100%;background:#2176ff;width:0%;transition:width 0.2s linear"; cdTrack.appendChild(cdFill); const cdHelp = document.createElement("p"); cdHelp.style.cssText = "margin:6px 0 0;font:400 11px/1.4 system-ui,-apple-system,sans-serif"; mCount.appendChild(cdLabel); mCount.appendChild(cdTrack); mCount.appendChild(cdHelp); mathBody.appendChild(mCount); const ppGrid = document.createElement("div"); ppGrid.style.cssText = "display:grid;grid-template-columns:repeat(auto-fit,minmax(220px,1fr));gap:10px"; ppGridWrap.appendChild(ppGrid); const pcRefs = {}; E.PARAMS.forEach(p => { const c = document.createElement("div"); c.style.cssText = "padding:10px 12px;border-radius:6px;display:flex;flex-direction:column;gap:4px;min-width:0"; const lbl = document.createElement("code"); lbl.style.cssText = "font:500 12px ui-monospace,Menlo,monospace;letter-spacing:-0.28px;display:block;overflow-wrap:anywhere"; lbl.textContent = p[1]; const valRow = document.createElement("div"); valRow.style.cssText = "display:flex;align-items:center;gap:10px;margin:2px 0"; const val = document.createElement("span"); val.style.cssText = "font:500 14px ui-monospace,Menlo,monospace;letter-spacing:-0.28px;min-width:48px"; const inp = document.createElement("input"); inp.type = "range"; inp.min = p[2]; inp.max = p[3]; inp.style.cssText = "flex:1;height:3px;accent-color:#0e863f;margin:0;min-width:0"; valRow.appendChild(val); valRow.appendChild(inp); const sum = document.createElement("p"); sum.style.cssText = "margin:2px 0 0;font:400 12px/1.4 system-ui,-apple-system,sans-serif"; E.setRich(sum, p[5]); const desc = document.createElement("p"); desc.style.cssText = "margin:0;font:400 11px/1.4 system-ui,-apple-system,sans-serif;color:#869089"; E.setRich(desc, p[6]); c.appendChild(lbl); c.appendChild(valRow); c.appendChild(sum); c.appendChild(desc); ppGrid.appendChild(c); inp.oninput = () => { const cfg = window._asState && window._asState.cfg; if (!cfg) return; let v = parseInt(inp.value, 10); if (p[0] === "minR") v = Math.min(v, cfg.maxR); if (p[0] === "maxR") v = Math.max(v, cfg.minR); cfg[p[0]] = v; val.textContent = v + p[4]; }; pcRefs[p[0]] = { c, lbl, val, sum, desc, inp, p }; }); root.appendChild(card); let visible = true, raf = 0; const obs = new IntersectionObserver(e => visible = e[0].isIntersecting, { threshold: 0.05 }); obs.observe(card); const tObs = new MutationObserver(() => { applyTheme(); prev.dirty = true; }); tObs.observe(document.documentElement, { attributes: true, attributeFilter: ["class"] }); function applyTheme() { const c = E.P(); card.style.cssText = "border:1px solid " + c.brd + ";border-radius:10px;padding:18px 20px;margin:14px 0;background:" + c.bg + ";max-width:" + W + "px"; const panelStyle = "padding:14px 16px;border-radius:8px;border:1px solid " + c.brdM + ";background:" + c.surf; live.style.cssText = panelStyle; mathP.style.cssText = panelStyle; params.style.cssText = panelStyle; segR.style.background = c.brdM; Object.values(pcRefs).forEach(r => { r.c.style.background = c.bg; r.c.style.border = "1px solid " + c.brd; r.lbl.style.color = c.txt; r.val.style.color = c.txt; r.sum.style.color = c.txt; E.setRich(r.sum, r.p[5]); E.setRich(r.desc, r.p[6]); }); } let prev = { ready: -1, starting: -1, cap: -1, conc: -1, sec: -1, dirty: true }; let dIF = 0, dProc = 0, dQueued = 0, dRoom = 0, dEta = 0; function smoothFrame() { const st = window._asState; if (!st) return; const sim = st.sim, cfg = st.cfg; const cap = sim.replicas * cfg.concurrency; const tP = Math.min(sim.inFlight, cap), tQ = Math.max(0, sim.inFlight - cap), tR = Math.max(0, cap - sim.inFlight); const k = 0.18; dIF += (sim.inFlight - dIF) * k; dProc += (tP - dProc) * k; dQueued += (tQ - dQueued) * k; dRoom += (tR - dRoom) * k; const tEta = sim.scaleDownStartedAt != null ? Math.max(0, cfg.delay - (sim.t - sim.scaleDownStartedAt)) : 0; dEta += (tEta - dEta) * k; segP.style.flex = Math.max(0.001, dProc); segQ.style.flex = Math.max(0.001, dQueued); segR.style.flex = Math.max(0.001, dRoom); legP.textContent = Math.round(dProc) + " processing"; legQ.textContent = Math.round(dQueued) + " queued"; legR.textContent = Math.round(dRoom) + " room"; if (sim.scaleDownStartedAt != null) cdFill.style.width = 100 * (1 - dEta / cfg.delay) + "%"; } function update() { const st = window._asState; if (!st) return; const sim = st.sim, cfg = st.cfg, c = E.P(); const cap = sim.replicas * cfg.concurrency; const sec = Math.floor(sim.t); if (sec !== prev.sec || prev.dirty) { lsSub.textContent = Math.round(dIF) + " in-flight · t = " + E.fmtTime(sim.t); prev.sec = sec; } if (sim.replicas !== prev.ready || sim.starting.length !== prev.starting || cap !== prev.cap || cfg.concurrency !== prev.conc || prev.dirty) { E.setRich(lsCap, sim.replicas + " ready · " + sim.starting.length + " starting · capacity " + cap + " = " + sim.replicas + " × `concurrency_target` (" + cfg.concurrency + ")"); lsCap.style.color = c.sub; prev.ready = sim.replicas; prev.starting = sim.starting.length; prev.cap = cap; prev.conc = cfg.concurrency; } prev.dirty = false; const m = E.computeMath(sim, cfg); mSub.textContent = "Next decision in " + E.fmt(m.nextDecision, 0) + "s"; E.setRich(mRows[0].a, "1. avg in-flight over `autoscaling_window`"); mRows[0].b.textContent = E.fmt(m.avg, 1); mRows[1].a.textContent = "2. effective per-replica capacity"; mRows[1].b.textContent = cfg.concurrency + " × " + E.fmt(cfg.utilization / 100, 2) + " = " + E.fmt(m.eff, 2); mRows[2].a.textContent = "3. desired = ⌈avg / effective⌉"; mRows[2].b.textContent = "⌈" + E.fmt(m.avg, 1) + " / " + E.fmt(m.eff, 2) + "⌉ = " + m.desired; mRows[3].a.textContent = "Current → desired"; const badge = m.desired > m.cur ? " +" + (m.desired - m.cur) + " starting" : m.desired === m.cur ? " stable" : " excess " + m.excess + ", halve to " + (m.cur - m.willHalve); mRows[3].b.textContent = m.cur + " → " + m.desired + badge; mRows[3].b.style.transition = "color 0.3s ease"; mRows[3].b.style.color = m.desired > m.cur ? c.p : m.desired < m.cur ? c.q : c.sub; mRows.forEach(r => { r.a.style.color = c.sub; r.r.style.borderBottom = r === mRows[3] ? "0" : "1px dashed " + c.brdM; }); mCount.style.opacity = m.sdEta != null ? "1" : "0"; if (m.sdEta != null) { E.setRich(cdL, "`scale_down_delay` countdown"); cdR.textContent = E.fmt(m.sdEta, 0) + "s remaining"; cdHelp.textContent = "Then removes ⌈excess / 2⌉ = " + m.willHalve + " replica" + (m.willHalve === 1 ? "" : "s") + " and waits " + cfg.delay + "s again."; cdHelp.style.color = c.sub; } ppSub.textContent = "Changes apply at the next decision (≤ " + cfg.window + "s)"; Object.values(pcRefs).forEach(r => { if (parseInt(r.inp.value, 10) !== cfg[r.p[0]]) r.inp.value = cfg[r.p[0]]; r.val.textContent = cfg[r.p[0]] + r.p[4]; }); } Object.values(pcRefs).forEach(r => { const cfg = window._asState.cfg; r.inp.value = cfg[r.p[0]]; r.val.textContent = cfg[r.p[0]] + r.p[4]; }); const initSim = window._asState.sim, initCfg = window._asState.cfg; const initCap = initSim.replicas * initCfg.concurrency; dIF = initSim.inFlight; dProc = Math.min(initSim.inFlight, initCap); dQueued = Math.max(0, initSim.inFlight - initCap); dRoom = Math.max(0, initCap - initSim.inFlight); let lastUpdate = 0; function loop(now) { raf = requestAnimationFrame(loop); if (!visible) return; smoothFrame(); if (now - lastUpdate < 250 && !prev.dirty) return; lastUpdate = now; update(); } applyTheme(); smoothFrame(); update(); raf = requestAnimationFrame(loop); return () => { cancelAnimationFrame(raf); obs.disconnect(); tObs.disconnect(); card.remove(); }; } tryMount(); return () => { destroyed = true; if (timer) clearTimeout(timer); if (cleanup) cleanup(); init.current = false; }; }, []); return
; }; export const AutoscalerSim = () => { const ref = React.useRef(null); const init = React.useRef(false); React.useEffect(() => { if (!ref.current || init.current) return; init.current = true; const root = ref.current; let cleanup = null, destroyed = false, timer = null, retries = 0; const tryMount = () => { if (destroyed || !ref.current) return; if (window._asEngine) { cleanup = mount(); } else if (retries++ < 60) { timer = setTimeout(tryMount, 30); } }; function mount() { const E = window._asEngine; const cfg = { concurrency: 4, utilization: 70, window: 60, delay: 120, minR: 0, maxR: 8, coldStart: 30, procTime: 1.0 }; let pattern = "steady"; let knobs = Object.fromEntries(E.PATTERNS.steady.knobs.map(k => [k[0], k[5]])); let sim = E.prefill(pattern, knobs, cfg); window._asState = { get sim() { return sim; }, get cfg() { return cfg; }, get pattern() { return pattern; }, get knobs() { return knobs; } }; const W = 1100, CH = 260; const card = document.createElement("div"); const tValue = document.createElement("span"); tValue.style.cssText = "display:block;text-align:right;font:500 12px ui-monospace,Menlo,monospace;color:#869089;font-variant-numeric:tabular-nums;margin:0 0 8px"; card.appendChild(tValue); const strip = document.createElement("div"); const stripL = document.createElement("div"); stripL.style.cssText = "display:flex;flex-direction:column;gap:6px;flex:1;min-width:280px"; const stripLabel = document.createElement("span"); stripLabel.style.cssText = "font:500 10px ui-monospace,Menlo,monospace;text-transform:lowercase;color:#869089"; stripLabel.textContent = "traffic shape"; const tabRow = document.createElement("div"); tabRow.style.cssText = "display:flex;gap:6px;flex-wrap:wrap"; const stripDesc = document.createElement("span"); stripDesc.style.cssText = "font:400 12px system-ui,-apple-system,sans-serif;color:#869089"; stripL.appendChild(stripLabel); stripL.appendChild(tabRow); stripL.appendChild(stripDesc); const knobsBox = document.createElement("div"); knobsBox.style.cssText = "display:flex;gap:14px;flex-wrap:wrap"; strip.appendChild(stripL); strip.appendChild(knobsBox); card.appendChild(strip); function rebuildTabs() { tabRow.replaceChildren(); Object.keys(E.PATTERNS).forEach(k => { const b = document.createElement("button"); b.style.cssText = "padding:5px 12px;border-radius:6px;cursor:pointer;font:500 12px system-ui,-apple-system,sans-serif"; b.textContent = E.PATTERNS[k].label; b.onclick = () => { pattern = k; knobs = Object.fromEntries(E.PATTERNS[k].knobs.map(x => [x[0], x[5]])); sim = E.prefill(pattern, knobs, cfg); rebuildTabs(); rebuildKnobs(); applyTheme(); }; tabRow.appendChild(b); }); } function rebuildKnobs() { knobsBox.replaceChildren(); const SLOTS = 3; for (let i = 0; i < SLOTS; i++) { const kd = E.PATTERNS[pattern].knobs[i]; const lbl = document.createElement("label"); lbl.style.cssText = "display:flex;flex-direction:column;gap:2px;width:140px;font:500 11px ui-monospace,Menlo,monospace;color:#869089"; if (!kd) { lbl.style.visibility = "hidden"; knobsBox.appendChild(lbl); continue; } const top = document.createElement("div"); top.style.cssText = "display:flex;justify-content:space-between"; const name = document.createElement("span"); name.textContent = kd[1]; const valS = document.createElement("span"); valS.style.cssText = "font-weight:500"; valS.textContent = knobs[kd[0]] + kd[2]; top.appendChild(name); top.appendChild(valS); const inp = document.createElement("input"); inp.type = "range"; inp.min = kd[3]; inp.max = kd[4]; inp.value = knobs[kd[0]]; inp.style.cssText = "width:100%;height:3px;accent-color:#0e863f"; inp.oninput = () => { const v = parseInt(inp.value, 10); knobs[kd[0]] = v; valS.textContent = v + kd[2]; }; lbl.appendChild(top); lbl.appendChild(inp); knobsBox.appendChild(lbl); } stripDesc.textContent = E.PATTERNS[pattern].desc; } const cv = document.createElement("canvas"); cv.style.cssText = "display:block;width:100%;max-width:" + W + "px;touch-action:pan-y"; card.appendChild(cv); const ctx = cv.getContext("2d"); const dpr = window.devicePixelRatio || 1; cv.width = W * dpr; cv.height = CH * dpr; cv.style.height = CH + "px"; ctx.scale(dpr, dpr); root.appendChild(card); let visible = true, raf = 0; const obs = new IntersectionObserver(e => visible = e[0].isIntersecting, { threshold: 0.05 }); obs.observe(cv); const tObs = new MutationObserver(() => applyTheme()); tObs.observe(document.documentElement, { attributes: true, attributeFilter: ["class"] }); function applyTheme() { const c = E.P(); card.style.cssText = "border:1px solid " + c.brd + ";border-radius:10px;padding:18px 20px;margin:14px 0;background:" + c.bg + ";max-width:" + W + "px"; strip.style.cssText = "display:flex;flex-wrap:wrap;align-items:center;gap:14px;padding:12px 14px;border:1px solid " + c.brdM + ";background:" + c.surf + ";border-radius:8px;margin:0 0 12px"; Array.from(tabRow.children).forEach((b, i) => { const k = Object.keys(E.PATTERNS)[i], active = k === pattern; b.style.background = active ? E.isDark() ? "#005934" : "#0e863f" : E.isDark() ? "#0C1D13" : "#fff"; b.style.color = active ? "#fff" : c.txt; b.style.border = "1px solid " + (active ? E.isDark() ? "#005934" : "#0e863f" : c.brd); }); } function drawChart() { const c = E.P(); const PAD = { l: 50, r: 50, t: 16, b: 26 }; const iW = W - PAD.l - PAD.r, iH = CH - PAD.t - PAD.b; const t1 = sim.t, t0 = Math.max(0, sim.t - E.HISTORY_S); const xS = t => PAD.l + (t - t0) / Math.max(1, t1 - t0) * iW; let maxLoad = 8; for (const h of sim.history) { const v = (h.inFlight || 0) + (h.queue || 0); if (v > maxLoad) maxLoad = v; if ((h.rps || 0) > maxLoad) maxLoad = h.rps; } if (!Number.isFinite(maxLoad) || maxLoad < 1) maxLoad = 8; const maxR = Math.max(cfg.maxR, 4); const yL = v => PAD.t + iH - v / maxLoad * iH; const yR = v => PAD.t + iH - v / maxR * iH; ctx.clearRect(0, 0, W, CH); const wx0 = xS(Math.max(t0, sim.t - cfg.window)), wx1 = xS(sim.t); ctx.fillStyle = c.qBg; ctx.fillRect(wx0, PAD.t, Math.max(0, wx1 - wx0), iH); ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.fillStyle = c.q; ctx.textAlign = "left"; ctx.textBaseline = "top"; ctx.fillText(cfg.window + "s window", wx0 + 4, PAD.t + 4); ctx.strokeStyle = c.grid; ctx.lineWidth = 0.6; ctx.setLineDash([2, 3]); for (let g = 0; g <= 4; g++) { const y = PAD.t + iH * (1 - g / 4); ctx.beginPath(); ctx.moveTo(PAD.l, y); ctx.lineTo(W - PAD.r, y); ctx.stroke(); ctx.fillStyle = c.sub; ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.textAlign = "end"; ctx.textBaseline = "middle"; ctx.fillText(Math.round(maxLoad * g / 4), PAD.l - 6, y); ctx.textAlign = "start"; ctx.fillText(Math.round(maxR * g / 4), W - PAD.r + 6, y); } ctx.setLineDash([]); ctx.fillStyle = c.sub; ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.textAlign = "end"; ctx.textBaseline = "alphabetic"; ctx.fillText("requests", PAD.l - 6, PAD.t - 4); ctx.textAlign = "start"; ctx.fillText("replicas", W - PAD.r + 6, PAD.t - 4); if (sim.history.length > 1) { ctx.fillStyle = c.pBg; ctx.beginPath(); ctx.moveTo(xS(sim.history[0].t), PAD.t + iH); let prevY = yR(sim.history[0].replicas); ctx.lineTo(xS(sim.history[0].t), prevY); for (let i = 1; i < sim.history.length; i++) { const x = xS(sim.history[i].t); ctx.lineTo(x, prevY); prevY = yR(sim.history[i].replicas); ctx.lineTo(x, prevY); } ctx.lineTo(xS(sim.history[sim.history.length - 1].t), PAD.t + iH); ctx.closePath(); ctx.fill(); ctx.strokeStyle = c.p; ctx.lineWidth = 2; ctx.beginPath(); ctx.moveTo(xS(sim.history[0].t), yR(sim.history[0].replicas)); let pY = yR(sim.history[0].replicas); for (let i = 1; i < sim.history.length; i++) { const x = xS(sim.history[i].t); ctx.lineTo(x, pY); pY = yR(sim.history[i].replicas); ctx.lineTo(x, pY); } ctx.stroke(); } if (sim.history.length > 0) { ctx.strokeStyle = c.q; ctx.lineWidth = 1.6; ctx.beginPath(); for (let i = 0; i < sim.history.length; i++) { const h = sim.history[i], x = xS(h.t), y = yL(h.inFlight); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); } ctx.stroke(); ctx.strokeStyle = c.sub; ctx.lineWidth = 1; ctx.setLineDash([3, 2]); ctx.globalAlpha = 0.7; ctx.beginPath(); for (let i = 0; i < sim.history.length; i++) { const h = sim.history[i], x = xS(h.t), y = yL(h.rps); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); } ctx.stroke(); ctx.setLineDash([]); ctx.globalAlpha = 1; } for (const e of sim.events) { if (e.t < t0 || e.type !== "scale-up" && e.type !== "scale-down") continue; const ex = xS(e.t); if (e.type === "scale-up") { ctx.fillStyle = c.p; ctx.beginPath(); ctx.moveTo(ex, PAD.t + iH + 3); ctx.lineTo(ex - 3, PAD.t + iH + 9); ctx.lineTo(ex + 3, PAD.t + iH + 9); ctx.closePath(); ctx.fill(); } else { ctx.fillStyle = c.sub; ctx.beginPath(); ctx.moveTo(ex - 3, PAD.t + iH + 3); ctx.lineTo(ex + 3, PAD.t + iH + 3); ctx.lineTo(ex, PAD.t + iH + 9); ctx.closePath(); ctx.fill(); } } ctx.fillStyle = c.sub; ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.textAlign = "center"; ctx.textBaseline = "alphabetic"; for (let g = 0; g <= 2; g++) { const x = PAD.l + g / 2 * iW, tval = t0 + g / 2 * (t1 - t0); ctx.fillText(E.fmtTime(tval), x, CH - 8); } } let last = performance.now(); function loop(now) { raf = requestAnimationFrame(loop); if (!visible) { last = now; return; } const dt = Math.min(200, now - last) / 1000; last = now; const k = E.safeKnobs(pattern, knobs); const rps = E.PATTERNS[pattern].fn(sim.t, k); E.tickSim(sim, dt * E.SIM_SPEED, cfg, Number.isFinite(rps) ? rps : 0); tValue.textContent = "t = " + E.fmtTime(sim.t) + " · " + E.SIM_SPEED + "× wall time"; drawChart(); } rebuildTabs(); rebuildKnobs(); applyTheme(); drawChart(); raf = requestAnimationFrame(loop); return () => { cancelAnimationFrame(raf); obs.disconnect(); tObs.disconnect(); card.remove(); delete window._asState; }; } tryMount(); return () => { destroyed = true; if (timer) clearTimeout(timer); if (cleanup) cleanup(); init.current = false; }; }, []); return
; }; export const AutoscalerSimEngine = () => { React.useEffect(() => { if (window._asEngine) return; const SIM_SPEED = 30, HISTORY_S = 300; const PATTERNS = { steady: { label: "Steady", desc: "Constant rate with mild jitter", knobs: [["rate", "rate", " rps", 1, 30, 8], ["noise", "noise", "%", 0, 50, 10]], fn: (t, k) => { const n = k.noise / 100 * Math.sin(t * 0.7) + k.noise / 100 * 0.5 * Math.sin(t * 1.9); return Math.max(0, k.rate * (1 + n)); } }, bursty: { label: "Bursty", desc: "Baseline with sharp peaks", knobs: [["baseline", "baseline", " rps", 0, 20, 3], ["peak", "peak", " rps", 5, 60, 25], ["period", "every", "s", 30, 300, 90]], fn: (t, k) => { const ph = t % k.period / k.period; const burst = Math.exp(-Math.pow((ph - 0.3) / 0.08, 2)); return k.baseline + (k.peak - k.baseline) * burst; } }, scheduled: { label: "Scheduled", desc: "Business-hours wave", knobs: [["low", "off-peak", " rps", 0, 10, 1], ["high", "peak", " rps", 5, 50, 18], ["cycle", "cycle", "s", 60, 600, 240]], fn: (t, k) => { const ph = t % k.cycle / k.cycle; const wave = 0.5 - 0.5 * Math.cos(ph * Math.PI * 2); return k.low + (k.high - k.low) * wave; } }, jittery: { label: "Jittery", desc: "Unpredictable, large variance", knobs: [["mean", "mean", " rps", 1, 30, 10], ["variance", "variance", "%", 20, 200, 100]], fn: (t, k) => { const n = Math.sin(t * 0.3) * 0.4 + Math.sin(t * 1.7 + 1.3) * 0.3 + Math.sin(t * 4.1 + 0.7) * 0.2 + Math.sin(t * 0.11) * 0.5; return Math.max(0, k.mean * (1 + k.variance / 100 * n)); } } }; function makeSim() { return { t: 0, inFlight: 0, replicas: 0, starting: [], target: 0, lastDecisionAt: -1e9, scaleDownStartedAt: null, arrAcc: 0, comAcc: 0, history: [], rpsAcc: { sum: 0 }, lastSec: 0, events: [] }; } function safeKnobs(pattern, knobs) { const out = {}; for (const k of PATTERNS[pattern].knobs) { const v = knobs?.[k[0]]; out[k[0]] = Number.isFinite(v) ? v : k[5]; } return out; } function fmt(v, d) { return (Number.isFinite(v) ? v : 0).toFixed(d == null ? 1 : d); } function fmtTime(s) { const m = Math.floor(s / 60), sec = Math.floor(s % 60); return m + ":" + String(sec).padStart(2, "0"); } function tickSim(sim, dt, cfg, rps) { sim.t += dt; sim.arrAcc += rps * dt; while (sim.arrAcc >= 1) { sim.inFlight++; sim.arrAcc -= 1; } const cap = sim.replicas * cfg.concurrency; const proc = Math.min(sim.inFlight, cap); sim.comAcc += proc / cfg.procTime * dt; while (sim.comAcc >= 1) { if (sim.inFlight > 0) sim.inFlight--; sim.comAcc -= 1; } sim.starting = sim.starting.filter(s => { if (sim.t >= s.readyAt) { sim.replicas++; sim.events.push({ t: sim.t, type: "ready" }); return false; } return true; }); sim.rpsAcc.sum += rps * dt; if (sim.t - sim.lastSec >= 1) { const queue = Math.max(0, sim.inFlight - cap); sim.history.push({ t: sim.t, rps: sim.rpsAcc.sum / Math.max(0.001, sim.t - sim.lastSec), inFlight: sim.inFlight, replicas: sim.replicas + sim.starting.length, ready: sim.replicas, queue: queue }); while (sim.history.length > 0 && sim.history[0].t < sim.t - HISTORY_S) sim.history.shift(); sim.rpsAcc = { sum: 0 }; sim.lastSec = sim.t; } if (sim.t - sim.lastDecisionAt >= cfg.window) { sim.lastDecisionAt = sim.t; const winStart = sim.t - cfg.window; const winS = sim.history.filter(h => h.t >= winStart); const avg = winS.length > 0 ? winS.reduce((s, h) => s + h.inFlight, 0) / winS.length : sim.inFlight; const eff = cfg.concurrency * (cfg.utilization / 100); let desired = Math.max(cfg.minR, Math.ceil(avg / Math.max(0.01, eff))); desired = Math.min(desired, cfg.maxR); if (avg > 0 && desired < 1) desired = Math.min(1, cfg.maxR); const cur = sim.replicas + sim.starting.length; sim.target = desired; if (desired > cur) { for (let i = 0; i < desired - cur; i++) sim.starting.push({ readyAt: sim.t + cfg.coldStart }); sim.events.push({ t: sim.t, type: "scale-up" }); sim.scaleDownStartedAt = null; } else if (desired < cur) { if (sim.scaleDownStartedAt == null) sim.scaleDownStartedAt = sim.t; } else { sim.scaleDownStartedAt = null; } } if (sim.scaleDownStartedAt != null && sim.t - sim.scaleDownStartedAt >= cfg.delay) { const cur = sim.replicas + sim.starting.length; const excess = cur - sim.target; if (excess > 0) { const toRemove = Math.ceil(excess / 2); let removed = 0; while (removed < toRemove && sim.starting.length > 0) { sim.starting.pop(); removed++; } while (removed < toRemove && sim.replicas > sim.target) { sim.replicas--; removed++; } sim.events.push({ t: sim.t, type: "scale-down" }); sim.scaleDownStartedAt = sim.replicas + sim.starting.length > sim.target ? sim.t : null; } else { sim.scaleDownStartedAt = null; } } if (sim.events.length > 80) sim.events.splice(0, sim.events.length - 80); } function prefill(pattern, knobs, cfg) { const sim = makeSim(); const k = safeKnobs(pattern, knobs); const fn = PATTERNS[pattern].fn; let rpsSum = 0; for (let i = 0; i < 60; i++) rpsSum += Math.max(0, fn(i / 60 * 600, k)); const avgRps = rpsSum / 60; const avgInF = avgRps * cfg.procTime; const eff = Math.max(0.01, cfg.concurrency * (cfg.utilization / 100)); const seed = Math.min(cfg.maxR, Math.max(cfg.minR, Math.ceil(avgInF / eff))); sim.replicas = seed; sim.target = seed; const dt = 0.5; const steps = Math.floor(HISTORY_S / dt); for (let i = 0; i < steps; i++) { const r = fn(sim.t, k); tickSim(sim, dt, cfg, Number.isFinite(r) ? r : 0); } sim.events = []; sim.lastDecisionAt = sim.t; return sim; } function computeMath(sim, cfg) { const winStart = sim.t - cfg.window; const winS = sim.history.filter(h => h.t >= winStart); const avg = winS.length > 0 ? winS.reduce((s, h) => s + h.inFlight, 0) / winS.length : sim.inFlight; const eff = cfg.concurrency * (cfg.utilization / 100); const desired = Math.min(cfg.maxR, Math.max(cfg.minR, Math.ceil(avg / Math.max(0.01, eff)))); const cur = sim.replicas + sim.starting.length; const excess = Math.max(0, cur - desired); const willHalve = Math.ceil(excess / 2); const nextDecision = Math.max(0, cfg.window - (sim.t - sim.lastDecisionAt)); const sdEta = sim.scaleDownStartedAt != null ? Math.max(0, cfg.delay - (sim.t - sim.scaleDownStartedAt)) : null; return { avg, eff, desired, cur, excess, willHalve, nextDecision, sdEta }; } const isDark = () => document.documentElement.classList.contains("dark"); function P() { const d = isDark(); return { bg: d ? "#021309" : "#fff", surf: d ? "#0C1D13" : "#f4f9f3", sub: "#869089", brd: d ? "#344339" : "#dee4de", brdM: d ? "#203026" : "#f4f9f3", txt: d ? "#dee4de" : "#0c1d13", q: d ? "#4a90ff" : "#2176ff", qBg: d ? "rgba(74,144,255,0.16)" : "rgba(199,220,255,0.6)", w: d ? "#f7c42f" : "#9c7400", p: d ? "#19E76E" : "#0e863f", pBg: d ? "rgba(25,231,110,0.18)" : "rgba(178,247,207,0.45)", rb: "#005934", rbf: d ? "rgba(25,231,110,0.22)" : "rgba(178,247,207,0.55)", rsf: d ? "rgba(247,196,47,0.18)" : "rgba(253,237,188,0.55)", grid: d ? "#203026" : "#dee4de" }; } function setRich(el, s) { el.replaceChildren(); const parts = s.split("`"); for (let i = 0; i < parts.length; i++) { if (i % 2 === 0) { if (parts[i]) el.appendChild(document.createTextNode(parts[i])); } else { const c = document.createElement("code"); c.textContent = parts[i]; c.style.cssText = "font-family:ui-monospace,Menlo,monospace;font-size:0.92em;background:" + (isDark() ? "rgba(255,255,255,0.08)" : "rgba(0,0,0,0.05)") + ";padding:1px 4px;border-radius:3px"; el.appendChild(c); } } } const PARAMS = [["concurrency", "concurrency_target", 1, 32, "", "How many requests each replica can handle simultaneously. This directly determines replica count for a given load.", "Controls requests sent to a replica, not requests processed inside it. `predict_concurrency` in `config.yaml` controls the inside-the-container limit."], ["utilization", "target_utilization_percentage", 1, 100, "%", "Headroom before scaling triggers. The autoscaler scales when utilization reaches this percentage of `concurrency_target`, not when replicas are fully loaded.", "Lower values (50 to 60%) absorb spikes but cost more. Higher values (80%+) are cost-efficient for steady traffic but absorb spikes less effectively."], ["window", "autoscaling_window", 10, 300, "s", "How far back (in seconds) the autoscaler looks when measuring traffic. Traffic is averaged over this window to make scaling decisions.", "Shorter windows (30 to 60s) react quickly, which suits bursty workloads. Longer windows (2 to 5 minutes) ignore short-lived fluctuations."], ["delay", "scale_down_delay", 0, 600, "s", "How long (in seconds) the autoscaler waits after load drops before removing replicas.", "When the timer elapses, the autoscaler removes replicas using exponential back-off: ⌈excess / 2⌉, wait, halve again. This is the primary lever against oscillation."], ["minR", "min_replica", 0, 8, "", "The floor for your deployment's capacity. The autoscaler will not scale below this number.", "The default of 0 enables scale-to-zero. For production, set this to 2 or more to eliminate cold starts and add redundancy."], ["maxR", "max_replica", 1, 16, "", "The ceiling for your deployment's capacity. The autoscaler will not scale above this number.", "Protects against runaway scaling. If traffic exceeds `max_replica` × `concurrency_target`, requests queue rather than triggering new replicas."]]; window._asEngine = { SIM_SPEED, HISTORY_S, PATTERNS, PARAMS, makeSim, safeKnobs, fmt, fmtTime, tickSim, prefill, computeMath, isDark, P, setRich }; }, []); return ; }; export const MiniScaleDown = () => { const ref = React.useRef(null); const init = React.useRef(false); React.useEffect(() => { if (!ref.current || init.current) return; init.current = true; const PER = 7000; function traffic(p) { if (p < 0.20) return 0.7 + Math.sin(p * 40) * 0.05; if (p < 0.30) return 0.7 * (1 - (p - 0.20) / 0.10); return 0; } const opts = { title: "`scale_down_delay`", desc: "Replicas must stay idle this long before they are removed. Smooths oscillation.", formula: "idle > `scale_down_delay` → terminate", W: 580, H: 180, draw: function (ctx, t, p) { const h = window._mdHelpers; const ph = t % PER / PER; const x0 = 40, x1 = 540; const cx = h.lerp(x0, x1, ph); const idleStart = 0.30, idleEnd = 0.78; const idleProg = ph < idleStart ? 0 : ph > idleEnd ? 1 : (ph - idleStart) / (idleEnd - idleStart); const repAlive = ph < idleEnd; const repStop = ph >= idleEnd && ph < idleEnd + 0.06; ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.sub; ctx.textAlign = "left"; ctx.textBaseline = "middle"; ctx.fillText("in_flight", 20, 22); ctx.strokeStyle = p.brdM; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(x0, 54); ctx.lineTo(x1, 54); ctx.stroke(); ctx.fillStyle = p.qFill; ctx.beginPath(); ctx.moveTo(x0, 54); for (let q = 0; q <= 1; q += 0.01) ctx.lineTo(h.lerp(x0, x1, q), h.lerp(54, 14, traffic(q))); ctx.lineTo(x1, 54); ctx.closePath(); ctx.fill(); ctx.strokeStyle = p.q; ctx.lineWidth = 1.5; ctx.beginPath(); for (let q = 0; q <= 1; q += 0.01) { const px = h.lerp(x0, x1, q), py = h.lerp(54, 14, traffic(q)); if (q === 0) ctx.moveTo(px, py); else ctx.lineTo(px, py); } ctx.stroke(); ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.sub; ctx.fillText("idle timer", 20, 84); ctx.strokeStyle = p.brd; ctx.lineWidth = 0.8; ctx.beginPath(); ctx.roundRect(x0, 88, x1 - x0, 8, 2); ctx.stroke(); if (idleProg > 0) { ctx.fillStyle = idleProg >= 1 ? p.qDark : p.q; ctx.beginPath(); ctx.roundRect(x0, 88, (x1 - x0) * idleProg, 8, 2); ctx.fill(); } ctx.setLineDash([2, 2]); ctx.strokeStyle = p.qDark; ctx.beginPath(); ctx.moveTo(h.lerp(x0, x1, idleEnd), 84); ctx.lineTo(h.lerp(x0, x1, idleEnd), 100); ctx.stroke(); ctx.setLineDash([]); ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.sub; ctx.fillText("replica", 20, 122); if (repAlive) { ctx.globalAlpha = repStop ? 1 - h.fade(ph, idleEnd, idleEnd + 0.06) : 1; h.repBox(ctx, x0, 106, 130, 24, ph < 0.30 ? "busy" : "ready", "Replica", 0, p, "ACTIVE"); ctx.globalAlpha = 1; } if (ph >= idleEnd && ph < idleEnd + 0.10) { ctx.globalAlpha = h.fade(ph, idleEnd, idleEnd + 0.04); ctx.font = "500 10px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.sub; ctx.textAlign = "left"; ctx.textBaseline = "middle"; ctx.fillText("Deployment is now SCALED_TO_ZERO", x0 + 150, 118); ctx.globalAlpha = 1; } ctx.strokeStyle = p.txt; ctx.globalAlpha = 0.25; ctx.lineWidth = 0.5; ctx.beginPath(); ctx.moveTo(cx, 10); ctx.lineTo(cx, 140); ctx.stroke(); ctx.globalAlpha = 1; ctx.font = "500 10px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.sub; ctx.textAlign = "left"; ctx.textBaseline = "middle"; let cap = ""; if (ph < 0.30) cap = "Serving traffic, deployment is ACTIVE"; else if (ph < idleEnd) { const rem = Math.round((idleEnd - ph) / (idleEnd - idleStart) * 20) * 5; cap = "Idle, " + rem + "% of scale_down_delay remains, still ACTIVE"; } else cap = "Replica reclaimed, deployment is now SCALED_TO_ZERO"; ctx.fillText(cap, x0, 168); } }; let cleanup = null, destroyed = false, timer = null, retries = 0; const tryMount = () => { if (destroyed || !ref.current) return; if (window._mdMount) { cleanup = window._mdMount(ref.current, opts); } else if (retries++ < 60) { timer = setTimeout(tryMount, 30); } }; tryMount(); return () => { destroyed = true; if (timer) clearTimeout(timer); if (cleanup) cleanup(); init.current = false; }; }, []); return
; }; export const MiniConcurrency = () => { const ref = React.useRef(null); const init = React.useRef(false); React.useEffect(() => { if (!ref.current || init.current) return; init.current = true; const PER = 7000, N = 8, UTIL = 0.5; const THRESHOLD = N * UTIL; const STAGES = [["Capacity available", 0.08, 0.08], ["At threshold", 0.36, 0.32], ["Scaling up", 0.55, 0.50], ["Two replicas", 0.85, 0.80]]; let frozen = null; const opts = { title: "`concurrency_target`", desc: "Click any stage below to freeze on it. Click again to resume the loop.", formula: "load > replicas × `concurrency_target` × `target_utilization` → scale up", W: 580, H: 224, onClick: function (x, y) { if (y > 6 && y < 30) { let nearest = null, nearestDist = Infinity; STAGES.forEach(s => { const sx = 40 + s[1] * 500; const d = Math.abs(x - sx); if (d < nearestDist) { nearestDist = d; nearest = s[0]; } }); if (nearestDist < 75) frozen = frozen === nearest ? null : nearest; } else { frozen = null; } }, draw: function (ctx, t, p) { const h = window._mdHelpers; const ph = frozen != null ? (STAGES.find(s => s[0] === frozen) || [null, 0, 0])[2] : t % PER / PER; const yOff = 28; STAGES.forEach(s => { const sx = 40 + s[1] * 500; let cur = false; if (s[0] === "Capacity available") cur = ph < 0.32; else if (s[0] === "At threshold") cur = ph >= 0.32 && ph < 0.42; else if (s[0] === "Scaling up") cur = ph >= 0.42 && ph < 0.65; else cur = ph >= 0.65; ctx.font = (cur ? "600 " : "500 ") + "9.5px ui-monospace,Menlo,monospace"; ctx.fillStyle = cur ? p.txt : p.sub; ctx.textAlign = "center"; ctx.textBaseline = "middle"; ctx.fillText(s[0], sx, 18); }); const slotY = 72 + yOff, slotH = 16, slotW = 12, slotGap = 3; const slotXFor = i => 244 + i * (slotW + slotGap); const slotFill = i => { if (i >= THRESHOLD) return 0; const s = 0.06 + i * 0.07, e = s + 0.07; return ph < s ? 0 : ph > e ? 1 : (ph - s) / (e - s); }; const overflowVis = ph > 0.32 && ph < 0.55; const overflowX = overflowVis ? h.lerp(40, 200, (ph - 0.32) / 0.10) : 40; const r2Vis = ph > 0.42; const r2Cold = ph > 0.42 && ph < 0.62; const r2Prog = h.fade(ph, 0.42, 0.62); const r2Busy = ph >= 0.65 && ph < 0.95; h.repBox(ctx, 200, 28 + yOff, 140, 32, ph < 0.10 ? "ready" : "busy", "Replica 1", 0, p, "Active"); for (let i = 0; i < N; i++) { const f = slotFill(i); ctx.strokeStyle = p.brd; ctx.globalAlpha = 0.7; ctx.lineWidth = 0.8; ctx.beginPath(); ctx.roundRect(slotXFor(i), slotY, slotW, slotH, 2); ctx.stroke(); ctx.globalAlpha = 1; ctx.fillStyle = p.p; ctx.globalAlpha = 0.8; ctx.fillRect(slotXFor(i), slotY + slotH * (1 - f), slotW, slotH * f); ctx.globalAlpha = 1; } const thrX = slotXFor(THRESHOLD) - slotGap / 2; ctx.strokeStyle = p.q; ctx.globalAlpha = 0.85; ctx.lineWidth = 1.3; ctx.setLineDash([3, 3]); ctx.beginPath(); ctx.moveTo(thrX, slotY - 4); ctx.lineTo(thrX, slotY + slotH + 4); ctx.stroke(); ctx.setLineDash([]); ctx.globalAlpha = 1; ctx.font = "500 8.5px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.q; ctx.textAlign = "center"; ctx.textBaseline = "bottom"; ctx.fillText("50% threshold", thrX, slotY - 6); ctx.font = "500 9.5px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.sub; ctx.textAlign = "left"; ctx.textBaseline = "middle"; ctx.fillText("concurrency_target = " + N, slotXFor(N) + 8, slotY + slotH / 2); if (r2Vis) { ctx.globalAlpha = h.fade(ph, 0.42, 0.46); h.repBox(ctx, 200, 122 + yOff, 140, 32, r2Cold ? "starting" : r2Busy ? "busy" : "ready", "Replica 2", r2Cold ? r2Prog : 0, p, r2Cold ? "Waking up" : "Active"); ctx.globalAlpha = 1; } if (overflowVis) { ctx.globalAlpha = h.fade(ph, 0.32, 0.36) * (1 - h.fade(ph, 0.50, 0.55)); ctx.fillStyle = p.q; ctx.beginPath(); ctx.arc(overflowX, 44 + yOff, 4.5, 0, Math.PI * 2); ctx.fill(); ctx.globalAlpha = 1; } if (ph > 0.36 && ph < 0.50) { ctx.globalAlpha = h.fade(ph, 0.36, 0.40) * (1 - h.fade(ph, 0.46, 0.50)); ctx.strokeStyle = p.p; ctx.lineWidth = 1.5; ctx.beginPath(); ctx.moveTo(200, 60 + yOff); ctx.quadraticCurveTo(180, 100 + yOff, 200, 130 + yOff); ctx.stroke(); const tx = 200 - 180, ty = 130 + yOff - (100 + yOff), tl = Math.hypot(tx, ty); const ux = tx / tl, uy = ty / tl, px = -uy, py = ux; const tipX = 200, tipY = 130 + yOff, headLen = 9, headW = 4; const baseX = tipX - ux * headLen, baseY = tipY - uy * headLen; ctx.fillStyle = p.p; ctx.beginPath(); ctx.moveTo(tipX, tipY); ctx.lineTo(baseX + px * headW, baseY + py * headW); ctx.lineTo(baseX - px * headW, baseY - py * headW); ctx.closePath(); ctx.fill(); ctx.font = "500 10px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.p; ctx.textAlign = "left"; ctx.fillText("scale up", 134, 100 + yOff); ctx.globalAlpha = 1; } for (let i = 0; i < 4; i++) { const off = i * 0.07; const local = ((ph - off) % 1 + 1) % 1; if (local > 0.3) continue; if (ph > 0.10 + i * 0.07) continue; const x = h.lerp(40, 220, local / 0.3); ctx.fillStyle = p.q; ctx.beginPath(); ctx.arc(x, 44 + yOff, 4.5, 0, Math.PI * 2); ctx.fill(); } ctx.font = "500 10px ui-monospace,Menlo,monospace"; ctx.fillStyle = p.sub; ctx.textAlign = "left"; ctx.textBaseline = "middle"; const reps = r2Vis ? 2 : 1; ctx.fillText("threshold " + reps * THRESHOLD + " (" + reps + " × concurrency_target " + N + " × target_utilization " + UTIL + ")", 20, 216); } }; let cleanup = null, destroyed = false, timer = null, retries = 0; const tryMount = () => { if (destroyed || !ref.current) return; if (window._mdMount) { cleanup = window._mdMount(ref.current, opts); } else if (retries++ < 60) { timer = setTimeout(tryMount, 30); } }; tryMount(); return () => { destroyed = true; if (timer) clearTimeout(timer); if (cleanup) cleanup(); init.current = false; }; }, []); return
; }; export const MiniDiagramEngine = () => { React.useEffect(() => { if (window._mdMount) return; const isDark = () => document.documentElement.classList.contains("dark"); const lerp = (a, b, t) => a + (b - a) * Math.min(1, Math.max(0, t)); const fade = (p, a, b) => Math.min(1, Math.max(0, (p - a) / (b - a))); function P() { const d = isDark(); return { bg: d ? "#021309" : "#fff", sub: "#869089", brd: d ? "#344339" : "#dee4de", brdM: d ? "#203026" : "#f4f9f3", q: d ? "#4a90ff" : "#2176ff", qFill: d ? "rgba(74,144,255,0.18)" : "rgba(199,220,255,0.7)", qDark: "#114aa6", w: d ? "#f7c42f" : "#9c7400", p: d ? "#19E76E" : "#0e863f", rb: "#005934", rbf: d ? "rgba(25,231,110,0.22)" : "rgba(178,247,207,0.55)", rsf: d ? "rgba(247,196,47,0.18)" : "rgba(253,237,188,0.55)", txt: d ? "#dee4de" : "#0c1d13" }; } function repBox(ctx, x, y, w, h, state, label, prog, p, displayState) { let fl = p.bg, st = p.p, tc = p.p, ds = []; if (state === "stopped") { fl = "transparent"; st = p.brd; tc = p.sub; ds = [3, 3]; } else if (state === "starting") { fl = p.rsf; st = p.w; tc = p.w; } else if (state === "busy") { fl = p.rbf; st = p.rb; tc = p.rb; } else if (state === "stopping") { st = p.sub; tc = p.sub; } ctx.globalAlpha = state === "stopped" || state === "stopping" ? 0.55 : 1; ctx.setLineDash(ds); ctx.beginPath(); ctx.roundRect(x, y, w, h, 6); ctx.fillStyle = fl; ctx.fill(); ctx.strokeStyle = st; ctx.lineWidth = 1.3; ctx.stroke(); ctx.setLineDash([]); ctx.globalAlpha = 1; ctx.font = "500 11px ui-monospace,Menlo,monospace"; ctx.fillStyle = tc; ctx.textAlign = "left"; ctx.textBaseline = "middle"; ctx.fillText(label, x + 10, y + h / 2); ctx.font = "500 9.5px ui-monospace,Menlo,monospace"; ctx.textAlign = "right"; ctx.fillText(displayState || state, x + w - 10, y + h / 2); if (state === "starting" && prog > 0) { ctx.fillStyle = p.w; ctx.fillRect(x, y + h - 2, w * prog, 2); } } function setRich(el, s) { el.replaceChildren(); const parts = s.split("`"); for (let i = 0; i < parts.length; i++) { if (i % 2 === 0) { if (parts[i]) el.appendChild(document.createTextNode(parts[i])); } else { const code = document.createElement("code"); code.textContent = parts[i]; code.style.cssText = "font-family:ui-monospace,Menlo,monospace;font-size:0.92em;background:" + (isDark() ? "rgba(255,255,255,0.08)" : "rgba(0,0,0,0.05)") + ";padding:1px 4px;border-radius:3px"; el.appendChild(code); } } } window._mdHelpers = { lerp, fade, repBox }; window._mdMount = function (root, opts) { const W = opts.W || 580, H = opts.H || 200; const card = document.createElement("div"); const tit = document.createElement("div"); tit.style.cssText = "font:500 12px ui-monospace,Menlo,monospace;letter-spacing:-0.28px;margin:0 0 4px"; const desc = document.createElement("p"); desc.style.cssText = "margin:0 0 10px;font-size:12px;line-height:1.45;font-family:system-ui,-apple-system,sans-serif"; const cv = document.createElement("canvas"); cv.style.cssText = "display:block;width:100%;max-width:" + W + "px"; const formula = document.createElement("div"); formula.style.cssText = "margin:8px 0 0;font:500 11px ui-monospace,Menlo,monospace;letter-spacing:-0.28px;border-radius:4px;padding:4px 8px;display:inline-block"; setRich(tit, opts.title); setRich(desc, opts.desc); if (opts.formula) setRich(formula, opts.formula); card.appendChild(tit); card.appendChild(desc); card.appendChild(cv); if (opts.formula) card.appendChild(formula); root.appendChild(card); const ctx = cv.getContext("2d"); const dpr = window.devicePixelRatio || 1; cv.width = W * dpr; cv.height = H * dpr; cv.style.height = H + "px"; ctx.scale(dpr, dpr); if (opts.onClick) { cv.style.cursor = "pointer"; cv.addEventListener("click", e => { const r = cv.getBoundingClientRect(); opts.onClick((e.clientX - r.left) / r.width * W, (e.clientY - r.top) / r.height * H); }); } function applyTheme() { const d = isDark(); card.style.cssText = "border:1px solid " + (d ? "#344339" : "#f4f9f3") + ";border-radius:8px;padding:16px 18px;margin:12px 0;background:" + (d ? "#021309" : "#fff") + ";max-width:" + W + "px"; tit.style.color = d ? "#dee4de" : "#0c1d13"; desc.style.color = d ? "#9CA59E" : "#5a675e"; formula.style.background = d ? "#0C1D13" : "#f4f9f3"; formula.style.borderColor = d ? "#203026" : "#dee4de"; formula.style.border = "1px solid " + (d ? "#203026" : "#dee4de"); formula.style.color = d ? "#19E76E" : "#0e863f"; if (opts.title) setRich(tit, opts.title); if (opts.desc) setRich(desc, opts.desc); if (opts.formula) setRich(formula, opts.formula); } applyTheme(); let visible = true, raf = 0, t0 = performance.now(), dirty = true; const obs = new IntersectionObserver(e => visible = e[0].isIntersecting, { threshold: 0.15 }); obs.observe(cv); const themeObs = new MutationObserver(() => { dirty = true; applyTheme(); }); themeObs.observe(document.documentElement, { attributes: true, attributeFilter: ["class"] }); function loop(ts) { raf = requestAnimationFrame(loop); if (!visible) { dirty = true; return; } const t = ts - t0; ctx.clearRect(0, 0, W, H); opts.draw(ctx, t, P()); dirty = false; } raf = requestAnimationFrame(loop); return () => { cancelAnimationFrame(raf); obs.disconnect(); themeObs.disconnect(); card.remove(); }; }; }, []); return ; }; Without autoscaling, you'd choose between two bad options: pay for enough GPUs to handle your peak traffic 24/7, or accept that requests fail when load exceeds your fixed capacity. Autoscaling eliminates this tradeoff by adjusting the number of **replicas** backing a deployment based on demand. When traffic rises, the autoscaler adds replicas. When it falls, it removes them. The goal is to match capacity to load so you pay for what you use without sacrificing latency. Baseten [bills per minute](/observability/usage) while a replica is deploying, scaling, or serving requests. A deployment scaled to zero replicas incurs no charges, but the wake-up period when a new request arrives is billable. For details on minimizing that startup cost, see [Cold starts](/deployment/autoscaling/cold-starts). Baseten provides default settings that work for most workloads. Tune your autoscaling settings based on your model and traffic. | Parameter | Default | Range | What it controls | | ------------------ | ------- | -------- | ---------------------------------------- | | Min replicas | 0 | ≥ 0 | Baseline capacity (0 = scale to zero). | | Max replicas | 1 | ≥ 1 | Cost/capacity ceiling. | | Autoscaling window | 60s | 10–3600s | Time window for traffic analysis. | | Scale-down delay | 900s | 0–3600s | Wait time before removing idle replicas. | | Concurrency target | 1 | ≥ 1 | Requests per replica before scaling. | | Target utilization | 70% | 1–100% | Headroom before scaling triggers. | You can configure autoscaling settings through the Baseten UI or API. 1. Select your deployment. 2. Under **Replicas** for your production environment, choose **Configure**. 3. Configure the autoscaling settings and choose **Update**. UI view to configure autoscaling ```bash theme={"system"} curl -X PATCH \ https://api.baseten.co/v1/models/{model_id}/deployments/{deployment_id}/autoscaling_settings \ -H "Authorization: Api-Key $BASETEN_API_KEY" \ -d '{ "min_replica": 2, "max_replica": 10, "concurrency_target": 32, "target_utilization_percentage": 70, "autoscaling_window": 60, "scale_down_delay": 900 }' ``` For more information, see the [API reference](/reference/management-api/deployments/autoscaling/updates-a-deployments-autoscaling-settings). ```python theme={"system"} import requests import os API_KEY = os.environ.get("BASETEN_API_KEY") response = requests.patch( "https://api.baseten.co/v1/models/{model_id}/deployments/{deployment_id}/autoscaling_settings", headers={"Authorization": f"Api-Key {API_KEY}"}, json={ "min_replica": 2, "max_replica": 10, "concurrency_target": 32, "target_utilization_percentage": 70, "autoscaling_window": 60, "scale_down_delay": 900 } ) print(response.json()) ``` For more information, see the [API reference](/reference/management-api/deployments/autoscaling/updates-a-deployments-autoscaling-settings). *** ## How autoscaling works The autoscaler matches replica count to demand by sampling in-flight requests once every `autoscaling_window` (60 seconds by default). It averages the load over that window, divides by each replica's effective capacity (`concurrency_target` × `target_utilization_percentage`), and rounds up to set the desired replica count. Scaling up happens at the next decision, but scaling down is deliberately patient: load has to stay below the threshold for an entire `scale_down_delay` before the autoscaler halves the excess, and it keeps halving on each subsequent delay rather than dropping replicas all at once. That asymmetry is what keeps the deployment from oscillating when traffic dips and recovers. Shape a traffic pattern in the simulator below to watch each of those decisions land on the chart. To put numbers on it, consider a deployment with `concurrency_target` set to 10 and `target_utilization_percentage` at 70%. Each replica's effective capacity is 7 concurrent requests (10 × 0.70), which is the threshold the autoscaler compares the windowed average against. If traffic jumps from 5 to 25 in-flight requests over an `autoscaling_window`, the autoscaler computes ⌈25 / 7⌉ = 4 replicas at the next decision boundary and begins provisioning them; on the chart above, each scale-up shows up as a triangle on the time axis and a step in the green replica line. Replicas keep coming online until the deployment hits `max_replica`, after which any additional load queues until capacity frees up. Scaling down works in the opposite direction at a deliberately slower pace. When the windowed load drops below the threshold, the autoscaler waits an entire `scale_down_delay` (900 seconds by default in production) before acting, and then removes only ⌈excess / 2⌉ replicas rather than the whole surplus. The timer resets after that halving, so a deployment with eight excess replicas drains to four, then two, then one, with a full delay between every step. That exponential back-off is what prevents oscillation: if traffic briefly dips and recovers within the delay, the replicas stay warm and no scale event fires. Scale-down also clamps at `min_replica`, which most production deployments hold at two or more so a healthy replica is always available. *** ## Replicas Each replica is an independent instance of your model, running on its own hardware and capable of serving requests in parallel with other replicas. The autoscaler controls how many replicas are active at any given time, but you set the boundaries. The floor for your deployment's capacity. The autoscaler won't scale below this number. **Range:** ≥ 0 The default of 0 enables *scale-to-zero*: when no requests arrive for long enough, all replicas shut down and your deployment incurs no charges. The tradeoff is that the next request triggers a [cold start](/deployment/autoscaling/cold-starts), which can take minutes for large models. During that wake-up period, [billing is per minute](/observability/usage) even though the replica isn't yet serving responses. For production deployments, set `min_replica` to at least 2. This eliminates cold starts and provides redundancy if one replica fails. The ceiling for your deployment's capacity. The autoscaler won't scale above this number. **Range:** ≥ 1 This setting protects against runaway scaling and unexpected costs. If traffic exceeds what your maximum replicas can handle, requests queue rather than triggering new replicas. See [Request lifecycle](/deployment/autoscaling/request-lifecycle) for details on queuing and load shedding behavior. The default of 1 effectively disables autoscaling: you get exactly one replica regardless of load. Estimate max replicas: $$ (peak\_requests\_per\_second / throughput\_per\_replica) + buffer $$ For high-volume workloads requiring guaranteed capacity, [contact Baseten](mailto:support@baseten.co) about reserved capacity options. *** ## Scaling triggers The autoscaler decides when a replica is "full" by comparing in-flight requests against a per-replica threshold. `concurrency_target` caps how many simultaneous requests each replica accepts, and `target_utilization_percentage` cuts the threshold lower so the autoscaler can trigger a scale-up before any replica is completely saturated, leaving room for new replicas to come online without queueing requests in the meantime. Scale-up fires when: $$ load > replicas \times concurrency\_target \times target\_utilization $$ The diagram below shows a replica with `concurrency_target` of 8 and `target_utilization` of 50%, so the per-replica threshold sits at 4. The first four requests fill capacity within headroom; the fifth crosses the threshold, and the autoscaler provisions a second replica to absorb the overflow before the remaining slots saturate. How many requests each replica can handle simultaneously. This directly determines replica count for a given load. **Range:** ≥ 1 Given the current load, the autoscaler calculates desired replicas: $$ desired\_replicas = \lceil in\_flight\_requests / (concurrency\_target \times target\_utilization) \rceil $$ *In-flight requests* are requests sent to your model that haven't returned a response (for streaming, until the stream completes). This count is exposed as [`baseten_concurrent_requests`](/observability/export-metrics/supported-metrics#baseten_concurrent_requests) in the metrics dashboard and metrics export. The right value depends on how your model uses hardware. Image generation models that consume all GPU memory per request can only process one at a time, so a concurrency target of 1 is correct. LLMs and embedding models batch requests internally and can handle dozens simultaneously, so higher targets (32 or more) reduce cost by packing more work onto each replica. **Tradeoff:** Higher concurrency = fewer replicas (lower cost) but more per-replica queueing (higher latency). Lower concurrency = more replicas (higher cost) but less queueing (lower latency). **Starting points by model type:** | Model type | Starting concurrency | | ----------------------- | -------------------- | | Standard Truss model | 1 | | vLLM / LLM inference | 32–128 | | SGLang | 32 | | Text embeddings (TEI) | 32 | | BEI embeddings | 96+ (min ≥ 8) | | Whisper (async batch) | 256 | | Image generation (SDXL) | 1 | For engine-specific guidance, see [Autoscaling engines](/engines/performance-concepts/autoscaling-engines). **Concurrency target** controls requests sent *to* a replica and triggers autoscaling. **predict\_concurrency** (Truss config.yaml) controls requests processed *inside* the container. Concurrency target should be less than or equal to predict\_concurrency. See the `predict_concurrency` field in the [Truss configuration reference](/reference/truss-configuration) for details. Headroom before scaling triggers. The autoscaler scales when utilization reaches this percentage of the concurrency target, not when replicas are fully loaded. **Range:** 1–100% The effective threshold is: $$ concurrency\_target × target\_utilization $$ With a concurrency target of 10 and utilization of 70%, scaling triggers at 7 concurrent requests (10 x 0.70), leaving 30% headroom for absorbing spikes while new replicas start. Lower values (50-60%) provide more headroom for spikes but cost more. Higher values (80%+) are cost-efficient for steady traffic but absorb spikes less effectively. Target utilization is **not** GPU utilization. It measures request slot usage relative to your concurrency target, not hardware utilization. *** ## Scaling dynamics Once the autoscaler decides to scale, two settings control the pace. `autoscaling_window` determines how much history feeds into each decision, so a longer window averages out short spikes while a shorter one reacts to traffic changes faster. `scale_down_delay` gates removal in the other direction by holding replicas warm even after load drops, so a brief dip does not trigger a teardown that the next request would have to wait through. Together, the two settings tune the tradeoff between responsiveness and stability. The diagram below shows traffic falling to zero, the idle timer filling up, and the replica being reclaimed only once the timer crosses the `scale_down_delay` threshold. How far back (in seconds) the autoscaler looks when measuring traffic. Traffic is averaged over this window to make scaling decisions. **Range:** 10–3600 seconds A 60-second window smooths out momentary spikes by averaging load over the past minute. Shorter windows (30-60s) react quickly to traffic changes, which suits bursty workloads. Longer windows (2-5 min) ignore short-lived fluctuations and prevent the autoscaler from chasing noise. How long (in seconds) the autoscaler waits after load drops before removing replicas. **Range:** 0–3600 seconds When load drops, the autoscaler starts a countdown. If load stays low for the full delay, it removes replicas using exponential back-off (half the excess, wait, half again). If traffic returns before the countdown finishes, the replicas stay active and the countdown resets. This is your primary lever for preventing *oscillation*. If replicas repeatedly scale up and down, increase this value first. A **short window** with a **long delay** gives you fast scale-up while maintaining capacity during temporary dips. This is a good starting configuration for most workloads. *** ## Development deployments Development deployments are designed for iteration, not production traffic. Replicas are fixed at 0-1 to match the [`truss watch`](/reference/cli/truss/watch) workflow, where you're testing changes on a single instance rather than handling concurrent users. You can still adjust timing and concurrency settings. | Setting | Value | Modifiable | | ------------------ | ----------- | ---------- | | Min replicas | 0 | No | | Max replicas | 1 | No | | Autoscaling window | 60 seconds | Yes | | Scale-down delay | 900 seconds | Yes | | Concurrency target | 1 | Yes | | Target utilization | 70% | Yes | To enable full autoscaling with configurable replica settings, [promote the deployment to production](/deployment/deployments). *** ## Next steps Identify your traffic pattern and get recommended starting settings. Understand cold starts and how to minimize their impact. Complete autoscaling API documentation. Recommended settings for BEI and Engine-Builder-LLM with dynamic batching. *** ## Troubleshooting Having issues with autoscaling? See [Autoscaling troubleshooting](/troubleshooting/deployments#autoscaling-issues) for solutions to common problems like oscillation, slow scale-up, and unexpected costs.