> ## Documentation Index > Fetch the complete documentation index at: https://docs.baseten.co/llms.txt > Use this file to discover all available pages before exploring further. # Metrics > Understand the load and performance of your model export const MiniEngineThroughput = () => { const ref = React.useRef(null); const init = React.useRef(false); React.useEffect(() => { if (!ref.current || init.current) return; init.current = true; const W = 620, H = 208, padL = 12, padR = 14, padT = 66, padB = 22, N = 90, YMIN = 2700, YMAX = 3300; const GRID = [2800, 3000, 3200]; const isDark = () => document.documentElement.classList.contains("dark"); const C = () => isDark() ? { sub: "#869089", body: "#aeb6b0", grid: "rgba(255,255,255,0.07)", line: "#17D465", fill0: "rgba(23,212,101,0.24)", fill1: "rgba(23,212,101,0.0)", big: "#f3f6f4", unit: "#869089" } : { sub: "#869089", body: "#5a675e", grid: "rgba(0,0,0,0.06)", line: "#0e863f", fill0: "rgba(14,134,63,0.16)", fill1: "rgba(14,134,63,0.0)", big: "#021309", unit: "#5a675e" }; const g = u => 3000 + 80 * Math.sin(u * 0.30) + 45 * Math.sin(u * 0.72 + 1.3) + 25 * Math.sin(u * 1.9 + 0.4); const fmt = v => Math.round(v).toLocaleString("en-US"); let phase = 0, visible = true, raf = 0, last = 0; const cv = document.createElement("canvas"); cv.style.cssText = "display:block;width:100%;max-width:" + W + "px;touch-action:pan-y"; const ctx = cv.getContext("2d"); const dpr = window.devicePixelRatio || 1; cv.width = W * dpr; cv.height = H * dpr; cv.style.height = H + "px"; ctx.scale(dpr, dpr); const cap = document.createElement("div"); cap.style.cssText = "font:400 12px/1.4 system-ui,-apple-system,sans-serif;margin:8px 2px 0"; cap.textContent = "An illustrative live view. Your deployment's own graphs appear in the Metrics tab."; ref.current.appendChild(cv); ref.current.appendChild(cap); const px0 = padL, px1 = W - padR, pw = px1 - px0, py0 = padT, py1 = H - padB, ph = py1 - py0; const yOf = v => py1 - (Math.min(YMAX, Math.max(YMIN, v)) - YMIN) / (YMAX - YMIN) * ph; const xOf = i => px0 + i / (N - 1) * pw; function draw() { const col = C(); ctx.clearRect(0, 0, W, H); ctx.font = "500 9px ui-monospace,Menlo,monospace"; ctx.textBaseline = "middle"; for (let k = 0; k < GRID.length; k++) { const gv = GRID[k], y = yOf(gv); ctx.strokeStyle = col.grid; ctx.lineWidth = 1; ctx.beginPath(); ctx.moveTo(px0, y); ctx.lineTo(px1, y); ctx.stroke(); ctx.fillStyle = col.sub; ctx.textAlign = "left"; ctx.fillText((gv / 1000).toFixed(1) + "k", px0 + 2, y - 6); } ctx.beginPath(); for (let i = 0; i < N; i++) { const x = xOf(i), y = yOf(g(phase + i)); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); } ctx.lineTo(px1, py1); ctx.lineTo(px0, py1); ctx.closePath(); const grad = ctx.createLinearGradient(0, py0, 0, py1); grad.addColorStop(0, col.fill0); grad.addColorStop(1, col.fill1); ctx.fillStyle = grad; ctx.fill(); ctx.beginPath(); for (let i = 0; i < N; i++) { const x = xOf(i), y = yOf(g(phase + i)); if (i === 0) ctx.moveTo(x, y); else ctx.lineTo(x, y); } ctx.strokeStyle = col.line; ctx.lineWidth = 2; ctx.lineJoin = "round"; ctx.stroke(); const cur = g(phase + (N - 1)), cx = xOf(N - 1), cy = yOf(cur); ctx.fillStyle = col.line; ctx.beginPath(); ctx.arc(cx, cy, 3, 0, Math.PI * 2); ctx.fill(); ctx.textAlign = "left"; ctx.textBaseline = "alphabetic"; ctx.font = "500 11px ui-monospace,Menlo,monospace"; ctx.fillStyle = col.sub; ctx.fillText("Generation throughput", px0, 18); ctx.font = "700 30px system-ui,-apple-system,sans-serif"; ctx.fillStyle = col.big; const big = fmt(cur); ctx.fillText(big, px0, 50); const bw = ctx.measureText(big).width; ctx.font = "500 13px system-ui,-apple-system,sans-serif"; ctx.fillStyle = col.unit; ctx.fillText(" tokens/sec", px0 + bw, 50); cap.style.color = col.body; } const io = new IntersectionObserver(en => visible = en[0].isIntersecting, { threshold: 0.1 }); io.observe(cv); const themeObs = new MutationObserver(() => draw()); themeObs.observe(document.documentElement, { attributes: true, attributeFilter: ["class"] }); function loop(ts) { raf = requestAnimationFrame(loop); if (!visible) { last = ts; return; } const dt = last ? Math.min(0.05, (ts - last) / 1000) : 0; last = ts; phase += dt * 0.6; draw(); } draw(); raf = requestAnimationFrame(loop); return () => { cancelAnimationFrame(raf); io.disconnect(); themeObs.disconnect(); cv.remove(); cap.remove(); init.current = false; }; }, []); return
; }; The Metrics tab in the model dashboard tracks model load and performance. Use the dropdowns at the top of the tab to scope by environment, deployment, or time range. Environment scope aggregates metrics across every deployment in that environment, which helps you watch a rollout or compare trends across the whole environment. Deployment scope restricts metrics to a single deployment ID for diagnosing one version in isolation. ## Customize your view By default the Metrics tab shows a standard set of graphs. Use the **Customize view** button at the top of the tab to show, hide, and reorder any graph, and your layout is saved per model. A hidden graph stays in the Customize view panel, so you can turn it back on at any time. ## Events Turn on the **Events** toggle at the top of the Metrics tab to overlay platform events on your graphs. When response time jumps or replica count changes, a marker shows whether a deployment, promotion, or settings change caused it. Events are available for models, not for shared Model API endpoints or training jobs. The toggle is off by default. Baseten marks these events: * **Deployed:** a new deployment, with its target environment. * **Promoted:** a deployment promoted to an environment. * **Promotion control action:** a pause, resume, or roll-forward during a promotion. * **Autoscaling changed:** a new replica range or concurrency target. * **Activated** and **Deactivated:** a deployment turned on or off. * **Instance type changed:** a move to a new instance type. * **Replica terminated:** an individual replica shut down. * **Environment updated:** a change to an environment. ## Inference volume Tracks the response rate over time, segmented by HTTP status codes: * `2xx`: 🟢 Successful requests * `4xx`: 🟡 Client errors * `5xx`: 🔴 Server errors (includes model prediction exceptions) For non-HTTP models and Chains (WebSockets and gRPC), the status codes reflect the status codes for those protocols. For a full list of the WebSocket close codes surfaced here, see [WebSocket status codes](/development/model/websockets#inference-volume). *** ## Incoming request volume Tracks the rate of requests arriving at your deployment, before they're routed to the model. Comparing this against [Inference volume](#inference-volume), which counts responses by status code, surfaces requests that were received but never produced a response. To export the same series, see [`baseten_incoming_inference_requests_total`](/observability/export-metrics/supported-metrics#baseten_incoming_inference_requests_total). *** ## Response time Measured at different percentiles (p50, p90, p95, p99): * **End-to-end response time:** Includes cold starts, queuing, and inference (excludes client-side latency). Reflects real-world performance. * **Inference time:** Covers only model execution, including pre/post-processing. Useful for optimizing single-replica performance. * **Time to first byte:** Measures the time from request receipt to the first byte Baseten writes back, including any queueing and routing time. It approximates time to first token (TTFT), because Baseten doesn't parse tokens out of your model's response. [Model API endpoints](#model-apis-metrics) graph time to first token instead. *** ## Request and response size Measured at different percentiles (p50, p90, p95, p99): * **Request size:** Tracks the request size distribution. A proxy for input tokens. * **Response size:** Tracks the response size distribution. A proxy for generated tokens. *** ## Replicas Tracks how many replicas the autoscaler is targeting and how many you have: * **Desired:** The number of replicas your [autoscaling settings](/deployment/autoscaling/overview) call for at the current [concurrent request](#concurrent-requests) volume. BIS-LLM deployments target [in-flight tokens](/engines/performance-concepts/autoscaling-engines) instead. * **Starting:** Waiting for resources or loading the model. * **Active:** Ready to serve requests. For development deployments, a replica counts as active while it runs the live reload server. When desired sits above your current replica count (active + starting), your settings are working as configured and Baseten is provisioning the difference. Scale-up takes longer when GPU capacity is in high demand, so expect a gap during a large scale-up rather than a misconfiguration. If the gap holds longer than you'd expect, [contact support](mailto:support@baseten.co). Desired stays inside your `min_replica` and `max_replica` range, so expect a flat line at your minimum while traffic is idle. When desired sits at `max_replica` and latency still degrades, your ceiling is the limit: raise `max_replica`, or raise `concurrency_target` if each replica can handle more traffic. Export [`baseten_replicas_desired`](/observability/export-metrics/supported-metrics#baseten_replicas_desired), [`baseten_replicas_active`](/observability/export-metrics/supported-metrics#baseten_replicas_active) and [`baseten_replicas_starting`](/observability/export-metrics/supported-metrics#baseten_replicas_starting) to track replica counts in your own [observability stack](/observability/export-metrics/overview), and use this graph when you need to compare them against desired. To see pods split by their Kubernetes Ready condition, for example when a [readiness probe](/development/model/health-checks#readiness-probe) pulls a replica out of traffic, export [`baseten_pod_readiness`](/observability/export-metrics/supported-metrics#baseten_pod_readiness). *** ## Restarts Tracks the cumulative number of times the model container has been restarted. Restarts are typically caused by application crashes, out-of-memory kills, or failed [liveness probes](/development/model/health-checks#liveness-probe). Frequent restarts usually indicate one of: * A crash in `load()` or in your model code. * An out-of-memory event: check the **Memory usage** graph. * A liveness probe failing under load: review `restart_threshold_seconds` and any [custom health check logic](/development/model/health-checks#custom-health-check-logic). *** ## Concurrent requests Total in-flight inference requests across replicas, including both requests currently being serviced and requests waiting to be processed. [Async inference requests](/inference/async) are not included in this metric. This is the primary signal that drives [autoscaling](/deployment/autoscaling/overview) decisions. For the full metric definition and labels, see [`baseten_concurrent_requests`](/observability/export-metrics/supported-metrics#baseten_concurrent_requests). This metric is a point-in-time gauge, sampled roughly every 30 seconds, while inference volume counts every request over the full minute. The two relate through Little's Law: `average concurrency ≈ requests per second × average end-to-end latency` When requests are fast, that product stays well below 1 even at high volume, so most samples catch the system empty and the gauge reads 0. For example, 600 requests per minute at 80 ms latency averages about 0.8 requests in flight. Autoscaling still responds correctly, because it acts on sustained concurrency rather than sub-second bursts. *** ## CPU usage and memory Displays resource utilization across replicas. Metrics are averaged and may not capture short spikes. ### Considerations: * **High CPU/memory usage**: May degrade performance. Consider upgrading to a larger instance type. * **Low CPU/memory usage**: Possible overprovisioning. Switch to a smaller instance to reduce costs. *** ## GPU usage and memory Shows GPU utilization across replicas. * **GPU usage**: Percentage of time a kernel function occupies the GPU. * **GPU memory**: Total memory used. ### Considerations: * **High GPU load**: Can slow inference. Check response time metrics. * **High memory usage**: May cause out-of-memory failures. * **Low utilization**: May indicate overprovisioning. Consider a smaller GPU. *** ## Organization GPU usage The GPU graphs above cover a single deployment. To see how many GPUs your whole workspace is using at once, organization admins can open the **GPU usage** tab in Organization settings. This view counts *active GPUs* (ready replicas multiplied by the GPUs each replica uses), aggregated across every model and deployment in the organization. **To view organization GPU usage**: 1. Sign in to your workspace at [app.baseten.co](https://app.baseten.co) and open **Organization settings**. 2. Choose the **GPU usage** tab. Use **Group by** to group usage by **GPU type** (the default) or by **Model**. Use the **GPU** filter to limit the chart to specific GPU types, and the time-range selector to choose a window up to the last 7 days. Choose **Reset** to clear the GPU filter. *** ## vLLM and SGLang metrics When your deployment serves an LLM with [vLLM](/examples/vllm) or [SGLang](/examples/sglang), Baseten surfaces engine-native metrics in the Metrics tab alongside the standard ones. These graphs report what the inference engine itself measures: metrics like tokens per second, time to first token, KV cache usage, and the number of requests running or queued. ### How detection works You don't turn these graphs on manually. Baseten scrapes your container's `/metrics` endpoint and looks for metrics that match the vLLM or SGLang format. When it finds them, the matching graphs appear in the Metrics tab automatically. No configuration or redeploy is required. If you don't see the graphs, and they don't appear in the **Customize view** panel either, Baseten was most likely unable to read your container's metrics endpoint. Common causes are that the endpoint isn't exposed, it's blocking Baseten's scrape, or the engine isn't emitting metrics yet. Confirm that your engine serves Prometheus metrics on its `/metrics` route. For [custom servers](/development/model/custom-server), routes like `/metrics` pass through to your server unchanged. Detection runs on a periodic scrape and results are cached, so a deployment that just started exporting metrics may take a few minutes to show its graphs. ### Show and hide graphs Many of these engine graphs are hidden by default. Turn them on with [Customize your view](#customize-your-view). The exact graphs depend on what your engine version emits. The latency graphs are shown at the p50, p90, p95, and p99 percentiles, and counters are summed over the selected time range. ### Export engine metrics The Metrics tab shows a curated set of graphs. You can also export the underlying vLLM and SGLang metrics, along with a few that aren't graphed in the dashboard, to your own observability stack through the [metrics export endpoint](/observability/export-metrics/overview). See [vLLM and SGLang metrics](/observability/export-metrics/supported-metrics#vllm-and-sglang-metrics) for the labels Baseten adds. *** ## BIS-LLM metrics When your deployment runs on the [BIS-LLM engine](/engines/bis-llm/overview), the Metrics tab adds engine graphs alongside the standard metrics above. They appear automatically for BIS-LLM deployments; no configuration is required. * **Token throughput**: input and output tokens over time, with per-request distributions for input tokens, output tokens, and tokens per second. * **KV cache hit rate**: the cache hit rates your workers observe. Falling hit rates mean more prefill work per request and higher latency. * **Time to first byte**: on BIS-LLM deployments this graph is measured inside the engine, covering engine queueing and prefill but not platform routing, unlike the [standard time to first byte](#response-time), which includes routing time. [Response time](#response-time) still covers the end-to-end view. * **Inflight tokens per worker**: the load signal the BIS-LLM autoscaler acts on, covering tokens being processed and queued at the router. Deployments with speculative decoding active also show an acceptance rate graph. Hide or show individual graphs with [Customize your view](#customize-your-view). ### Export engine metrics The underlying series carry the `baseten_llm_*` prefix and export to your own observability stack through the [metrics export endpoint](/observability/export-metrics/overview). That includes the raw accepted and draft token counters behind the acceptance rate graph, so you can rebuild it in your own dashboards. For the full list with types and labels, see [BIS-LLM metrics](/observability/export-metrics/supported-metrics#bis-llm-metrics). *** ## Model APIs metrics [Model API](/inference/model-apis/overview) endpoints have their own Metrics tab. Because you call a shared endpoint rather than run your own replicas, the tab graphs request and token metrics, not the replica and hardware metrics above. * **Inference volume**: response rate over time, segmented by HTTP status code. * **Rate limit**: rate of requests the endpoint rejects for exceeding your [rate limits](/inference/model-apis/pricing-and-limits). * **End-to-end response time**: time from request receipt to the last byte of the response, including queueing and inference. * **Time to first token**: time from request receipt to the first generated token, including queueing and routing. The inference engine reports this timing directly, unlike [time to first byte](#response-time) on your own deployments. * **Request size** and **Response size**: request and response payload size distributions. * **Tokens usage**: tokens per second the endpoint processes for your workspace. The latency and size graphs report the p50, p90, p95, and p99 percentiles. *** ## Async queue metrics * **Time in Async Queue**: Time spent in the async queue before execution (p50, p90, p95, p99). * **Async Queue Size**: Number of queued async requests. * **Webhook requests**: Number of [async webhook](/inference/async) delivery requests sent. * **Webhook latency**: Latency of async webhook delivery requests (p50, p90, p95, p99). ### Considerations: * Large queue size indicates requests are queued faster than they are processed. * To improve async throughput, increase the max replicas or adjust autoscaling concurrency. * Async Queue Size is a point-in-time gauge, like [concurrent requests](#concurrent-requests). When requests spend little time queued, most samples catch an empty queue and it reads 0 even under steady load. *** ## Use metrics for autoscaling Use these metrics to diagnose autoscaling behavior and tune your settings. ### Key metrics to watch | Metric | What it tells you | | -------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------- | | **Concurrent requests** | Shows total demand (queued + active). This is the signal driving autoscaling. | | **Replicas** (desired vs active vs starting) | Shows what the autoscaler is targeting against what you have. A persistent gap means Baseten is still provisioning replicas. | | **Inference volume** | Shows traffic patterns. Use to identify if you have noisy, bursty, or steady traffic. | | **Response time** (p95, p99) | Shows if scaling is keeping up. Spikes aligned with replica changes indicate thrash. | | **Async queue size** | Shows backpressure. Growing queue means you need more capacity. | ### Diagnose autoscaling issues | You see... | Likely cause | Fix | | --------------------------------------------------- | -------------------------------------- | ----------------------------------------------------------------------------------- | | Latency spikes aligned with replica count changes | Oscillation (thrash) | Increase scale-down delay | | Desired pinned at max, latency still degrading | Replica ceiling too low | Increase max replicas or concurrency target | | Desired above active + starting for several minutes | Baseten is still provisioning replicas | Settings are correct; [contact support](mailto:support@baseten.co) if the gap holds | | Replicas stuck in starting for several minutes | Cold start delays | Increase min replicas, check image optimization | | Traffic high but replicas staying low | Concurrency target too high | Lower concurrency target or target utilization | | Replicas scaling down too quickly | Scale-down delay too short | Increase scale-down delay | For solutions to common autoscaling problems, see [Autoscaling troubleshooting](/troubleshooting/deployments#autoscaling-issues).