> ## Documentation Index > Fetch the complete documentation index at: https://docs.baseten.co/llms.txt > Use this file to discover all available pages before exploring further. # Model recipes > Run popular models through optimized inference engines on Baseten. export const RecipeBuilder = () => { const CATEGORIES = [{ id: "llm", label: "LLMs" }, { id: "image-gen", label: "Image" }, { id: "transcription", label: "Transcription" }, { id: "embedding", label: "Embedding" }]; const CAPABILITIES = ["Reasoning", "Tool calling", "Long context", "Agentic", "Speech to text", "Text-to-image", "Embeddings", "Reranking", "Cross-encoder", "Multimodal (image)"]; const RECIPES = [{ id: "qwen3.6--27b--default", category: "llm", family: "qwen3.6", familyLabel: "Qwen 3.6", variant: "27B", preset: null, presetLabel: "27B", hardware: "H100 × 4", engine: "vLLM 0.20.0", context: "256K", concurrency: "64", capabilities: ["Reasoning", "Tool calling", "Agentic", "Long context"], recipePath: "/examples/models/llm/qwen3.6", configYaml: `model_name: "model:qwen3.6-27b preset:latency" model_metadata: example_model_input: model: "Qwen/Qwen3.6-27B" messages: - role: user content: "What is the capital of France?" stream: true max_tokens: 512 temperature: 1.0 top_p: 0.95 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.20.0 weights: - source: "hf://Qwen/Qwen3.6-27B@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" resources: accelerator: H100:4 use_gpu: true runtime: predict_concurrency: 64 environment_variables: HF_HUB_ENABLE_HF_TRANSFER: "1" VLLM_LOGGING_LEVEL: WARNING secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --served-model-name Qwen/Qwen3.6-27B --host 0.0.0.0 --port 8000 --trust-remote-code --tensor-parallel-size $GPU_COUNT --max-model-len 262144 --language-model-only --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --speculative_config.method mtp --speculative_config.num_speculative_tokens 2' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 ` }, { id: "qwen3.6--35b-a3b--latency", category: "llm", family: "qwen3.6", familyLabel: "Qwen 3.6", variant: "35B-A3B", preset: "Latency", presetLabel: "35B-A3B Latency", hardware: "H100 × 4", engine: "vLLM 0.20.0", context: "256K", concurrency: "64", capabilities: ["Reasoning", "Tool calling", "Agentic", "Long context"], recipePath: "/examples/models/llm/qwen3.6", configYaml: `model_name: "model:qwen3.6-35b-a3b preset:latency" model_metadata: example_model_input: model: "Qwen/Qwen3.6-35B-A3B" messages: - role: user content: "What is the capital of France?" stream: true max_tokens: 512 temperature: 1.0 top_p: 0.95 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.20.0 weights: - source: "hf://Qwen/Qwen3.6-35B-A3B@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" resources: accelerator: H100:4 use_gpu: true runtime: predict_concurrency: 64 environment_variables: HF_HUB_ENABLE_HF_TRANSFER: "1" VLLM_LOGGING_LEVEL: WARNING secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --served-model-name Qwen/Qwen3.6-35B-A3B --host 0.0.0.0 --port 8000 --trust-remote-code --tensor-parallel-size $GPU_COUNT --max-model-len 262144 --language-model-only --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --speculative_config.method mtp --speculative_config.num_speculative_tokens 2' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 ` }, { id: "qwen3.6--35b-a3b--throughput", category: "llm", family: "qwen3.6", familyLabel: "Qwen 3.6", variant: "35B-A3B", preset: "Throughput", presetLabel: "35B-A3B Throughput", hardware: "B200", engine: "vLLM (nightly build)", context: "256K", concurrency: "1000", capabilities: ["Reasoning", "Tool calling", "Agentic", "Long context"], recipePath: "/examples/models/llm/qwen3.6", configYaml: `model_name: "model:qwen3.6-35b-a3b preset:throughput" model_metadata: example_model_input: model: "RedHatAI/Qwen3.6-35B-A3B-NVFP4" messages: - role: user content: "What is the capital of France?" max_tokens: 100 temperature: 0.7 tags: - openai-compatible - vllm - qwen3.6 - nvfp4 - b200 base_image: image: vllm/vllm-openai:nightly weights: - source: "hf://RedHatAI/Qwen3.6-35B-A3B-NVFP4@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" build_commands: [] environment_variables: PYTORCH_ALLOC_CONF: "expandable_segments:True" VLLM_FLASHINFER_MOE_BACKEND: throughput VLLM_USE_FLASHINFER_MOE_FP4: 1 VLLM_USE_FLASHINFER_MOE_FP8: 1 docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name RedHatAI/Qwen3.6-35B-A3B-NVFP4 --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.95 --max-model-len 262144 --max-num-batched-tokens 32768 --dtype auto --enable-chunked-prefill --enable-prefix-caching --max-num-seqs 512 --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --moe_backend flashinfer_cutlass --speculative-config '\''{"method":"qwen3_5_mtp","num_speculative_tokens":3}'\'' --trust-remote-code' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 runtime: predict_concurrency: 1000 health_checks: restart_check_delay_seconds: 1500 restart_threshold_seconds: 1500 stop_traffic_threshold_seconds: 120 resources: accelerator: B200 use_gpu: true secrets: hf_access_token: null ` }, { id: "qwen3.5--4b--default", category: "llm", family: "qwen3.5", familyLabel: "Qwen 3.5", variant: "4B", preset: null, presetLabel: "4B", hardware: "H100 × 1", engine: "vLLM (0.22.0-cu129 build)", context: "32K", concurrency: "128", capabilities: ["Reasoning", "Tool calling", "Long context", "Agentic"], recipePath: "/examples/models/llm/qwen3.5", configYaml: `model_name: "model:qwen3.5-4b preset:latency" model_metadata: description: >- Qwen 3.5 4B instruct (dense), OpenAI-compatible chat using vLLM with Qwen tooling parsers. repo_id: Qwen/Qwen3.5-4B example_model_input: model: "Qwen/Qwen3.5-4B" messages: - role: user content: "What is the capital of France?" stream: true max_tokens: 100 temperature: 0.7 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://Qwen/Qwen3.5-4B@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name Qwen/Qwen3.5-4B --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.95 --max-model-len 32768 --dtype bfloat16 --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --trust-remote-code --enable-prefix-caching --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" runtime: predict_concurrency: 128 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 resources: accelerator: H100:1 use_gpu: true ` }, { id: "qwen3.5--9b--default", category: "llm", family: "qwen3.5", familyLabel: "Qwen 3.5", variant: "9B", preset: null, presetLabel: "9B", hardware: "H100 × 1", engine: "vLLM (0.22.0-cu129 build)", context: "32K", concurrency: "128", capabilities: ["Reasoning", "Tool calling", "Long context", "Agentic"], recipePath: "/examples/models/llm/qwen3.5", configYaml: `model_name: "model:qwen3.5-9b preset:latency" model_metadata: description: >- Qwen 3.5 9B instruct (dense), OpenAI-compatible chat using vLLM with Qwen tooling parsers. repo_id: Qwen/Qwen3.5-9B example_model_input: model: "Qwen/Qwen3.5-9B" messages: - role: user content: "What is the capital of France?" stream: true max_tokens: 100 temperature: 0.7 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://Qwen/Qwen3.5-9B@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name Qwen/Qwen3.5-9B --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.95 --max-model-len 32768 --dtype bfloat16 --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --trust-remote-code --enable-prefix-caching --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" runtime: predict_concurrency: 128 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 resources: accelerator: H100:1 use_gpu: true ` }, { id: "qwen3.5--35b--latency", category: "llm", family: "qwen3.5", familyLabel: "Qwen 3.5", variant: "35B", preset: "Latency", presetLabel: "35B Latency", hardware: "H100 × 2", engine: "vLLM (0.22.0-cu129 build)", context: "32K", concurrency: "128", capabilities: ["Reasoning", "Tool calling", "Long context", "Agentic"], recipePath: "/examples/models/llm/qwen3.5", configYaml: `model_name: "model:qwen3.5-35b preset:latency" model_metadata: description: >- Qwen 3.5 35B A3B MoE instruct FP8 weights, TP=2 latency preset with Qwen parsers. repo_id: Qwen/Qwen3.5-35B-A3B-FP8 example_model_input: model: "Qwen/Qwen3.5-35B-A3B-FP8" messages: - role: user content: "What is the capital of France?" stream: true max_tokens: 100 temperature: 0.7 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://Qwen/Qwen3.5-35B-A3B-FP8@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --served-model-name Qwen/Qwen3.5-35B-A3B-FP8 --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.95 --max-model-len 32768 --kv-cache-dtype fp8 --tensor-parallel-size $GPU_COUNT --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --trust-remote-code --enable-prefix-caching --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" runtime: predict_concurrency: 128 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 resources: accelerator: H100:2 use_gpu: true ` }, { id: "qwen3.5--35b--throughput", category: "llm", family: "qwen3.5", familyLabel: "Qwen 3.5", variant: "35B", preset: "Throughput", presetLabel: "35B Throughput", hardware: "B200", engine: "vLLM 0.22.0", context: "256K", concurrency: "1000", capabilities: ["Reasoning", "Tool calling", "Long context", "Agentic"], recipePath: "/examples/models/llm/qwen3.5", configYaml: `######################################################## # Remove ( --language-model-only ) from the start command to turn on multimodal mode ######################################################## model_name: "model:qwen3.5-35b preset:throughput" model_metadata: description: >- Qwen 3.5 35B A3B FP8 MoE throughput on B200, language-only mode (--language-model-only) optional. repo_id: Qwen/Qwen3.5-35B-A3B-FP8 example_model_input: model: "Qwen/Qwen3.5-35B-A3B-FP8" messages: - role: user content: "What is the capital of France?" stream: true max_tokens: 100 temperature: 0.7 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0 weights: - source: "hf://Qwen/Qwen3.5-35B-A3B-FP8@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_USE_FLASHINFER_MOE_FP8: "0" PYTORCH_ALLOC_CONF: "expandable_segments:True" docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name Qwen/Qwen3.5-35B-A3B-FP8 --host 0.0.0.0 --port 8000 --language-model-only --gpu-memory-utilization 0.95 --kv-cache-dtype fp8 --reasoning-parser qwen3 --enable-chunked-prefill --enable-prefix-caching --max-num-seqs 512 --trust-remote-code --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 runtime: predict_concurrency: 1000 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 resources: accelerator: B200 use_gpu: true ` }, { id: "qwen3.5--122b--default", category: "llm", family: "qwen3.5", familyLabel: "Qwen 3.5", variant: "122B", preset: null, presetLabel: "122B", hardware: "H100 × 4", engine: "vLLM (0.22.0-cu129 build)", context: "32K", concurrency: "128", capabilities: ["Reasoning", "Tool calling", "Long context", "Agentic"], recipePath: "/examples/models/llm/qwen3.5", configYaml: `model_name: "model:qwen3.5-122b preset:latency" model_metadata: description: >- Qwen 3.5 122B A10B MoE instruct FP8 weights, TP=4 latency preset using vLLM with Qwen parsers. repo_id: Qwen/Qwen3.5-122B-A10B-FP8 example_model_input: model: "Qwen/Qwen3.5-122B-A10B-FP8" messages: - role: user content: "What is the capital of France?" stream: true max_tokens: 100 temperature: 0.7 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://Qwen/Qwen3.5-122B-A10B-FP8@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --served-model-name Qwen/Qwen3.5-122B-A10B-FP8 --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.95 --max-model-len 32768 --kv-cache-dtype fp8 --tensor-parallel-size $GPU_COUNT --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --trust-remote-code --enable-prefix-caching --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" runtime: predict_concurrency: 128 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 resources: accelerator: H100:4 use_gpu: true ` }, { id: "qwen3--default--default", category: "llm", family: "qwen3", familyLabel: "Qwen 3", variant: null, preset: null, presetLabel: "Default", hardware: "H100 × 8", engine: "TRT-LLM v2", context: "256K", concurrency: "256", capabilities: ["Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/qwen3", configYaml: `model_metadata: example_model_input: # Loads sample request into Baseten playground messages: - role: system content: "You are a helpful assistant." - role: user content: "What does Tongyi Qianwen mean?" stream: false model: "Qwen/Qwen3-235B-A22B-Instruct-2507-FP8" max_tokens: 512 temperature: 0.6 tags: - openai-compatible repo_id: Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 model_name: "model:qwen3-235b preset:latency" weights: - source: "hf://Qwen/Qwen3-235B-A22B-Instruct-2507-FP8@main" mount_location: "/app/model_cache/trt_model" resources: accelerator: H100:8 cpu: "1" memory: 10Gi use_gpu: true trt_llm: build: checkpoint_repository: repo: michaelfeil/empty-model revision: main source: HF inference_stack: v2 runtime: enable_chunked_prefill: true max_batch_size: 256 max_num_tokens: 8192 max_seq_len: 262144 served_model_name: Qwen/Qwen3-235B-A22B-Instruct-2507-FP8 tensor_parallel_size: 8 patch_kwargs: disable_overlap_scheduler: True model_path: /app/model_cache/trt_model moe_expert_parallel_size: 8 cuda_graph_config: enable_padding: true max_batch_size: 256 enable_autotune: false guided_decoding_backend: "xgrammar" enable_iter_perf_stats: 0 kv_cache_config: enable_block_reuse: true free_gpu_memory_fraction: 0.8 version_overrides: v2_llm_version: null ` }, { id: "glm-5--default--default", category: "llm", family: "glm-5", familyLabel: "GLM 5", variant: null, preset: null, presetLabel: "Default", hardware: "B200 × 8", engine: "vLLM 0.22.0", context: "128K", concurrency: "64", capabilities: ["Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/glm-5", configYaml: `model_metadata: example_model_input: messages: - role: system content: "You are a helpful assistant." - role: user content: "What is the meaning of life?" stream: true model: zai-org/GLM-5 max_tokens: 32768 temperature: 0.7 tags: - openai-compatible model_name: "model:glm-5 preset:latency" base_image: image: vllm/vllm-openai:v0.22.0 docker_server: start_command: > sh -c "VLLM_DEEP_GEMM_WARMUP=relax python3 -m vllm.entrypoints.openai.api_server --model /models/GLM-5-FP8 --chat-template /models/GLM-5-FP8/chat_template.jinja --host 0.0.0.0 --port 8000 --served-model-name zai-org/GLM-5 --tensor-parallel-size 8 --trust-remote-code --load-format runai_streamer --disable-log-stats --max-num-seqs 64 --max-num-batched-tokens 8192 --tool-call-parser glm47 --reasoning-parser glm45 --enable-auto-tool-choice --speculative-config.method mtp --speculative-config.num_speculative_tokens 1" readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 weights: - source: "hf://zai-org/GLM-5-FP8@main" mount_location: "/models/GLM-5-FP8" ignore_patterns: - "*.md" - "*.txt" resources: accelerator: B200:8 use_gpu: true runtime: predict_concurrency: 64 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 ` }, { id: "glm-4.7--standard--default", category: "llm", family: "glm-4.7", familyLabel: "GLM 4.7", variant: "Standard", preset: null, presetLabel: "Standard", hardware: "B200 × 4", engine: "TRT-LLM v2", context: "198K", concurrency: "64", capabilities: ["Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/glm-4.7", configYaml: `model_name: "model:glm-4.7 preset:latency" resources: accelerator: B200:4 cpu: "1" memory: 10Gi use_gpu: true model_metadata: tags: - openai-compatible example_model_input: model: glm47 messages: - role: user content: "Given an array of integers nums and an integer target, return indices of the two numbers such that they add up to target. You may assume that each input would have exactly one solution, and you may not use the same element twice. You can return the answer in any order. class Solution: def twoSum(self, nums: List[int], target: int) -> List[int]:" stream: true max_tokens: 2048 temperature: 0.5 trt_llm: build: checkpoint_repository: repo: baseten-admin/glm-4.7-fp4 revision: main source: HF # runtime_secret_name: "hf_access_token_custom" inference_stack: v2 runtime: enable_chunked_prefill: true max_batch_size: 64 max_num_tokens: 8192 max_seq_len: 202752 tensor_parallel_size: 4 served_model_name: glm47 patch_kwargs: disable_overlap_scheduler: True moe_expert_parallel_size: 4 moe_config: use_low_precision_moe_combine: true backend: TRTLLM kv_cache_config: free_gpu_memory_fraction: 0.8 enable_block_reuse: true cuda_graph_config: enable_padding: true max_batch_size: 64 speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 autotuner_enabled: false reasoning_parser: glm47 tool_call_parser: glm47 ` }, { id: "glm-4.7--flash--default", category: "llm", family: "glm-4.7", familyLabel: "GLM 4.7", variant: "Flash", preset: null, presetLabel: "Flash", hardware: "H100 × 2", engine: "vLLM (0.22.0-cu129 build)", context: "128K", concurrency: "32", capabilities: ["Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/glm-4.7", configYaml: `model_name: "model:glm-4.7-flash preset:latency" model_metadata: description: >- Zhipu GLM-4.7 Flash using vLLM (H100 × 2 TP), fast GLM tool calling and auto tool choice from BDN-mounted weights. repo_id: zai-org/GLM-4.7-Flash example_model_input: model: zai-org/GLM-4.7-Flash messages: - role: system content: "You are a helpful assistant." - role: user content: "What is the meaning of life?" stream: true max_tokens: 32768 temperature: 0.7 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://zai-org/GLM-4.7-Flash@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --gpu-memory-utilization 0.8 --tool-call-parser glm47 --enable-auto-tool-choice --served-model-name zai-org/GLM-4.7-Flash --host 0.0.0.0 --port 8000 --trust-remote-code --max-model-len auto --enable-prefix-caching --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 resources: accelerator: H100:2 cpu: "1" memory: 2Gi use_gpu: true runtime: is_websocket_endpoint: false predict_concurrency: 32 transport: kind: http health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 ` }, { id: "gpt-oss--20b--default", category: "llm", family: "gpt-oss", familyLabel: "GPT-OSS", variant: "20B", preset: null, presetLabel: "20B", hardware: "H100", engine: "TRT-LLM v2", context: "128K", concurrency: "64", capabilities: ["Reasoning", "Tool calling", "Agentic", "Long context"], recipePath: "/examples/models/llm/gpt-oss", configYaml: `model_name: "model:gpt-oss-20b preset:latency" build_commands: - python -c 'from openai_harmony import load_harmony_encoding; load_harmony_encoding("HarmonyGptOss")' model_metadata: repo_id: openai/gpt-oss-20b example_model_input: { "model": "openai/gpt-oss-20b", "messages": [ { "role": "user", "content": "Given an array of integers nums and an integer target, return indices of the two numbers such that they add up to target. You may assume that each input would have exactly one solution, and you may not use the same element twice. You can return the answer in any order. class Solution: def twoSum(self, nums: List[int], target: int) -> List[int]:", }, ], "stream": true, "max_tokens": 4096, "temperature": 0.5, } tags: - openai-compatible resources: accelerator: H100 cpu: "1" memory: 10Gi use_gpu: true weights: - source: "hf://openai/gpt-oss-20b@main" mount_location: "/app/model_cache/trt_model" trt_llm: build: checkpoint_repository: repo: michaelfeil/empty-model revision: main source: HF inference_stack: v2 runtime: enable_chunked_prefill: true max_batch_size: 64 max_num_tokens: 8192 max_seq_len: 131072 patch_kwargs: model_path: /app/model_cache/trt_model chat_processor: harmony moe_expert_parallel_size: 1 backend: pytorch cuda_graph_config: enable_padding: true disable_overlap_scheduler: 1 enable_autotuner: 0 enable_iter_perf_stats: 0 enable_trtllm_sampler: 1 guided_decoding_backend: xgrammar kv_cache_config: enable_block_reuse: true free_gpu_memory_fraction: 0.8 event_buffer_max_size: 1024 max_beam_width: 1 max_input_len: 131072 model_level_stop_words: - "<|call|>" tokenizer_limit_length: 131072 trust_remote_code: 1 moe_config: backend: CUTLASS served_model_name: openai/gpt-oss-20b tensor_parallel_size: 1 version_overrides: v2_llm_version: null ` }, { id: "gpt-oss--120b--h100-throughput", category: "llm", family: "gpt-oss", familyLabel: "GPT-OSS", variant: "120B", preset: "H100 Throughput", presetLabel: "120B H100 Throughput", hardware: "H100 × 4", engine: "vLLM (0.22.0-cu129 build)", context: "16K", concurrency: "256", capabilities: ["Reasoning", "Tool calling", "Agentic", "Long context"], recipePath: "/examples/models/llm/gpt-oss", configYaml: `model_name: "model:gpt-oss-120b preset:h100-throughput" model_metadata: description: >- GPT-OSS 120B on vLLM H100 × 4 throughput; weights from BDN, async scheduling and prefix caching. repo_id: openai/gpt-oss-120b tags: - openai-compatible example_model_input: messages: - role: system content: "You are a helpful assistant." - role: user content: "Write FizzBuzz in Python" stream: true model: "openai/gpt-oss-120b" max_tokens: 4096 temperature: 0.5 base_image: image: vllm/vllm-openai:v0.22.0-cu129 build_commands: - mkdir -p /opt/tiktoken - curl -fsSL https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken -o /opt/tiktoken/o200k_base.tiktoken - curl -fsSL https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken -o /opt/tiktoken/cl100k_base.tiktoken weights: - source: "hf://openai/gpt-oss-120b@b5c939de8f754692c1647ca79fbf85e8c1e70f8a" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" ignore_patterns: ["original/*", "metal/model.bin"] secrets: hf_access_token: null environment_variables: TIKTOKEN_ENCODINGS_BASE: "/opt/tiktoken" TIKTOKEN_RS_CACHE_DIR: "/opt/tiktoken" VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --host 0.0.0.0 --port 8000 --served-model-name openai/gpt-oss-120b --tensor-parallel-size $GPU_COUNT --gpu-memory-utilization 0.90 --max-model-len 16384 --max-num-batched-tokens 16384 --max-num-seqs 256 --stream-interval 20 --enable-chunked-prefill --enable-prefix-caching --async-scheduling --trust-remote-code --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 resources: accelerator: H100:4 use_gpu: true runtime: predict_concurrency: 256 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 ` }, { id: "gpt-oss--120b--throughput", category: "llm", family: "gpt-oss", familyLabel: "GPT-OSS", variant: "120B", preset: "Throughput", presetLabel: "120B Throughput", hardware: "B200 × 4", engine: "vLLM 0.22.0", context: "8K", concurrency: "256", capabilities: ["Reasoning", "Tool calling", "Agentic", "Long context"], recipePath: "/examples/models/llm/gpt-oss", configYaml: `model_name: "model:gpt-oss-120b preset:throughput" model_metadata: description: >- GPT-OSS 120B on vLLM Blackwell (B200 × 4), Harmony recipe with FlashInfer MoE MXFP paths. repo_id: openai/gpt-oss-120b tags: - openai-compatible example_model_input: messages: - role: user content: "Write FizzBuzz in Python." stream: true model: openai/gpt-oss-120b max_tokens: 4096 temperature: 0.5 base_image: image: vllm/vllm-openai:v0.22.0 build_commands: - mkdir -p /opt/tiktoken - curl -fsSL https://openaipublic.blob.core.windows.net/encodings/o200k_base.tiktoken -o /opt/tiktoken/o200k_base.tiktoken - curl -fsSL https://openaipublic.blob.core.windows.net/encodings/cl100k_base.tiktoken -o /opt/tiktoken/cl100k_base.tiktoken weights: - source: "hf://openai/gpt-oss-120b@b5c939de8f754692c1647ca79fbf85e8c1e70f8a" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" ignore_patterns: ["original/*", "metal/model.bin"] secrets: hf_access_token: null environment_variables: TIKTOKEN_ENCODINGS_BASE: "/opt/tiktoken" TIKTOKEN_RS_CACHE_DIR: "/opt/tiktoken" VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" VLLM_USE_FLASHINFER_MOE_MXFP4_MXFP8: "1" docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --host 0.0.0.0 --port 8000 --served-model-name gpt-oss-120b --tensor-parallel-size $GPU_COUNT --gpu-memory-utilization 0.95 --max-model-len 8192 --max-num-batched-tokens 8192 --max-num-seqs 256 --cuda-graph-capture-size 2048 --stream-interval 20 --kv-cache-dtype fp8 --enable-prefix-caching --async-scheduling --trust-remote-code --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 resources: accelerator: B200:4 use_gpu: true runtime: predict_concurrency: 256 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 ` }, { id: "minimax-m2.5--default--default", category: "llm", family: "minimax-m2.5", familyLabel: "MiniMax M2.5", variant: null, preset: null, presetLabel: "Default", hardware: "H100 × 4", engine: "vLLM (0.22.0-cu129 build)", context: "200K", concurrency: "64", capabilities: ["Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/minimax-m2.5", configYaml: `model_name: "model:minimax-m2.5 preset:throughput" model_metadata: description: >- MiniMax-M2.5 Mixture-of-Experts (Run:AI streamer loading), throughput on H100 × 4 with MiniMax parsers. repo_id: MiniMaxAI/MiniMax-M2.5 example_model_input: messages: - role: system content: "You are a helpful assistant." - role: user content: "What is the meaning of life?" stream: true model: MiniMaxAI/MiniMax-M2.5 max_tokens: 32768 temperature: 0.7 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://MiniMaxAI/MiniMax-M2.5@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" ignore_patterns: - "*.md" - "*.txt" secrets: hf_access_token: null environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && SAFETENSORS_FAST_GPU=1 vllm serve /app/checkpoint/model --host 0.0.0.0 --port 8000 --served-model-name MiniMaxAI/MiniMax-M2.5 --tensor-parallel-size $GPU_COUNT --enable-expert-parallel --trust-remote-code --load-format runai_streamer --disable-log-stats --max-num-seqs 64 --max-num-batched-tokens 8192 --tool-call-parser minimax_m2 --reasoning-parser minimax_m2_append_think --enable-auto-tool-choice --enable-prefix-caching' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 resources: accelerator: H100:4 use_gpu: true runtime: predict_concurrency: 64 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 ` }, { id: "gemma-4--e2b--default", category: "llm", family: "gemma-4", familyLabel: "Gemma 4", variant: "E2B", preset: null, presetLabel: "E2B", hardware: "L4", engine: "vLLM (0.22.0-cu129 build)", context: "125K", concurrency: "8", capabilities: ["Reasoning", "Tool calling", "Multimodal (image)", "Long context"], recipePath: "/examples/models/llm/gemma-4", configYaml: `model_name: model:gemma-4-E2B-it preset:latency model_metadata: description: >- Gemma 4 multimodal instruct (E2B), OpenAI-compatible chat with vision using vLLM on L4. repo_id: google/gemma-4-E2B-it example_model_input: model: google/gemma-4-E2B-it messages: - role: user content: - type: text text: "Describe this image in one sentence." - type: image_url image_url: url: "https://picsum.photos/id/237/200/300" stream: true max_tokens: 512 temperature: 1.0 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://google/gemma-4-E2B-it@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name google/gemma-4-E2B-it --max-num-seqs 16 --max-model-len auto --limit-mm-per-prompt.image 1 --gpu-memory-utilization 0.9 --async-scheduling --trust-remote-code --enable-auto-tool-choice --enable-prefix-caching --reasoning-parser gemma4 --tool-call-parser gemma4 --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" resources: accelerator: L4 use_gpu: true runtime: health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 predict_concurrency: 8 ` }, { id: "gemma-4--e4b--default", category: "llm", family: "gemma-4", familyLabel: "Gemma 4", variant: "E4B", preset: null, presetLabel: "E4B", hardware: "H100", engine: "vLLM (0.22.0-cu129 build)", context: "125K", concurrency: "8", capabilities: ["Reasoning", "Tool calling", "Multimodal (image)", "Long context"], recipePath: "/examples/models/llm/gemma-4", configYaml: `model_name: model:gemma-4-E4B-it preset:latency model_metadata: description: >- Gemma 4 multimodal instruct (E4B), OpenAI-compatible chat with vision using vLLM on H100. repo_id: google/gemma-4-E4B-it example_model_input: model: google/gemma-4-E4B-it messages: - role: user content: - type: text text: "Describe this image in one sentence." - type: image_url image_url: url: "https://picsum.photos/id/237/200/300" stream: true max_tokens: 512 temperature: 1.0 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://google/gemma-4-E4B-it@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name google/gemma-4-E4B-it --max-num-seqs 16 --max-model-len auto --limit-mm-per-prompt.image 1 --gpu-memory-utilization 0.9 --async-scheduling --trust-remote-code --enable-auto-tool-choice --enable-prefix-caching --reasoning-parser gemma4 --tool-call-parser gemma4 --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" resources: accelerator: H100 use_gpu: true runtime: health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 predict_concurrency: 8 ` }, { id: "gemma-4--26b-a4b--default", category: "llm", family: "gemma-4", familyLabel: "Gemma 4", variant: "26B A4B", preset: null, presetLabel: "26B A4B", hardware: "H100 × 2", engine: "vLLM (0.22.0-cu129 build)", context: "256K", concurrency: "8", capabilities: ["Reasoning", "Tool calling", "Multimodal (image)", "Long context"], recipePath: "/examples/models/llm/gemma-4", configYaml: `model_name: model:gemma-4-26B-A4B-it preset:latency model_metadata: description: >- Gemma 4 multimodal instruct (26B MOE FP8 dynamique), speculative decoding Eagle3 using vLLM. repo_id: RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic example_model_input: model: google/gemma-4-26B-A4B-it messages: - role: user content: - type: text text: "Describe this image in one sentence." - type: image_url image_url: url: "https://picsum.photos/id/237/200/300" stream: true max_tokens: 512 temperature: 1.0 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://RedHatAI/gemma-4-26B-A4B-it-FP8-Dynamic@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name google/gemma-4-26B-A4B-it --max-num-seqs 16 --max-model-len auto --limit-mm-per-prompt.image 1 --gpu-memory-utilization 0.9 --enable-prefix-caching --speculative-config.model RedHatAI/gemma-4-26B-A4B-it-speculator.eagle3 --speculative-config.num_speculative_tokens 3 --speculative-config.method eagle3 --trust-remote-code --enable-auto-tool-choice --reasoning-parser gemma4 --tool-call-parser gemma4 --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" resources: accelerator: H100:2 use_gpu: true runtime: health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 predict_concurrency: 8 ` }, { id: "gemma-4--31b--latency", category: "llm", family: "gemma-4", familyLabel: "Gemma 4", variant: "31B", preset: "Latency", presetLabel: "31B Latency", hardware: "H100 × 2", engine: "vLLM (0.22.0-cu129 build)", context: "256K", concurrency: "8", capabilities: ["Reasoning", "Tool calling", "Multimodal (image)", "Long context"], recipePath: "/examples/models/llm/gemma-4", configYaml: `model_name: model:gemma-4-31B-it preset:latency model_metadata: description: >- Gemma 4 multimodal instruct (FP8), OpenAI-compatible chat with vision using vLLM. repo_id: RedHatAI/gemma-4-31B-it-FP8-block example_model_input: model: google/gemma-4-31B-it messages: - role: user content: - type: text text: "Describe this image in one sentence." - type: image_url image_url: url: "https://picsum.photos/id/237/200/300" stream: true max_tokens: 512 temperature: 1.0 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://RedHatAI/gemma-4-31B-it-FP8-block@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name google/gemma-4-31B-it --max-num-seqs 16 --max-model-len auto --limit-mm-per-prompt.image 1 --gpu-memory-utilization 0.9 --enable-prefix-caching --speculative-config.model RedHatAI/gemma-4-31B-it-speculator.eagle3 --speculative-config.num_speculative_tokens 3 --speculative-config.method eagle3 --trust-remote-code --enable-auto-tool-choice --reasoning-parser gemma4 --tool-call-parser gemma4 --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" resources: accelerator: H100:2 use_gpu: true runtime: health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 predict_concurrency: 8 ` }, { id: "gemma-4--31b--throughput", category: "llm", family: "gemma-4", familyLabel: "Gemma 4", variant: "31B", preset: "Throughput", presetLabel: "31B Throughput", hardware: "RTX_PRO_6000", engine: "vLLM 0.22.1", context: "128K", concurrency: "64", capabilities: ["Reasoning", "Tool calling", "Multimodal (image)", "Long context"], recipePath: "/examples/models/llm/gemma-4", configYaml: `model_name: model:gemma-4-31B-it preset:throughput model_metadata: description: >- Gemma 4 multimodal instruct (NVFP4), OpenAI-compatible chat with vision using vLLM on RTX PRO 6000. repo_id: nvidia/Gemma-4-31B-IT-NVFP4 example_model_input: model: google/gemma-4-31B-it messages: - role: user content: - type: text text: "Describe this image in one sentence." - type: image_url image_url: url: "https://picsum.photos/id/237/200/300" stream: true max_tokens: 512 temperature: 1.0 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.1 weights: - source: "hf://nvidia/Gemma-4-31B-IT-NVFP4@main" mount_location: "/app/checkpoint/gemma" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/gemma --tensor-parallel-size $GPU_COUNT --served-model-name google/gemma-4-31B-it --max-num-seqs 64 --max-model-len 131072 --kv-cache-dtype fp8 --enable-chunked-prefill --limit-mm-per-prompt.image 1 --enable-prefix-caching --trust-remote-code --enable-auto-tool-choice --reasoning-parser gemma4 --tool-call-parser gemma4 --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 resources: accelerator: RTX_PRO_6000 use_gpu: true runtime: health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 predict_concurrency: 64 ` }, { id: "llama-4--default--default", category: "llm", family: "llama-4", familyLabel: "Llama 4", variant: null, preset: null, presetLabel: "Default", hardware: "H100 × 4", engine: "vLLM (0.22.0-cu129 build)", context: "128K", concurrency: "256", capabilities: ["Tool calling", "Long context", "Multimodal (image)"], recipePath: "/examples/models/llm/llama-4", configYaml: `model_name: "model:llama-4-scout preset:latency" model_metadata: description: >- Llama 4 Scout 17B multimodal instruct (RedHat FP8-dynamic), long-context with TP=4 FP8 KV, OpenAI-compatible chat using vLLM. repo_id: RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic example_model_input: model: llama messages: - role: user content: "Given an array of integers nums and an integer target, return indices of the two numbers such that they add up to target. You may assume that each input would have exactly one solution, and you may not use the same element twice. You can return the answer in any order. class Solution: def twoSum(self, nums: List[int], target: int) -> List[int]:" stream: true max_tokens: 512 temperature: 0.5 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://RedHatAI/Llama-4-Scout-17B-16E-Instruct-FP8-dynamic@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" # Cap Run:ai streamer CPU buffering (10 GiB per TP worker); unbounded # buffering across 4 workers exceeds the pod memory limit (OOMKilled). RUNAI_STREAMER_MEMORY_LIMIT: "10737418240" docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --served-model-name llama --host 0.0.0.0 --port 8000 --trust-remote-code --max-model-len 131072 --tensor-parallel-size $GPU_COUNT --distributed-executor-backend mp --gpu-memory-utilization 0.95 --kv-cache-dtype fp8 --limit-mm-per-prompt.image 10 --override-generation-config.attn_temperature_tuning true --enable-prefix-caching --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 resources: accelerator: H100:4 use_gpu: true runtime: predict_concurrency: 256 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 ` }, { id: "llama-3.3--latency--default", category: "llm", family: "llama-3.3", familyLabel: "Llama 3.3", variant: "Latency", preset: null, presetLabel: "Latency", hardware: "H100 × 4", engine: "vLLM 0.26.0", context: "128K", concurrency: "128", capabilities: ["Tool calling", "Long context"], recipePath: "/examples/models/llm/llama-3.3", configYaml: `model_name: "model:llama-3.3-70b-instruct preset:latency" model_metadata: repo_id: nvidia/Llama-3.3-70B-Instruct-FP8 tags: - openai-compatible - vllm example_model_input: stream: true model: nvidia/Llama-3.3-70B-Instruct-FP8 messages: - role: user content: Tell me everything you know about optimized inference. max_tokens: 512 temperature: 0.5 base_image: image: vllm/vllm-openai:v0.26.0 docker_server: start_command: >- vllm serve /app/model_cache/llama-3-3-70b-instruct --served-model-name nvidia/Llama-3.3-70B-Instruct-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 4 --distributed-executor-backend mp --max-model-len 131072 --max-num-seqs 128 --max-num-batched-tokens 8192 --enable-chunked-prefill --enable-prefix-caching --gpu-memory-utilization 0.90 readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 secrets: hf_access_token: null weights: - source: hf://nvidia/Llama-3.3-70B-Instruct-FP8@main allow_patterns: - "*.safetensors" - "*.json" - "*.model" - tokenizer.model - "*.tiktoken" - "*.jinja" mount_location: /app/model_cache/llama-3-3-70b-instruct ignore_patterns: - original/* - "*.pth" auth_secret_name: hf_access_token resources: cpu: "4" memory: 40Gi use_gpu: true accelerator: H100:4 runtime: predict_concurrency: 128 streaming_read_timeout: 60 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 environment_variables: VLLM_LOGGING_LEVEL: INFO VLLM_ENGINE_READY_TIMEOUT_S: "3600" ` }, { id: "llama-3.3--throughput--default", category: "llm", family: "llama-3.3", familyLabel: "Llama 3.3", variant: "Throughput", preset: null, presetLabel: "Throughput", hardware: "H100 × 4", engine: "vLLM 0.26.0", context: "128K", concurrency: "256", capabilities: ["Tool calling", "Long context"], recipePath: "/examples/models/llm/llama-3.3", configYaml: `######################################################## # Throughput preset for Llama 3.3 70B Instruct (FP8), H100:4, vLLM. # # Tuned for high total token throughput on long-context workloads. # Key throughput levers compared with the latency preset: # gpu-memory-utilization 0.95 and max-num-seqs 256 for more KV cache headroom # and larger batches, plus prefix caching for shared prefixes. FP8 (ModelOpt) # weights and fp8 KV cache are auto-detected from the checkpoint. # Raising max-num-batched-tokens beyond 16384 added no further throughput. ######################################################## model_name: "model:llama-3.3-70b-instruct preset:throughput" model_metadata: repo_id: nvidia/Llama-3.3-70B-Instruct-FP8 tags: - openai-compatible - vllm example_model_input: stream: true model: nvidia/Llama-3.3-70B-Instruct-FP8 messages: - role: user content: Tell me everything you know about optimized inference. max_tokens: 512 temperature: 0.5 base_image: image: vllm/vllm-openai:v0.26.0 docker_server: start_command: >- vllm serve /app/model_cache/llama-3-3-70b-instruct --served-model-name nvidia/Llama-3.3-70B-Instruct-FP8 --host 0.0.0.0 --port 8000 --tensor-parallel-size 4 --distributed-executor-backend mp --max-model-len 131072 --max-num-seqs 256 --max-num-batched-tokens 16384 --enable-chunked-prefill --enable-prefix-caching --gpu-memory-utilization 0.95 readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 weights: - source: hf://nvidia/Llama-3.3-70B-Instruct-FP8@main allow_patterns: - "*.safetensors" - "*.json" - "*.model" - tokenizer.model - "*.tiktoken" - "*.jinja" mount_location: /app/model_cache/llama-3-3-70b-instruct ignore_patterns: - original/* - "*.pth" auth_secret_name: hf_access_token secrets: hf_access_token: null resources: cpu: "4" memory: 40Gi use_gpu: true accelerator: H100:4 runtime: predict_concurrency: 256 streaming_read_timeout: 60 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 environment_variables: VLLM_LOGGING_LEVEL: INFO VLLM_ENGINE_READY_TIMEOUT_S: "3600" ` }, { id: "llama-3.2--default--default", category: "llm", family: "llama-3.2", familyLabel: "Llama 3.2", variant: null, preset: null, presetLabel: "Default", hardware: "H100_40GB", engine: "TRT-LLM", context: "128K", concurrency: null, capabilities: ["Tool calling", "Long context"], recipePath: "/examples/models/llm/llama-3.2", configYaml: `model_metadata: example_model_input: max_tokens: 512 messages: - content: Tell me everything you know about optimized inference. role: user stream: true temperature: 0.5 tags: - openai-compatible model_name: "model:llama-3.2-3b-instruct preset:latency" python_version: py39 resources: accelerator: H100_40GB cpu: "1" memory: 10Gi use_gpu: true trt_llm: build: base_model: decoder checkpoint_repository: repo: meta-llama/Llama-3.2-3B-Instruct revision: main source: HF max_seq_len: 131072 quantization_type: fp8_kv tensor_parallel_count: 1 runtime: enable_chunked_context: true ` }, { id: "llama-3.1--default--default", category: "llm", family: "llama-3.1", familyLabel: "Llama 3.1", variant: null, preset: null, presetLabel: "Default", hardware: "B200", engine: "TRT-LLM v2", context: "128K", concurrency: "512", capabilities: ["Tool calling", "Long context"], recipePath: "/examples/models/llm/llama-3.1", configYaml: `model_name: "model:llama-3.1-8b-instruct preset:throughput" model_metadata: example_model_input: messages: - role: user content: "Write FizzBuzz in Python" stream: true model: "nvidia/Llama-3.1-8B-Instruct-FP8" max_tokens: 512 temperature: 0.5 tags: - openai-compatible resources: accelerator: B200 cpu: "1" memory: 10Gi use_gpu: true weights: - source: "hf://nvidia/Llama-3.1-8B-Instruct-FP8@main" mount_location: "/app/model_cache/trt_model" auth_secret_name: "hf_access_token" - source: "hf://yuhuili/EAGLE3-LLaMA3.1-Instruct-8B@main" mount_location: "/app/model_cache/eagle3_draft" auth_secret_name: "hf_access_token" secrets: hf_access_token: null trt_llm: build: checkpoint_repository: repo: michaelfeil/empty-model revision: main source: HF inference_stack: v2 runtime: enable_chunked_prefill: true max_batch_size: 512 max_num_tokens: 16384 max_seq_len: 131072 tensor_parallel_size: 1 served_model_name: nvidia/Llama-3.1-8B-Instruct-FP8 patch_kwargs: model_path: /app/model_cache/trt_model backend: pytorch sampler_type: TorchSampler guided_decoding_backend: xgrammar max_beam_width: 1 max_input_len: 131072 trust_remote_code: 1 cuda_graph_config: enable_padding: true max_batch_size: 512 kv_cache_config: dtype: fp8 enable_block_reuse: true free_gpu_memory_fraction: 0.9 speculative_config: decoding_type: Eagle max_draft_len: 3 speculative_model_dir: /app/model_cache/eagle3_draft eagle3_one_model: true version_overrides: v2_llm_version: null runtime: predict_concurrency: 512 ` }, { id: "nemotron-3--default--default", category: "llm", family: "nemotron-3", familyLabel: "Nemotron 3", variant: null, preset: null, presetLabel: "Default", hardware: "B200 × 4", engine: "TRT-LLM v2", context: "128K", concurrency: "32", capabilities: ["Reasoning", "Tool calling", "Agentic"], recipePath: "/examples/models/llm/nemotron-3", configYaml: `model_name: model:nemotron-3-super-120b-a12b preset:throughput model_metadata: example_model_input: model: "nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4" max_tokens: 512 messages: - role: user content: Tell me everything you know about optimized inference. stream: true temperature: 0.5 tags: - openai-compatible resources: accelerator: B200:4 cpu: "1" memory: 10Gi use_gpu: true environment_variables: PYTORCH_CUDA_ALLOC_CONF: "expandable_segments:True" TRTLLM_ENABLE_PDL: "1" BAD_TOKEN_ID_SEQ_CHECK_ENABLED: "1" ENABLE_B10_LOOKAHEAD: "0" secrets: hf_access_token: null trt_llm: inference_stack: v2 build: checkpoint_repository: repo: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 revision: main source: HF runtime_secret_name: hf_access_token runtime: enable_chunked_prefill: true max_batch_size: 32 max_num_tokens: 16384 max_seq_len: 131072 tensor_parallel_size: 4 served_model_name: nvidia/NVIDIA-Nemotron-3-Super-120B-A12B-NVFP4 patch_kwargs: reasoning_parser: nemotron3 tool_call_parser: qwen3_coder tokenizer_limit_length: 131072 arguments_as_json: true engine_config: backend: pytorch enable_chunked_prefill: true enable_iter_perf_stats: true max_batch_size: 32 max_beam_width: 1 max_input_len: 131072 max_num_tokens: 16384 max_seq_len: 131072 trust_remote_code: true moe_expert_parallel_size: 4 cuda_graph_config: enable_padding: true max_batch_size: 32 kv_cache_config: dtype: fp8 enable_block_reuse: false free_gpu_memory_fraction: 0.8 mamba_ssm_cache_dtype: float32 moe_config: backend: TRTLLM speculative_config: decoding_type: MTP num_nextn_predict_layers: 3 allow_advanced_sampling: true ` }, { id: "diffusiongemma--default--default", category: "llm", family: "diffusiongemma", familyLabel: "DiffusionGemma", variant: null, preset: null, presetLabel: "Default", hardware: "H100", engine: "vLLM (nightly-2c9c07c8... build)", context: "8K", concurrency: "8", capabilities: [], recipePath: "/examples/models/llm/diffusiongemma", configYaml: `model_name: "model:diffusiongemma-26B-A4B-it preset:latency" base_image: image: vllm/vllm-openai:nightly-2c9c07c85e56c799afffd5a671a8a0bace377a39 model_metadata: repo_id: RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic example_model_input: model: google/diffusiongemma-26B-A4B-it messages: - role: user content: Explain how diffusion language models differ from autoregressive ones. stream: true max_tokens: 512 tags: - openai-compatible weights: - source: "hf://RedHatAI/diffusiongemma-26B-A4B-it-FP8-dynamic@main" mount_location: "/app/checkpoint/diffusiongemma" auth_secret_name: "hf_access_token" build_commands: - apt-get update && apt-get install -y --no-install-recommends git ca-certificates && git clone --filter=blob:none https://github.com/vllm-project/vllm.git /opt/vllm-dgemma && cd /opt/vllm-dgemma && git checkout d25326b1fcfbcdfdc4133e7263b0d95ec31c9b87 - cd /opt/vllm-dgemma && VLLM_USE_PRECOMPILED=1 VLLM_PRECOMPILED_WHEEL_LOCATION='https://wheels.vllm.ai/3d300aecb1e6639872b698bd74ed38fb81d9603e/vllm-0.22.1rc1.dev373%2Bg3d300aecb-cp38-abi3-manylinux_2_28_x86_64.whl' pip install --no-deps --force-reinstall . docker_server: start_command: >- sh -c "vllm serve /app/checkpoint/diffusiongemma --served-model-name google/diffusiongemma-26B-A4B-it --tensor-parallel-size 1 --attention-backend TRITON_ATTN --generation-config vllm --hf-overrides.diffusion_sampler entropy_bound --hf-overrides.diffusion_entropy_bound 0.1 --diffusion-config.canvas_length 256 --enable-chunked-prefill --enable-prefix-caching --max-model-len 8192 --max-num-seqs 8 --gpu-memory-utilization 0.85 --trust-remote-code" readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_USE_V2_MODEL_RUNNER: "1" VLLM_LOGGING_LEVEL: INFO PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True resources: accelerator: H100 use_gpu: true secrets: hf_access_token: null runtime: health_checks: restart_check_delay_seconds: 300 restart_threshold_seconds: 300 stop_traffic_threshold_seconds: 120 predict_concurrency: 8 ` }, { id: "holo-3.1--default--default", category: "llm", family: "holo-3.1", familyLabel: "Holo 3.1", variant: null, preset: null, presetLabel: "Default", hardware: "H100", engine: "vLLM (0.20.2-cu129 build)", context: "256K", concurrency: "1000", capabilities: ["Agentic", "Multimodal (image)", "Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/holo-3.1", configYaml: `model_name: "model:holo-3.1-35b-a3b preset:throughput" model_metadata: description: >- Holo-3.1-35B-A3B (FP8), H Company's computer-use / GUI-agent vision-language model built on the Qwen3.6-35B-A3B MoE base. OpenAI-compatible multimodal chat with image input and native function calling, served using vLLM. repo_id: Hcompany/Holo-3.1-35B-A3B-FP8 example_model_input: model: Hcompany/Holo-3.1-35B-A3B-FP8 messages: - role: user content: - type: text text: "Describe this image in one sentence." - type: image_url image_url: url: "https://picsum.photos/id/237/200/300" stream: true max_tokens: 512 temperature: 1.0 tags: - openai-compatible - vllm - holo3.1 - fp8 - h100 - multimodal base_image: image: vllm/vllm-openai:v0.20.2-cu129 weights: - source: "hf://Hcompany/Holo-3.1-35B-A3B-FP8@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" build_commands: [] environment_variables: PYTORCH_ALLOC_CONF: "expandable_segments:True" VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name Hcompany/Holo-3.1-35B-A3B-FP8 --host 0.0.0.0 --port 8000 --gpu-memory-utilization 0.95 --max-model-len 262144 --max-num-batched-tokens 32768 --dtype auto --enable-chunked-prefill --enable-prefix-caching --max-num-seqs 512 --limit-mm-per-prompt.image 2 --reasoning-parser qwen3 --enable-auto-tool-choice --tool-call-parser qwen3_coder --trust-remote-code' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 runtime: predict_concurrency: 1000 health_checks: restart_check_delay_seconds: 1500 restart_threshold_seconds: 1500 stop_traffic_threshold_seconds: 120 resources: accelerator: H100 use_gpu: true secrets: hf_access_token: null ` }, { id: "laguna--m-1--default", category: "llm", family: "laguna", familyLabel: "Laguna", variant: "M.1", preset: null, presetLabel: "M.1", hardware: "H100 × 4", engine: "vLLM 0.21.0", context: "256K", concurrency: "64", capabilities: ["Agentic", "Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/laguna", configYaml: `model_name: "model:laguna-m.1 preset:latency" model_metadata: description: >- Laguna M.1 FP8 MoE reasoning model from Poolside, served with vLLM (H100 TP=4), OpenAI-compatible chat with tool calling and extended reasoning support. Latency-optimized: low max-num-seqs to minimize head-of-line blocking from long thinking traces. repo_id: poolside/Laguna-M.1-FP8 trust_remote_code: true tags: - openai-compatible - vllm - moe - reasoning - agentic-coding - fp8 example_model_input: model: poolside/laguna-m.1 messages: - role: user content: "Write a Python retry wrapper with exponential backoff." stream: true temperature: 1.0 top_k: 20 # --------------------------------------------------------------------------- # Base image — vLLM with Laguna support (requires vLLM >= 0.21.0) # --------------------------------------------------------------------------- base_image: image: vllm/vllm-openai:v0.21.0 python_executable_path: /usr/bin/python3 # --------------------------------------------------------------------------- # Weights — FP8 quantized checkpoint (~225 GB, fits in 4× H100 / 320 GB) # Quantization is detected automatically from the checkpoint's # quantization_config — no extra vLLM flags needed. # --------------------------------------------------------------------------- weights: - source: "hf://poolside/Laguna-M.1-FP8" mount_location: "/models/laguna-m1" environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" # --------------------------------------------------------------------------- # Docker server — vLLM OpenAI-compatible endpoint # --------------------------------------------------------------------------- docker_server: start_command: > vllm serve /models/laguna-m1 --served-model-name poolside/laguna-m.1 --host 0.0.0.0 --port 8000 --tool-call-parser poolside_v1 --reasoning-parser poolside_v1 --enable-auto-tool-choice --default-chat-template-kwargs '{"enable_thinking": true}' --tensor-parallel-size 4 --max-model-len 262144 --max-num-seqs 64 --gpu-memory-utilization 0.95 --trust-remote-code readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 # --------------------------------------------------------------------------- # Resources # FP8 ~225 GB → 4× H100 (320 GB total VRAM) with comfortable headroom # --------------------------------------------------------------------------- resources: accelerator: H100:4 cpu: "8" memory: 32Gi use_gpu: true # --------------------------------------------------------------------------- # Runtime # --------------------------------------------------------------------------- runtime: predict_concurrency: 64 health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 600 stop_traffic_threshold_seconds: 180 ` }, { id: "laguna--s-2-1--default", category: "llm", family: "laguna", familyLabel: "Laguna", variant: "S 2.1", preset: null, presetLabel: "S 2.1", hardware: "H100 × 8", engine: null, context: "256K", concurrency: null, capabilities: ["Agentic", "Reasoning", "Tool calling", "Long context"], recipePath: "/examples/models/llm/laguna", configYaml: `model_name: "model:laguna-s.2.1 preset:latency" model_metadata: description: >- Poolside Laguna S 2.1 FP8 agentic coding model served with TensorRT-LLM on eight H100 GPUs, with OpenAI-compatible chat, tool calling, reasoning, and a 262K-token context window. repo_id: poolside/Laguna-S-2.1-FP8 trust_remote_code: true tags: - openai-compatible - trt-llm - moe - reasoning - agentic-coding - fp8 example_model_input: model: poolside/Laguna-S-2.1-FP8 messages: - role: user content: "Write a Python retry wrapper with exponential backoff." stream: true max_tokens: 32768 temperature: 1.0 environment_variables: {} resources: accelerator: H100:8 use_gpu: true bis_llm: version: 0.0.1-20260601190849-691c46ce config: additional_environment_variables: Worker: BAD_TOKEN_ID_SEQ_CHECK_ENABLED: "1" LD_LIBRARY_PATH: /usr/local/mpi/lib:/src/.venv-3.12/lib/python3.12/site-packages/tensorrt_llm/libs:/src/.venv-3.12/lib/python3.12/site-packages/tensorrt_libs:/usr/local/cuda/targets/x86_64-linux/lib:/usr/local/lib/python3.12/dist-packages/torch/lib:/usr/local/lib/python3.12/dist-packages/torch_tensorrt/lib:/usr/local/cuda/compat/lib:/usr/local/nvidia/lib:/usr/local/nvidia/lib64 PATH: /src/.venv-3.12/bin:/usr/local/lib/python3.12/dist-packages/torch_tensorrt/bin:/usr/local/nvidia/bin:/usr/local/cuda/bin:/usr/local/mpi/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/local/ucx/bin:/opt/amazon/efa/bin:/opt/tensorrt/bin PYTHONPATH: /src/.venv-3.12/lib/python3.12/site-packages:/workspace/trtllm PYTORCH_CUDA_ALLOC_CONF: expandable_segments:True TRTLLM_ENABLE_PDL: "0" arguments_as_json: true b10_autoscaling_config: additional_autoscaling_config: metrics: - name: in_flight_tokens target: 130000 max_replica: 1 min_replica: 1 b10_routing_config: algo_selector: B10 router_active_replicas: 1 router_cache_miss_min_isl: 15000 router_cache_miss_weight: 0.03 router_decode_token_discount: 1.5 router_disable_snapshots_in_primary: true router_overlap_score_weight: 6.5 router_prefill_token_discount: 0.15 router_queue_threshold: 2 router_replica_sync: true router_snapshot_threshold: 200000 router_temperature: 0.015 checkpoint_name: poolside/Laguna-S-2.1-FP8 default_sampling_params: max_tokens: 32768 default_sampling_params_thinking: max_tokens: 32768 default_thinking_enabled: true engine_config: backend: pytorch cuda_graph_config: batch_sizes: - 1 - 2 - 4 - 8 enable_padding: true enable_chunked_prefill: true guided_decoding_backend: xgrammar kv_cache_config: dtype: fp8 enable_block_reuse: true enable_partial_reuse: false event_buffer_max_size: 16384 free_gpu_memory_fraction: 0.95 max_attention_window: - 262144 - 512 - 512 - 512 max_batch_size: 8 max_input_len: 262144 max_num_tokens: 16384 max_seq_len: 262144 moe_config: backend: CUTLASS trust_remote_code: true gpuTRTImage: baseten/dynamo-cache-aware-routing:maxtokclamp-20260715--trtllm-laguna-dflash-laguna-pr15666-dflash-ac9980ff16-3ee6c64a9-969251ab0@sha256:921b680c6a7c8b343ddd88ceb4099b304bcc4069d1ebd6b0f98f83651b4928c8 model_level_stop_words: - model_name: poolside/Laguna-S-2.1-FP8 model_path: /models/laguna model_path_for_tokenizer: /models/laguna reasoning_parser: laguna served_model_name: poolside/Laguna-S-2.1-FP8 served_model_name_response: poolside/Laguna-S-2.1-FP8 tensor_parallel_size: 8 tokenizer_limit_length: 262144 tokenizer_max_new_tokens_limit: 32768 tool_call_parser: laguna weights: # Pinned to the SpinQuant-rotated FP8 checkpoint the gpuTRTImage was built # for. Later upstream revisions (17cacdc6 "spinquantless FP8", 9e0b8ba6 # "RC2 1M release config") changed the quantization format and crash this # engine at weight load (MoE w3_w1 shape mismatch), looping until the # deploy times out. - source: hf://poolside/Laguna-S-2.1-FP8@610e62523e5ae1261d6b4d3ae7974479041d6027 mount_location: /models/laguna auth_secret_name: hf_access_token ` }, { id: "mellum2--default--default", category: "llm", family: "mellum2", familyLabel: "Mellum2", variant: null, preset: null, presetLabel: "Default", hardware: "H100", engine: "vLLM 0.23.0", context: "128K", concurrency: "128", capabilities: ["Tool calling", "Long context"], recipePath: "/examples/models/llm/mellum2", configYaml: `model_name: model:mellum2-12b-a2.5b-instruct preset:latency model_metadata: example_model_input: model: "JetBrains/Mellum2-12B-A2.5B-Instruct" messages: - role: user content: "Write a Python function to reverse a string." stream: true max_tokens: 4096 temperature: 0.6 top_p: 0.95 tags: - openai-compatible - code - moe base_image: image: vllm/vllm-openai:v0.23.0 weights: - source: "hf://JetBrains/Mellum2-12B-A2.5B-Instruct@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" resources: accelerator: H100 use_gpu: true runtime: predict_concurrency: 128 health_checks: startup_threshold_seconds: 1800 restart_threshold_seconds: 600 stop_traffic_threshold_seconds: 120 environment_variables: HF_HUB_ENABLE_HF_TRANSFER: "1" VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" secrets: hf_access_token: null docker_server: # No --reasoning-parser: Instruct answers directly without blocks. # Remove --enable-auto-tool-choice / --tool-call-parser if you don't need tool use. start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name JetBrains/Mellum2-12B-A2.5B-Instruct --host 0.0.0.0 --port 8000 --max-model-len auto --enable-prefix-caching --enable-auto-tool-choice --tool-call-parser hermes --trust-remote-code --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 ` }, { id: "qwen3-vl--default--default", category: "llm", family: "qwen3-vl", familyLabel: "Qwen3-VL", variant: null, preset: null, presetLabel: "Default", hardware: "RTX_PRO_6000", engine: "vLLM (0.22.0-cu129 build)", context: null, concurrency: "8", capabilities: ["Multimodal (image)", "Tool calling"], recipePath: "/examples/models/llm/qwen3-vl", configYaml: `model_name: "model:qwen3-vl-32b preset:throughput" model_metadata: description: >- Qwen3-VL-32B-Instruct (NVFP4), an OpenAI-compatible multimodal chat model with vision served using vLLM. repo_id: RedHatAI/Qwen3-VL-32B-Instruct-NVFP4 example_model_input: model: Qwen/Qwen3-VL-32B-Instruct messages: - role: user content: - type: text text: "Describe this image in one sentence." - type: image_url image_url: url: "https://picsum.photos/id/237/200/300" stream: true max_tokens: 512 temperature: 1.0 tags: - openai-compatible base_image: image: vllm/vllm-openai:v0.22.0-cu129 weights: - source: "hf://RedHatAI/Qwen3-VL-32B-Instruct-NVFP4@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" secrets: hf_access_token: null docker_server: start_command: >- sh -c 'GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/model --tensor-parallel-size $GPU_COUNT --served-model-name Qwen/Qwen3-VL-32B-Instruct --max-num-seqs 16 --max-model-len auto --limit-mm-per-prompt.image 2 --gpu-memory-utilization 0.9 --enable-prefix-caching --trust-remote-code --enable-auto-tool-choice --tool-call-parser hermes --load-format runai_streamer' readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 environment_variables: VLLM_LOGGING_LEVEL: WARNING VLLM_ENGINE_READY_TIMEOUT_S: "3600" resources: accelerator: RTX_PRO_6000 use_gpu: true runtime: health_checks: restart_check_delay_seconds: 1800 restart_threshold_seconds: 1200 stop_traffic_threshold_seconds: 120 predict_concurrency: 8 ` }, { id: "flux1--dev--default", category: "image-gen", family: "flux1", familyLabel: "FLUX.1", variant: "dev", preset: null, presetLabel: "dev", hardware: "H100_40GB", engine: null, context: null, concurrency: null, capabilities: ["Text-to-image"], recipePath: "/examples/models/image-gen/flux1", configYaml: `external_package_dirs: [] model_metadata: output_media: - json_path: "data" media_type: "image/jpeg" encoding: "base64" label: "Generated Image" example_model_input: {"prompt": 'black forest gateau cake spelling out the words "FLUX DEV", tasty, food photography, dynamic shot'} repo_id: black-forest-labs/FLUX.1-dev model_name: "model:flux1-dev preset:throughput" python_version: py311 requirements: - git+https://github.com/huggingface/diffusers.git@fc6a91e3834c35e57b398ad1c0d99f6f83557e04 - transformers>=4.0.0,<5.0.0 - accelerate - sentencepiece - protobuf weights: - source: "hf://black-forest-labs/FLUX.1-dev@main" mount_location: "/models/FLUX.1-dev" auth_secret_name: "hf_access_token" resources: accelerator: H100_40GB use_gpu: true secrets: hf_access_token: null system_packages: - ffmpeg - libsm6 - libxext6 ` }, { id: "flux1--schnell--default", category: "image-gen", family: "flux1", familyLabel: "FLUX.1", variant: "schnell", preset: null, presetLabel: "schnell", hardware: "H100_40GB", engine: null, context: null, concurrency: null, capabilities: ["Text-to-image"], recipePath: "/examples/models/image-gen/flux1", configYaml: `external_package_dirs: [] model_metadata: output_media: - json_path: "data" media_type: "image/jpeg" encoding: "base64" label: "Generated Image" example_model_input: {"prompt": 'black forest gateau cake spelling out the words "FLUX SCHNELL", tasty, food photography, dynamic shot'} repo_id: black-forest-labs/FLUX.1-schnell model_name: "model:flux1-schnell preset:throughput" python_version: py311 requirements: - git+https://github.com/huggingface/diffusers.git@fc6a91e3834c35e57b398ad1c0d99f6f83557e04 - transformers>=4.0.0,<5.0.0 - accelerate - sentencepiece - protobuf - b10-transfer weights: - source: "hf://black-forest-labs/FLUX.1-schnell@main" mount_location: "/models/FLUX.1-schnell" auth_secret_name: "hf_access_token" resources: accelerator: H100_40GB use_gpu: true secrets: hf_access_token: null system_packages: - ffmpeg - libsm6 - libxext6 ` }, { id: "krea-2-turbo--default--default", category: "image-gen", family: "krea-2-turbo", familyLabel: "Krea 2 Turbo", variant: null, preset: null, presetLabel: "Default", hardware: "H100", engine: "SGLang 1.3", context: null, concurrency: null, capabilities: ["Text-to-image"], recipePath: "/examples/models/image-gen/krea-2-turbo", configYaml: `model_name: "model:krea-2-turbo preset:lossy" base_image: image: baseten/sglang-diffusion-h100:v1.3 weights: - source: hf://krea/Krea-2-Turbo mount_location: /app/model_cache/krea-2-turbo auth_secret_name: hf_access_token docker_server: start_command: /app/start_sglang.sh readiness_endpoint: /health_generate liveness_endpoint: /health_generate predict_endpoint: /v1/images/generations server_port: 8000 model_metadata: output_media: - json_path: "data[*].b64_json" media_type: "image/png" encoding: "base64" label: "Generated Image" visual_gen: extra_sglang_args: enable_fp8: "0" enable_fp4_dit: "0" enable_cache_dit: "0" dit_cpu_offload: "false" text_encoder_cpu_offload: "false" image_encoder_cpu_offload: "false" vae_cpu_offload: "false" trust_remote_code: "false" b10_cpu_memory_saving: "0" num_gpus: "1" warmup_resolutions: "1024x1024 2048x2048" lora_path: "" example_model_input: prompt: "immense rocket launch exhaust as seen from extremely close up" n: 1 size: "1024x1024" response_format: "b64_json" resources: accelerator: H100 use_gpu: true secrets: hf_access_token: null runtime: health_checks: startup_threshold_seconds: 1200 restart_threshold_seconds: 300 stop_traffic_threshold_seconds: 300 ` }, { id: "qwen3-asr--default--default", category: "transcription", family: "qwen3-asr", familyLabel: "Qwen 3 ASR", variant: null, preset: null, presetLabel: "Default", hardware: "H100_40GB × 1", engine: "vLLM (0.22.0-cu129 build)", context: null, concurrency: "256", capabilities: ["Speech-to-text"], recipePath: "/examples/models/transcription/qwen3-asr", configYaml: `model_name: "model:qwen3-asr-1.7b preset:latency" model_metadata: repo_id: Qwen/Qwen3-ASR-1.7B example_model_input: stream: false messages: - role: user content: - type: audio_url audio_url: url: https://qianwen-res.oss-cn-beijing.aliyuncs.com/Qwen3-ASR-Repo/asr_en.wav tags: - openai-compatible secrets: hf_access_token: null weights: - source: "hf://Qwen/Qwen3-ASR-1.7B@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" base_image: image: vllm/vllm-openai:v0.22.0-cu129 docker_server: start_command: sh -c "vllm serve /app/checkpoint/model --tensor-parallel-size 1 --served-model-name Qwen/Qwen3-ASR-1.7B --gpu-memory-utilization 0.8 --host 0.0.0.0 --port 8000 --load-format runai_streamer" readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/chat/completions server_port: 8000 resources: accelerator: H100_40GB:1 cpu: "1" memory: 10Gi use_gpu: true requirements: - vllm[audio] - librosa - torch - torchaudio - pynvml - ffmpeg-python system_packages: - python3.10-venv - ffmpeg - openmpi-bin - libopenmpi-dev runtime: predict_concurrency: 256 ` }, { id: "vibevoice--default--default", category: "transcription", family: "vibevoice", familyLabel: "VibeVoice", variant: null, preset: null, presetLabel: "Default", hardware: "H100", engine: "vLLM 0.14.1", context: "32K", concurrency: "32", capabilities: ["Speech-to-text"], recipePath: "/examples/models/transcription/vibevoice", configYaml: `model_name: "model:vibevoice-asr preset:latency" python_version: py310 model_metadata: repo_id: microsoft/VibeVoice-ASR tags: - openai-compatible - audio - asr - speech-to-text example_model_input: model: vibevoice messages: - role: system content: You are a helpful assistant that transcribes audio input into text output in JSON format. - role: user content: - type: audio_url audio_url: url: https://github.com/ggerganov/whisper.cpp/raw/master/samples/jfk.wav - type: text text: Transcribe this audio. max_tokens: 64 temperature: 0.0 base_image: image: vllm/vllm-openai:v0.14.1 python_executable_path: /usr/bin/python3 environment_variables: HF_HOME: /cache/org HF_HUB_CACHE: /cache/org TRANSFORMERS_CACHE: /cache/org VIBEVOICE_FFMPEG_MAX_CONCURRENCY: "64" VLLM_MEDIA_LOADING_THREAD_COUNT: "16" PYTORCH_ALLOC_CONF: "expandable_segments:True" requirements: - transformers==4.57.6 - accelerate>=0.30.0 - safetensors - huggingface-hub>=0.23.0 - librosa>=0.10.0 - soundfile - scipy - pydub - diffusers - git+https://github.com/microsoft/VibeVoice.git@main resources: accelerator: H100 cpu: "4" memory: 32Gi use_gpu: true runtime: predict_concurrency: 32 secrets: hf_access_token: null system_packages: - ffmpeg - git # Weights are pre-downloaded at build time and mounted at /models/vibevoice-asr, # so cold starts skip the 9.2 GB HF download entirely. weights: - source: "hf://microsoft/VibeVoice-ASR@main" mount_location: "/models/vibevoice-asr" auth_secret_name: "hf_access_token" # Pass-through mode: no model.py, Truss just runs vllm serve and proxies # /predict requests to /v1/chat/completions on the container's localhost. docker_server: server_port: 8000 predict_endpoint: /v1/chat/completions readiness_endpoint: /v1/models liveness_endpoint: /v1/models start_command: | bash -c ' set -e echo "[entrypoint] Applying microsoft/VibeVoice plugin patches..." python3 /app/data/patch.py echo "[entrypoint] Generating tokenizer files..." python3 -m vllm_plugin.tools.generate_tokenizer_files --output /models/vibevoice-asr echo "[entrypoint] Starting vLLM serve..." exec vllm serve /models/vibevoice-asr \ --served-model-name vibevoice \ --trust-remote-code \ --dtype bfloat16 \ --max-num-seqs 16 \ --max-model-len 32768 \ --gpu-memory-utilization 0.85 \ --num-gpu-blocks-override 4096 \ --no-enable-prefix-caching \ --enable-chunked-prefill \ --chat-template-content-format openai \ --allowed-local-media-path /app \ --media-io-kwargs "{\"audio\": {\"target_sr\": 24000}}" \ --enforce-eager \ --skip-mm-profiling \ --host 0.0.0.0 \ --port 8000 ' ` }, { id: "voxtral--default--default", category: "transcription", family: "voxtral", familyLabel: "Voxtral", variant: null, preset: null, presetLabel: "Default", hardware: "H100", engine: "vLLM (0.22.0 custom build)", context: "10K", concurrency: "48", capabilities: ["Speech-to-text"], recipePath: "/examples/models/transcription/voxtral", configYaml: `model_name: "model:voxtral-mini-4b preset:latency" model_metadata: repo_id: mistralai/Voxtral-Mini-4B-Realtime-2602 secrets: hf_access_token: null weights: - source: "hf://mistralai/Voxtral-Mini-4B-Realtime-2602@main" mount_location: "/app/checkpoint/model" auth_secret_name: "hf_access_token" environment_variables: VLLM_CACHE_ROOT: /cache/org/vllm TORCHINDUCTOR_CACHE_DIR: /cache/org/inductor TRITON_CACHE_DIR: /cache/org/triton base_image: image: "baseten/vllm-openai:0.22.0-voxtral-realtime-fixes" docker_server: start_command: >- sh -c "vllm serve /app/checkpoint/model --tensor-parallel-size 1 --api-server-count 8 --enable-realtime-unbounded --no-enable-prefix-caching --realtime-reanchor-margin-tokens 1024 --hf-overrides '{\"text_config\": {\"sliding_window\": 4096}}' --served-model-name mistralai/Voxtral-Mini-4B-Realtime-2602 --host 0.0.0.0 --port 8000 --max-num-seqs 48 --max-model-len 10240 --compilation-config '{\"cudagraph_mode\": \"PIECEWISE\", \"cudagraph_capture_sizes\": [1, 2, 4, 8, 16, 24, 32, 48], \"max_cudagraph_capture_size\": 48}'" readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/realtime server_port: 8000 resources: accelerator: H100 cpu: "8" memory: 32Gi use_gpu: true requirements: - librosa - pynvml - ffmpeg-python - websockets system_packages: - python3.10-venv - ffmpeg - openmpi-bin - libopenmpi-dev runtime: predict_concurrency: 48 is_websocket_endpoint: true transport: kind: websocket ping_interval_seconds: null ping_timeout_seconds: null ` }, { id: "nemotron-3-embed--1b--default", category: "embedding", family: "nemotron-3-embed", familyLabel: "Nemotron 3 Embed", variant: "1B", preset: null, presetLabel: "1B", hardware: "H100", engine: "vLLM 0.25.0", context: "32K", concurrency: "16", capabilities: ["Embeddings"], recipePath: "/examples/models/embedding/nemotron-3-embed", configYaml: `# yaml-language-server: $schema=https://raw.githubusercontent.com/basetenlabs/truss/main/truss/config.schema.json model_name: model:Nemotron-3-Embed-1B preset:bf16 model_metadata: example_model_input: model: nvidia/Nemotron-3-Embed-1B-BF16 input: - "query: What is the capital of France?" - "passage: Paris is the capital of France." encoding_format: float dimensions: 2048 truncate_prompt_tokens: 32768 truncation_side: right repo_id: nvidia/Nemotron-3-Embed-1B-BF16 tags: - openai-compatible - embedding - vllm base_image: image: vllm/vllm-openai:v0.25.0 docker_server: start_command: >- vllm serve /models/nemotron-1b --host 0.0.0.0 --port 8000 --served-model-name nvidia/Nemotron-3-Embed-1B-BF16 --runner pooling --hf-overrides.is_matryoshka true --max-model-len 32768 --max-num-batched-tokens 131072 --max-num-seqs 64 --enable-chunked-prefill --gpu-memory-utilization 0.95 readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/embeddings server_port: 8000 weights: - source: "hf://nvidia/Nemotron-3-Embed-1B-BF16@c5e9806ae078a32aedc0829038410c7e94f5a748" mount_location: "/models/nemotron-1b" resources: accelerator: H100 use_gpu: true runtime: predict_concurrency: 16 health_checks: restart_threshold_seconds: 600 stop_traffic_threshold_seconds: 240 environment_variables: VLLM_LOGGING_LEVEL: INFO secrets: {} system_packages: [] requirements: [] ` }, { id: "nemotron-3-embed--8b--default", category: "embedding", family: "nemotron-3-embed", familyLabel: "Nemotron 3 Embed", variant: "8B", preset: null, presetLabel: "8B", hardware: "H100", engine: "vLLM 0.25.0", context: "32K", concurrency: "4", capabilities: ["Embeddings"], recipePath: "/examples/models/embedding/nemotron-3-embed", configYaml: `# yaml-language-server: $schema=https://raw.githubusercontent.com/basetenlabs/truss/main/truss/config.schema.json model_name: model:Nemotron-3-Embed-8B preset:bf16 model_metadata: example_model_input: model: nvidia/Nemotron-3-Embed-8B-BF16 input: - "query: What is the capital of France?" - "passage: Paris is the capital of France." encoding_format: float dimensions: 4096 truncate_prompt_tokens: 32768 truncation_side: right repo_id: nvidia/Nemotron-3-Embed-8B-BF16 tags: - openai-compatible - embedding - vllm base_image: image: vllm/vllm-openai:v0.25.0 docker_server: start_command: >- vllm serve /models/nemotron-8b --host 0.0.0.0 --port 8000 --served-model-name nvidia/Nemotron-3-Embed-8B-BF16 --runner pooling --hf-overrides.is_matryoshka true --max-model-len 32768 --max-num-batched-tokens 65536 --max-num-seqs 32 --enable-chunked-prefill --gpu-memory-utilization 0.95 readiness_endpoint: /health liveness_endpoint: /health predict_endpoint: /v1/embeddings server_port: 8000 weights: - source: "hf://nvidia/Nemotron-3-Embed-8B-BF16@c663c3bd4f1f7792928dfd91db0b136d57631fca" mount_location: "/models/nemotron-8b" resources: accelerator: H100 use_gpu: true runtime: predict_concurrency: 4 health_checks: restart_threshold_seconds: 600 stop_traffic_threshold_seconds: 240 environment_variables: VLLM_LOGGING_LEVEL: INFO secrets: {} system_packages: [] requirements: [] ` }, { id: "qwen3-embedding--0-6b--default", category: "embedding", family: "qwen3-embedding", familyLabel: "Qwen 3 Embedding", variant: "0.6B", preset: null, presetLabel: "0.6B", hardware: "L4", engine: "TRT-LLM", context: null, concurrency: null, capabilities: ["Embeddings"], recipePath: "/examples/models/embedding/qwen3-embedding", configYaml: `model_metadata: example_model_input: input: - Baseten is a fast inference provider - Embeddings let you do semantic search. model: qwen3-embedding-0.6b model_name: "model:qwen3-embedding-0.6b preset:throughput" python_version: py39 resources: accelerator: L4 cpu: '1' memory: 10Gi use_gpu: true trt_llm: build: base_model: encoder checkpoint_repository: repo: michaelfeil/Qwen3-Embedding-0.6B-auto revision: main source: HF max_num_tokens: 32768 num_builder_gpus: 1 quantization_type: fp8 runtime: webserver_default_route: /v1/embeddings ` }, { id: "qwen3-embedding--4b--default", category: "embedding", family: "qwen3-embedding", familyLabel: "Qwen 3 Embedding", variant: "4B", preset: null, presetLabel: "4B", hardware: "H100", engine: "TRT-LLM", context: null, concurrency: null, capabilities: ["Embeddings"], recipePath: "/examples/models/embedding/qwen3-embedding", configYaml: `model_metadata: example_model_input: input: - Baseten is a fast inference provider - Embeddings let you do semantic search. model: qwen3-embedding-4b model_name: "model:qwen3-embedding-4b preset:throughput" python_version: py39 resources: accelerator: H100 cpu: '1' memory: 10Gi use_gpu: true trt_llm: build: base_model: encoder checkpoint_repository: repo: michaelfeil/Qwen3-Embedding-4B-auto revision: main source: HF max_num_tokens: 32768 num_builder_gpus: 1 quantization_type: fp8 runtime: webserver_default_route: /v1/embeddings ` }, { id: "qwen3-embedding--8b--default", category: "embedding", family: "qwen3-embedding", familyLabel: "Qwen 3 Embedding", variant: "8B", preset: null, presetLabel: "8B", hardware: "H100", engine: "TRT-LLM", context: null, concurrency: null, capabilities: ["Embeddings"], recipePath: "/examples/models/embedding/qwen3-embedding", configYaml: `model_metadata: example_model_input: input: - Baseten is a fast inference provider - Embeddings let you do semantic search. model: qwen3-embedding-8b model_name: "model:qwen3-embedding-8b preset:throughput" python_version: py39 resources: accelerator: H100 cpu: '1' memory: 10Gi use_gpu: true trt_llm: build: base_model: encoder checkpoint_repository: repo: michaelfeil/Qwen3-Embedding-8B-auto revision: main source: HF max_num_tokens: 40960 num_builder_gpus: 1 quantization_type: fp8 runtime: webserver_default_route: /v1/embeddings ` }, { id: "qwen3-reranker--0-6b--default", category: "embedding", family: "qwen3-reranker", familyLabel: "Qwen 3 Reranker", variant: "0.6B", preset: null, presetLabel: "0.6B", hardware: "L4", engine: "TRT-LLM", context: null, concurrency: null, capabilities: ["Reranking", "Cross-encoder"], recipePath: "/examples/models/embedding/qwen3-reranker", configYaml: `# this file was autogenerated by \`generate_templates.py\` - please do change via template only model_metadata: example_model_input: inputs: - - Baseten is a fast inference provider - - Classify this separately. raw_scores: true truncate: true truncation_direction: Right model_name: "model:qwen3-reranker-0.6b preset:throughput" python_version: py39 resources: accelerator: L4 cpu: '1' memory: 10Gi use_gpu: true trt_llm: build: base_model: encoder checkpoint_repository: repo: michaelfeil/Qwen3-Reranker-0.6B-seq revision: main source: HF max_num_tokens: 32768 num_builder_gpus: 1 quantization_type: fp8 runtime: webserver_default_route: /predict ` }, { id: "qwen3-reranker--4b--default", category: "embedding", family: "qwen3-reranker", familyLabel: "Qwen 3 Reranker", variant: "4B", preset: null, presetLabel: "4B", hardware: "H100", engine: "TRT-LLM", context: null, concurrency: null, capabilities: ["Reranking", "Cross-encoder"], recipePath: "/examples/models/embedding/qwen3-reranker", configYaml: `# this file was autogenerated by \`generate_templates.py\` - please do change via template only model_metadata: example_model_input: inputs: - - Baseten is a fast inference provider - - Classify this separately. raw_scores: true truncate: true truncation_direction: Right model_name: "model:qwen3-reranker-4b preset:throughput" python_version: py39 resources: accelerator: H100 cpu: '1' memory: 10Gi use_gpu: true trt_llm: build: base_model: encoder checkpoint_repository: repo: michaelfeil/Qwen3-Reranker-4B-seq revision: main source: HF max_num_tokens: 32768 num_builder_gpus: 1 quantization_type: fp8 runtime: webserver_default_route: /predict ` }, { id: "qwen3-reranker--8b--default", category: "embedding", family: "qwen3-reranker", familyLabel: "Qwen 3 Reranker", variant: "8B", preset: null, presetLabel: "8B", hardware: "H100", engine: "TRT-LLM", context: null, concurrency: null, capabilities: ["Reranking", "Cross-encoder"], recipePath: "/examples/models/embedding/qwen3-reranker", configYaml: `# this file was autogenerated by \`generate_templates.py\` - please do change via template only model_metadata: example_model_input: inputs: - - Baseten is a fast inference provider - - Classify this separately. raw_scores: true truncate: true truncation_direction: Right model_name: "model:qwen3-reranker-8b preset:throughput" python_version: py39 resources: accelerator: H100 cpu: '1' memory: 10Gi use_gpu: true trt_llm: build: base_model: encoder checkpoint_repository: repo: michaelfeil/Qwen3-Reranker-8B-seq revision: main source: HF max_num_tokens: 40960 num_builder_gpus: 1 quantization_type: fp8 runtime: webserver_default_route: /predict ` }]; const initialRecipe = (RECIPES.find(r => r.category === "llm") ?? RECIPES[0]) ?? null; const [category, setCategory] = useState(initialRecipe?.category ?? "llm"); const [activeCaps, setActiveCaps] = useState([]); const [selectedId, setSelectedId] = useState(initialRecipe?.id ?? null); const [copied, setCopied] = useState(false); const visibleByCategory = useMemo(() => RECIPES.filter(r => r.category === category), [category]); const visibleAfterCaps = useMemo(() => { if (activeCaps.length === 0) return visibleByCategory; return visibleByCategory.filter(r => activeCaps.every(c => r.capabilities.includes(c))); }, [visibleByCategory, activeCaps]); const families = useMemo(() => { const seen = new Map(); for (const r of visibleAfterCaps) { if (!seen.has(r.family)) { seen.set(r.family, { family: r.family, label: r.familyLabel }); } } return [...seen.values()]; }, [visibleAfterCaps]); const selectedRecipe = useMemo(() => (visibleAfterCaps.find(r => r.id === selectedId) ?? visibleAfterCaps[0]) ?? null, [visibleAfterCaps, selectedId]); const variantsForFamily = useMemo(() => { if (!selectedRecipe) return []; return visibleAfterCaps.filter(r => r.family === selectedRecipe.family); }, [visibleAfterCaps, selectedRecipe]); const visibleCaps = useMemo(() => { const present = new Set(); for (const r of visibleByCategory) { for (const c of r.capabilities) present.add(c); } return CAPABILITIES.filter(c => present.has(c)); }, [visibleByCategory]); const deadEndCaps = useMemo(() => { const dead = new Set(); for (const cap of visibleCaps) { if (activeCaps.includes(cap)) continue; const next = [...activeCaps, cap]; const reachable = visibleByCategory.some(r => next.every(c => r.capabilities.includes(c))); if (!reachable) dead.add(cap); } return dead; }, [visibleByCategory, visibleCaps, activeCaps]); const switchCategory = id => { if (id === category) return; setCategory(id); setActiveCaps([]); const first = RECIPES.find(r => r.category === id); if (first) setSelectedId(first.id); }; const toggleCap = cap => { setActiveCaps(prev => prev.includes(cap) ? prev.filter(c => c !== cap) : [...prev, cap]); }; const pickFamily = family => { if (selectedRecipe?.family === family) return; const first = visibleAfterCaps.find(r => r.family === family); if (first) setSelectedId(first.id); }; const copy = async () => { if (!selectedRecipe) return; try { await navigator.clipboard.writeText(selectedRecipe.configYaml); setCopied(true); setTimeout(() => setCopied(false), 1600); } catch {} }; return
Category
{CATEGORIES.map(c => )}
{visibleCaps.length > 0 &&
Capability
{visibleCaps.map(cap => { const disabled = deadEndCaps.has(cap); return ; })}
} {families.length > 0 &&
Family
{families.map(f => )}
} {variantsForFamily.length > 1 &&
Preset
{variantsForFamily.map(v => )}
} {selectedRecipe ?
{selectedRecipe.familyLabel} {(selectedRecipe.variant || selectedRecipe.preset) && {selectedRecipe.presetLabel}}
{selectedRecipe.hardware && {selectedRecipe.hardware}} {selectedRecipe.engine && {selectedRecipe.engine}} {selectedRecipe.context && {selectedRecipe.context} context} {selectedRecipe.concurrency && {selectedRecipe.concurrency} concurrency} 
              {selectedRecipe.configYaml}
Open the full recipe ↗
:
No recipes match this filter. Clear a capability to see recipes again.
}
; }; Each page in this section is a deploy-ready Truss config for an open-weight model family, with hardware, engine version, quantization, and serving flags chosen for a sensible default. If your model is a fine-tune of one of these base checkpoints, [bring over your weights](#bring-over-fine-tune-weights) rather than starting from scratch. If you are new to Baseten, work through [Deploy your first model](/examples/deploy-your-first-model) first. ## Build a recipe Pick a category, narrow by capability, then choose a family and preset to see the matching `config.yaml`. Every recipe links to its full page for deploy and inference steps. ## Browse recipes by category Pick a category to land on a representative family. The sidebar lists every family under each group. Chat-completions models served with vLLM or TensorRT-LLM, including Qwen, GLM, GPT-OSS, Gemma, Llama, MiniMax, and Nemotron. Text-to-image and text-to-video diffusion models. Speech-to-text with Voxtral and Qwen3-ASR. Dense embeddings and cross-encoder rerankers served with BEI. ## Browse by capability Every family page tags its capabilities. Open a capability page to see every recipe that supports it. ## Bring over fine-tune weights Most recipes in this section work as-is with a fine-tuned version of the base model. If you have your own weights from fine-tuning one of these base checkpoints, the only parameter you usually change is the Hugging Face ID pointing at your weights. The rest of the config (engine version, parsers, prefix caching, health checks) stays as written. Interested in fine-tuning? See [training on Baseten](/training/getting-started) to set up a workspace and start a run. ### Swap the Hugging Face repo Three parameters in the config name the checkpoint. Change all three to point at your fine-tuned repo: | Parameter | Change to | | ---------------------------------------- | --------------------------------------------------------------------------------------------------------------- | | `model_metadata.repo_id` | Your fine-tuned Hugging Face repo. | | `weights.source` | The `hf://` URI for the same repo with a branch or revision, like `hf://your-org/gemma-4-31B-it-finetune@main`. | | `weights.auth_secret_name` | Keep `hf_access_token` for gated repos and add the secret in your workspace before pushing. | | `--served-model-name` in `start_command` | The `model` string your clients pass in `chat.completions.create(model=...)`. | For example, here is how you'd adapt the [Gemma 4 31B recipe](/examples/models/llm/gemma-4) to point at a fine-tune: ```diff config.yaml theme={"system"} model_metadata: - repo_id: RedHatAI/gemma-4-31B-it-FP8-block + repo_id: your-org/gemma-4-31B-it-finetune example_model_input: model: google/gemma-4-31B-it ... weights: - - source: "hf://RedHatAI/gemma-4-31B-it-FP8-block@main" + - source: "hf://your-org/gemma-4-31B-it-finetune@main" mount_location: "/app/checkpoint/gemma" auth_secret_name: "hf_access_token" docker_server: start_command: >- sh -c "GPU_COUNT=$(nvidia-smi --list-gpus | wc -l) && vllm serve /app/checkpoint/gemma --tensor-parallel-size $GPU_COUNT - --served-model-name google/gemma-4-31B-it + --served-model-name your-org/gemma-4-31B-it-finetune ..." ``` For checkpoints in S3, GCS, or mirrored through Baseten, see [Baseten Delivery Network](/development/model/bdn) for `s3://`, `gs://`, and `bdn://` source URIs. ### Match hardware to model size Pick the variant tab whose base model matches the size and architecture of your fine-tune, then copy that config. The `--tensor-parallel-size $GPU_COUNT` flag reads the GPU count at runtime, so the start command adapts automatically when you change `resources.accelerator`. See [resources](/deployment/resources) for the accelerator list. ### Drop speculative decoding unless you trained a matching speculator Several recipes ship with an EAGLE3 draft model trained against the base checkpoint: ```text config.yaml theme={"system"} --speculative-config.model RedHatAI/gemma-4-31B-it-speculator.eagle3 --speculative-config.num_speculative_tokens 3 --speculative-config.method eagle3 ``` A fine-tune shifts the output distribution, so the base speculator's acceptance rate drops and latency gets worse, not better. Remove all three `--speculative-config.*` flags unless you have trained a speculator on your fine-tune. ### Disable multimodal for text-only fine-tunes Multimodal recipes (Gemma 4, Llama 4) include `--limit-mm-per-prompt.image 1` to cap image inputs per prompt. If your fine-tune dropped the vision tower or you only need text serving, swap that flag: ```diff config.yaml theme={"system"} - --limit-mm-per-prompt.image 1 + --language-model-only ``` `--language-model-only` skips the multimodal preprocessing path and avoids loading the vision encoder weights. ## When a recipe is not the right starting point The recipes assume open-weight models served over the OpenAI-compatible API. For other cases: | Case | Start here | | ------------------------------------------------------------------ | --------------------------------------------------------------------------------------------------------------------- | | Custom inference logic, or pre- or post-processing in Python | [Custom Python models](/examples/deploy-a-llm) | | A serving stack with no recipe yet | [vLLM](/examples/vllm), [SGLang](/examples/sglang), [Ollama](/examples/ollama), or [generic Docker](/examples/docker) | | Embeddings, rerankers, or classification beyond the listed recipes | [Embeddings with BEI](/examples/bei) | | TensorRT-LLM engines from scratch | [TensorRT-LLM](/examples/tensorrt-llm) | For a fine-tune of a base model not listed here, [contact support](mailto:support@baseten.co) and we can suggest a starting recipe.