Model APIs
Create a BIS-LLM deployment
POST
/
v1
/
llm_models
cURL
curl --request POST \
--url https://api.baseten.co/v1/llm_models \
--header "Authorization: Bearer $BASETEN_API_KEY" \
--data '{
"resources": null,
"region": null,
"llm_version": null,
"model_metadata": null,
"autoscaling_settings": {
"autoscaling_window": 600,
"concurrency_target": null,
"max_replica": 5,
"max_scale_down_rate": null,
"min_replica": 1,
"scale_down_delay": 300,
"target_in_flight_tokens": null,
"target_utilization_percentage": null
},
"additional_autoscaling_config": {
"metrics": [
{
"name": "in_flight_tokens",
"target": 40000
}
]
},
"metadata": {
"environment": "production",
"git_sha": "abc123"
},
"weights": [
{
"mount_location": "/models/base",
"source": "hf://meta-llama/Llama-3-8B"
}
],
"name": null
}'import requests
import os
API_KEY = os.environ.get("BASETEN_API_KEY", "<YOUR_API_KEY>")
url = "https://api.baseten.co/v1/llm_models"
headers = {"Authorization": f"Bearer {API_KEY}"}
response = requests.request(
"POST",
url,
headers=headers,
json={'resources': None, 'region': None, 'llm_version': None, 'model_metadata': None, 'autoscaling_settings': {'autoscaling_window': 600, 'concurrency_target': None, 'max_replica': 5, 'max_scale_down_rate': None, 'min_replica': 1, 'scale_down_delay': 300, 'target_in_flight_tokens': None, 'target_utilization_percentage': None}, 'additional_autoscaling_config': {'metrics': [{'name': 'in_flight_tokens', 'target': 40000}]}, 'metadata': {'environment': 'production', 'git_sha': 'abc123'}, 'weights': [{'mount_location': '/models/base', 'source': 'hf://meta-llama/Llama-3-8B'}], 'name': None}
)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
resources: {},
name: '<string>',
region: '<string>',
llm_version: '<string>',
llm_config: {},
environment_variables: {},
model_metadata: {},
autoscaling_settings: {
autoscaling_window: 600,
concurrency_target: null,
max_replica: 5,
max_scale_down_rate: null,
min_replica: 1,
scale_down_delay: 300,
target_in_flight_tokens: null,
target_utilization_percentage: null
},
additional_autoscaling_config: {metrics: [{name: 'in_flight_tokens', target: 40000}]},
metadata: {environment: 'production', git_sha: 'abc123'},
weights: [{mount_location: '/models/base', source: 'hf://meta-llama/Llama-3-8B'}]
})
};
fetch('https://api.baseten.co/v1/llm_models', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.baseten.co/v1/llm_models",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'resources' => [
],
'name' => '<string>',
'region' => '<string>',
'llm_version' => '<string>',
'llm_config' => [
],
'environment_variables' => [
],
'model_metadata' => [
],
'autoscaling_settings' => [
'autoscaling_window' => 600,
'concurrency_target' => null,
'max_replica' => 5,
'max_scale_down_rate' => null,
'min_replica' => 1,
'scale_down_delay' => 300,
'target_in_flight_tokens' => null,
'target_utilization_percentage' => null
],
'additional_autoscaling_config' => [
'metrics' => [
[
'name' => 'in_flight_tokens',
'target' => 40000
]
]
],
'metadata' => [
'environment' => 'production',
'git_sha' => 'abc123'
],
'weights' => [
[
'mount_location' => '/models/base',
'source' => 'hf://meta-llama/Llama-3-8B'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.baseten.co/v1/llm_models"
payload := strings.NewReader("{\n \"resources\": {},\n \"name\": \"<string>\",\n \"region\": \"<string>\",\n \"llm_version\": \"<string>\",\n \"llm_config\": {},\n \"environment_variables\": {},\n \"model_metadata\": {},\n \"autoscaling_settings\": {\n \"autoscaling_window\": 600,\n \"concurrency_target\": null,\n \"max_replica\": 5,\n \"max_scale_down_rate\": null,\n \"min_replica\": 1,\n \"scale_down_delay\": 300,\n \"target_in_flight_tokens\": null,\n \"target_utilization_percentage\": null\n },\n \"additional_autoscaling_config\": {\n \"metrics\": [\n {\n \"name\": \"in_flight_tokens\",\n \"target\": 40000\n }\n ]\n },\n \"metadata\": {\n \"environment\": \"production\",\n \"git_sha\": \"abc123\"\n },\n \"weights\": [\n {\n \"mount_location\": \"/models/base\",\n \"source\": \"hf://meta-llama/Llama-3-8B\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.baseten.co/v1/llm_models")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"resources\": {},\n \"name\": \"<string>\",\n \"region\": \"<string>\",\n \"llm_version\": \"<string>\",\n \"llm_config\": {},\n \"environment_variables\": {},\n \"model_metadata\": {},\n \"autoscaling_settings\": {\n \"autoscaling_window\": 600,\n \"concurrency_target\": null,\n \"max_replica\": 5,\n \"max_scale_down_rate\": null,\n \"min_replica\": 1,\n \"scale_down_delay\": 300,\n \"target_in_flight_tokens\": null,\n \"target_utilization_percentage\": null\n },\n \"additional_autoscaling_config\": {\n \"metrics\": [\n {\n \"name\": \"in_flight_tokens\",\n \"target\": 40000\n }\n ]\n },\n \"metadata\": {\n \"environment\": \"production\",\n \"git_sha\": \"abc123\"\n },\n \"weights\": [\n {\n \"mount_location\": \"/models/base\",\n \"source\": \"hf://meta-llama/Llama-3-8B\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.baseten.co/v1/llm_models")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"resources\": {},\n \"name\": \"<string>\",\n \"region\": \"<string>\",\n \"llm_version\": \"<string>\",\n \"llm_config\": {},\n \"environment_variables\": {},\n \"model_metadata\": {},\n \"autoscaling_settings\": {\n \"autoscaling_window\": 600,\n \"concurrency_target\": null,\n \"max_replica\": 5,\n \"max_scale_down_rate\": null,\n \"min_replica\": 1,\n \"scale_down_delay\": 300,\n \"target_in_flight_tokens\": null,\n \"target_utilization_percentage\": null\n },\n \"additional_autoscaling_config\": {\n \"metrics\": [\n {\n \"name\": \"in_flight_tokens\",\n \"target\": 40000\n }\n ]\n },\n \"metadata\": {\n \"environment\": \"production\",\n \"git_sha\": \"abc123\"\n },\n \"weights\": [\n {\n \"mount_location\": \"/models/base\",\n \"source\": \"hf://meta-llama/Llama-3-8B\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"model_id": "<string>",
"version_id": "<string>",
"hostname": "<string>",
"instance_type_name": "<string>"
}Authorizations
Send Authorization: Bearer <api_key>. The legacy Authorization: Api-Key <api_key> scheme is also accepted.
Body
application/json
A request to create a BIS-LLM model
Resources allocated to the model
Name of the model
Region in which to deploy the model
Version of the helm chart to use.
Configuration specific to the LLM model
Environment variables for the model
Model metadata persisted into model_config
Autoscaling settings for the model
Show child attributes
Show child attributes
Example:
{
"autoscaling_window": 600,
"concurrency_target": null,
"max_replica": 5,
"max_scale_down_rate": null,
"min_replica": 1,
"scale_down_delay": 300,
"target_in_flight_tokens": null,
"target_utilization_percentage": null
}
Additional autoscaling configuration (e.g. target in-flight tokens)
Example:
{
"metrics": [
{
"name": "in_flight_tokens",
"target": 40000
}
]
}
User-defined metadata for the deployment
Example:
{
"environment": "production",
"git_sha": "abc123"
}
Weight configurations for BDN model weight distribution
Example:
[
{
"mount_location": "/models/base",
"source": "hf://meta-llama/Llama-3-8B"
}
]
Response
200 - application/json
Handle for a BIS-LLM model deployment.
Was this page helpful?
⌘I
cURL
curl --request POST \
--url https://api.baseten.co/v1/llm_models \
--header "Authorization: Bearer $BASETEN_API_KEY" \
--data '{
"resources": null,
"region": null,
"llm_version": null,
"model_metadata": null,
"autoscaling_settings": {
"autoscaling_window": 600,
"concurrency_target": null,
"max_replica": 5,
"max_scale_down_rate": null,
"min_replica": 1,
"scale_down_delay": 300,
"target_in_flight_tokens": null,
"target_utilization_percentage": null
},
"additional_autoscaling_config": {
"metrics": [
{
"name": "in_flight_tokens",
"target": 40000
}
]
},
"metadata": {
"environment": "production",
"git_sha": "abc123"
},
"weights": [
{
"mount_location": "/models/base",
"source": "hf://meta-llama/Llama-3-8B"
}
],
"name": null
}'import requests
import os
API_KEY = os.environ.get("BASETEN_API_KEY", "<YOUR_API_KEY>")
url = "https://api.baseten.co/v1/llm_models"
headers = {"Authorization": f"Bearer {API_KEY}"}
response = requests.request(
"POST",
url,
headers=headers,
json={'resources': None, 'region': None, 'llm_version': None, 'model_metadata': None, 'autoscaling_settings': {'autoscaling_window': 600, 'concurrency_target': None, 'max_replica': 5, 'max_scale_down_rate': None, 'min_replica': 1, 'scale_down_delay': 300, 'target_in_flight_tokens': None, 'target_utilization_percentage': None}, 'additional_autoscaling_config': {'metrics': [{'name': 'in_flight_tokens', 'target': 40000}]}, 'metadata': {'environment': 'production', 'git_sha': 'abc123'}, 'weights': [{'mount_location': '/models/base', 'source': 'hf://meta-llama/Llama-3-8B'}], 'name': None}
)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
resources: {},
name: '<string>',
region: '<string>',
llm_version: '<string>',
llm_config: {},
environment_variables: {},
model_metadata: {},
autoscaling_settings: {
autoscaling_window: 600,
concurrency_target: null,
max_replica: 5,
max_scale_down_rate: null,
min_replica: 1,
scale_down_delay: 300,
target_in_flight_tokens: null,
target_utilization_percentage: null
},
additional_autoscaling_config: {metrics: [{name: 'in_flight_tokens', target: 40000}]},
metadata: {environment: 'production', git_sha: 'abc123'},
weights: [{mount_location: '/models/base', source: 'hf://meta-llama/Llama-3-8B'}]
})
};
fetch('https://api.baseten.co/v1/llm_models', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.baseten.co/v1/llm_models",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'resources' => [
],
'name' => '<string>',
'region' => '<string>',
'llm_version' => '<string>',
'llm_config' => [
],
'environment_variables' => [
],
'model_metadata' => [
],
'autoscaling_settings' => [
'autoscaling_window' => 600,
'concurrency_target' => null,
'max_replica' => 5,
'max_scale_down_rate' => null,
'min_replica' => 1,
'scale_down_delay' => 300,
'target_in_flight_tokens' => null,
'target_utilization_percentage' => null
],
'additional_autoscaling_config' => [
'metrics' => [
[
'name' => 'in_flight_tokens',
'target' => 40000
]
]
],
'metadata' => [
'environment' => 'production',
'git_sha' => 'abc123'
],
'weights' => [
[
'mount_location' => '/models/base',
'source' => 'hf://meta-llama/Llama-3-8B'
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.baseten.co/v1/llm_models"
payload := strings.NewReader("{\n \"resources\": {},\n \"name\": \"<string>\",\n \"region\": \"<string>\",\n \"llm_version\": \"<string>\",\n \"llm_config\": {},\n \"environment_variables\": {},\n \"model_metadata\": {},\n \"autoscaling_settings\": {\n \"autoscaling_window\": 600,\n \"concurrency_target\": null,\n \"max_replica\": 5,\n \"max_scale_down_rate\": null,\n \"min_replica\": 1,\n \"scale_down_delay\": 300,\n \"target_in_flight_tokens\": null,\n \"target_utilization_percentage\": null\n },\n \"additional_autoscaling_config\": {\n \"metrics\": [\n {\n \"name\": \"in_flight_tokens\",\n \"target\": 40000\n }\n ]\n },\n \"metadata\": {\n \"environment\": \"production\",\n \"git_sha\": \"abc123\"\n },\n \"weights\": [\n {\n \"mount_location\": \"/models/base\",\n \"source\": \"hf://meta-llama/Llama-3-8B\"\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.baseten.co/v1/llm_models")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"resources\": {},\n \"name\": \"<string>\",\n \"region\": \"<string>\",\n \"llm_version\": \"<string>\",\n \"llm_config\": {},\n \"environment_variables\": {},\n \"model_metadata\": {},\n \"autoscaling_settings\": {\n \"autoscaling_window\": 600,\n \"concurrency_target\": null,\n \"max_replica\": 5,\n \"max_scale_down_rate\": null,\n \"min_replica\": 1,\n \"scale_down_delay\": 300,\n \"target_in_flight_tokens\": null,\n \"target_utilization_percentage\": null\n },\n \"additional_autoscaling_config\": {\n \"metrics\": [\n {\n \"name\": \"in_flight_tokens\",\n \"target\": 40000\n }\n ]\n },\n \"metadata\": {\n \"environment\": \"production\",\n \"git_sha\": \"abc123\"\n },\n \"weights\": [\n {\n \"mount_location\": \"/models/base\",\n \"source\": \"hf://meta-llama/Llama-3-8B\"\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.baseten.co/v1/llm_models")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"resources\": {},\n \"name\": \"<string>\",\n \"region\": \"<string>\",\n \"llm_version\": \"<string>\",\n \"llm_config\": {},\n \"environment_variables\": {},\n \"model_metadata\": {},\n \"autoscaling_settings\": {\n \"autoscaling_window\": 600,\n \"concurrency_target\": null,\n \"max_replica\": 5,\n \"max_scale_down_rate\": null,\n \"min_replica\": 1,\n \"scale_down_delay\": 300,\n \"target_in_flight_tokens\": null,\n \"target_utilization_percentage\": null\n },\n \"additional_autoscaling_config\": {\n \"metrics\": [\n {\n \"name\": \"in_flight_tokens\",\n \"target\": 40000\n }\n ]\n },\n \"metadata\": {\n \"environment\": \"production\",\n \"git_sha\": \"abc123\"\n },\n \"weights\": [\n {\n \"mount_location\": \"/models/base\",\n \"source\": \"hf://meta-llama/Llama-3-8B\"\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"model_id": "<string>",
"version_id": "<string>",
"hostname": "<string>",
"instance_type_name": "<string>"
}