Evaluation
Evaluate
Single-shot evaluation endpoint. Submit a complete conversation and receive a reward score.
POST
/
api
/
v1
/
evaluate
cURL
curl --request POST \
--url https://labs.tacitintelligence.co/api/v1/evaluate \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"collection_slug": "<string>",
"scenario_slug": "<string>",
"messages": [
{
"content": "<string>",
"tool_calls": [
{
"name": "<string>",
"input": {}
}
]
}
],
"normalization_key": "<string>",
"normalization_windows": [
"<string>"
],
"reward_config": {
"quality_weights": {
"success_metrics": 0.5,
"failure_metrics": 0.5,
"best_practices": 0.5,
"rubrics": 0.5,
"discovery": 0.5,
"output_similarity": 0.5,
"decision_match": 0.5
},
"skip_safety_gate": true
}
}
'import requests
url = "https://labs.tacitintelligence.co/api/v1/evaluate"
payload = {
"collection_slug": "<string>",
"scenario_slug": "<string>",
"messages": [
{
"content": "<string>",
"tool_calls": [
{
"name": "<string>",
"input": {}
}
]
}
],
"normalization_key": "<string>",
"normalization_windows": ["<string>"],
"reward_config": {
"quality_weights": {
"success_metrics": 0.5,
"failure_metrics": 0.5,
"best_practices": 0.5,
"rubrics": 0.5,
"discovery": 0.5,
"output_similarity": 0.5,
"decision_match": 0.5
},
"skip_safety_gate": True
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
collection_slug: '<string>',
scenario_slug: '<string>',
messages: [{content: '<string>', tool_calls: [{name: '<string>', input: {}}]}],
normalization_key: '<string>',
normalization_windows: ['<string>'],
reward_config: {
quality_weights: {
success_metrics: 0.5,
failure_metrics: 0.5,
best_practices: 0.5,
rubrics: 0.5,
discovery: 0.5,
output_similarity: 0.5,
decision_match: 0.5
},
skip_safety_gate: true
}
})
};
fetch('https://labs.tacitintelligence.co/api/v1/evaluate', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://labs.tacitintelligence.co/api/v1/evaluate",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'collection_slug' => '<string>',
'scenario_slug' => '<string>',
'messages' => [
[
'content' => '<string>',
'tool_calls' => [
[
'name' => '<string>',
'input' => [
]
]
]
]
],
'normalization_key' => '<string>',
'normalization_windows' => [
'<string>'
],
'reward_config' => [
'quality_weights' => [
'success_metrics' => 0.5,
'failure_metrics' => 0.5,
'best_practices' => 0.5,
'rubrics' => 0.5,
'discovery' => 0.5,
'output_similarity' => 0.5,
'decision_match' => 0.5
],
'skip_safety_gate' => true
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://labs.tacitintelligence.co/api/v1/evaluate"
payload := strings.NewReader("{\n \"collection_slug\": \"<string>\",\n \"scenario_slug\": \"<string>\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_calls\": [\n {\n \"name\": \"<string>\",\n \"input\": {}\n }\n ]\n }\n ],\n \"normalization_key\": \"<string>\",\n \"normalization_windows\": [\n \"<string>\"\n ],\n \"reward_config\": {\n \"quality_weights\": {\n \"success_metrics\": 0.5,\n \"failure_metrics\": 0.5,\n \"best_practices\": 0.5,\n \"rubrics\": 0.5,\n \"discovery\": 0.5,\n \"output_similarity\": 0.5,\n \"decision_match\": 0.5\n },\n \"skip_safety_gate\": true\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://labs.tacitintelligence.co/api/v1/evaluate")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"collection_slug\": \"<string>\",\n \"scenario_slug\": \"<string>\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_calls\": [\n {\n \"name\": \"<string>\",\n \"input\": {}\n }\n ]\n }\n ],\n \"normalization_key\": \"<string>\",\n \"normalization_windows\": [\n \"<string>\"\n ],\n \"reward_config\": {\n \"quality_weights\": {\n \"success_metrics\": 0.5,\n \"failure_metrics\": 0.5,\n \"best_practices\": 0.5,\n \"rubrics\": 0.5,\n \"discovery\": 0.5,\n \"output_similarity\": 0.5,\n \"decision_match\": 0.5\n },\n \"skip_safety_gate\": true\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://labs.tacitintelligence.co/api/v1/evaluate")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"collection_slug\": \"<string>\",\n \"scenario_slug\": \"<string>\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_calls\": [\n {\n \"name\": \"<string>\",\n \"input\": {}\n }\n ]\n }\n ],\n \"normalization_key\": \"<string>\",\n \"normalization_windows\": [\n \"<string>\"\n ],\n \"reward_config\": {\n \"quality_weights\": {\n \"success_metrics\": 0.5,\n \"failure_metrics\": 0.5,\n \"best_practices\": 0.5,\n \"rubrics\": 0.5,\n \"discovery\": 0.5,\n \"output_similarity\": 0.5,\n \"decision_match\": 0.5\n },\n \"skip_safety_gate\": true\n }\n}"
response = http.request(request)
puts response.read_body{
"reward": 0.5,
"scoring_version": "<string>",
"scores": {},
"score_breakdown": {},
"normalized": {},
"metadata": {
"scenario_name": "<string>",
"evaluation_time_ms": 123
}
}{
"error": "<string>",
"details": {}
}Authorizations
API key obtained from Labs Portal
Body
application/json
The collection slug containing the scenario
The scenario slug to evaluate against
The conversation history to evaluate
Show child attributes
Show child attributes
Optional key used to group evaluations in scoring logs (e.g. training run ID, experiment name)
Windows for z-normalization (e.g. "7d", "30d", "100n"). Requires normalization_key. Max 5 windows.
Maximum array length:
5Window string: number + unit (d=days, w=weeks, m=months, n=last N evaluations)
Pattern:
^\d+[dwmn]$Reward configuration. Controls which scorers run and their weights. Omitted quality weight keys default to 0, so only specified scorers execute.
Show child attributes
Show child attributes
Response
Success
Reward score (0 to 1)
Required range:
0 <= x <= 1Version of the scoring algorithm used
Optional breakdown of individual score dimensions. When omitted, only the combined reward is provided.
Show child attributes
Show child attributes
Textual reasoning per score dimension. Only included for subscriptions with full score access.
Show child attributes
Show child attributes
Z-normalized scores per requested time window. Only present when normalization_key and normalization_windows are provided.
Show child attributes
Show child attributes
Additional evaluation metadata
Show child attributes
Show child attributes
Previous
Batch EvaluateHigh-throughput batch evaluation. Submit 1-100 conversations and receive rewards for each.
Next
⌘I
cURL
curl --request POST \
--url https://labs.tacitintelligence.co/api/v1/evaluate \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data '
{
"collection_slug": "<string>",
"scenario_slug": "<string>",
"messages": [
{
"content": "<string>",
"tool_calls": [
{
"name": "<string>",
"input": {}
}
]
}
],
"normalization_key": "<string>",
"normalization_windows": [
"<string>"
],
"reward_config": {
"quality_weights": {
"success_metrics": 0.5,
"failure_metrics": 0.5,
"best_practices": 0.5,
"rubrics": 0.5,
"discovery": 0.5,
"output_similarity": 0.5,
"decision_match": 0.5
},
"skip_safety_gate": true
}
}
'import requests
url = "https://labs.tacitintelligence.co/api/v1/evaluate"
payload = {
"collection_slug": "<string>",
"scenario_slug": "<string>",
"messages": [
{
"content": "<string>",
"tool_calls": [
{
"name": "<string>",
"input": {}
}
]
}
],
"normalization_key": "<string>",
"normalization_windows": ["<string>"],
"reward_config": {
"quality_weights": {
"success_metrics": 0.5,
"failure_metrics": 0.5,
"best_practices": 0.5,
"rubrics": 0.5,
"discovery": 0.5,
"output_similarity": 0.5,
"decision_match": 0.5
},
"skip_safety_gate": True
}
}
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
collection_slug: '<string>',
scenario_slug: '<string>',
messages: [{content: '<string>', tool_calls: [{name: '<string>', input: {}}]}],
normalization_key: '<string>',
normalization_windows: ['<string>'],
reward_config: {
quality_weights: {
success_metrics: 0.5,
failure_metrics: 0.5,
best_practices: 0.5,
rubrics: 0.5,
discovery: 0.5,
output_similarity: 0.5,
decision_match: 0.5
},
skip_safety_gate: true
}
})
};
fetch('https://labs.tacitintelligence.co/api/v1/evaluate', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://labs.tacitintelligence.co/api/v1/evaluate",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'collection_slug' => '<string>',
'scenario_slug' => '<string>',
'messages' => [
[
'content' => '<string>',
'tool_calls' => [
[
'name' => '<string>',
'input' => [
]
]
]
]
],
'normalization_key' => '<string>',
'normalization_windows' => [
'<string>'
],
'reward_config' => [
'quality_weights' => [
'success_metrics' => 0.5,
'failure_metrics' => 0.5,
'best_practices' => 0.5,
'rubrics' => 0.5,
'discovery' => 0.5,
'output_similarity' => 0.5,
'decision_match' => 0.5
],
'skip_safety_gate' => true
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://labs.tacitintelligence.co/api/v1/evaluate"
payload := strings.NewReader("{\n \"collection_slug\": \"<string>\",\n \"scenario_slug\": \"<string>\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_calls\": [\n {\n \"name\": \"<string>\",\n \"input\": {}\n }\n ]\n }\n ],\n \"normalization_key\": \"<string>\",\n \"normalization_windows\": [\n \"<string>\"\n ],\n \"reward_config\": {\n \"quality_weights\": {\n \"success_metrics\": 0.5,\n \"failure_metrics\": 0.5,\n \"best_practices\": 0.5,\n \"rubrics\": 0.5,\n \"discovery\": 0.5,\n \"output_similarity\": 0.5,\n \"decision_match\": 0.5\n },\n \"skip_safety_gate\": true\n }\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://labs.tacitintelligence.co/api/v1/evaluate")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"collection_slug\": \"<string>\",\n \"scenario_slug\": \"<string>\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_calls\": [\n {\n \"name\": \"<string>\",\n \"input\": {}\n }\n ]\n }\n ],\n \"normalization_key\": \"<string>\",\n \"normalization_windows\": [\n \"<string>\"\n ],\n \"reward_config\": {\n \"quality_weights\": {\n \"success_metrics\": 0.5,\n \"failure_metrics\": 0.5,\n \"best_practices\": 0.5,\n \"rubrics\": 0.5,\n \"discovery\": 0.5,\n \"output_similarity\": 0.5,\n \"decision_match\": 0.5\n },\n \"skip_safety_gate\": true\n }\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://labs.tacitintelligence.co/api/v1/evaluate")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"collection_slug\": \"<string>\",\n \"scenario_slug\": \"<string>\",\n \"messages\": [\n {\n \"content\": \"<string>\",\n \"tool_calls\": [\n {\n \"name\": \"<string>\",\n \"input\": {}\n }\n ]\n }\n ],\n \"normalization_key\": \"<string>\",\n \"normalization_windows\": [\n \"<string>\"\n ],\n \"reward_config\": {\n \"quality_weights\": {\n \"success_metrics\": 0.5,\n \"failure_metrics\": 0.5,\n \"best_practices\": 0.5,\n \"rubrics\": 0.5,\n \"discovery\": 0.5,\n \"output_similarity\": 0.5,\n \"decision_match\": 0.5\n },\n \"skip_safety_gate\": true\n }\n}"
response = http.request(request)
puts response.read_body{
"reward": 0.5,
"scoring_version": "<string>",
"scores": {},
"score_breakdown": {},
"normalized": {},
"metadata": {
"scenario_name": "<string>",
"evaluation_time_ms": 123
}
}{
"error": "<string>",
"details": {}
}