Evaluation
Batch Evaluate
High-throughput batch evaluation. Submit 1-100 conversations and receive rewards for each.
POST
/
api
/
v1
/
batch
/
evaluate
cURL
curl --request POST \
--url https://labs.tacitintelligence.co/api/v1/batch/evaluate \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data @- <<EOF
{
"items": [
{
"id": "eval_001",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "I need a refund"
},
{
"role": "assistant",
"content": "I'll process that for you right away."
}
]
},
{
"id": "eval_002",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "Where is my order?"
},
{
"role": "assistant",
"content": "Let me check the tracking for you.",
"tool_calls": [
{
"type": "artifact",
"name": "check_tracking",
"input": {
"order_id": "12345"
}
}
]
}
]
}
]
}
EOFimport requests
url = "https://labs.tacitintelligence.co/api/v1/batch/evaluate"
payload = { "items": [
{
"id": "eval_001",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "I need a refund"
},
{
"role": "assistant",
"content": "I'll process that for you right away."
}
]
},
{
"id": "eval_002",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "Where is my order?"
},
{
"role": "assistant",
"content": "Let me check the tracking for you.",
"tool_calls": [
{
"type": "artifact",
"name": "check_tracking",
"input": { "order_id": "12345" }
}
]
}
]
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
items: [
{
id: 'eval_001',
collection_slug: 'case-management',
scenario_slug: 'back-injury-claim',
messages: [
{role: 'user', content: 'I need a refund'},
{role: 'assistant', content: 'I\'ll process that for you right away.'}
]
},
{
id: 'eval_002',
collection_slug: 'case-management',
scenario_slug: 'back-injury-claim',
messages: [
{role: 'user', content: 'Where is my order?'},
{
role: 'assistant',
content: 'Let me check the tracking for you.',
tool_calls: [{type: 'artifact', name: 'check_tracking', input: {order_id: '12345'}}]
}
]
}
]
})
};
fetch('https://labs.tacitintelligence.co/api/v1/batch/evaluate', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://labs.tacitintelligence.co/api/v1/batch/evaluate",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'items' => [
[
'id' => 'eval_001',
'collection_slug' => 'case-management',
'scenario_slug' => 'back-injury-claim',
'messages' => [
[
'role' => 'user',
'content' => 'I need a refund'
],
[
'role' => 'assistant',
'content' => 'I\'ll process that for you right away.'
]
]
],
[
'id' => 'eval_002',
'collection_slug' => 'case-management',
'scenario_slug' => 'back-injury-claim',
'messages' => [
[
'role' => 'user',
'content' => 'Where is my order?'
],
[
'role' => 'assistant',
'content' => 'Let me check the tracking for you.',
'tool_calls' => [
[
'type' => 'artifact',
'name' => 'check_tracking',
'input' => [
'order_id' => '12345'
]
]
]
]
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://labs.tacitintelligence.co/api/v1/batch/evaluate"
payload := strings.NewReader("{\n \"items\": [\n {\n \"id\": \"eval_001\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I need a refund\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I'll process that for you right away.\"\n }\n ]\n },\n {\n \"id\": \"eval_002\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Where is my order?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Let me check the tracking for you.\",\n \"tool_calls\": [\n {\n \"type\": \"artifact\",\n \"name\": \"check_tracking\",\n \"input\": {\n \"order_id\": \"12345\"\n }\n }\n ]\n }\n ]\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://labs.tacitintelligence.co/api/v1/batch/evaluate")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"items\": [\n {\n \"id\": \"eval_001\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I need a refund\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I'll process that for you right away.\"\n }\n ]\n },\n {\n \"id\": \"eval_002\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Where is my order?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Let me check the tracking for you.\",\n \"tool_calls\": [\n {\n \"type\": \"artifact\",\n \"name\": \"check_tracking\",\n \"input\": {\n \"order_id\": \"12345\"\n }\n }\n ]\n }\n ]\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://labs.tacitintelligence.co/api/v1/batch/evaluate")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"items\": [\n {\n \"id\": \"eval_001\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I need a refund\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I'll process that for you right away.\"\n }\n ]\n },\n {\n \"id\": \"eval_002\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Where is my order?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Let me check the tracking for you.\",\n \"tool_calls\": [\n {\n \"type\": \"artifact\",\n \"name\": \"check_tracking\",\n \"input\": {\n \"order_id\": \"12345\"\n }\n }\n ]\n }\n ]\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"id": "eval_001",
"status": "success",
"reward": 0.65
},
{
"id": "eval_002",
"status": "success",
"reward": 0.82
}
],
"scoring_version": "2.0.0",
"total_time_ms": 438
}{
"error": "<string>",
"details": {}
}Authorizations
API key obtained from Labs Portal
Body
application/json
Array of evaluation items (1-100 items per batch)
Required array length:
1 - 100 elementsShow child attributes
Show child attributes
Response
Success
Results for each item in the batch
Each result is either a success (with reward and scores) or an error (with error message). Check the status field to distinguish - error items have no reward and must not be used for training.
- Option 1
- Option 2
Show child attributes
Show child attributes
Version of the scoring algorithm used
Total time to process the batch in milliseconds
⌘I
cURL
curl --request POST \
--url https://labs.tacitintelligence.co/api/v1/batch/evaluate \
--header 'Authorization: Bearer <token>' \
--header 'Content-Type: application/json' \
--data @- <<EOF
{
"items": [
{
"id": "eval_001",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "I need a refund"
},
{
"role": "assistant",
"content": "I'll process that for you right away."
}
]
},
{
"id": "eval_002",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "Where is my order?"
},
{
"role": "assistant",
"content": "Let me check the tracking for you.",
"tool_calls": [
{
"type": "artifact",
"name": "check_tracking",
"input": {
"order_id": "12345"
}
}
]
}
]
}
]
}
EOFimport requests
url = "https://labs.tacitintelligence.co/api/v1/batch/evaluate"
payload = { "items": [
{
"id": "eval_001",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "I need a refund"
},
{
"role": "assistant",
"content": "I'll process that for you right away."
}
]
},
{
"id": "eval_002",
"collection_slug": "case-management",
"scenario_slug": "back-injury-claim",
"messages": [
{
"role": "user",
"content": "Where is my order?"
},
{
"role": "assistant",
"content": "Let me check the tracking for you.",
"tool_calls": [
{
"type": "artifact",
"name": "check_tracking",
"input": { "order_id": "12345" }
}
]
}
]
}
] }
headers = {
"Authorization": "Bearer <token>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: 'Bearer <token>', 'Content-Type': 'application/json'},
body: JSON.stringify({
items: [
{
id: 'eval_001',
collection_slug: 'case-management',
scenario_slug: 'back-injury-claim',
messages: [
{role: 'user', content: 'I need a refund'},
{role: 'assistant', content: 'I\'ll process that for you right away.'}
]
},
{
id: 'eval_002',
collection_slug: 'case-management',
scenario_slug: 'back-injury-claim',
messages: [
{role: 'user', content: 'Where is my order?'},
{
role: 'assistant',
content: 'Let me check the tracking for you.',
tool_calls: [{type: 'artifact', name: 'check_tracking', input: {order_id: '12345'}}]
}
]
}
]
})
};
fetch('https://labs.tacitintelligence.co/api/v1/batch/evaluate', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://labs.tacitintelligence.co/api/v1/batch/evaluate",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'items' => [
[
'id' => 'eval_001',
'collection_slug' => 'case-management',
'scenario_slug' => 'back-injury-claim',
'messages' => [
[
'role' => 'user',
'content' => 'I need a refund'
],
[
'role' => 'assistant',
'content' => 'I\'ll process that for you right away.'
]
]
],
[
'id' => 'eval_002',
'collection_slug' => 'case-management',
'scenario_slug' => 'back-injury-claim',
'messages' => [
[
'role' => 'user',
'content' => 'Where is my order?'
],
[
'role' => 'assistant',
'content' => 'Let me check the tracking for you.',
'tool_calls' => [
[
'type' => 'artifact',
'name' => 'check_tracking',
'input' => [
'order_id' => '12345'
]
]
]
]
]
]
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: Bearer <token>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://labs.tacitintelligence.co/api/v1/batch/evaluate"
payload := strings.NewReader("{\n \"items\": [\n {\n \"id\": \"eval_001\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I need a refund\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I'll process that for you right away.\"\n }\n ]\n },\n {\n \"id\": \"eval_002\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Where is my order?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Let me check the tracking for you.\",\n \"tool_calls\": [\n {\n \"type\": \"artifact\",\n \"name\": \"check_tracking\",\n \"input\": {\n \"order_id\": \"12345\"\n }\n }\n ]\n }\n ]\n }\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "Bearer <token>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://labs.tacitintelligence.co/api/v1/batch/evaluate")
.header("Authorization", "Bearer <token>")
.header("Content-Type", "application/json")
.body("{\n \"items\": [\n {\n \"id\": \"eval_001\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I need a refund\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I'll process that for you right away.\"\n }\n ]\n },\n {\n \"id\": \"eval_002\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Where is my order?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Let me check the tracking for you.\",\n \"tool_calls\": [\n {\n \"type\": \"artifact\",\n \"name\": \"check_tracking\",\n \"input\": {\n \"order_id\": \"12345\"\n }\n }\n ]\n }\n ]\n }\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://labs.tacitintelligence.co/api/v1/batch/evaluate")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = 'Bearer <token>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"items\": [\n {\n \"id\": \"eval_001\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"I need a refund\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"I'll process that for you right away.\"\n }\n ]\n },\n {\n \"id\": \"eval_002\",\n \"collection_slug\": \"case-management\",\n \"scenario_slug\": \"back-injury-claim\",\n \"messages\": [\n {\n \"role\": \"user\",\n \"content\": \"Where is my order?\"\n },\n {\n \"role\": \"assistant\",\n \"content\": \"Let me check the tracking for you.\",\n \"tool_calls\": [\n {\n \"type\": \"artifact\",\n \"name\": \"check_tracking\",\n \"input\": {\n \"order_id\": \"12345\"\n }\n }\n ]\n }\n ]\n }\n ]\n}"
response = http.request(request)
puts response.read_body{
"results": [
{
"id": "eval_001",
"status": "success",
"reward": 0.65
},
{
"id": "eval_002",
"status": "success",
"reward": 0.82
}
],
"scoring_version": "2.0.0",
"total_time_ms": 438
}{
"error": "<string>",
"details": {}
}