Create custom evaluator
curl --request POST \
--url https://api.traceloop.com/v2/evaluators/custom-evaluator \
--header 'Authorization: <api-key>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"provider": "<string>",
"model": "<string>",
"messages": [
{
"role": "<string>",
"content": "<string>"
}
],
"input_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": [
"<string>"
]
}
],
"output_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": [
"<string>"
]
}
],
"slug": "<string>",
"description": "<string>",
"temperature": 123,
"top_p": 123,
"max_tokens": 123,
"frequency_penalty": 123,
"presence_penalty": 123,
"stop": [
"<string>"
]
}
'import requests
url = "https://api.traceloop.com/v2/evaluators/custom-evaluator"
payload = {
"name": "<string>",
"provider": "<string>",
"model": "<string>",
"messages": [
{
"role": "<string>",
"content": "<string>"
}
],
"input_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": ["<string>"]
}
],
"output_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": ["<string>"]
}
],
"slug": "<string>",
"description": "<string>",
"temperature": 123,
"top_p": 123,
"max_tokens": 123,
"frequency_penalty": 123,
"presence_penalty": 123,
"stop": ["<string>"]
}
headers = {
"Authorization": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
provider: '<string>',
model: '<string>',
messages: [{role: '<string>', content: '<string>'}],
input_schema: [
{
name: '<string>',
description: '<string>',
label: '<string>',
enum_values: ['<string>']
}
],
output_schema: [
{
name: '<string>',
description: '<string>',
label: '<string>',
enum_values: ['<string>']
}
],
slug: '<string>',
description: '<string>',
temperature: 123,
top_p: 123,
max_tokens: 123,
frequency_penalty: 123,
presence_penalty: 123,
stop: ['<string>']
})
};
fetch('https://api.traceloop.com/v2/evaluators/custom-evaluator', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.traceloop.com/v2/evaluators/custom-evaluator",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'provider' => '<string>',
'model' => '<string>',
'messages' => [
[
'role' => '<string>',
'content' => '<string>'
]
],
'input_schema' => [
[
'name' => '<string>',
'description' => '<string>',
'label' => '<string>',
'enum_values' => [
'<string>'
]
]
],
'output_schema' => [
[
'name' => '<string>',
'description' => '<string>',
'label' => '<string>',
'enum_values' => [
'<string>'
]
]
],
'slug' => '<string>',
'description' => '<string>',
'temperature' => 123,
'top_p' => 123,
'max_tokens' => 123,
'frequency_penalty' => 123,
'presence_penalty' => 123,
'stop' => [
'<string>'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: <api-key>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.traceloop.com/v2/evaluators/custom-evaluator"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"provider\": \"<string>\",\n \"model\": \"<string>\",\n \"messages\": [\n {\n \"role\": \"<string>\",\n \"content\": \"<string>\"\n }\n ],\n \"input_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"output_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"slug\": \"<string>\",\n \"description\": \"<string>\",\n \"temperature\": 123,\n \"top_p\": 123,\n \"max_tokens\": 123,\n \"frequency_penalty\": 123,\n \"presence_penalty\": 123,\n \"stop\": [\n \"<string>\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.traceloop.com/v2/evaluators/custom-evaluator")
.header("Authorization", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"provider\": \"<string>\",\n \"model\": \"<string>\",\n \"messages\": [\n {\n \"role\": \"<string>\",\n \"content\": \"<string>\"\n }\n ],\n \"input_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"output_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"slug\": \"<string>\",\n \"description\": \"<string>\",\n \"temperature\": 123,\n \"top_p\": 123,\n \"max_tokens\": 123,\n \"frequency_penalty\": 123,\n \"presence_penalty\": 123,\n \"stop\": [\n \"<string>\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.traceloop.com/v2/evaluators/custom-evaluator")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"provider\": \"<string>\",\n \"model\": \"<string>\",\n \"messages\": [\n {\n \"role\": \"<string>\",\n \"content\": \"<string>\"\n }\n ],\n \"input_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"output_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"slug\": \"<string>\",\n \"description\": \"<string>\",\n \"temperature\": 123,\n \"top_p\": 123,\n \"max_tokens\": 123,\n \"frequency_penalty\": 123,\n \"presence_penalty\": 123,\n \"stop\": [\n \"<string>\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"evaluator_id": "<string>",
"version_id": "<string>",
"slug": "<string>",
"type": "<string>"
}{
"error": "error message"
}{
"error": "error message"
}Evaluators
Create custom evaluator
A custom evaluator requires three main components: a prompt, an input schema, and an output schema.
Prompt
The prompt serves as an LLM-as-a-judge, instructing the model to evaluate your data against the criteria you define. Use Jinja2 templating syntax {{var_name}} to reference variables from your input schema.
Input Schema
Defines the variables available to your prompt template. Each variable declared here can be referenced in your prompt using {{var_name}}.
Output Schema
Defines the structure of the evaluation result. The model returns its assessment as structured output matching this schema.
Additionally, you’ll specify an LLM provider, model, and any provider-specific settings.
POST
/
v2
/
evaluators
/
custom-evaluator
Create custom evaluator
curl --request POST \
--url https://api.traceloop.com/v2/evaluators/custom-evaluator \
--header 'Authorization: <api-key>' \
--header 'Content-Type: application/json' \
--data '
{
"name": "<string>",
"provider": "<string>",
"model": "<string>",
"messages": [
{
"role": "<string>",
"content": "<string>"
}
],
"input_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": [
"<string>"
]
}
],
"output_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": [
"<string>"
]
}
],
"slug": "<string>",
"description": "<string>",
"temperature": 123,
"top_p": 123,
"max_tokens": 123,
"frequency_penalty": 123,
"presence_penalty": 123,
"stop": [
"<string>"
]
}
'import requests
url = "https://api.traceloop.com/v2/evaluators/custom-evaluator"
payload = {
"name": "<string>",
"provider": "<string>",
"model": "<string>",
"messages": [
{
"role": "<string>",
"content": "<string>"
}
],
"input_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": ["<string>"]
}
],
"output_schema": [
{
"name": "<string>",
"description": "<string>",
"label": "<string>",
"enum_values": ["<string>"]
}
],
"slug": "<string>",
"description": "<string>",
"temperature": 123,
"top_p": 123,
"max_tokens": 123,
"frequency_penalty": 123,
"presence_penalty": 123,
"stop": ["<string>"]
}
headers = {
"Authorization": "<api-key>",
"Content-Type": "application/json"
}
response = requests.post(url, json=payload, headers=headers)
print(response.text)const options = {
method: 'POST',
headers: {Authorization: '<api-key>', 'Content-Type': 'application/json'},
body: JSON.stringify({
name: '<string>',
provider: '<string>',
model: '<string>',
messages: [{role: '<string>', content: '<string>'}],
input_schema: [
{
name: '<string>',
description: '<string>',
label: '<string>',
enum_values: ['<string>']
}
],
output_schema: [
{
name: '<string>',
description: '<string>',
label: '<string>',
enum_values: ['<string>']
}
],
slug: '<string>',
description: '<string>',
temperature: 123,
top_p: 123,
max_tokens: 123,
frequency_penalty: 123,
presence_penalty: 123,
stop: ['<string>']
})
};
fetch('https://api.traceloop.com/v2/evaluators/custom-evaluator', options)
.then(res => res.json())
.then(res => console.log(res))
.catch(err => console.error(err));<?php
$curl = curl_init();
curl_setopt_array($curl, [
CURLOPT_URL => "https://api.traceloop.com/v2/evaluators/custom-evaluator",
CURLOPT_RETURNTRANSFER => true,
CURLOPT_ENCODING => "",
CURLOPT_MAXREDIRS => 10,
CURLOPT_TIMEOUT => 30,
CURLOPT_HTTP_VERSION => CURL_HTTP_VERSION_1_1,
CURLOPT_CUSTOMREQUEST => "POST",
CURLOPT_POSTFIELDS => json_encode([
'name' => '<string>',
'provider' => '<string>',
'model' => '<string>',
'messages' => [
[
'role' => '<string>',
'content' => '<string>'
]
],
'input_schema' => [
[
'name' => '<string>',
'description' => '<string>',
'label' => '<string>',
'enum_values' => [
'<string>'
]
]
],
'output_schema' => [
[
'name' => '<string>',
'description' => '<string>',
'label' => '<string>',
'enum_values' => [
'<string>'
]
]
],
'slug' => '<string>',
'description' => '<string>',
'temperature' => 123,
'top_p' => 123,
'max_tokens' => 123,
'frequency_penalty' => 123,
'presence_penalty' => 123,
'stop' => [
'<string>'
]
]),
CURLOPT_HTTPHEADER => [
"Authorization: <api-key>",
"Content-Type: application/json"
],
]);
$response = curl_exec($curl);
$err = curl_error($curl);
curl_close($curl);
if ($err) {
echo "cURL Error #:" . $err;
} else {
echo $response;
}package main
import (
"fmt"
"strings"
"net/http"
"io"
)
func main() {
url := "https://api.traceloop.com/v2/evaluators/custom-evaluator"
payload := strings.NewReader("{\n \"name\": \"<string>\",\n \"provider\": \"<string>\",\n \"model\": \"<string>\",\n \"messages\": [\n {\n \"role\": \"<string>\",\n \"content\": \"<string>\"\n }\n ],\n \"input_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"output_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"slug\": \"<string>\",\n \"description\": \"<string>\",\n \"temperature\": 123,\n \"top_p\": 123,\n \"max_tokens\": 123,\n \"frequency_penalty\": 123,\n \"presence_penalty\": 123,\n \"stop\": [\n \"<string>\"\n ]\n}")
req, _ := http.NewRequest("POST", url, payload)
req.Header.Add("Authorization", "<api-key>")
req.Header.Add("Content-Type", "application/json")
res, _ := http.DefaultClient.Do(req)
defer res.Body.Close()
body, _ := io.ReadAll(res.Body)
fmt.Println(string(body))
}HttpResponse<String> response = Unirest.post("https://api.traceloop.com/v2/evaluators/custom-evaluator")
.header("Authorization", "<api-key>")
.header("Content-Type", "application/json")
.body("{\n \"name\": \"<string>\",\n \"provider\": \"<string>\",\n \"model\": \"<string>\",\n \"messages\": [\n {\n \"role\": \"<string>\",\n \"content\": \"<string>\"\n }\n ],\n \"input_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"output_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"slug\": \"<string>\",\n \"description\": \"<string>\",\n \"temperature\": 123,\n \"top_p\": 123,\n \"max_tokens\": 123,\n \"frequency_penalty\": 123,\n \"presence_penalty\": 123,\n \"stop\": [\n \"<string>\"\n ]\n}")
.asString();require 'uri'
require 'net/http'
url = URI("https://api.traceloop.com/v2/evaluators/custom-evaluator")
http = Net::HTTP.new(url.host, url.port)
http.use_ssl = true
request = Net::HTTP::Post.new(url)
request["Authorization"] = '<api-key>'
request["Content-Type"] = 'application/json'
request.body = "{\n \"name\": \"<string>\",\n \"provider\": \"<string>\",\n \"model\": \"<string>\",\n \"messages\": [\n {\n \"role\": \"<string>\",\n \"content\": \"<string>\"\n }\n ],\n \"input_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"output_schema\": [\n {\n \"name\": \"<string>\",\n \"description\": \"<string>\",\n \"label\": \"<string>\",\n \"enum_values\": [\n \"<string>\"\n ]\n }\n ],\n \"slug\": \"<string>\",\n \"description\": \"<string>\",\n \"temperature\": 123,\n \"top_p\": 123,\n \"max_tokens\": 123,\n \"frequency_penalty\": 123,\n \"presence_penalty\": 123,\n \"stop\": [\n \"<string>\"\n ]\n}"
response = http.request(request)
puts response.read_body{
"evaluator_id": "<string>",
"version_id": "<string>",
"slug": "<string>",
"type": "<string>"
}{
"error": "error message"
}{
"error": "error message"
}Authorizations
Type "Bearer" followed by a space and JWT token.
Body
application/json
Custom evaluator configuration
Display name of the evaluator
LLM provider (e.g., openai, anthropic)
Model to use for evaluation
Prompt messages for the LLM judge
Show child attributes
Show child attributes
Schema defining evaluator inputs
Show child attributes
Show child attributes
Schema defining evaluator outputs
Show child attributes
Show child attributes
URL-safe identifier (auto-generated if not provided)
Description of what the evaluator does
Temperature setting for the LLM
Top P (nucleus sampling) setting
Maximum tokens in the response
Frequency penalty to reduce repetition
Presence penalty for topic diversity
Stop sequences
Was this page helpful?
⌘I