Create

1 import requests 2 3 url = "https://api.humanloop.com/v4/evaluators" 4 5 payload = { 6 "name": "name", 7 "description": "description", 8 "arguments_type": "target_free", 9 "return_type": "boolean", 10 "type": "python" 11 } 12 headers = { 13 "X-API-KEY": "<apiKey>", 14 "Content-Type": "application/json" 15 } 16 17 response = requests.post(url, json=payload, headers=headers) 18 19 print(response.json())

1 { 2 "name": "name", 3 "description": "description", 4 "arguments_type": "target_free", 5 "return_type": "boolean", 6 "type": "python", 7 "id": "id", 8 "created_at": "2024-01-15T09:30:00Z", 9 "updated_at": "2024-01-15T09:30:00Z", 10 "code": "code", 11 "model_config": { 12 "id": "id", 13 "model": "model", 14 "other": { 15 "key": "value" 16 }, 17 "name": "name", 18 "description": "description", 19 "provider": "anthropic", 20 "max_tokens": 1, 21 "temperature": 1.1, 22 "top_p": 1.1, 23 "stop": "stop", 24 "presence_penalty": 1.1, 25 "frequency_penalty": 1.1, 26 "seed": 1, 27 "response_format": { 28 "type": "json_object", 29 "json_schema": { 30 "key": "value" 31 } 32 }, 33 "reasoning_effort": "high", 34 "template_language": "default", 35 "prompt_template": "prompt_template", 36 "chat_template": [ 37 { 38 "role": "user" 39 } 40 ], 41 "tools": [ 42 { 43 "id": "id", 44 "name": "name" 45 } 46 ], 47 "endpoint": "complete", 48 "tool_configs": [ 49 { 50 "id": "id", 51 "status": "status", 52 "name": "name" 53 } 54 ] 55 }, 56 "logging_project": { 57 "id": "id", 58 "name": "name", 59 "users": [ 60 { 61 "id": "id", 62 "email_address": "email_address" 63 } 64 ], 65 "data_count": 1, 66 "feedback_types": [ 67 { 68 "type": "rating" 69 } 70 ], 71 "team_id": "team_id", 72 "created_at": "2024-01-15T09:30:00Z", 73 "updated_at": "2024-01-15T09:30:00Z", 74 "active_config": { 75 "project_id": "project_id", 76 "project_name": "project_name", 77 "created_at": "2024-01-15T09:30:00Z", 78 "updated_at": "2024-01-15T09:30:00Z", 79 "last_used": "2024-01-15T09:30:00Z", 80 "config": { 81 "type": "model", 82 "id": "id", 83 "model": "model" 84 }, 85 "num_datapoints": 1, 86 "evaluation_aggregates": [ 87 { 88 "model_config_id": "model_config_id", 89 "evaluator_id": "evaluator_id", 90 "evaluator_version_id": "evaluator_version_id" 91 } 92 ] 93 }, 94 "config_type": "generic", 95 "directory_id": "directory_id" 96 } 97 }

Create an evaluator within your organization.

Request

This endpoint expects an object.

namestringRequired>=1 character<=255 characters

The name of the evaluator.

descriptionstringRequired<=255 characters

The description of the evaluator.

arguments_typeenumRequired

Whether this evaluator is target-free or target-required.

Allowed values:

return_typeenumRequired

The type of the return value of the evaluator.

Allowed values:

typeenumRequired

The type of the evaluator.

Allowed values:

codestringOptional

The code for the evaluator. This code will be executed in a sandboxed environment.

model_configobjectOptional

The model configuration used to generate.

Response

Successful Response

namestring>=1 character<=255 characters

The name of the evaluator.

descriptionstring<=255 characters

The description of the evaluator.

arguments_typeenum

Whether this evaluator is target-free or target-required.

Allowed values:

return_typeenum

The type of the return value of the evaluator.

Allowed values:

typeenum

The type of the evaluator.

Allowed values:

idstring

Unique ID for the evaluator. Starts with evfn_.

created_atdatetime

updated_atdatetime

codestring or null

The code for the evaluator. This code will be executed in a sandboxed environment.

model_configobject or null

The model config defining the LLM evaluator.

logging_projectobject or null

The project where the evaluator logs are stored.

Headers

Request

Response

Errors