Add Evaluators

1 import requests 2 3 url = "https://api.humanloop.com/v4/evaluations/id/evaluators" 4 5 payload = {} 6 headers = { 7 "X-API-KEY": "<apiKey>", 8 "Content-Type": "application/json" 9 } 10 11 response = requests.patch(url, json=payload, headers=headers) 12 13 print(response.json())

1 { 2 "id": "id", 3 "status": "pending", 4 "config": { 5 "type": "model", 6 "id": "id", 7 "model": "model", 8 "chat_template": [ 9 { 10 "role": "user" 11 } 12 ], 13 "description": "description", 14 "endpoint": "complete", 15 "frequency_penalty": 1.1, 16 "max_tokens": 1, 17 "name": "name", 18 "other": { 19 "key": "value" 20 }, 21 "presence_penalty": 1.1, 22 "prompt_template": "prompt_template", 23 "provider": "openai", 24 "reasoning_effort": "high", 25 "response_format": { 26 "type": "json_object", 27 "json_schema": { 28 "key": "value" 29 } 30 }, 31 "seed": 1, 32 "stop": "stop", 33 "temperature": 1.1, 34 "template_language": "default", 35 "tools": [ 36 { 37 "id": "id", 38 "name": "name" 39 } 40 ], 41 "top_p": 1.1, 42 "tool_configs": [ 43 { 44 "id": "id", 45 "status": "status", 46 "name": "name" 47 } 48 ] 49 }, 50 "created_at": "2024-01-15T09:30:00Z", 51 "updated_at": "2024-01-15T09:30:00Z", 52 "evaluators": [ 53 { 54 "name": "name", 55 "description": "description", 56 "arguments_type": "target_free", 57 "return_type": "boolean", 58 "type": "python", 59 "id": "id", 60 "created_at": "2024-01-15T09:30:00Z", 61 "updated_at": "2024-01-15T09:30:00Z", 62 "code": "code", 63 "model_config": { 64 "id": "id", 65 "model": "model" 66 }, 67 "logging_project": { 68 "id": "id", 69 "name": "name", 70 "users": [ 71 { 72 "id": "id", 73 "email_address": "email_address" 74 } 75 ], 76 "data_count": 1, 77 "feedback_types": [ 78 { 79 "type": "rating" 80 } 81 ], 82 "team_id": "team_id", 83 "created_at": "2024-01-15T09:30:00Z", 84 "updated_at": "2024-01-15T09:30:00Z" 85 } 86 } 87 ], 88 "dataset": { 89 "id": "id", 90 "name": "name", 91 "datapoint_count": 1, 92 "created_at": "2024-01-15T09:30:00Z", 93 "updated_at": "2024-01-15T09:30:00Z", 94 "project_id": "project_id", 95 "description": "description" 96 }, 97 "dataset_version_id": "dataset_version_id", 98 "dataset_snapshot": { 99 "id": "id", 100 "name": "name", 101 "datapoint_count": 1, 102 "created_at": "2024-01-15T09:30:00Z", 103 "updated_at": "2024-01-15T09:30:00Z", 104 "project_id": "project_id", 105 "description": "description" 106 }, 107 "evaluator_aggregates": [ 108 { 109 "model_config_id": "model_config_id", 110 "evaluator_id": "evaluator_id", 111 "evaluator_version_id": "evaluator_version_id", 112 "aggregate_value": 1.1 113 } 114 ] 115 }

Add evaluators to an existing evaluation run.

Path parameters

idstringRequired

String ID of evaluation run. Starts with ev_.

Request

This endpoint expects an object.

evaluator_idslist of stringsOptional

IDs of evaluators to add to the evaluation run. IDs start with evfn_

evaluator_version_idslist of stringsOptional

IDs of the evaluator versions to add to the evaluation run. IDs start with evv_

Response

Successful Response

idstring

Unique ID for the evaluation. Starts with ev_.

statusenum

Status of an evaluation.

Allowed values:

configobject

created_atdatetime

updated_atdatetime

evaluatorslist of objects

datasetobject

dataset_version_idstring

dataset_snapshotobjectOptional

evaluator_aggregateslist of objectsOptional

Path parameters

Headers

Request

Response

Errors