Humanloop Documentation and API Reference

Update an Evaluation.

Update the setup of an Evaluation by specifying the Dataset, versions to be evaluated (Evaluatees), and which Evaluators to provide judgments.

Path parameters

idstringRequired

Unique identifier for Evaluation.

Request

This endpoint expects an object.

datasetobjectOptional

Dataset to use in this Evaluation.

evaluateeslist of objectsOptional

Unique identifiers for the Prompt/Tool Versions to include in the Evaluation. Can be left unpopulated if you wish to add evaluatees to this Evaluation by specifying evaluation_id in Log calls.

evaluatorslist of objectsOptional

The Evaluators used to evaluate.

namestringOptional

Name of the Evaluation to help identify it. Must be unique within the associated File.

fileobjectOptional

The File to associate with the Evaluation.

Response

Successful Response

idstring

Unique identifier for the Evaluation. Starts with evr.

datasetobject

The Dataset used in the Evaluation.

evaluateeslist of objects

The Prompt/Tool Versions included in the Evaluation.

evaluatorslist of objects

The Evaluator Versions used to evaluate.

statusenum

Allowed values: pendingrunningcompletedcancelled

The current status of the Evaluation.

"pending": The Evaluation has been created but is not actively being worked on by Humanloop.
"running": Humanloop is checking for any missing Logs and Evaluator Logs, and will generate them where appropriate.
"completed": All Logs an Evaluator Logs have been generated.
"cancelled": The Evaluation has been cancelled by the user. Humanloop will stop generating Logs and Evaluator Logs.

created_atdatetime

updated_atdatetime

namestringOptional

Name of the Evaluation to help identify it. Must be unique among Evaluations associated with File.

file_idstringOptional

Unique identifier for the File associated with the Evaluation.

created_byanyOptional

urlstringOptional

URL to view the Evaluation on the Humanloop.

Errors

1 curl -X PATCH https://api.humanloop.com/v5/evaluations/ev_567yza \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{ 5 "dataset": { 6 "version_id": "dsv_6L78pqrdFi2xa" 7 }, 8 "evaluatees": [ 9 { 10 "version_id": "prv_7ZlQREDScH0xkhUwtXruN", 11 "orchestrated": false 12 } 13 ], 14 "evaluators": [ 15 { 16 "version_id": "evv_012def", 17 "orchestrated": false 18 } 19 ] 20 }'

1 { 2 "id": "ev_567yza", 3 "dataset": { 4 "path": "test-questions", 5 "id": "ds_345mno", 6 "name": "test-questions", 7 "version_id": "dsv_678pqr", 8 "created_at": "2024-05-01T12:00:00Z", 9 "updated_at": "2024-05-01T12:00:00Z", 10 "status": "committed", 11 "last_used_at": "2024-05-01T12:00:00Z", 12 "datapoints_count": 2, 13 "type": "dataset", 14 "created_by": { 15 "id": "usr_v23rSVAgas2a", 16 "full_name": "Jordan Burges", 17 "email_address": "jordan@humanloop.com" 18 }, 19 "commit_message": "initial commit" 20 }, 21 "evaluatees": [ 22 { 23 "version": { 24 "path": "Personal Projects/Coding Assistant", 25 "id": "pr_30gco7dx6JDq4200GVOHa", 26 "name": "Coding Assistant", 27 "version_id": "prv_7ZlQREDScH0xkhUwtXruN", 28 "type": "prompt", 29 "environments": [ 30 { 31 "id": "env_ffSVxEBzJcBZ1H5jcNMVj", 32 "created_at": "2023-06-27T23:16:07.992339", 33 "name": "development", 34 "tag": "default" 35 } 36 ], 37 "created_at": "2024-07-08T22:40:35.656915", 38 "updated_at": "2024-07-08T22:40:35.656915", 39 "created_by": { 40 "id": "usr_01RJO1k2spBVqNUt1ASef", 41 "email_address": "raza@humanloop.com", 42 "full_name": "Raza Habib" 43 }, 44 "status": "committed", 45 "last_used_at": "2024-07-08T22:40:35.656915", 46 "model": "gpt-4o", 47 "endpoint": "chat", 48 "template": [ 49 { 50 "content": "You are a helpful coding assistant specialising in {{language}}", 51 "role": "system" 52 } 53 ], 54 "provider": "openai", 55 "max_tokens": -1, 56 "temperature": 0.7, 57 "top_p": 1, 58 "presence_penalty": 0, 59 "frequency_penalty": 0, 60 "other": {}, 61 "tools": [], 62 "linked_tools": [], 63 "commit_message": "Initial commit", 64 "version_logs_count": 0, 65 "total_logs_count": 0, 66 "inputs": [ 67 { 68 "name": "messages" 69 } 70 ] 71 }, 72 "orchestrated": false, 73 "pinned": true, 74 "added_at": "2024-10-12T12:00:00Z" 75 } 76 ], 77 "evaluators": [ 78 { 79 "version": { 80 "path": "Shared Evaluators/Accuracy Evaluator", 81 "id": "ev_890bcd", 82 "spec": { 83 "arguments_type": "target_required", 84 "return_type": "number", 85 "evaluator_type": "python", 86 "code": "def evaluate(answer, target):\n return 0.5" 87 }, 88 "name": "Accuracy Evaluator", 89 "version_id": "evv_012def", 90 "created_at": "2024-05-01T12:00:00Z", 91 "updated_at": "2024-05-01T12:00:00Z", 92 "status": "committed", 93 "last_used_at": "2024-05-01T12:00:00Z", 94 "version_logs_count": 1, 95 "total_logs_count": 1, 96 "inputs": [ 97 { 98 "name": "answer" 99 } 100 ], 101 "type": "evaluator" 102 }, 103 "orchestrated": false, 104 "added_at": "2024-10-12T12:00:00Z" 105 } 106 ], 107 "status": "completed", 108 "created_at": "2024-05-01T12:00:00Z", 109 "updated_at": "2024-05-01T12:00:00Z" 110 }

1	curl -X PATCH https://api.humanloop.com/v5/evaluations/ev_567yza \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{
5	"dataset": {
6	"version_id": "dsv_6L78pqrdFi2xa"
7	},
8	"evaluatees": [
9	{
10	"version_id": "prv_7ZlQREDScH0xkhUwtXruN",
11	"orchestrated": false
12	}
13	],
14	"evaluators": [
15	{
16	"version_id": "evv_012def",
17	"orchestrated": false
18	}
19	]
20	}'

1	{
2	"id": "ev_567yza",
3	"dataset": {
4	"path": "test-questions",
5	"id": "ds_345mno",
6	"name": "test-questions",
7	"version_id": "dsv_678pqr",
8	"created_at": "2024-05-01T12:00:00Z",
9	"updated_at": "2024-05-01T12:00:00Z",
10	"status": "committed",
11	"last_used_at": "2024-05-01T12:00:00Z",
12	"datapoints_count": 2,
13	"type": "dataset",
14	"created_by": {
15	"id": "usr_v23rSVAgas2a",
16	"full_name": "Jordan Burges",
17	"email_address": "jordan@humanloop.com"
18	},
19	"commit_message": "initial commit"
20	},
21	"evaluatees": [
22	{
23	"version": {
24	"path": "Personal Projects/Coding Assistant",
25	"id": "pr_30gco7dx6JDq4200GVOHa",
26	"name": "Coding Assistant",
27	"version_id": "prv_7ZlQREDScH0xkhUwtXruN",
28	"type": "prompt",
29	"environments": [
30	{
31	"id": "env_ffSVxEBzJcBZ1H5jcNMVj",
32	"created_at": "2023-06-27T23:16:07.992339",
33	"name": "development",
34	"tag": "default"
35	}
36	],
37	"created_at": "2024-07-08T22:40:35.656915",
38	"updated_at": "2024-07-08T22:40:35.656915",
39	"created_by": {
40	"id": "usr_01RJO1k2spBVqNUt1ASef",
41	"email_address": "raza@humanloop.com",
42	"full_name": "Raza Habib"
43	},
44	"status": "committed",
45	"last_used_at": "2024-07-08T22:40:35.656915",
46	"model": "gpt-4o",
47	"endpoint": "chat",
48	"template": [
49	{
50	"content": "You are a helpful coding assistant specialising in {{language}}",
51	"role": "system"
52	}
53	],
54	"provider": "openai",
55	"max_tokens": -1,
56	"temperature": 0.7,
57	"top_p": 1,
58	"presence_penalty": 0,
59	"frequency_penalty": 0,
60	"other": {},
61	"tools": [],
62	"linked_tools": [],
63	"commit_message": "Initial commit",
64	"version_logs_count": 0,
65	"total_logs_count": 0,
66	"inputs": [
67	{
68	"name": "messages"
69	}
70	]
71	},
72	"orchestrated": false,
73	"pinned": true,
74	"added_at": "2024-10-12T12:00:00Z"
75	}
76	],
77	"evaluators": [
78	{
79	"version": {
80	"path": "Shared Evaluators/Accuracy Evaluator",
81	"id": "ev_890bcd",
82	"spec": {
83	"arguments_type": "target_required",
84	"return_type": "number",
85	"evaluator_type": "python",
86	"code": "def evaluate(answer, target):\n return 0.5"
87	},
88	"name": "Accuracy Evaluator",
89	"version_id": "evv_012def",
90	"created_at": "2024-05-01T12:00:00Z",
91	"updated_at": "2024-05-01T12:00:00Z",
92	"status": "committed",
93	"last_used_at": "2024-05-01T12:00:00Z",
94	"version_logs_count": 1,
95	"total_logs_count": 1,
96	"inputs": [
97	{
98	"name": "answer"
99	}
100	],
101	"type": "evaluator"
102	},
103	"orchestrated": false,
104	"added_at": "2024-10-12T12:00:00Z"
105	}
106	],
107	"status": "completed",
108	"created_at": "2024-05-01T12:00:00Z",
109	"updated_at": "2024-05-01T12:00:00Z"
110	}