Humanloop Documentation and API Reference

Update the status of an Evaluation.

Can be used to cancel a running Evaluation, or mark an Evaluation that uses external or human evaluators as completed.

Path parameters

idstringRequired

Unique identifier for Evaluation.

Request

This endpoint expects an object.

statusenumRequired

Allowed values: pendingrunningcompletedcancelled

Status of an evaluation.

Response

Successful Response

idstring

Unique identifier for the Evaluation. Starts with evr.

datasetobject

The Dataset used in the Evaluation.

evaluateeslist of objects

The Prompt/Tool Versions included in the Evaluation.

evaluatorslist of objects

The Evaluator Versions used to evaluate.

statusenum

Allowed values: pendingrunningcompletedcancelled

The current status of the Evaluation.

"pending": The Evaluation has been created but is not actively being worked on by Humanloop.
"running": Humanloop is checking for any missing Logs and Evaluator Logs, and will generate them where appropriate.
"completed": All Logs an Evaluator Logs have been generated.
"cancelled": The Evaluation has been cancelled by the user. Humanloop will stop generating Logs and Evaluator Logs.

created_atdatetime

updated_atdatetime

namestringOptional

Name of the Evaluation to help identify it. Must be unique among Evaluations associated with File.

file_idstringOptional

Unique identifier for the File associated with the Evaluation.

created_byanyOptional

urlstringOptional

URL to view the Evaluation on the Humanloop.

Errors

1 curl -X PATCH https://api.humanloop.com/v5/evaluations/id/status \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{ 5 "status": "pending" 6 }'

1 { 2 "id": "id", 3 "dataset": { 4 "path": "path", 5 "id": "id", 6 "name": "name", 7 "version_id": "version_id", 8 "created_at": "2024-01-15T09:30:00Z", 9 "updated_at": "2024-01-15T09:30:00Z", 10 "status": "uncommitted", 11 "last_used_at": "2024-01-15T09:30:00Z", 12 "datapoints_count": 1, 13 "directory_id": "directory_id", 14 "type": "dataset", 15 "environments": [ 16 { 17 "id": "id", 18 "created_at": "2024-01-15T09:30:00Z", 19 "name": "name", 20 "tag": "default" 21 } 22 ], 23 "created_by": { 24 "id": "id", 25 "email_address": "email_address", 26 "full_name": "full_name" 27 }, 28 "commit_message": "commit_message", 29 "datapoints": [ 30 { 31 "id": "id" 32 } 33 ], 34 "attributes": { 35 "key": "value" 36 } 37 }, 38 "evaluatees": [ 39 { 40 "version": { 41 "path": "path", 42 "id": "id", 43 "model": "model", 44 "name": "name", 45 "version_id": "version_id", 46 "created_at": "2024-01-15T09:30:00Z", 47 "updated_at": "2024-01-15T09:30:00Z", 48 "status": "uncommitted", 49 "last_used_at": "2024-01-15T09:30:00Z", 50 "version_logs_count": 1, 51 "total_logs_count": 1, 52 "inputs": [ 53 { 54 "name": "name" 55 } 56 ] 57 }, 58 "orchestrated": true, 59 "pinned": true, 60 "batch_id": "batch_id", 61 "added_at": "2024-01-15T09:30:00Z" 62 } 63 ], 64 "evaluators": [ 65 { 66 "version": { 67 "path": "path", 68 "id": "id", 69 "spec": { 70 "arguments_type": "target_free", 71 "return_type": "boolean", 72 "evaluator_type": "llm" 73 }, 74 "name": "name", 75 "version_id": "version_id", 76 "created_at": "2024-01-15T09:30:00Z", 77 "updated_at": "2024-01-15T09:30:00Z", 78 "status": "uncommitted", 79 "last_used_at": "2024-01-15T09:30:00Z", 80 "version_logs_count": 1, 81 "total_logs_count": 1, 82 "inputs": [ 83 { 84 "name": "name" 85 } 86 ] 87 }, 88 "orchestrated": true, 89 "added_at": "2024-01-15T09:30:00Z" 90 } 91 ], 92 "status": "pending", 93 "created_at": "2024-01-15T09:30:00Z", 94 "updated_at": "2024-01-15T09:30:00Z", 95 "name": "name", 96 "file_id": "file_id", 97 "created_by": { 98 "id": "id", 99 "email_address": "email_address", 100 "full_name": "full_name" 101 }, 102 "url": "url" 103 }

1	curl -X PATCH https://api.humanloop.com/v5/evaluations/id/status \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{
5	"status": "pending"
6	}'

1	{
2	"id": "id",
3	"dataset": {
4	"path": "path",
5	"id": "id",
6	"name": "name",
7	"version_id": "version_id",
8	"created_at": "2024-01-15T09:30:00Z",
9	"updated_at": "2024-01-15T09:30:00Z",
10	"status": "uncommitted",
11	"last_used_at": "2024-01-15T09:30:00Z",
12	"datapoints_count": 1,
13	"directory_id": "directory_id",
14	"type": "dataset",
15	"environments": [
16	{
17	"id": "id",
18	"created_at": "2024-01-15T09:30:00Z",
19	"name": "name",
20	"tag": "default"
21	}
22	],
23	"created_by": {
24	"id": "id",
25	"email_address": "email_address",
26	"full_name": "full_name"
27	},
28	"commit_message": "commit_message",
29	"datapoints": [
30	{
31	"id": "id"
32	}
33	],
34	"attributes": {
35	"key": "value"
36	}
37	},
38	"evaluatees": [
39	{
40	"version": {
41	"path": "path",
42	"id": "id",
43	"model": "model",
44	"name": "name",
45	"version_id": "version_id",
46	"created_at": "2024-01-15T09:30:00Z",
47	"updated_at": "2024-01-15T09:30:00Z",
48	"status": "uncommitted",
49	"last_used_at": "2024-01-15T09:30:00Z",
50	"version_logs_count": 1,
51	"total_logs_count": 1,
52	"inputs": [
53	{
54	"name": "name"
55	}
56	]
57	},
58	"orchestrated": true,
59	"pinned": true,
60	"batch_id": "batch_id",
61	"added_at": "2024-01-15T09:30:00Z"
62	}
63	],
64	"evaluators": [
65	{
66	"version": {
67	"path": "path",
68	"id": "id",
69	"spec": {
70	"arguments_type": "target_free",
71	"return_type": "boolean",
72	"evaluator_type": "llm"
73	},
74	"name": "name",
75	"version_id": "version_id",
76	"created_at": "2024-01-15T09:30:00Z",
77	"updated_at": "2024-01-15T09:30:00Z",
78	"status": "uncommitted",
79	"last_used_at": "2024-01-15T09:30:00Z",
80	"version_logs_count": 1,
81	"total_logs_count": 1,
82	"inputs": [
83	{
84	"name": "name"
85	}
86	]
87	},
88	"orchestrated": true,
89	"added_at": "2024-01-15T09:30:00Z"
90	}
91	],
92	"status": "pending",
93	"created_at": "2024-01-15T09:30:00Z",
94	"updated_at": "2024-01-15T09:30:00Z",
95	"name": "name",
96	"file_id": "file_id",
97	"created_by": {
98	"id": "id",
99	"email_address": "email_address",
100	"full_name": "full_name"
101	},
102	"url": "url"
103	}