Humanloop Documentation and API Reference

Update the status of an Evaluation.

Can be used to cancel a running Evaluation, or mark an Evaluation that uses external or human evaluators as completed.

Path parameters

idstringRequired

Unique identifier for Evaluation.

Request

This endpoint expects an object.

statusenumRequired

Allowed values: pendingrunningcompletedcancelled

Status of an evaluation.

Response

This endpoint returns an object.

idstring

Unique identifier for the Evaluation. Starts with evr.

datasetobject

The Dataset used in the Evaluation.

evaluateeslist of objects

The Prompt/Tool Versions included in the Evaluation.

evaluatorslist of objects

The Evaluator Versions used to evaluate.

statusenum

Allowed values: pendingrunningcompletedcancelled

The current status of the Evaluation.

"pending": The Evaluation has been created but is not actively being worked on by Humanloop.
"running": Humanloop is checking for any missing Logs and Evaluator Logs, and will generate them where appropriate.
"completed": All Logs an Evaluator Logs have been generated.
"cancelled": The Evaluation has been cancelled by the user. Humanloop will stop generating Logs and Evaluator Logs.

created_atdatetime

updated_atdatetime

created_byobjectOptional

urlstringOptional

URL to view the Evaluation on the Humanloop.

Errors

1 curl -X PATCH https://api.humanloop.com/v5/evaluations/id/status \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{ 5 "status": "pending" 6 }'

1 { 2 "id": "id", 3 "dataset": { 4 "path": "path", 5 "id": "id", 6 "directory_id": "directory_id", 7 "name": "name", 8 "version_id": "version_id", 9 "type": "dataset", 10 "environments": [ 11 { 12 "id": "id", 13 "created_at": "2024-01-15T09:30:00Z", 14 "name": "name", 15 "tag": "default" 16 } 17 ], 18 "created_at": "2024-01-15T09:30:00Z", 19 "updated_at": "2024-01-15T09:30:00Z", 20 "created_by": { 21 "id": "id", 22 "email_address": "email_address", 23 "full_name": "full_name" 24 }, 25 "status": "uncommitted", 26 "last_used_at": "2024-01-15T09:30:00Z", 27 "commit_message": "commit_message", 28 "datapoints_count": 1, 29 "datapoints": [ 30 { 31 "id": "id" 32 } 33 ], 34 "attributes": { 35 "key": "value" 36 } 37 }, 38 "evaluatees": [ 39 { 40 "version": { 41 "path": "path", 42 "id": "id", 43 "model": "model", 44 "name": "name", 45 "version_id": "version_id", 46 "created_at": "2024-01-15T09:30:00Z", 47 "updated_at": "2024-01-15T09:30:00Z", 48 "status": "uncommitted", 49 "last_used_at": "2024-01-15T09:30:00Z", 50 "version_logs_count": 1, 51 "total_logs_count": 1, 52 "inputs": [ 53 { 54 "name": "name" 55 } 56 ] 57 }, 58 "batch_id": "batch_id", 59 "orchestrated": true 60 } 61 ], 62 "evaluators": [ 63 { 64 "version": { 65 "path": "path", 66 "id": "id", 67 "spec": { 68 "arguments_type": "target_free", 69 "return_type": "boolean", 70 "evaluator_type": "llm" 71 }, 72 "name": "name", 73 "version_id": "version_id", 74 "created_at": "2024-01-15T09:30:00Z", 75 "updated_at": "2024-01-15T09:30:00Z", 76 "status": "uncommitted", 77 "last_used_at": "2024-01-15T09:30:00Z", 78 "version_logs_count": 1, 79 "total_logs_count": 1, 80 "inputs": [ 81 { 82 "name": "name" 83 } 84 ] 85 }, 86 "orchestrated": true 87 } 88 ], 89 "status": "pending", 90 "created_at": "2024-01-15T09:30:00Z", 91 "created_by": { 92 "id": "id", 93 "email_address": "email_address", 94 "full_name": "full_name" 95 }, 96 "updated_at": "2024-01-15T09:30:00Z", 97 "url": "url" 98 }

1	curl -X PATCH https://api.humanloop.com/v5/evaluations/id/status \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{
5	"status": "pending"
6	}'

1	{
2	"id": "id",
3	"dataset": {
4	"path": "path",
5	"id": "id",
6	"directory_id": "directory_id",
7	"name": "name",
8	"version_id": "version_id",
9	"type": "dataset",
10	"environments": [
11	{
12	"id": "id",
13	"created_at": "2024-01-15T09:30:00Z",
14	"name": "name",
15	"tag": "default"
16	}
17	],
18	"created_at": "2024-01-15T09:30:00Z",
19	"updated_at": "2024-01-15T09:30:00Z",
20	"created_by": {
21	"id": "id",
22	"email_address": "email_address",
23	"full_name": "full_name"
24	},
25	"status": "uncommitted",
26	"last_used_at": "2024-01-15T09:30:00Z",
27	"commit_message": "commit_message",
28	"datapoints_count": 1,
29	"datapoints": [
30	{
31	"id": "id"
32	}
33	],
34	"attributes": {
35	"key": "value"
36	}
37	},
38	"evaluatees": [
39	{
40	"version": {
41	"path": "path",
42	"id": "id",
43	"model": "model",
44	"name": "name",
45	"version_id": "version_id",
46	"created_at": "2024-01-15T09:30:00Z",
47	"updated_at": "2024-01-15T09:30:00Z",
48	"status": "uncommitted",
49	"last_used_at": "2024-01-15T09:30:00Z",
50	"version_logs_count": 1,
51	"total_logs_count": 1,
52	"inputs": [
53	{
54	"name": "name"
55	}
56	]
57	},
58	"batch_id": "batch_id",
59	"orchestrated": true
60	}
61	],
62	"evaluators": [
63	{
64	"version": {
65	"path": "path",
66	"id": "id",
67	"spec": {
68	"arguments_type": "target_free",
69	"return_type": "boolean",
70	"evaluator_type": "llm"
71	},
72	"name": "name",
73	"version_id": "version_id",
74	"created_at": "2024-01-15T09:30:00Z",
75	"updated_at": "2024-01-15T09:30:00Z",
76	"status": "uncommitted",
77	"last_used_at": "2024-01-15T09:30:00Z",
78	"version_logs_count": 1,
79	"total_logs_count": 1,
80	"inputs": [
81	{
82	"name": "name"
83	}
84	]
85	},
86	"orchestrated": true
87	}
88	],
89	"status": "pending",
90	"created_at": "2024-01-15T09:30:00Z",
91	"created_by": {
92	"id": "id",
93	"email_address": "email_address",
94	"full_name": "full_name"
95	},
96	"updated_at": "2024-01-15T09:30:00Z",
97	"url": "url"
98	}