Humanloop Documentation and API Reference

Update an Evaluation Run.

Specify control=true to use this Run as the control Run for the Evaluation. You can cancel a running/pending Run, or mark a Run that uses external or human Evaluators as completed.

Path parameters

idstringRequired

Unique identifier for Evaluation.

run_idstringRequired

Unique identifier for Run.

Request

This endpoint expects an object.

controlbooleanOptional

If True, this Run will be used as the control in the Evaluation. Stats for other Runs will be compared to this Run. This will replace any existing control Run.

statusenumOptional

Allowed values: pendingrunningcompletedcancelled

Used to set the Run to cancelled or completed. Can only be used if the Run is currently pending or running.

Response

Successful Response

idstring

Unique identifier for the Run.

orchestratedboolean

Whether the Run is orchestrated by Humanloop.

added_atdatetime

When the Run was added to the Evaluation.

created_atdatetime

When the Run was created.

statusenum

Allowed values: pendingrunningcompletedcancelled

The status of the Run.

controlboolean

Stats for other Runs will be displayed in comparison to the control Run.

datasetobjectOptional

The Dataset used in the Run.

versionobjectOptional

The version used in the Run.

created_byanyOptional

The User who created the Run.

Errors

1 curl -X PATCH https://api.humanloop.com/v5/evaluations/id/runs/run_id \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{}'

1 { 2 "id": "id", 3 "orchestrated": true, 4 "added_at": "2024-01-15T09:30:00Z", 5 "created_at": "2024-01-15T09:30:00Z", 6 "status": "pending", 7 "control": true, 8 "dataset": { 9 "path": "path", 10 "id": "id", 11 "name": "name", 12 "version_id": "version_id", 13 "created_at": "2024-01-15T09:30:00Z", 14 "updated_at": "2024-01-15T09:30:00Z", 15 "status": "uncommitted", 16 "last_used_at": "2024-01-15T09:30:00Z", 17 "datapoints_count": 1, 18 "directory_id": "directory_id", 19 "type": "dataset", 20 "environments": [ 21 { 22 "id": "id", 23 "created_at": "2024-01-15T09:30:00Z", 24 "name": "name", 25 "tag": "default" 26 } 27 ], 28 "created_by": { 29 "id": "id", 30 "email_address": "email_address", 31 "full_name": "full_name" 32 }, 33 "committed_by": { 34 "id": "id", 35 "email_address": "email_address", 36 "full_name": "full_name" 37 }, 38 "committed_at": "2024-01-15T09:30:00Z", 39 "commit_message": "commit_message", 40 "datapoints": [ 41 { 42 "id": "id" 43 } 44 ], 45 "attributes": { 46 "key": "value" 47 } 48 }, 49 "version": { 50 "path": "path", 51 "id": "id", 52 "directory_id": "directory_id", 53 "model": "model", 54 "endpoint": "complete", 55 "template": "template", 56 "provider": "openai", 57 "max_tokens": 1, 58 "temperature": 1.1, 59 "top_p": 1.1, 60 "stop": "stop", 61 "presence_penalty": 1.1, 62 "frequency_penalty": 1.1, 63 "other": { 64 "key": "value" 65 }, 66 "seed": 1, 67 "response_format": { 68 "type": "json_object", 69 "json_schema": { 70 "key": "value" 71 } 72 }, 73 "tools": [ 74 { 75 "name": "name", 76 "description": "description" 77 } 78 ], 79 "linked_tools": [ 80 { 81 "name": "name", 82 "description": "description", 83 "id": "id", 84 "version_id": "version_id" 85 } 86 ], 87 "attributes": { 88 "key": "value" 89 }, 90 "commit_message": "commit_message", 91 "name": "name", 92 "version_id": "version_id", 93 "type": "prompt", 94 "environments": [ 95 { 96 "id": "id", 97 "created_at": "2024-01-15T09:30:00Z", 98 "name": "name", 99 "tag": "default" 100 } 101 ], 102 "created_at": "2024-01-15T09:30:00Z", 103 "updated_at": "2024-01-15T09:30:00Z", 104 "created_by": { 105 "id": "id", 106 "email_address": "email_address", 107 "full_name": "full_name" 108 }, 109 "committed_by": { 110 "id": "id", 111 "email_address": "email_address", 112 "full_name": "full_name" 113 }, 114 "committed_at": "2024-01-15T09:30:00Z", 115 "status": "uncommitted", 116 "last_used_at": "2024-01-15T09:30:00Z", 117 "version_logs_count": 1, 118 "total_logs_count": 1, 119 "inputs": [ 120 { 121 "name": "name" 122 } 123 ], 124 "evaluator_aggregates": [ 125 { 126 "value": 1.1, 127 "evaluator_id": "evaluator_id", 128 "evaluator_version_id": "evaluator_version_id", 129 "created_at": "2024-01-15T09:30:00Z", 130 "updated_at": "2024-01-15T09:30:00Z" 131 } 132 ] 133 }, 134 "created_by": { 135 "id": "id", 136 "email_address": "email_address", 137 "full_name": "full_name" 138 } 139 }

1	curl -X PATCH https://api.humanloop.com/v5/evaluations/id/runs/run_id \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{}'

1	{
2	"id": "id",
3	"orchestrated": true,
4	"added_at": "2024-01-15T09:30:00Z",
5	"created_at": "2024-01-15T09:30:00Z",
6	"status": "pending",
7	"control": true,
8	"dataset": {
9	"path": "path",
10	"id": "id",
11	"name": "name",
12	"version_id": "version_id",
13	"created_at": "2024-01-15T09:30:00Z",
14	"updated_at": "2024-01-15T09:30:00Z",
15	"status": "uncommitted",
16	"last_used_at": "2024-01-15T09:30:00Z",
17	"datapoints_count": 1,
18	"directory_id": "directory_id",
19	"type": "dataset",
20	"environments": [
21	{
22	"id": "id",
23	"created_at": "2024-01-15T09:30:00Z",
24	"name": "name",
25	"tag": "default"
26	}
27	],
28	"created_by": {
29	"id": "id",
30	"email_address": "email_address",
31	"full_name": "full_name"
32	},
33	"committed_by": {
34	"id": "id",
35	"email_address": "email_address",
36	"full_name": "full_name"
37	},
38	"committed_at": "2024-01-15T09:30:00Z",
39	"commit_message": "commit_message",
40	"datapoints": [
41	{
42	"id": "id"
43	}
44	],
45	"attributes": {
46	"key": "value"
47	}
48	},
49	"version": {
50	"path": "path",
51	"id": "id",
52	"directory_id": "directory_id",
53	"model": "model",
54	"endpoint": "complete",
55	"template": "template",
56	"provider": "openai",
57	"max_tokens": 1,
58	"temperature": 1.1,
59	"top_p": 1.1,
60	"stop": "stop",
61	"presence_penalty": 1.1,
62	"frequency_penalty": 1.1,
63	"other": {
64	"key": "value"
65	},
66	"seed": 1,
67	"response_format": {
68	"type": "json_object",
69	"json_schema": {
70	"key": "value"
71	}
72	},
73	"tools": [
74	{
75	"name": "name",
76	"description": "description"
77	}
78	],
79	"linked_tools": [
80	{
81	"name": "name",
82	"description": "description",
83	"id": "id",
84	"version_id": "version_id"
85	}
86	],
87	"attributes": {
88	"key": "value"
89	},
90	"commit_message": "commit_message",
91	"name": "name",
92	"version_id": "version_id",
93	"type": "prompt",
94	"environments": [
95	{
96	"id": "id",
97	"created_at": "2024-01-15T09:30:00Z",
98	"name": "name",
99	"tag": "default"
100	}
101	],
102	"created_at": "2024-01-15T09:30:00Z",
103	"updated_at": "2024-01-15T09:30:00Z",
104	"created_by": {
105	"id": "id",
106	"email_address": "email_address",
107	"full_name": "full_name"
108	},
109	"committed_by": {
110	"id": "id",
111	"email_address": "email_address",
112	"full_name": "full_name"
113	},
114	"committed_at": "2024-01-15T09:30:00Z",
115	"status": "uncommitted",
116	"last_used_at": "2024-01-15T09:30:00Z",
117	"version_logs_count": 1,
118	"total_logs_count": 1,
119	"inputs": [
120	{
121	"name": "name"
122	}
123	],
124	"evaluator_aggregates": [
125	{
126	"value": 1.1,
127	"evaluator_id": "evaluator_id",
128	"evaluator_version_id": "evaluator_version_id",
129	"created_at": "2024-01-15T09:30:00Z",
130	"updated_at": "2024-01-15T09:30:00Z"
131	}
132	]
133	},
134	"created_by": {
135	"id": "id",
136	"email_address": "email_address",
137	"full_name": "full_name"
138	}
139	}