Humanloop Documentation and API Reference

Log an evaluation result to an evaluation run.

The run must have status ‘running’. One of result or error must be provided.

Path parameters

evaluation_idstringRequired

ID of the evaluation run. Starts with evrun_.

Query parameters

evaluatee_idstringOptional

String ID of evaluatee version to return. If not defined, the first evaluatee will be returned. Starts with evv_.

Request

This endpoint expects an object.

log_idstringRequired

The log that was evaluated. Must have as its source_datapoint_id one of the datapoints in the dataset being evaluated.

evaluator_idstringRequired

ID of the evaluator that evaluated the log. Starts with evfn_. Must be one of the evaluator IDs associated with the evaluation run being logged to.

resultboolean or integer or doubleOptional

The result value of the evaluation.

errorstringOptional

An error that occurred during evaluation.

Response

Successful Response

idstring

evaluator_idstring

evaluator_version_idstring

log_idstring

updated_atdatetime

created_atdatetime

evaluation_idstringOptional

logobjectOptional

Request model for logging a datapoint.

version_idstringOptional

versionanyOptional

valueboolean or double or string or list of stringsOptional

errorstringOptional

evaluator_logobjectOptional

Request model for logging a datapoint.

Errors

1 curl -X POST https://api.humanloop.com/v4/evaluations/evaluation_id/result \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{ 5 "log_id": "log_id", 6 "evaluator_id": "evaluator_id" 7 }'

1 { 2 "id": "id", 3 "evaluator_id": "evaluator_id", 4 "evaluator_version_id": "evaluator_version_id", 5 "log_id": "log_id", 6 "updated_at": "2024-01-15T09:30:00Z", 7 "created_at": "2024-01-15T09:30:00Z", 8 "evaluation_id": "evaluation_id", 9 "log": { 10 "id": "id", 11 "config": { 12 "type": "generic", 13 "id": "config", 14 "name": "config", 15 "status": "config", 16 "created_by": { 17 "id": "id", 18 "email_address": "email_address", 19 "verified": true 20 }, 21 "description": "config", 22 "other": { 23 "key": "value" 24 } 25 }, 26 "evaluation_results": [], 27 "observability_status": "pending", 28 "updated_at": "2024-01-15T09:30:00Z", 29 "project": "project", 30 "project_id": "project_id", 31 "session_id": "session_id", 32 "session_reference_id": "session_reference_id", 33 "parent_id": "parent_id", 34 "parent_reference_id": "parent_reference_id", 35 "inputs": { 36 "key": "value" 37 }, 38 "source": "source", 39 "metadata": { 40 "key": "value" 41 }, 42 "save": true, 43 "source_datapoint_id": "source_datapoint_id", 44 "reference_id": "reference_id", 45 "messages": [ 46 { 47 "role": "user" 48 } 49 ], 50 "output": "output", 51 "judgment": true, 52 "config_id": "config_id", 53 "environment": "environment", 54 "feedback": [ 55 { 56 "type": "rating", 57 "value": true, 58 "id": "id" 59 } 60 ], 61 "created_at": "2024-01-15T09:30:00Z", 62 "error": "error", 63 "stdout": "stdout", 64 "duration": 1.1, 65 "output_message": { 66 "role": "user", 67 "content": "content", 68 "name": "name", 69 "tool_call_id": "tool_call_id", 70 "tool_calls": [ 71 { 72 "id": "id", 73 "type": "function", 74 "function": { 75 "name": "name" 76 } 77 } 78 ], 79 "tool_call": { 80 "name": "name" 81 } 82 }, 83 "prompt_tokens": 1, 84 "output_tokens": 1, 85 "prompt_cost": 1.1, 86 "output_cost": 1.1, 87 "provider_request": { 88 "key": "value" 89 }, 90 "provider_response": { 91 "key": "value" 92 }, 93 "user": "user", 94 "provider_latency": 1.1, 95 "tokens": 1, 96 "raw_output": "raw_output", 97 "finish_reason": "finish_reason", 98 "tools": [ 99 { 100 "id": "id", 101 "name": "name", 102 "signature": "signature", 103 "result": "result" 104 } 105 ], 106 "tool_choice": "none", 107 "batch_ids": [ 108 "batch_ids" 109 ] 110 }, 111 "version_id": "version_id", 112 "version": { 113 "key": "value" 114 }, 115 "value": true, 116 "error": "error", 117 "evaluator_log": { 118 "id": "id", 119 "config": { 120 "type": "generic", 121 "id": "config", 122 "name": "config", 123 "status": "config", 124 "created_by": { 125 "id": "id", 126 "email_address": "email_address", 127 "verified": true 128 }, 129 "description": "config", 130 "other": { 131 "key": "value" 132 } 133 }, 134 "evaluation_results": [], 135 "observability_status": "pending", 136 "updated_at": "2024-01-15T09:30:00Z", 137 "project": "project", 138 "project_id": "project_id", 139 "session_id": "session_id", 140 "session_reference_id": "session_reference_id", 141 "parent_id": "parent_id", 142 "parent_reference_id": "parent_reference_id", 143 "inputs": { 144 "key": "value" 145 }, 146 "source": "source", 147 "metadata": { 148 "key": "value" 149 }, 150 "save": true, 151 "source_datapoint_id": "source_datapoint_id", 152 "reference_id": "reference_id", 153 "messages": [ 154 { 155 "role": "user" 156 } 157 ], 158 "output": "output", 159 "judgment": true, 160 "config_id": "config_id", 161 "environment": "environment", 162 "feedback": [ 163 { 164 "type": "rating", 165 "value": true, 166 "id": "id" 167 } 168 ], 169 "created_at": "2024-01-15T09:30:00Z", 170 "error": "error", 171 "stdout": "stdout", 172 "duration": 1.1, 173 "output_message": { 174 "role": "user", 175 "content": "content", 176 "name": "name", 177 "tool_call_id": "tool_call_id", 178 "tool_calls": [ 179 { 180 "id": "id", 181 "type": "function", 182 "function": { 183 "name": "name" 184 } 185 } 186 ], 187 "tool_call": { 188 "name": "name" 189 } 190 }, 191 "prompt_tokens": 1, 192 "output_tokens": 1, 193 "prompt_cost": 1.1, 194 "output_cost": 1.1, 195 "provider_request": { 196 "key": "value" 197 }, 198 "provider_response": { 199 "key": "value" 200 }, 201 "user": "user", 202 "provider_latency": 1.1, 203 "tokens": 1, 204 "raw_output": "raw_output", 205 "finish_reason": "finish_reason", 206 "tools": [ 207 { 208 "id": "id", 209 "name": "name", 210 "signature": "signature", 211 "result": "result" 212 } 213 ], 214 "tool_choice": "none", 215 "batch_ids": [ 216 "batch_ids" 217 ] 218 } 219 }

1	curl -X POST https://api.humanloop.com/v4/evaluations/evaluation_id/result \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{
5	"log_id": "log_id",
6	"evaluator_id": "evaluator_id"
7	}'

1	{
2	"id": "id",
3	"evaluator_id": "evaluator_id",
4	"evaluator_version_id": "evaluator_version_id",
5	"log_id": "log_id",
6	"updated_at": "2024-01-15T09:30:00Z",
7	"created_at": "2024-01-15T09:30:00Z",
8	"evaluation_id": "evaluation_id",
9	"log": {
10	"id": "id",
11	"config": {
12	"type": "generic",
13	"id": "config",
14	"name": "config",
15	"status": "config",
16	"created_by": {
17	"id": "id",
18	"email_address": "email_address",
19	"verified": true
20	},
21	"description": "config",
22	"other": {
23	"key": "value"
24	}
25	},
26	"evaluation_results": [],
27	"observability_status": "pending",
28	"updated_at": "2024-01-15T09:30:00Z",
29	"project": "project",
30	"project_id": "project_id",
31	"session_id": "session_id",
32	"session_reference_id": "session_reference_id",
33	"parent_id": "parent_id",
34	"parent_reference_id": "parent_reference_id",
35	"inputs": {
36	"key": "value"
37	},
38	"source": "source",
39	"metadata": {
40	"key": "value"
41	},
42	"save": true,
43	"source_datapoint_id": "source_datapoint_id",
44	"reference_id": "reference_id",
45	"messages": [
46	{
47	"role": "user"
48	}
49	],
50	"output": "output",
51	"judgment": true,
52	"config_id": "config_id",
53	"environment": "environment",
54	"feedback": [
55	{
56	"type": "rating",
57	"value": true,
58	"id": "id"
59	}
60	],
61	"created_at": "2024-01-15T09:30:00Z",
62	"error": "error",
63	"stdout": "stdout",
64	"duration": 1.1,
65	"output_message": {
66	"role": "user",
67	"content": "content",
68	"name": "name",
69	"tool_call_id": "tool_call_id",
70	"tool_calls": [
71	{
72	"id": "id",
73	"type": "function",
74	"function": {
75	"name": "name"
76	}
77	}
78	],
79	"tool_call": {
80	"name": "name"
81	}
82	},
83	"prompt_tokens": 1,
84	"output_tokens": 1,
85	"prompt_cost": 1.1,
86	"output_cost": 1.1,
87	"provider_request": {
88	"key": "value"
89	},
90	"provider_response": {
91	"key": "value"
92	},
93	"user": "user",
94	"provider_latency": 1.1,
95	"tokens": 1,
96	"raw_output": "raw_output",
97	"finish_reason": "finish_reason",
98	"tools": [
99	{
100	"id": "id",
101	"name": "name",
102	"signature": "signature",
103	"result": "result"
104	}
105	],
106	"tool_choice": "none",
107	"batch_ids": [
108	"batch_ids"
109	]
110	},
111	"version_id": "version_id",
112	"version": {
113	"key": "value"
114	},
115	"value": true,
116	"error": "error",
117	"evaluator_log": {
118	"id": "id",
119	"config": {
120	"type": "generic",
121	"id": "config",
122	"name": "config",
123	"status": "config",
124	"created_by": {
125	"id": "id",
126	"email_address": "email_address",
127	"verified": true
128	},
129	"description": "config",
130	"other": {
131	"key": "value"
132	}
133	},
134	"evaluation_results": [],
135	"observability_status": "pending",
136	"updated_at": "2024-01-15T09:30:00Z",
137	"project": "project",
138	"project_id": "project_id",
139	"session_id": "session_id",
140	"session_reference_id": "session_reference_id",
141	"parent_id": "parent_id",
142	"parent_reference_id": "parent_reference_id",
143	"inputs": {
144	"key": "value"
145	},
146	"source": "source",
147	"metadata": {
148	"key": "value"
149	},
150	"save": true,
151	"source_datapoint_id": "source_datapoint_id",
152	"reference_id": "reference_id",
153	"messages": [
154	{
155	"role": "user"
156	}
157	],
158	"output": "output",
159	"judgment": true,
160	"config_id": "config_id",
161	"environment": "environment",
162	"feedback": [
163	{
164	"type": "rating",
165	"value": true,
166	"id": "id"
167	}
168	],
169	"created_at": "2024-01-15T09:30:00Z",
170	"error": "error",
171	"stdout": "stdout",
172	"duration": 1.1,
173	"output_message": {
174	"role": "user",
175	"content": "content",
176	"name": "name",
177	"tool_call_id": "tool_call_id",
178	"tool_calls": [
179	{
180	"id": "id",
181	"type": "function",
182	"function": {
183	"name": "name"
184	}
185	}
186	],
187	"tool_call": {
188	"name": "name"
189	}
190	},
191	"prompt_tokens": 1,
192	"output_tokens": 1,
193	"prompt_cost": 1.1,
194	"output_cost": 1.1,
195	"provider_request": {
196	"key": "value"
197	},
198	"provider_response": {
199	"key": "value"
200	},
201	"user": "user",
202	"provider_latency": 1.1,
203	"tokens": 1,
204	"raw_output": "raw_output",
205	"finish_reason": "finish_reason",
206	"tools": [
207	{
208	"id": "id",
209	"name": "name",
210	"signature": "signature",
211	"result": "result"
212	}
213	],
214	"tool_choice": "none",
215	"batch_ids": [
216	"batch_ids"
217	]
218	}
219	}