Humanloop Documentation and API Reference

Create an Evaluation Run.

Optionally specify the Dataset and version to be evaluated.

Humanloop will automatically start generating Logs and running Evaluators where orchestrated=true. If you are generating Logs yourself, you can set orchestrated=false and then generate and submit the required Logs via the API.

If dataset and version are provided, you can set use_existing_logs=True to reuse existing Logs, avoiding generating new Logs unnecessarily. Logs that are associated with the specified Version and have source_datapoint_id referencing a datapoint in the specified Dataset will be associated with the Run.

To keep updated on the progress of the Run, you can poll the Run using the GET /evaluations/{id}/runs endpoint and check its status.

Path parameters

idstringRequired

Unique identifier for Evaluation.

Request

This endpoint expects an object.

datasetobjectOptional

Dataset to use in this Run.

versionobjectOptional

Version to use in this Run.

orchestratedbooleanOptional

Whether the Run is orchestrated by Humanloop. If True, Humanloop will generate Logs for the Run; dataset and version must be provided. If False, a log for the Prompt/Tool should be submitted by the user via the API.

use_existing_logsbooleanOptional

If True, the Run will be initialized with existing Logs associated with the Dataset and Version. If False, the Run will be initialized with no Logs. Can only be set to True when both dataset and version are provided.

Response

Successful Response

idstring

Unique identifier for the Run.

orchestratedboolean

Whether the Run is orchestrated by Humanloop.

added_atdatetime

When the Run was added to the Evaluation.

created_atdatetime

When the Run was created.

statusenum

Allowed values: pendingrunningcompletedcancelled

The status of the Run.

controlboolean

Stats for other Runs will be displayed in comparison to the control Run.

datasetobjectOptional

The Dataset used in the Run.

versionobjectOptional

The version used in the Run.

created_byanyOptional

The User who created the Run.

Errors

1 curl -X POST https://api.humanloop.com/v5/evaluations/id/runs \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{}'

1 { 2 "id": "id", 3 "orchestrated": true, 4 "added_at": "2024-01-15T09:30:00Z", 5 "created_at": "2024-01-15T09:30:00Z", 6 "status": "pending", 7 "control": true, 8 "dataset": { 9 "path": "path", 10 "id": "id", 11 "name": "name", 12 "version_id": "version_id", 13 "created_at": "2024-01-15T09:30:00Z", 14 "updated_at": "2024-01-15T09:30:00Z", 15 "status": "uncommitted", 16 "last_used_at": "2024-01-15T09:30:00Z", 17 "datapoints_count": 1, 18 "directory_id": "directory_id", 19 "type": "dataset", 20 "environments": [ 21 { 22 "id": "id", 23 "created_at": "2024-01-15T09:30:00Z", 24 "name": "name", 25 "tag": "default" 26 } 27 ], 28 "created_by": { 29 "id": "id", 30 "email_address": "email_address", 31 "full_name": "full_name" 32 }, 33 "committed_by": { 34 "id": "id", 35 "email_address": "email_address", 36 "full_name": "full_name" 37 }, 38 "committed_at": "2024-01-15T09:30:00Z", 39 "commit_message": "commit_message", 40 "datapoints": [ 41 { 42 "id": "id" 43 } 44 ], 45 "attributes": { 46 "key": "value" 47 } 48 }, 49 "version": { 50 "path": "path", 51 "id": "id", 52 "directory_id": "directory_id", 53 "model": "model", 54 "endpoint": "complete", 55 "template": "template", 56 "provider": "openai", 57 "max_tokens": 1, 58 "temperature": 1.1, 59 "top_p": 1.1, 60 "stop": "stop", 61 "presence_penalty": 1.1, 62 "frequency_penalty": 1.1, 63 "other": { 64 "key": "value" 65 }, 66 "seed": 1, 67 "response_format": { 68 "type": "json_object", 69 "json_schema": { 70 "key": "value" 71 } 72 }, 73 "tools": [ 74 { 75 "name": "name", 76 "description": "description" 77 } 78 ], 79 "linked_tools": [ 80 { 81 "name": "name", 82 "description": "description", 83 "id": "id", 84 "version_id": "version_id" 85 } 86 ], 87 "attributes": { 88 "key": "value" 89 }, 90 "commit_message": "commit_message", 91 "name": "name", 92 "version_id": "version_id", 93 "type": "prompt", 94 "environments": [ 95 { 96 "id": "id", 97 "created_at": "2024-01-15T09:30:00Z", 98 "name": "name", 99 "tag": "default" 100 } 101 ], 102 "created_at": "2024-01-15T09:30:00Z", 103 "updated_at": "2024-01-15T09:30:00Z", 104 "created_by": { 105 "id": "id", 106 "email_address": "email_address", 107 "full_name": "full_name" 108 }, 109 "committed_by": { 110 "id": "id", 111 "email_address": "email_address", 112 "full_name": "full_name" 113 }, 114 "committed_at": "2024-01-15T09:30:00Z", 115 "status": "uncommitted", 116 "last_used_at": "2024-01-15T09:30:00Z", 117 "version_logs_count": 1, 118 "total_logs_count": 1, 119 "inputs": [ 120 { 121 "name": "name" 122 } 123 ], 124 "evaluator_aggregates": [ 125 { 126 "value": 1.1, 127 "evaluator_id": "evaluator_id", 128 "evaluator_version_id": "evaluator_version_id", 129 "created_at": "2024-01-15T09:30:00Z", 130 "updated_at": "2024-01-15T09:30:00Z" 131 } 132 ] 133 }, 134 "created_by": { 135 "id": "id", 136 "email_address": "email_address", 137 "full_name": "full_name" 138 } 139 }

1	curl -X POST https://api.humanloop.com/v5/evaluations/id/runs \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{}'

1	{
2	"id": "id",
3	"orchestrated": true,
4	"added_at": "2024-01-15T09:30:00Z",
5	"created_at": "2024-01-15T09:30:00Z",
6	"status": "pending",
7	"control": true,
8	"dataset": {
9	"path": "path",
10	"id": "id",
11	"name": "name",
12	"version_id": "version_id",
13	"created_at": "2024-01-15T09:30:00Z",
14	"updated_at": "2024-01-15T09:30:00Z",
15	"status": "uncommitted",
16	"last_used_at": "2024-01-15T09:30:00Z",
17	"datapoints_count": 1,
18	"directory_id": "directory_id",
19	"type": "dataset",
20	"environments": [
21	{
22	"id": "id",
23	"created_at": "2024-01-15T09:30:00Z",
24	"name": "name",
25	"tag": "default"
26	}
27	],
28	"created_by": {
29	"id": "id",
30	"email_address": "email_address",
31	"full_name": "full_name"
32	},
33	"committed_by": {
34	"id": "id",
35	"email_address": "email_address",
36	"full_name": "full_name"
37	},
38	"committed_at": "2024-01-15T09:30:00Z",
39	"commit_message": "commit_message",
40	"datapoints": [
41	{
42	"id": "id"
43	}
44	],
45	"attributes": {
46	"key": "value"
47	}
48	},
49	"version": {
50	"path": "path",
51	"id": "id",
52	"directory_id": "directory_id",
53	"model": "model",
54	"endpoint": "complete",
55	"template": "template",
56	"provider": "openai",
57	"max_tokens": 1,
58	"temperature": 1.1,
59	"top_p": 1.1,
60	"stop": "stop",
61	"presence_penalty": 1.1,
62	"frequency_penalty": 1.1,
63	"other": {
64	"key": "value"
65	},
66	"seed": 1,
67	"response_format": {
68	"type": "json_object",
69	"json_schema": {
70	"key": "value"
71	}
72	},
73	"tools": [
74	{
75	"name": "name",
76	"description": "description"
77	}
78	],
79	"linked_tools": [
80	{
81	"name": "name",
82	"description": "description",
83	"id": "id",
84	"version_id": "version_id"
85	}
86	],
87	"attributes": {
88	"key": "value"
89	},
90	"commit_message": "commit_message",
91	"name": "name",
92	"version_id": "version_id",
93	"type": "prompt",
94	"environments": [
95	{
96	"id": "id",
97	"created_at": "2024-01-15T09:30:00Z",
98	"name": "name",
99	"tag": "default"
100	}
101	],
102	"created_at": "2024-01-15T09:30:00Z",
103	"updated_at": "2024-01-15T09:30:00Z",
104	"created_by": {
105	"id": "id",
106	"email_address": "email_address",
107	"full_name": "full_name"
108	},
109	"committed_by": {
110	"id": "id",
111	"email_address": "email_address",
112	"full_name": "full_name"
113	},
114	"committed_at": "2024-01-15T09:30:00Z",
115	"status": "uncommitted",
116	"last_used_at": "2024-01-15T09:30:00Z",
117	"version_logs_count": 1,
118	"total_logs_count": 1,
119	"inputs": [
120	{
121	"name": "name"
122	}
123	],
124	"evaluator_aggregates": [
125	{
126	"value": 1.1,
127	"evaluator_id": "evaluator_id",
128	"evaluator_version_id": "evaluator_version_id",
129	"created_at": "2024-01-15T09:30:00Z",
130	"updated_at": "2024-01-15T09:30:00Z"
131	}
132	]
133	},
134	"created_by": {
135	"id": "id",
136	"email_address": "email_address",
137	"full_name": "full_name"
138	}
139	}