Humanloop Documentation and API Reference

Create an Evaluation.

Create a new Evaluation by specifying the Dataset, versions to be evaluated (Evaluatees), and which Evaluators to provide judgments.

Humanloop will automatically start generating Logs and running Evaluators where orchestrated=true. If you own the runtime for the Evaluatee or Evaluator, you can set orchestrated=false and then generate and submit the required logs using your runtime.

To keep updated on the progress of the Evaluation, you can poll the Evaluation using the GET /evaluations/:id endpoint and check its status.

Request

This endpoint expects an object.

datasetobjectRequired

Dataset to use in this Evaluation.

evaluatorslist of objectsRequired

The Evaluators used to evaluate.

evaluateeslist of objectsOptional

Unique identifiers for the Prompt/Tool Versions to include in the Evaluation. Can be left unpopulated if you wish to add Evaluatees to this Evaluation by specifying evaluation_id in Log calls.

namestringOptional

Name of the Evaluation to help identify it. Must be unique within the associated File.

fileobjectOptional

The File to associate with the Evaluation.

Response

Successful Response

idstring

Unique identifier for the Evaluation. Starts with evr.

datasetobject

The Dataset used in the Evaluation.

evaluateeslist of objects

The Prompt/Tool Versions included in the Evaluation.

evaluatorslist of objects

The Evaluator Versions used to evaluate.

statusenum

Allowed values: pendingrunningcompletedcancelled

The current status of the Evaluation.

"pending": The Evaluation has been created but is not actively being worked on by Humanloop.
"running": Humanloop is checking for any missing Logs and Evaluator Logs, and will generate them where appropriate.
"completed": All Logs an Evaluator Logs have been generated.
"cancelled": The Evaluation has been cancelled by the user. Humanloop will stop generating Logs and Evaluator Logs.

created_atdatetime

updated_atdatetime

namestringOptional

Name of the Evaluation to help identify it. Must be unique among Evaluations associated with File.

file_idstringOptional

Unique identifier for the File associated with the Evaluation.

created_byanyOptional

urlstringOptional

URL to view the Evaluation on the Humanloop.

Errors

1 curl -X POST https://api.humanloop.com/v5/evaluations \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{ 5 "dataset": { 6 "version_id": "dsv_6L78pqrdFi2xa" 7 }, 8 "evaluators": [ 9 { 10 "version_id": "evv_012def", 11 "orchestrated": false 12 } 13 ], 14 "evaluatees": [ 15 { 16 "version_id": "prv_7ZlQREDScH0xkhUwtXruN", 17 "orchestrated": false 18 } 19 ] 20 }'

1 { 2 "id": "ev_567yza", 3 "dataset": { 4 "path": "test-questions", 5 "id": "ds_345mno", 6 "name": "test-questions", 7 "version_id": "dsv_678pqr", 8 "created_at": "2024-05-01T12:00:00Z", 9 "updated_at": "2024-05-01T12:00:00Z", 10 "status": "committed", 11 "last_used_at": "2024-05-01T12:00:00Z", 12 "datapoints_count": 2, 13 "type": "dataset", 14 "created_by": { 15 "id": "usr_v23rSVAgas2a", 16 "full_name": "Jordan Burges", 17 "email_address": "jordan@humanloop.com" 18 }, 19 "commit_message": "initial commit" 20 }, 21 "evaluatees": [ 22 { 23 "version": { 24 "path": "Personal Projects/Coding Assistant", 25 "id": "pr_30gco7dx6JDq4200GVOHa", 26 "name": "Coding Assistant", 27 "version_id": "prv_7ZlQREDScH0xkhUwtXruN", 28 "type": "prompt", 29 "environments": [ 30 { 31 "id": "env_ffSVxEBzJcBZ1H5jcNMVj", 32 "created_at": "2023-06-27T23:16:07.992339", 33 "name": "development", 34 "tag": "default" 35 } 36 ], 37 "created_at": "2024-07-08T22:40:35.656915", 38 "updated_at": "2024-07-08T22:40:35.656915", 39 "created_by": { 40 "id": "usr_01RJO1k2spBVqNUt1ASef", 41 "email_address": "raza@humanloop.com", 42 "full_name": "Raza Habib" 43 }, 44 "status": "committed", 45 "last_used_at": "2024-07-08T22:40:35.656915", 46 "model": "gpt-4o", 47 "endpoint": "chat", 48 "template": [ 49 { 50 "content": "You are a helpful coding assistant specialising in {{language}}", 51 "role": "system" 52 } 53 ], 54 "provider": "openai", 55 "max_tokens": -1, 56 "temperature": 0.7, 57 "top_p": 1, 58 "presence_penalty": 0, 59 "frequency_penalty": 0, 60 "other": {}, 61 "tools": [], 62 "linked_tools": [], 63 "commit_message": "Initial commit", 64 "version_logs_count": 0, 65 "total_logs_count": 0, 66 "inputs": [ 67 { 68 "name": "messages" 69 } 70 ] 71 }, 72 "orchestrated": false, 73 "pinned": false 74 } 75 ], 76 "evaluators": [ 77 { 78 "version": { 79 "path": "Shared Evaluators/Accuracy Evaluator", 80 "id": "ev_890bcd", 81 "spec": { 82 "arguments_type": "target_required", 83 "return_type": "number", 84 "evaluator_type": "python", 85 "code": "def evaluate(answer, target):\n return 0.5" 86 }, 87 "name": "Accuracy Evaluator", 88 "version_id": "evv_012def", 89 "created_at": "2024-05-01T12:00:00Z", 90 "updated_at": "2024-05-01T12:00:00Z", 91 "status": "committed", 92 "last_used_at": "2024-05-01T12:00:00Z", 93 "version_logs_count": 1, 94 "total_logs_count": 1, 95 "inputs": [ 96 { 97 "name": "answer" 98 } 99 ], 100 "type": "evaluator" 101 }, 102 "orchestrated": false, 103 "added_at": "2024-05-01T12:00:00Z" 104 } 105 ], 106 "status": "completed", 107 "created_at": "2024-05-01T12:00:00Z", 108 "updated_at": "2024-05-01T12:00:00Z" 109 }

1	curl -X POST https://api.humanloop.com/v5/evaluations \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{
5	"dataset": {
6	"version_id": "dsv_6L78pqrdFi2xa"
7	},
8	"evaluators": [
9	{
10	"version_id": "evv_012def",
11	"orchestrated": false
12	}
13	],
14	"evaluatees": [
15	{
16	"version_id": "prv_7ZlQREDScH0xkhUwtXruN",
17	"orchestrated": false
18	}
19	]
20	}'

1	{
2	"id": "ev_567yza",
3	"dataset": {
4	"path": "test-questions",
5	"id": "ds_345mno",
6	"name": "test-questions",
7	"version_id": "dsv_678pqr",
8	"created_at": "2024-05-01T12:00:00Z",
9	"updated_at": "2024-05-01T12:00:00Z",
10	"status": "committed",
11	"last_used_at": "2024-05-01T12:00:00Z",
12	"datapoints_count": 2,
13	"type": "dataset",
14	"created_by": {
15	"id": "usr_v23rSVAgas2a",
16	"full_name": "Jordan Burges",
17	"email_address": "jordan@humanloop.com"
18	},
19	"commit_message": "initial commit"
20	},
21	"evaluatees": [
22	{
23	"version": {
24	"path": "Personal Projects/Coding Assistant",
25	"id": "pr_30gco7dx6JDq4200GVOHa",
26	"name": "Coding Assistant",
27	"version_id": "prv_7ZlQREDScH0xkhUwtXruN",
28	"type": "prompt",
29	"environments": [
30	{
31	"id": "env_ffSVxEBzJcBZ1H5jcNMVj",
32	"created_at": "2023-06-27T23:16:07.992339",
33	"name": "development",
34	"tag": "default"
35	}
36	],
37	"created_at": "2024-07-08T22:40:35.656915",
38	"updated_at": "2024-07-08T22:40:35.656915",
39	"created_by": {
40	"id": "usr_01RJO1k2spBVqNUt1ASef",
41	"email_address": "raza@humanloop.com",
42	"full_name": "Raza Habib"
43	},
44	"status": "committed",
45	"last_used_at": "2024-07-08T22:40:35.656915",
46	"model": "gpt-4o",
47	"endpoint": "chat",
48	"template": [
49	{
50	"content": "You are a helpful coding assistant specialising in {{language}}",
51	"role": "system"
52	}
53	],
54	"provider": "openai",
55	"max_tokens": -1,
56	"temperature": 0.7,
57	"top_p": 1,
58	"presence_penalty": 0,
59	"frequency_penalty": 0,
60	"other": {},
61	"tools": [],
62	"linked_tools": [],
63	"commit_message": "Initial commit",
64	"version_logs_count": 0,
65	"total_logs_count": 0,
66	"inputs": [
67	{
68	"name": "messages"
69	}
70	]
71	},
72	"orchestrated": false,
73	"pinned": false
74	}
75	],
76	"evaluators": [
77	{
78	"version": {
79	"path": "Shared Evaluators/Accuracy Evaluator",
80	"id": "ev_890bcd",
81	"spec": {
82	"arguments_type": "target_required",
83	"return_type": "number",
84	"evaluator_type": "python",
85	"code": "def evaluate(answer, target):\n return 0.5"
86	},
87	"name": "Accuracy Evaluator",
88	"version_id": "evv_012def",
89	"created_at": "2024-05-01T12:00:00Z",
90	"updated_at": "2024-05-01T12:00:00Z",
91	"status": "committed",
92	"last_used_at": "2024-05-01T12:00:00Z",
93	"version_logs_count": 1,
94	"total_logs_count": 1,
95	"inputs": [
96	{
97	"name": "answer"
98	}
99	],
100	"type": "evaluator"
101	},
102	"orchestrated": false,
103	"added_at": "2024-05-01T12:00:00Z"
104	}
105	],
106	"status": "completed",
107	"created_at": "2024-05-01T12:00:00Z",
108	"updated_at": "2024-05-01T12:00:00Z"
109	}