Humanloop Documentation and API Reference

Get an Evaluation.

Path parameters

idstringRequired

Unique identifier for Evaluation.

Response

Successful Response

idstring

Unique identifier for the Evaluation. Starts with evr.

datasetobject

The Dataset used in the Evaluation.

evaluateeslist of objects

The Prompt/Tool Versions included in the Evaluation.

evaluatorslist of objects

The Evaluator Versions used to evaluate.

statusenum

Allowed values: pendingrunningcompletedcancelled

The current status of the Evaluation.

"pending": The Evaluation has been created but is not actively being worked on by Humanloop.
"running": Humanloop is checking for any missing Logs and Evaluator Logs, and will generate them where appropriate.
"completed": All Logs an Evaluator Logs have been generated.
"cancelled": The Evaluation has been cancelled by the user. Humanloop will stop generating Logs and Evaluator Logs.

created_atdatetime

updated_atdatetime

namestringOptional

Name of the Evaluation to help identify it. Must be unique among Evaluations associated with File.

file_idstringOptional

Unique identifier for the File associated with the Evaluation.

created_byanyOptional

urlstringOptional

URL to view the Evaluation on the Humanloop.

Errors

1 curl https://api.humanloop.com/v5/evaluations/ev_567yza \ 2 -H "X-API-KEY: <apiKey>"

1 { 2 "id": "ev_567yza", 3 "dataset": { 4 "path": "test-questions", 5 "id": "ds_345mno", 6 "name": "test-questions", 7 "version_id": "dsv_678pqr", 8 "created_at": "2024-05-01T12:00:00Z", 9 "updated_at": "2024-05-01T12:00:00Z", 10 "status": "committed", 11 "last_used_at": "2024-05-01T12:00:00Z", 12 "datapoints_count": 2, 13 "type": "dataset", 14 "created_by": { 15 "id": "usr_v23rSVAgas2a", 16 "full_name": "Jordan Burges", 17 "email_address": "jordan@humanloop.com" 18 }, 19 "commit_message": "initial commit" 20 }, 21 "evaluatees": [ 22 { 23 "version": { 24 "path": "Personal Projects/Coding Assistant", 25 "id": "pr_30gco7dx6JDq4200GVOHa", 26 "name": "Coding Assistant", 27 "version_id": "prv_7ZlQREDScH0xkhUwtXruN", 28 "type": "prompt", 29 "environments": [ 30 { 31 "id": "env_ffSVxEBzJcBZ1H5jcNMVj", 32 "created_at": "2023-06-27T23:16:07.992339", 33 "name": "development", 34 "tag": "default" 35 } 36 ], 37 "created_at": "2024-07-08T22:40:35.656915", 38 "updated_at": "2024-07-08T22:40:35.656915", 39 "created_by": { 40 "id": "usr_01RJO1k2spBVqNUt1ASef", 41 "email_address": "raza@humanloop.com", 42 "full_name": "Raza Habib" 43 }, 44 "status": "committed", 45 "last_used_at": "2024-07-08T22:40:35.656915", 46 "model": "gpt-4o", 47 "endpoint": "chat", 48 "template": [ 49 { 50 "content": "You are a helpful coding assistant specialising in {{language}}", 51 "role": "system" 52 } 53 ], 54 "provider": "openai", 55 "max_tokens": -1, 56 "temperature": 0.7, 57 "top_p": 1, 58 "presence_penalty": 0, 59 "frequency_penalty": 0, 60 "other": {}, 61 "tools": [], 62 "linked_tools": [], 63 "commit_message": "Initial commit", 64 "version_logs_count": 0, 65 "total_logs_count": 0, 66 "inputs": [ 67 { 68 "name": "messages" 69 } 70 ] 71 }, 72 "orchestrated": false, 73 "pinned": true, 74 "added_at": "2024-10-12T12:00:00Z" 75 } 76 ], 77 "evaluators": [ 78 { 79 "version": { 80 "path": "Shared Evaluators/Accuracy Evaluator", 81 "id": "ev_890bcd", 82 "spec": { 83 "arguments_type": "target_required", 84 "return_type": "number", 85 "evaluator_type": "python", 86 "code": "def evaluate(answer, target):\n return 0.5" 87 }, 88 "name": "Accuracy Evaluator", 89 "version_id": "evv_012def", 90 "created_at": "2024-05-01T12:00:00Z", 91 "updated_at": "2024-05-01T12:00:00Z", 92 "status": "committed", 93 "last_used_at": "2024-05-01T12:00:00Z", 94 "version_logs_count": 1, 95 "total_logs_count": 1, 96 "inputs": [ 97 { 98 "name": "answer" 99 } 100 ], 101 "type": "evaluator" 102 }, 103 "orchestrated": false, 104 "added_at": "2024-10-12T12:00:00Z" 105 } 106 ], 107 "status": "completed", 108 "created_at": "2024-05-01T12:00:00Z", 109 "updated_at": "2024-05-01T12:00:00Z" 110 }

1	curl https://api.humanloop.com/v5/evaluations/ev_567yza \
2	-H "X-API-KEY: <apiKey>"

1	{
2	"id": "ev_567yza",
3	"dataset": {
4	"path": "test-questions",
5	"id": "ds_345mno",
6	"name": "test-questions",
7	"version_id": "dsv_678pqr",
8	"created_at": "2024-05-01T12:00:00Z",
9	"updated_at": "2024-05-01T12:00:00Z",
10	"status": "committed",
11	"last_used_at": "2024-05-01T12:00:00Z",
12	"datapoints_count": 2,
13	"type": "dataset",
14	"created_by": {
15	"id": "usr_v23rSVAgas2a",
16	"full_name": "Jordan Burges",
17	"email_address": "jordan@humanloop.com"
18	},
19	"commit_message": "initial commit"
20	},
21	"evaluatees": [
22	{
23	"version": {
24	"path": "Personal Projects/Coding Assistant",
25	"id": "pr_30gco7dx6JDq4200GVOHa",
26	"name": "Coding Assistant",
27	"version_id": "prv_7ZlQREDScH0xkhUwtXruN",
28	"type": "prompt",
29	"environments": [
30	{
31	"id": "env_ffSVxEBzJcBZ1H5jcNMVj",
32	"created_at": "2023-06-27T23:16:07.992339",
33	"name": "development",
34	"tag": "default"
35	}
36	],
37	"created_at": "2024-07-08T22:40:35.656915",
38	"updated_at": "2024-07-08T22:40:35.656915",
39	"created_by": {
40	"id": "usr_01RJO1k2spBVqNUt1ASef",
41	"email_address": "raza@humanloop.com",
42	"full_name": "Raza Habib"
43	},
44	"status": "committed",
45	"last_used_at": "2024-07-08T22:40:35.656915",
46	"model": "gpt-4o",
47	"endpoint": "chat",
48	"template": [
49	{
50	"content": "You are a helpful coding assistant specialising in {{language}}",
51	"role": "system"
52	}
53	],
54	"provider": "openai",
55	"max_tokens": -1,
56	"temperature": 0.7,
57	"top_p": 1,
58	"presence_penalty": 0,
59	"frequency_penalty": 0,
60	"other": {},
61	"tools": [],
62	"linked_tools": [],
63	"commit_message": "Initial commit",
64	"version_logs_count": 0,
65	"total_logs_count": 0,
66	"inputs": [
67	{
68	"name": "messages"
69	}
70	]
71	},
72	"orchestrated": false,
73	"pinned": true,
74	"added_at": "2024-10-12T12:00:00Z"
75	}
76	],
77	"evaluators": [
78	{
79	"version": {
80	"path": "Shared Evaluators/Accuracy Evaluator",
81	"id": "ev_890bcd",
82	"spec": {
83	"arguments_type": "target_required",
84	"return_type": "number",
85	"evaluator_type": "python",
86	"code": "def evaluate(answer, target):\n return 0.5"
87	},
88	"name": "Accuracy Evaluator",
89	"version_id": "evv_012def",
90	"created_at": "2024-05-01T12:00:00Z",
91	"updated_at": "2024-05-01T12:00:00Z",
92	"status": "committed",
93	"last_used_at": "2024-05-01T12:00:00Z",
94	"version_logs_count": 1,
95	"total_logs_count": 1,
96	"inputs": [
97	{
98	"name": "answer"
99	}
100	],
101	"type": "evaluator"
102	},
103	"orchestrated": false,
104	"added_at": "2024-10-12T12:00:00Z"
105	}
106	],
107	"status": "completed",
108	"created_at": "2024-05-01T12:00:00Z",
109	"updated_at": "2024-05-01T12:00:00Z"
110	}