Humanloop Documentation and API Reference

Pin the specified Evaluatee.

Pinned Evaluatees are always displayed in the Evaluation Overview, and serve as the baseline for comparison with other Evaluatees.

Path parameters

idstringRequired

Unique identifier for Evaluation.

Request

This endpoint expects an object.

version_idstringOptional

Unique identifier for the File Version. If provided, none of the other fields should be specified.

pathstringOptional

Path identifying a File. Provide either this or file_id if you want to specify a File.

file_idstringOptional

Unique identifier for the File. Provide either this or path if you want to specify a File.

environmentstringOptional

Name of the Environment a Version is deployed to. Only provide this when specifying a File. If not provided (and a File is specified), the default Environment is used.

batch_idstringOptional

Unique identifier for the batch of Logs to include in the Evaluation Report.

orchestratedbooleanOptional

Whether the Prompt/Tool is orchestrated by Humanloop. Default is True. If False, a log for the Prompt/Tool should be submitted by the user via the API.

Response

Successful Response

idstring

Unique identifier for the Evaluation. Starts with evr.

datasetobject

The Dataset used in the Evaluation.

evaluateeslist of objects

The Prompt/Tool Versions included in the Evaluation.

evaluatorslist of objects

The Evaluator Versions used to evaluate.

statusenum

Allowed values: pendingrunningcompletedcancelled

The current status of the Evaluation.

"pending": The Evaluation has been created but is not actively being worked on by Humanloop.
"running": Humanloop is checking for any missing Logs and Evaluator Logs, and will generate them where appropriate.
"completed": All Logs an Evaluator Logs have been generated.
"cancelled": The Evaluation has been cancelled by the user. Humanloop will stop generating Logs and Evaluator Logs.

created_atdatetime

updated_atdatetime

namestringOptional

Name of the Evaluation to help identify it. Must be unique among Evaluations associated with File.

file_idstringOptional

Unique identifier for the File associated with the Evaluation.

created_byanyOptional

urlstringOptional

URL to view the Evaluation on the Humanloop.

Errors

1 curl -X POST https://api.humanloop.com/v5/evaluations/id/pin-evaluatee \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{}'

1 { 2 "id": "id", 3 "dataset": { 4 "path": "path", 5 "id": "id", 6 "name": "name", 7 "version_id": "version_id", 8 "created_at": "2024-01-15T09:30:00Z", 9 "updated_at": "2024-01-15T09:30:00Z", 10 "status": "uncommitted", 11 "last_used_at": "2024-01-15T09:30:00Z", 12 "datapoints_count": 1, 13 "directory_id": "directory_id", 14 "type": "dataset", 15 "environments": [ 16 { 17 "id": "id", 18 "created_at": "2024-01-15T09:30:00Z", 19 "name": "name", 20 "tag": "default" 21 } 22 ], 23 "created_by": { 24 "id": "id", 25 "email_address": "email_address", 26 "full_name": "full_name" 27 }, 28 "commit_message": "commit_message", 29 "datapoints": [ 30 { 31 "id": "id" 32 } 33 ], 34 "attributes": { 35 "key": "value" 36 } 37 }, 38 "evaluatees": [ 39 { 40 "version": { 41 "path": "path", 42 "id": "id", 43 "model": "model", 44 "name": "name", 45 "version_id": "version_id", 46 "created_at": "2024-01-15T09:30:00Z", 47 "updated_at": "2024-01-15T09:30:00Z", 48 "status": "uncommitted", 49 "last_used_at": "2024-01-15T09:30:00Z", 50 "version_logs_count": 1, 51 "total_logs_count": 1, 52 "inputs": [ 53 { 54 "name": "name" 55 } 56 ] 57 }, 58 "orchestrated": true, 59 "pinned": true, 60 "batch_id": "batch_id", 61 "added_at": "2024-01-15T09:30:00Z" 62 } 63 ], 64 "evaluators": [ 65 { 66 "version": { 67 "path": "path", 68 "id": "id", 69 "spec": { 70 "arguments_type": "target_free", 71 "return_type": "boolean", 72 "evaluator_type": "llm" 73 }, 74 "name": "name", 75 "version_id": "version_id", 76 "created_at": "2024-01-15T09:30:00Z", 77 "updated_at": "2024-01-15T09:30:00Z", 78 "status": "uncommitted", 79 "last_used_at": "2024-01-15T09:30:00Z", 80 "version_logs_count": 1, 81 "total_logs_count": 1, 82 "inputs": [ 83 { 84 "name": "name" 85 } 86 ] 87 }, 88 "orchestrated": true, 89 "added_at": "2024-01-15T09:30:00Z" 90 } 91 ], 92 "status": "pending", 93 "created_at": "2024-01-15T09:30:00Z", 94 "updated_at": "2024-01-15T09:30:00Z", 95 "name": "name", 96 "file_id": "file_id", 97 "created_by": { 98 "id": "id", 99 "email_address": "email_address", 100 "full_name": "full_name" 101 }, 102 "url": "url" 103 }

1	curl -X POST https://api.humanloop.com/v5/evaluations/id/pin-evaluatee \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{}'

1	{
2	"id": "id",
3	"dataset": {
4	"path": "path",
5	"id": "id",
6	"name": "name",
7	"version_id": "version_id",
8	"created_at": "2024-01-15T09:30:00Z",
9	"updated_at": "2024-01-15T09:30:00Z",
10	"status": "uncommitted",
11	"last_used_at": "2024-01-15T09:30:00Z",
12	"datapoints_count": 1,
13	"directory_id": "directory_id",
14	"type": "dataset",
15	"environments": [
16	{
17	"id": "id",
18	"created_at": "2024-01-15T09:30:00Z",
19	"name": "name",
20	"tag": "default"
21	}
22	],
23	"created_by": {
24	"id": "id",
25	"email_address": "email_address",
26	"full_name": "full_name"
27	},
28	"commit_message": "commit_message",
29	"datapoints": [
30	{
31	"id": "id"
32	}
33	],
34	"attributes": {
35	"key": "value"
36	}
37	},
38	"evaluatees": [
39	{
40	"version": {
41	"path": "path",
42	"id": "id",
43	"model": "model",
44	"name": "name",
45	"version_id": "version_id",
46	"created_at": "2024-01-15T09:30:00Z",
47	"updated_at": "2024-01-15T09:30:00Z",
48	"status": "uncommitted",
49	"last_used_at": "2024-01-15T09:30:00Z",
50	"version_logs_count": 1,
51	"total_logs_count": 1,
52	"inputs": [
53	{
54	"name": "name"
55	}
56	]
57	},
58	"orchestrated": true,
59	"pinned": true,
60	"batch_id": "batch_id",
61	"added_at": "2024-01-15T09:30:00Z"
62	}
63	],
64	"evaluators": [
65	{
66	"version": {
67	"path": "path",
68	"id": "id",
69	"spec": {
70	"arguments_type": "target_free",
71	"return_type": "boolean",
72	"evaluator_type": "llm"
73	},
74	"name": "name",
75	"version_id": "version_id",
76	"created_at": "2024-01-15T09:30:00Z",
77	"updated_at": "2024-01-15T09:30:00Z",
78	"status": "uncommitted",
79	"last_used_at": "2024-01-15T09:30:00Z",
80	"version_logs_count": 1,
81	"total_logs_count": 1,
82	"inputs": [
83	{
84	"name": "name"
85	}
86	]
87	},
88	"orchestrated": true,
89	"added_at": "2024-01-15T09:30:00Z"
90	}
91	],
92	"status": "pending",
93	"created_at": "2024-01-15T09:30:00Z",
94	"updated_at": "2024-01-15T09:30:00Z",
95	"name": "name",
96	"file_id": "file_id",
97	"created_by": {
98	"id": "id",
99	"email_address": "email_address",
100	"full_name": "full_name"
101	},
102	"url": "url"
103	}