Humanloop Documentation and API Reference

Create an evaluation.

Path parameters

project_idstringRequired

String ID of project. Starts with pr_.

Request

This endpoint expects an object.

config_idstringRequired

ID of the config to evaluate. Starts with config_.

evaluator_idslist of stringsRequired

IDs of evaluators to run on the dataset. IDs start with evfn_

dataset_idstringRequired

ID of the dataset to use in this evaluation. Starts with evts_.

provider_api_keysobjectOptional

API keys required by each provider to make API calls. The API keys provided here are not stored by Humanloop. If not specified here, Humanloop will fall back to the key saved to your organization. Ensure you provide an API key for the provider for the model config you are evaluating, or have one saved to your organization.

hl_generatedbooleanOptional

Whether the log generations for this evaluation should be performed by Humanloop. If False, the log generations should be submitted by the user via the API.

Response

This endpoint returns an object

idstring

Unique ID for the evaluation. Starts with ev_.

statusenum

Allowed values: pendingrunningcompletedcancelled

Status of an evaluation.

configobject

created_atdatetime

updated_atdatetime

evaluatorslist of objects

datasetobject

dataset_version_idstring

dataset_snapshotobjectOptional

evaluator_aggregateslist of objectsOptional

Errors

1 curl -X POST https://api.humanloop.com/v4/projects/project_id/evaluations \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{ 5 "config_id": "config_id", 6 "evaluator_ids": [ 7 "evaluator_ids" 8 ], 9 "dataset_id": "dataset_id" 10 }'

1 { 2 "id": "id", 3 "status": "pending", 4 "config": { 5 "type": "generic", 6 "id": "config", 7 "other": { 8 "key": "value" 9 }, 10 "created_by": { 11 "id": "id", 12 "email_address": "email_address", 13 "full_name": "full_name", 14 "verified": true 15 }, 16 "status": "config", 17 "name": "config", 18 "description": "config" 19 }, 20 "created_at": "2024-01-15T09:30:00Z", 21 "updated_at": "2024-01-15T09:30:00Z", 22 "evaluators": [ 23 { 24 "name": "name", 25 "description": "description", 26 "arguments_type": "target_free", 27 "return_type": "boolean", 28 "code": "code", 29 "model_config": { 30 "id": "id", 31 "model": "model" 32 }, 33 "type": "python", 34 "id": "id", 35 "created_at": "2024-01-15T09:30:00Z", 36 "updated_at": "2024-01-15T09:30:00Z", 37 "logging_project": { 38 "id": "id", 39 "name": "name", 40 "users": [ 41 { 42 "id": "id", 43 "email_address": "email_address" 44 } 45 ], 46 "data_count": 1, 47 "feedback_types": [ 48 { 49 "type": "rating" 50 } 51 ], 52 "team_id": "team_id", 53 "created_at": "2024-01-15T09:30:00Z", 54 "updated_at": "2024-01-15T09:30:00Z" 55 } 56 } 57 ], 58 "dataset": { 59 "id": "id", 60 "project_id": "project_id", 61 "name": "name", 62 "description": "description", 63 "datapoint_count": 1, 64 "created_at": "2024-01-15T09:30:00Z", 65 "updated_at": "2024-01-15T09:30:00Z" 66 }, 67 "dataset_version_id": "dataset_version_id", 68 "dataset_snapshot": { 69 "id": "id", 70 "project_id": "project_id", 71 "name": "name", 72 "description": "description", 73 "datapoint_count": 1, 74 "created_at": "2024-01-15T09:30:00Z", 75 "updated_at": "2024-01-15T09:30:00Z" 76 }, 77 "evaluator_aggregates": [ 78 { 79 "model_config_id": "model_config_id", 80 "evaluator_id": "evaluator_id", 81 "evaluator_version_id": "evaluator_version_id", 82 "aggregate_value": 1.1 83 } 84 ] 85 }

1	curl -X POST https://api.humanloop.com/v4/projects/project_id/evaluations \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{
5	"config_id": "config_id",
6	"evaluator_ids": [
7	"evaluator_ids"
8	],
9	"dataset_id": "dataset_id"
10	}'

1	{
2	"id": "id",
3	"status": "pending",
4	"config": {
5	"type": "generic",
6	"id": "config",
7	"other": {
8	"key": "value"
9	},
10	"created_by": {
11	"id": "id",
12	"email_address": "email_address",
13	"full_name": "full_name",
14	"verified": true
15	},
16	"status": "config",
17	"name": "config",
18	"description": "config"
19	},
20	"created_at": "2024-01-15T09:30:00Z",
21	"updated_at": "2024-01-15T09:30:00Z",
22	"evaluators": [
23	{
24	"name": "name",
25	"description": "description",
26	"arguments_type": "target_free",
27	"return_type": "boolean",
28	"code": "code",
29	"model_config": {
30	"id": "id",
31	"model": "model"
32	},
33	"type": "python",
34	"id": "id",
35	"created_at": "2024-01-15T09:30:00Z",
36	"updated_at": "2024-01-15T09:30:00Z",
37	"logging_project": {
38	"id": "id",
39	"name": "name",
40	"users": [
41	{
42	"id": "id",
43	"email_address": "email_address"
44	}
45	],
46	"data_count": 1,
47	"feedback_types": [
48	{
49	"type": "rating"
50	}
51	],
52	"team_id": "team_id",
53	"created_at": "2024-01-15T09:30:00Z",
54	"updated_at": "2024-01-15T09:30:00Z"
55	}
56	}
57	],
58	"dataset": {
59	"id": "id",
60	"project_id": "project_id",
61	"name": "name",
62	"description": "description",
63	"datapoint_count": 1,
64	"created_at": "2024-01-15T09:30:00Z",
65	"updated_at": "2024-01-15T09:30:00Z"
66	},
67	"dataset_version_id": "dataset_version_id",
68	"dataset_snapshot": {
69	"id": "id",
70	"project_id": "project_id",
71	"name": "name",
72	"description": "description",
73	"datapoint_count": 1,
74	"created_at": "2024-01-15T09:30:00Z",
75	"updated_at": "2024-01-15T09:30:00Z"
76	},
77	"evaluator_aggregates": [
78	{
79	"model_config_id": "model_config_id",
80	"evaluator_id": "evaluator_id",
81	"evaluator_version_id": "evaluator_version_id",
82	"aggregate_value": 1.1
83	}
84	]
85	}