Humanloop Documentation and API Reference

Activate and deactivate Evaluators for monitoring the Evaluator.

An activated Evaluator will automatically be run on all new Logs within the Evaluator for monitoring purposes.

Path parameters

idstringRequired

Request

This endpoint expects an object.

activatelist of objectsOptional

Evaluators to activate for Monitoring. These will be automatically run on new Logs.

deactivatelist of objectsOptional

Evaluators to deactivate. These will not be run on new Logs.

Response

Successful Response

pathstring

Path of the Evaluator including the Evaluator name, which is used as a unique identifier.

idstring

Unique identifier for the Evaluator.

specobject

namestring

Name of the Evaluator, which is used as a unique identifier.

version_idstring

Unique identifier for the specific Evaluator Version. If no query params provided, the default deployed Evaluator Version is returned.

created_atdatetime

updated_atdatetime

statusenum

Allowed values: uncommittedcommitteddeleted

An enumeration.

last_used_atdatetime

version_logs_countinteger

The number of logs that have been generated for this Evaluator Version

total_logs_countinteger

The number of logs that have been generated across all Evaluator Versions

inputslist of objects

Inputs associated to the Evaluator. Inputs correspond to any of the variables used within the Evaluator template.

directory_idstringOptional

ID of the directory that the file is in on Humanloop.

commit_messagestringOptional

Message describing the changes made.

type"evaluator"OptionalDefaults to evaluator

environmentslist of objectsOptional

The list of environments the Evaluator Version is deployed to.

created_byanyOptional

The user who created the Evaluator.

committed_byanyOptional

The user who committed the Evaluator Version.

committed_atdatetimeOptional

The date and time the Evaluator Version was committed.

evaluatorslist of objectsOptional

Evaluators that have been attached to this Evaluator that are used for monitoring logs.

evaluator_aggregateslist of objectsOptional

Aggregation of Evaluator results for the Evaluator Version.

attributesmap from strings to anyOptional

Additional fields to describe the Evaluator. Helpful to separate Evaluator versions from each other with details on how they were created or used.

Errors

1 curl -X POST https://api.humanloop.com/v5/evaluators/id/evaluators \ 2 -H "X-API-KEY: <apiKey>" \ 3 -H "Content-Type: application/json" \ 4 -d '{}'

1 { 2 "path": "path", 3 "id": "id", 4 "spec": { 5 "arguments_type": "target_free", 6 "return_type": "boolean", 7 "attributes": { 8 "key": "value" 9 }, 10 "evaluator_type": "llm", 11 "prompt": { 12 "model": "model", 13 "endpoint": "complete", 14 "template": "template", 15 "provider": "openai", 16 "max_tokens": 1, 17 "temperature": 1.1, 18 "top_p": 1.1, 19 "stop": "stop", 20 "presence_penalty": 1.1, 21 "frequency_penalty": 1.1, 22 "other": { 23 "key": "value" 24 }, 25 "seed": 1, 26 "response_format": { 27 "type": "json_object" 28 }, 29 "tools": [ 30 { 31 "name": "name", 32 "description": "description" 33 } 34 ], 35 "linked_tools": [ 36 "linked_tools" 37 ], 38 "attributes": { 39 "key": "value" 40 } 41 } 42 }, 43 "name": "name", 44 "version_id": "version_id", 45 "created_at": "2024-01-15T09:30:00Z", 46 "updated_at": "2024-01-15T09:30:00Z", 47 "status": "uncommitted", 48 "last_used_at": "2024-01-15T09:30:00Z", 49 "version_logs_count": 1, 50 "total_logs_count": 1, 51 "inputs": [ 52 { 53 "name": "name" 54 } 55 ], 56 "directory_id": "directory_id", 57 "commit_message": "commit_message", 58 "type": "evaluator", 59 "environments": [ 60 { 61 "id": "id", 62 "created_at": "2024-01-15T09:30:00Z", 63 "name": "name", 64 "tag": "default" 65 } 66 ], 67 "created_by": { 68 "id": "id", 69 "email_address": "email_address", 70 "full_name": "full_name" 71 }, 72 "committed_by": { 73 "id": "id", 74 "email_address": "email_address", 75 "full_name": "full_name" 76 }, 77 "committed_at": "2024-01-15T09:30:00Z", 78 "evaluators": [ 79 { 80 "version_reference": { 81 "file": { 82 "path": "path", 83 "id": "id", 84 "model": "model", 85 "name": "name", 86 "version_id": "version_id", 87 "created_at": "2024-01-15T09:30:00Z", 88 "updated_at": "2024-01-15T09:30:00Z", 89 "status": "uncommitted", 90 "last_used_at": "2024-01-15T09:30:00Z", 91 "version_logs_count": 1, 92 "total_logs_count": 1, 93 "inputs": [ 94 { 95 "name": "name" 96 } 97 ] 98 }, 99 "environment": { 100 "id": "id", 101 "created_at": "2024-01-15T09:30:00Z", 102 "name": "name", 103 "tag": "default" 104 }, 105 "type": "environment" 106 }, 107 "state": "active", 108 "created_at": "2024-01-15T09:30:00Z", 109 "updated_at": "2024-01-15T09:30:00Z" 110 } 111 ], 112 "evaluator_aggregates": [ 113 { 114 "value": 1.1, 115 "evaluator_id": "evaluator_id", 116 "evaluator_version_id": "evaluator_version_id", 117 "created_at": "2024-01-15T09:30:00Z", 118 "updated_at": "2024-01-15T09:30:00Z" 119 } 120 ], 121 "attributes": { 122 "key": "value" 123 } 124 }

1	curl -X POST https://api.humanloop.com/v5/evaluators/id/evaluators \
2	-H "X-API-KEY: <apiKey>" \
3	-H "Content-Type: application/json" \
4	-d '{}'

1	{
2	"path": "path",
3	"id": "id",
4	"spec": {
5	"arguments_type": "target_free",
6	"return_type": "boolean",
7	"attributes": {
8	"key": "value"
9	},
10	"evaluator_type": "llm",
11	"prompt": {
12	"model": "model",
13	"endpoint": "complete",
14	"template": "template",
15	"provider": "openai",
16	"max_tokens": 1,
17	"temperature": 1.1,
18	"top_p": 1.1,
19	"stop": "stop",
20	"presence_penalty": 1.1,
21	"frequency_penalty": 1.1,
22	"other": {
23	"key": "value"
24	},
25	"seed": 1,
26	"response_format": {
27	"type": "json_object"
28	},
29	"tools": [
30	{
31	"name": "name",
32	"description": "description"
33	}
34	],
35	"linked_tools": [
36	"linked_tools"
37	],
38	"attributes": {
39	"key": "value"
40	}
41	}
42	},
43	"name": "name",
44	"version_id": "version_id",
45	"created_at": "2024-01-15T09:30:00Z",
46	"updated_at": "2024-01-15T09:30:00Z",
47	"status": "uncommitted",
48	"last_used_at": "2024-01-15T09:30:00Z",
49	"version_logs_count": 1,
50	"total_logs_count": 1,
51	"inputs": [
52	{
53	"name": "name"
54	}
55	],
56	"directory_id": "directory_id",
57	"commit_message": "commit_message",
58	"type": "evaluator",
59	"environments": [
60	{
61	"id": "id",
62	"created_at": "2024-01-15T09:30:00Z",
63	"name": "name",
64	"tag": "default"
65	}
66	],
67	"created_by": {
68	"id": "id",
69	"email_address": "email_address",
70	"full_name": "full_name"
71	},
72	"committed_by": {
73	"id": "id",
74	"email_address": "email_address",
75	"full_name": "full_name"
76	},
77	"committed_at": "2024-01-15T09:30:00Z",
78	"evaluators": [
79	{
80	"version_reference": {
81	"file": {
82	"path": "path",
83	"id": "id",
84	"model": "model",
85	"name": "name",
86	"version_id": "version_id",
87	"created_at": "2024-01-15T09:30:00Z",
88	"updated_at": "2024-01-15T09:30:00Z",
89	"status": "uncommitted",
90	"last_used_at": "2024-01-15T09:30:00Z",
91	"version_logs_count": 1,
92	"total_logs_count": 1,
93	"inputs": [
94	{
95	"name": "name"
96	}
97	]
98	},
99	"environment": {
100	"id": "id",
101	"created_at": "2024-01-15T09:30:00Z",
102	"name": "name",
103	"tag": "default"
104	},
105	"type": "environment"
106	},
107	"state": "active",
108	"created_at": "2024-01-15T09:30:00Z",
109	"updated_at": "2024-01-15T09:30:00Z"
110	}
111	],
112	"evaluator_aggregates": [
113	{
114	"value": 1.1,
115	"evaluator_id": "evaluator_id",
116	"evaluator_version_id": "evaluator_version_id",
117	"created_at": "2024-01-15T09:30:00Z",
118	"updated_at": "2024-01-15T09:30:00Z"
119	}
120	],
121	"attributes": {
122	"key": "value"
123	}
124	}