Evaluate an agent

1
from humanloop import Humanloop
2
from openai import OpenAI
3
from openai.types.chat.chat_completion_message import ChatCompletionMessage as Message
4
import wikipedia
5
import json
6
7
8
openai = OpenAI(api_key="ADD YOUR KEY HERE")
9
humanloop = Humanloop(api_key="ADD YOUR KEY HERE")
10
11
12
def search_wikipedia(query: str) -> dict:
13
    """Search Wikipedia to get up-to-date information for a query."""
14
    try:
15
        page = wikipedia.page(query)
16
        return {
17
            "title": page.title,
18
            "content": page.content,
19
            "url": page.url,
20
        }
21
    except Exception as _:
22
        return {
23
            "title": "",
24
            "content": "No results found",
25
            "url": "",
26
        }
27
28
29
def call_model(messages: list[Message]) -> Message:
30
    """Calls the model with the given messages"""
31
    system_message = {
32
        "role": "system",
33
        "content": (
34
            "You are an assistant that helps to answer user questions. "
35
            "You should leverage wikipedia to answer questions so that "
36
            "the information is up to date. If the response from "
37
            "Wikipedia does not seem relevant, rephrase the question "
38
            "and call the tool again. Then finally respond to the user."
39
        ),
40
    }
41
    response = openai.chat.completions.create(
42
        model="gpt-4o",
43
        messages=[system_message] + messages,
44
        tools=[
45
            {
46
                "type": "function",
47
                "function": {
48
                    "name": "search_wikipedia",
49
                    "description": "Search the internet to get up to date answers for a query.",
50
                    "parameters": {
51
                        "type": "object",
52
                        "required": ["query"],
53
                        "properties": {
54
                            "query": {"type": "string"},
55
                        },
56
                        "additionalProperties": False,
57
                    },
58
                },
59
            }
60
        ],
61
    )
62
    return response.choices[0].message.to_dict(exclude_unset=False)
63
64
65
def call_agent(query: str) -> str:
66
    """Calls the main agent loop and returns the final result"""
67
    messages = [{"role": "user", "content": query}]
68
    # Retry for a relevant response 3 times at most
69
    for _ in range(3):
70
        response = call_model(messages)
71
        messages.append(response)
72
        if response["tool_calls"]:
73
            # Call wikipedia to get up-to-date information
74
            for tool_call in response["tool_calls"]:
75
                source = search_wikipedia(
76
                    **json.loads(tool_call["function"]["arguments"])
77
                )
78
                messages.append(
79
                    {
80
                        "role": "tool",
81
                        "content": json.dumps(source),
82
                        "tool_call_id": tool_call["id"],
83
                    }
84
                )
85
        else:
86
            # Respond to the user
87
            return response["content"]
88
89
90
if __name__ == "__main__":
91
    result = call_agent("Where does the sun go at night?")
92
    print(result)

1
def call_model(messages: list[Message]) -> Message:
2
    """Calls the model with the given messages"""
3
    system_message = {
4
        "role": "system",
5
        "content": "You are an assistant that help to answer user questions. You should "
6
                   "leverage wikipedia to answer questions so that the information is up to date. "
7
                   "If the response from Wikipedia does not seem relevant, rephrase the question "
8
                   "and call the tool again. Then finally respond to the user."
9
                   "Formulate the response so that it is easy to understand for a 5 year old."
10
    }
11
    response = openai.chat.completions.create(
12
        model="gpt-4o",
13
        messages=[system_message] + messages,
14
        tools=[
15
            {
16
                "type": "function",
17
                "function": {
18
                    "name": "search_wikipedia",
19
                    "description": "Search the internet to get up to date answers for a query.",
20
                    "parameters": {
21
                        "type": "object",
22
                        "required": ["query"],
23
                        "properties": {
24
                            "query": {"type": "string"},
25
                        },
26
                        "additionalProperties": False,
27
                    },
28
                }
29
            }
30
        ],
31
    )
32
    return response.choices[0].message.to_dict(exclude_unset=False)

$
python agent.py
>
>
Flow Version ID: flv_9ECTrfeZYno2OIj9KAqlz
>
Run ID: rn_WnIwPSI7JFKEtwTS0l3mj
>
>
Navigate to your Evaluation:
>
https://app.humanloop.com/project/fl_9CCIoTzySPfUFeIxfYE6g/evaluations/rn_WnIwPSI7JFKEtwTS0l3mj/stats
>
>
Running 'Agent' over the Dataset 'Children Questions' using 8 workers 
>
[######################------------------] 34/50 (68.00%) | ETA: 14
>
>
...
>
>
+------------------------+---------------------+---------------------+
>
|                        |       Control       |        Latest       |
>
+------------------------+---------------------+---------------------+
>
|                 Run ID |        67tEc        |        WnIwP        |
>
+------------------------+---------------------+---------------------+
>
|             Version ID |        9ECTr        |        9ECTr        |
>
+------------------------+---------------------+---------------------+
>
|                  Added | 2024-11-19 22:05:17 | 2024-11-19 22:24:13 |
>
+------------------------+---------------------+---------------------+
>
|             Evaluators |                     |                     |
>
+------------------------+---------------------+---------------------+
>
| QA Agent/Comprehension |         3.24        |         8.04        |
>
+------------------------+---------------------+---------------------+
>
>
Change of [4.80] for Evaluator QA Agent/Comprehension

1
@humanloop.tool(path="QA Agent/Search Wikipedia")
2
def search_wikipedia(query: str) -> dict:
3
    ...
4
5
@humanloop.prompt(path="QA Agent/Prompt")
6
def call_model(messages: list[Message]) -> Message:
7
    ...
8
9
@humanloop.flow(path="QA Agent/Agent")
10
def call_agent(question: str) -> str:
11
    ...