Evaluate an Agent in code

1
from humanloop import Humanloop
2
from openai import OpenAI
3
from openai.types.chat.chat_completion_message import ChatCompletionMessage as Message
4
import wikipedia
5
import json
6
7
8
openai = OpenAI(api_key="ADD YOUR KEY HERE")
9
humanloop = Humanloop(api_key="ADD YOUR KEY HERE")
10
11
12
def search_wikipedia(query: str) -> dict:
13
    """Search Wikipedia to get up-to-date information for a query."""
14
    try:
15
        page = wikipedia.page(query)
16
        return {
17
            "title": page.title,
18
            "content": page.content,
19
            "url": page.url,
20
        }
21
    except Exception as _:
22
        return {
23
            "title": "",
24
            "content": "No results found",
25
            "url": "",
26
        }
27
28
29
def call_model(messages: list[Message]) -> Message:
30
    """Calls the model with the given messages"""
31
    system_message = {
32
        "role": "system",
33
        "content": (
34
            "You are an assistant that helps to answer user questions. "
35
            "You should leverage wikipedia to answer questions so that "
36
            "the information is up to date. If the response from "
37
            "Wikipedia does not seem relevant, rephrase the question "
38
            "and call the tool again. Then finally respond to the user."
39
        ),
40
    }
41
    response = openai.chat.completions.create(
42
        model="gpt-4o",
43
        messages=[system_message] + messages,
44
        tools=[
45
            {
46
                "type": "function",
47
                "function": {
48
                    "name": "search_wikipedia",
49
                    "description": "Search the internet to get up to date answers for a query.",
50
                    "parameters": {
51
                        "type": "object",
52
                        "required": ["query"],
53
                        "properties": {
54
                            "query": {"type": "string"},
55
                        },
56
                        "additionalProperties": False,
57
                    },
58
                },
59
            }
60
        ],
61
    )
62
    return response.choices[0].message.to_dict(exclude_unset=False)
63
64
65
def call_agent(question: str) -> str:
66
    """Calls the main agent loop and returns the final result"""
67
    messages = [{"role": "user", "content": question}]
68
    # Retry for a relevant response 3 times at most
69
    for _ in range(3):
70
        response = call_model(messages)
71
        messages.append(response)
72
        if response["tool_calls"]:
73
            # Call wikipedia to get up-to-date information
74
            for tool_call in response["tool_calls"]:
75
                source = search_wikipedia(
76
                    **json.loads(tool_call["function"]["arguments"])
77
                )
78
                messages.append(
79
                    {
80
                        "role": "tool",
81
                        "content": json.dumps(source),
82
                        "tool_call_id": tool_call["id"],
83
                    }
84
                )
85
        else:
86
            # Respond to the user
87
            return response["content"]
88
89
90
if __name__ == "__main__":
91
    result = call_agent("Where does the sun go at night?")
92
    print(result)