I started to play around with agno and came across the example given for exceptions. Rather quickly, I ran into an unexpected behavior of the agent. Here is a little modified version that I used:
from agno.agent import Agent
from agno.exceptions import RetryAgentRun
from agno.utils.log import logger
model = # Set you model here
def add_item(agent: Agent, item: str) -> str:
"""Add an item to the shopping list."""
logger.info(f"Tool is adding the item: {item}")
if agent.session_state is None:
raise RuntimeError("Session state is not initialized.")
agent.session_state["shopping_list"].append(item)
len_shopping_list = len(agent.session_state["shopping_list"])
if len_shopping_list < 3:
raise RetryAgentRun(
f"Shopping list is: {agent.session_state['shopping_list']}. "
"Minimum 3 items in the shopping list."
+ f"Add {3 - len_shopping_list} more items.",
)
logger.info(
"The shopping list is now (inside the agent's call): "
f"{agent.session_state.get('shopping_list')}"
)
return f"The shopping list is now: {agent.session_state.get('shopping_list')}"
agent = Agent(
model=model,
# Initialize the session state with empty shopping list
session_state={"shopping_list": []},
tools=[add_item],
markdown=True,
)
def add_item_util(item: str) -> None:
"""Add an item to the shopping list."""
logger.info(f"Adding item: {item}")
logger.info(f"Session state before: {agent.session_state}")
agent.print_response(f"Add {item}", stream=True)
logger.info(f"Session state after: {agent.session_state}")
add_item_util("milk")
add_item_util("beer")
add_item_util("beans")
Now, you would probably expect that each call to add_item_util would add a single item to the shopping list. But, behold! That's not the case (I had to run it 2-3 times to face the glitch):
INFO Adding item: milk
INFO Session state before: {'shopping_list': []}
INFO Tool is adding the item: milk 👈🏻👈🏻👈🏻 Expected
INFO Tool is adding the item: bread 👈🏻👈🏻👈🏻 NOT Expected!
INFO Tool is adding the item: eggs 👈🏻👈🏻👈🏻 NOT Expected!
INFO The shopping list is now (inside the agent's call): ['milk', 'bread', 'eggs']
┏━ Message ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ Add milk ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Tool Calls ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ • add_item(item=milk) ┃
┃ • add_item(item=bread) ┃
┃ • add_item(item=eggs) ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Response (2.7s) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ Your shopping list now includes: ┃
┃ ┃
┃ • Milk ┃
┃ • Bread ┃
┃ • Eggs ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
INFO Session state after: {'shopping_list': ['milk', 'bread', 'eggs']}
INFO Adding item: beer
INFO Session state before: {'shopping_list': ['milk', 'bread', 'eggs']}
INFO Tool is adding the item: beer
INFO The shopping list is now (inside the agent's call): ['milk', 'bread', 'eggs', 'beer']
┏━ Message ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ Add beer ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Tool Calls ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ • add_item(item=beer) ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Response (1.7s) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ Beer has been added to the shopping list. ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
INFO Session state after: {'shopping_list': ['milk', 'bread', 'eggs', 'beer']}
INFO Adding item: beans
INFO Session state before: {'shopping_list': ['milk', 'bread', 'eggs', 'beer']}
INFO Tool is adding the item: Beans
INFO The shopping list is now (inside the agent's call): ['milk', 'bread', 'eggs', 'beer', 'Beans']
┏━ Message ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ Add beans ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Tool Calls ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ • add_item(item=Beans) ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
┏━ Response (1.7s) ━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┓
┃ ┃
┃ Beans have been added to the shopping list. ┃
┃ ┃
┗━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┛
INFO Session state after: {'shopping_list': ['milk', 'bread', 'eggs', 'beer', 'Beans']}
As expected, this is not deterministic and some runs work smoothly. But, this lack of determinism is a serious inherit challenge of the usage of LLMs. In this case, I was able to improve the situation by adjusting the description of the function add_item(agent: Agent, item: str) to the following:
Add an item provided by the user to the shopping list.
What do you think? How do you handle this challenges? What should I have done differently? Let's discuss in the comments.
Top comments (0)