-
Notifications
You must be signed in to change notification settings - Fork 568
fix(ai): truncate messages in openai agents #5049
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: master
Are you sure you want to change the base?
Changes from 1 commit
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -4,6 +4,7 @@ | |
| normalize_message_roles, | ||
| set_data_normalized, | ||
| normalize_message_role, | ||
| truncate_and_annotate_messages, | ||
| ) | ||
| from sentry_sdk.consts import SPANDATA, SPANSTATUS, OP | ||
| from sentry_sdk.integrations import DidNotEnable | ||
|
|
@@ -111,36 +112,50 @@ def _set_input_data(span, get_response_kwargs): | |
| ) | ||
|
|
||
| for message in get_response_kwargs.get("input", []): | ||
| if "role" in message: | ||
| normalized_role = normalize_message_role(message.get("role")) | ||
| import json | ||
|
|
||
| serialized_str = safe_serialize(message) | ||
| try: | ||
| serialized_message = json.loads(serialized_str) | ||
| except (json.JSONDecodeError, TypeError): | ||
| continue | ||
|
Comment on lines
+116
to
+120
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Why do you serialize and deserialize here?
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. I haven't found a better way to be sure that whatever is in the messages is actually serialize-able down the line. I.e. if there is an object that is passed in the messages, and the object is not serializeable, it will fail. Do we have a better way of doing this?
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. what exactly fails down the line? I'm just trying to understand why we need this. Do we want to ensure we only have standard JSON types like strings, integers and so on in the dictionaries? Because then we can just directly check the types in the dictionary. |
||
|
|
||
| if "role" in serialized_message: | ||
| normalized_role = normalize_message_role(serialized_message.get("role")) | ||
| request_messages.append( | ||
| { | ||
| "role": normalized_role, | ||
| "content": [{"type": "text", "text": message.get("content")}], | ||
| "content": [ | ||
| {"type": "text", "text": serialized_message.get("content")} | ||
| ], | ||
| } | ||
| ) | ||
| else: | ||
| if message.get("type") == "function_call": | ||
| if serialized_message.get("type") == "function_call": | ||
| request_messages.append( | ||
| { | ||
| "role": GEN_AI_ALLOWED_MESSAGE_ROLES.ASSISTANT, | ||
| "content": [message], | ||
| "content": [serialized_message], | ||
| } | ||
| ) | ||
| elif message.get("type") == "function_call_output": | ||
| elif serialized_message.get("type") == "function_call_output": | ||
| request_messages.append( | ||
| { | ||
| "role": GEN_AI_ALLOWED_MESSAGE_ROLES.TOOL, | ||
| "content": [message], | ||
| "content": [serialized_message], | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Bug: BugThe |
||
| } | ||
| ) | ||
|
|
||
| set_data_normalized( | ||
| span, | ||
| SPANDATA.GEN_AI_REQUEST_MESSAGES, | ||
| normalize_message_roles(request_messages), | ||
| unpack=False, | ||
| ) | ||
| normalized_messages = normalize_message_roles(request_messages) | ||
| scope = sentry_sdk.get_current_scope() | ||
| messages_data = truncate_and_annotate_messages(normalized_messages, span, scope) | ||
| if messages_data is not None: | ||
| set_data_normalized( | ||
| span, | ||
| SPANDATA.GEN_AI_REQUEST_MESSAGES, | ||
| messages_data, | ||
| unpack=False, | ||
| ) | ||
|
|
||
|
|
||
| def _set_output_data(span, result): | ||
|
|
||
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,33 +1,35 @@ | ||
| import asyncio | ||
|
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Organized some imports here |
||
| import json | ||
| import os | ||
| import re | ||
| import pytest | ||
| from unittest.mock import MagicMock, patch | ||
| import os | ||
|
|
||
| from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration | ||
| from sentry_sdk.integrations.openai_agents.utils import safe_serialize | ||
| from sentry_sdk.utils import parse_version | ||
|
|
||
| import agents | ||
| import pytest | ||
| from agents import ( | ||
| Agent, | ||
| ModelResponse, | ||
| Usage, | ||
| ModelSettings, | ||
| Usage, | ||
| ) | ||
| from agents.items import ( | ||
| McpCall, | ||
| ResponseFunctionToolCall, | ||
| ResponseOutputMessage, | ||
| ResponseOutputText, | ||
| ResponseFunctionToolCall, | ||
| ) | ||
| from agents.version import __version__ as OPENAI_AGENTS_VERSION | ||
|
|
||
| from openai.types.responses.response_usage import ( | ||
| InputTokensDetails, | ||
| OutputTokensDetails, | ||
| ) | ||
|
|
||
| from sentry_sdk import start_span | ||
| from sentry_sdk.consts import SPANDATA | ||
| from sentry_sdk.integrations.openai_agents import OpenAIAgentsIntegration | ||
| from sentry_sdk.integrations.openai_agents.utils import _set_input_data, safe_serialize | ||
| from sentry_sdk.utils import parse_version | ||
|
|
||
| test_run_config = agents.RunConfig(tracing_disabled=True) | ||
|
|
||
|
|
||
|
|
@@ -1051,18 +1053,13 @@ def test_openai_agents_message_role_mapping(sentry_init, capture_events): | |
|
|
||
| get_response_kwargs = {"input": test_input} | ||
|
|
||
| from sentry_sdk.integrations.openai_agents.utils import _set_input_data | ||
| from sentry_sdk import start_span | ||
|
|
||
| with start_span(op="test") as span: | ||
| _set_input_data(span, get_response_kwargs) | ||
|
|
||
| # Verify that messages were processed and roles were mapped | ||
| from sentry_sdk.consts import SPANDATA | ||
|
|
||
| if SPANDATA.GEN_AI_REQUEST_MESSAGES in span._data: | ||
| import json | ||
|
|
||
| stored_messages = json.loads(span._data[SPANDATA.GEN_AI_REQUEST_MESSAGES]) | ||
|
|
||
| # Verify roles were properly mapped | ||
|
|
@@ -1184,3 +1181,48 @@ def failing_tool(message: str) -> str: | |
| # Verify error status was set (this is the key test for our patch) | ||
| # The span should be marked as error because the tool execution failed | ||
| assert execute_tool_span["tags"]["status"] == "error" | ||
|
|
||
|
|
||
| def test_openai_agents_message_truncation(sentry_init, capture_events): | ||
| """Test that large messages are truncated properly in OpenAI Agents integration.""" | ||
|
|
||
| large_content = ( | ||
| "This is a very long message that will exceed our size limits. " * 1000 | ||
| ) | ||
|
|
||
| sentry_init( | ||
| integrations=[OpenAIAgentsIntegration()], | ||
| traces_sample_rate=1.0, | ||
| send_default_pii=True, | ||
| ) | ||
|
|
||
| test_messages = [ | ||
| {"role": "system", "content": "small message 1"}, | ||
| {"role": "user", "content": large_content}, | ||
| {"role": "assistant", "content": large_content}, | ||
| {"role": "user", "content": "small message 4"}, | ||
| {"role": "assistant", "content": "small message 5"}, | ||
| ] | ||
|
|
||
| get_response_kwargs = {"input": test_messages} | ||
|
|
||
| with start_span(op="gen_ai.chat") as span: | ||
| import sentry_sdk | ||
shellmayr marked this conversation as resolved.
Outdated
Show resolved
Hide resolved
|
||
|
|
||
| scope = sentry_sdk.get_current_scope() | ||
| _set_input_data(span, get_response_kwargs) | ||
| if hasattr(scope, "_gen_ai_original_message_count"): | ||
| truncated_count = scope._gen_ai_original_message_count.get(span.span_id) | ||
| assert truncated_count == 5, ( | ||
| f"Expected 5 original messages, got {truncated_count}" | ||
| ) | ||
|
|
||
| assert SPANDATA.GEN_AI_REQUEST_MESSAGES in span._data | ||
| messages_data = span._data[SPANDATA.GEN_AI_REQUEST_MESSAGES] | ||
| assert isinstance(messages_data, str) | ||
|
|
||
| parsed_messages = json.loads(messages_data) | ||
| assert isinstance(parsed_messages, list) | ||
| assert len(parsed_messages) == 2 | ||
| assert "small message 4" in str(parsed_messages[0]) | ||
| assert "small message 5" in str(parsed_messages[1]) | ||
Uh oh!
There was an error while loading. Please reload this page.