diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-tool-loop-agent.mjs b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-tool-loop-agent.mjs new file mode 100644 index 000000000000..fe485ce29a90 --- /dev/null +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/scenario-tool-loop-agent.mjs @@ -0,0 +1,61 @@ +import * as Sentry from '@sentry/node'; +import { ToolLoopAgent, stepCountIs, tool } from 'ai'; +import { MockLanguageModelV3 } from 'ai/test'; +import { z } from 'zod'; + +async function run() { + await Sentry.startSpan({ op: 'function', name: 'main' }, async () => { + let callCount = 0; + + const agent = new ToolLoopAgent({ + experimental_telemetry: { isEnabled: true }, + model: new MockLanguageModelV3({ + doGenerate: async () => { + if (callCount++ === 0) { + return { + finishReason: { unified: 'tool-calls', raw: 'tool_calls' }, + usage: { + inputTokens: { total: 10, noCache: 10, cached: 0 }, + outputTokens: { total: 20, noCache: 20, cached: 0 }, + totalTokens: { total: 30, noCache: 30, cached: 0 }, + }, + content: [ + { + type: 'tool-call', + toolCallId: 'call-1', + toolName: 'getWeather', + input: JSON.stringify({ location: 'San Francisco' }), + }, + ], + warnings: [], + }; + } + return { + finishReason: { unified: 'stop', raw: 'stop' }, + usage: { + inputTokens: { total: 15, noCache: 15, cached: 0 }, + outputTokens: { total: 25, noCache: 25, cached: 0 }, + totalTokens: { total: 40, noCache: 40, cached: 0 }, + }, + content: [{ type: 'text', text: 'The weather in San Francisco is sunny, 72°F.' }], + warnings: [], + }; + }, + }), + tools: { + getWeather: tool({ + description: 'Get the current weather for a location', + inputSchema: z.object({ location: z.string() }), + execute: async ({ location }) => `Weather in ${location}: Sunny, 72°F`, + }), + }, + stopWhen: stepCountIs(3), + }); + + await agent.generate({ + prompt: 'What is the weather in San Francisco?', + }); + }); +} + +run(); diff --git a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts index 39ee00254373..bc08081e7fc8 100644 --- a/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts +++ b/dev-packages/node-integration-tests/suites/tracing/vercelai/v6/test.ts @@ -601,4 +601,81 @@ describe('Vercel AI integration (V6)', () => { }, }, ); + + createEsmAndCjsTests( + __dirname, + 'scenario-tool-loop-agent.mjs', + 'instrument.mjs', + (createRunner, test) => { + test('creates spans for ToolLoopAgent with tool calls', async () => { + const expectedTransaction = { + transaction: 'main', + spans: expect.arrayContaining([ + // ToolLoopAgent outer span + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'invoke_agent', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.invoke_agent', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + [GEN_AI_REQUEST_MODEL_ATTRIBUTE]: 'mock-model-id', + }), + op: 'gen_ai.invoke_agent', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // First doGenerate span (returns tool-calls) + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 10, + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 20, + [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['tool-calls'], + }), + op: 'gen_ai.generate_content', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Tool execution span + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_TOOL_CALL_ID_ATTRIBUTE]: 'call-1', + [GEN_AI_TOOL_NAME_ATTRIBUTE]: 'getWeather', + [GEN_AI_TOOL_TYPE_ATTRIBUTE]: 'function', + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'execute_tool', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.execute_tool', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + }), + description: 'execute_tool getWeather', + op: 'gen_ai.execute_tool', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + // Second doGenerate span (returns final text) + expect.objectContaining({ + data: expect.objectContaining({ + [GEN_AI_OPERATION_NAME_ATTRIBUTE]: 'generate_content', + [SEMANTIC_ATTRIBUTE_SENTRY_OP]: 'gen_ai.generate_content', + [SEMANTIC_ATTRIBUTE_SENTRY_ORIGIN]: 'auto.vercelai.otel', + [GEN_AI_USAGE_INPUT_TOKENS_ATTRIBUTE]: 15, + [GEN_AI_USAGE_OUTPUT_TOKENS_ATTRIBUTE]: 25, + [GEN_AI_RESPONSE_FINISH_REASONS_ATTRIBUTE]: ['stop'], + }), + op: 'gen_ai.generate_content', + origin: 'auto.vercelai.otel', + status: 'ok', + }), + ]), + }; + + await createRunner().expect({ transaction: expectedTransaction }).start().completed(); + }); + }, + { + additionalDependencies: { + ai: '^6.0.0', + }, + }, + ); });