diff --git a/core/src/agents/functions.ts b/core/src/agents/functions.ts index 386e4f54b..eb062b69d 100644 --- a/core/src/agents/functions.ts +++ b/core/src/agents/functions.ts @@ -190,6 +190,9 @@ async function callToolAsync( ), }); return result; + } catch (e: unknown) { + traceToolCall({tool, args, error: e}); + throw e; } finally { span.end(); } diff --git a/core/src/telemetry/tracing.ts b/core/src/telemetry/tracing.ts index 4d97440bc..ed556cdd2 100644 --- a/core/src/telemetry/tracing.ts +++ b/core/src/telemetry/tracing.ts @@ -16,7 +16,7 @@ */ import {Content} from '@google/genai'; -import {context, Context, trace} from '@opentelemetry/api'; +import {context, Context, SpanStatusCode, trace} from '@opentelemetry/api'; import {BaseAgent} from '../agents/base_agent.js'; import {InvocationContext} from '../agents/invocation_context.js'; @@ -24,8 +24,10 @@ import {Event} from '../events/event.js'; import {LlmRequest} from '../models/llm_request.js'; import {LlmResponse} from '../models/llm_response.js'; import {BaseTool} from '../tools/base_tool.js'; +import {resolveErrorType} from '../utils/error_utils.js'; import {version} from '../version.js'; +const ERROR_TYPE = 'error.type'; const GEN_AI_AGENT_DESCRIPTION = 'gen_ai.agent.description'; const GEN_AI_AGENT_NAME = 'gen_ai.agent.name'; const GEN_AI_CONVERSATION_ID = 'gen_ai.conversation.id'; @@ -93,18 +95,26 @@ export function traceAgentInvocation({ export interface TraceToolCallParams { tool: BaseTool; args: Record; - functionResponseEvent: Event; + /** Absent when the tool threw: there is no response to describe. */ + functionResponseEvent?: Event; + /** + * The error the tool threw, if any. A thrown error is the authoritative + * classification for the span. + */ + error?: unknown; } /** * Traces tool call. * - * @param params The parameters object containing tool, args, and function response event. + * @param params The parameters object containing tool, args, the function + * response event, and the error the tool threw. */ export function traceToolCall({ tool, args, functionResponseEvent, + error, }: TraceToolCallParams): void { const span = trace.getActiveSpan(); if (!span) return; @@ -124,11 +134,20 @@ export function traceToolCall({ : '{}', }); + if (error !== undefined) { + const failureType = resolveErrorType(error); + span.recordException(error instanceof Error ? error : String(error)); + span.setAttribute(ERROR_TYPE, failureType); + // The type rather than the message, so that no tool content lands in a + // field the content toggle cannot gate. + span.setStatus({code: SpanStatusCode.ERROR, message: failureType}); + } + // Tracing tool response let toolCallId = ''; let toolResponse: unknown = ''; - if (functionResponseEvent.content?.parts) { + if (functionResponseEvent?.content?.parts) { const responseParts = functionResponseEvent.content.parts; const functionResponse = responseParts[0]?.functionResponse; if (functionResponse?.id) { @@ -144,7 +163,7 @@ export function traceToolCall({ span.setAttributes({ [GEN_AI_TOOL_CALL_ID]: toolCallId, - 'gcp.vertex.agent.event_id': functionResponseEvent.id, + 'gcp.vertex.agent.event_id': functionResponseEvent?.id, 'gcp.vertex.agent.tool_response': shouldAddRequestResponseToSpans() ? safeJsonSerialize(toolResponse) : '{}', diff --git a/core/src/utils/error_utils.ts b/core/src/utils/error_utils.ts index f1f80589b..2ae22e94c 100644 --- a/core/src/utils/error_utils.ts +++ b/core/src/utils/error_utils.ts @@ -159,3 +159,41 @@ function formatErrorRecursive(err: unknown, seen: Set): string { export function formatError(err: unknown): string { return formatErrorRecursive(err, new Set()); } + +/** + * Derives the `error.type` telemetry label for a failure. + * + * Prefers, in order: an `errorType` the error classified itself with; an HTTP + * status the error carries (the `@google/genai` `ApiError` reports one, and the + * SDK collapses every 4xx into a single class and every 5xx into another, so + * the status is the only signal that tells them apart); finally the class name. + * + * The status is duck-typed rather than matched with `instanceof`, because two + * copies of `@google/genai` can share one runtime and an error raised by one is + * not an `instanceof` the class of the other. It is bounded to a plausible HTTP + * range so that an unrelated numeric `status` field is not reported as a status + * code. + * + * @param error The thrown value to classify. JavaScript allows any value to be + * thrown, so this accepts `unknown` rather than `Error`. + * @return The value to report as `error.type`. + */ +export function resolveErrorType(error: unknown): string { + const record = asRecord(error); + const errorType = record?.['errorType']; + if (typeof errorType === 'string') { + return errorType; + } + const status = record?.['status']; + if ( + typeof status === 'number' && + status >= MIN_HTTP_STATUS && + status <= MAX_HTTP_STATUS + ) { + return String(status); + } + // `constructor.name` rather than `name`, mirroring Python's + // `type(error).__name__`: a subclass that never assigns `this.name` still + // reports its own class. + return error instanceof Error ? error.constructor.name : String(error); +} diff --git a/core/test/telemetry/tool_exception_span_test.ts b/core/test/telemetry/tool_exception_span_test.ts new file mode 100644 index 000000000..a8196db3c --- /dev/null +++ b/core/test/telemetry/tool_exception_span_test.ts @@ -0,0 +1,192 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + BaseTool, + Event, + functionsExportedForTestingOnly, + InvocationContext, + LlmAgent, + PluginManager, + Session, +} from '@google/adk'; +import {FunctionCall} from '@google/genai'; +import {SpanStatusCode, trace} from '@opentelemetry/api'; +import { + InMemorySpanExporter, + ReadableSpan, + SimpleSpanProcessor, +} from '@opentelemetry/sdk-trace-base'; +import {NodeTracerProvider} from '@opentelemetry/sdk-trace-node'; +import { + afterAll, + afterEach, + beforeAll, + beforeEach, + describe, + expect, + it, +} from 'vitest'; + +const {handleFunctionCallList} = functionsExportedForTestingOnly; + +/** An error that classifies itself, the way an MCP tool failure does. */ +class ClassifiedError extends Error { + constructor( + message: string, + readonly errorType: string, + ) { + super(message); + } +} + +/** The error a plain throwing tool raises. */ +class InventoryLookupError extends Error {} + +/** + * A tool that throws. `FunctionTool` re-wraps a thrown error into a plain + * `Error`, which would flatten the class these tests assert on, so these + * fixtures subclass `BaseTool` directly. + */ +class ThrowingTool extends BaseTool { + constructor(private readonly error: unknown) { + super({name: 'throwingTool', description: 'always throws'}); + } + + override async runAsync(): Promise { + throw this.error; + } +} + +/** A tool that succeeds, used to pin the unchanged success path. */ +class SucceedingTool extends BaseTool { + constructor() { + super({name: 'succeedingTool', description: 'always succeeds'}); + } + + override async runAsync(): Promise { + return {result: 'tool executed'}; + } +} + +describe('execute_tool span for a throwing tool', () => { + let exporter: InMemorySpanExporter; + let provider: NodeTracerProvider; + let invocationContext: InvocationContext; + + // The tracer in tracing.ts caches its delegate on first use, so the provider + // is registered once for the whole file and only the exporter is reset. + beforeAll(() => { + exporter = new InMemorySpanExporter(); + provider = new NodeTracerProvider({ + spanProcessors: [new SimpleSpanProcessor(exporter)], + }); + provider.register(); + }); + + afterAll(async () => { + await provider.shutdown(); + trace.disable(); + }); + + beforeEach(() => { + invocationContext = new InvocationContext({ + invocationId: 'inv_123', + session: {} as Session, + agent: new LlmAgent({name: 'test_agent', model: 'test_model'}), + pluginManager: new PluginManager(), + }); + }); + + afterEach(() => { + exporter.reset(); + }); + + /** Runs one tool through the production call path and returns its event. */ + async function runTool(tool: BaseTool): Promise { + const functionCall: FunctionCall = { + id: 'call_1', + name: tool.name, + args: {}, + }; + return handleFunctionCallList({ + invocationContext, + functionCalls: [functionCall], + toolsDict: {[tool.name]: tool}, + beforeToolCallbacks: [], + afterToolCallbacks: [], + }); + } + + /** Returns the single exported `execute_tool` span. */ + function toolSpan(tool: BaseTool): ReadableSpan { + const spans = exporter + .getFinishedSpans() + .filter((span) => span.name === `execute_tool ${tool.name}`); + expect(spans).toHaveLength(1); + return spans[0]; + } + + it('marks the span as failed and records the exception', async () => { + const tool = new ThrowingTool(new InventoryLookupError('sku not found')); + + await runTool(tool); + + const span = toolSpan(tool); + expect(span.status.code).toBe(SpanStatusCode.ERROR); + expect(span.status.message).toBe('InventoryLookupError'); + expect(span.attributes['error.type']).toBe('InventoryLookupError'); + expect(span.attributes['gen_ai.tool.name']).toBe('throwingTool'); + const exceptions = span.events.filter((e) => e.name === 'exception'); + expect(exceptions).toHaveLength(1); + expect(exceptions[0].attributes?.['exception.message']).toBe( + 'sku not found', + ); + }); + + it('reports no response event on the error path', async () => { + const tool = new ThrowingTool(new InventoryLookupError('sku not found')); + + await runTool(tool); + + const span = toolSpan(tool); + expect('gcp.vertex.agent.event_id' in span.attributes).toBe(false); + expect(span.attributes['gen_ai.tool.call.id']).toBe(''); + }); + + it('lets the exception reach the caller unchanged', async () => { + const tool = new ThrowingTool(new InventoryLookupError('sku not found')); + + const event = await runTool(tool); + + expect(event?.content?.parts?.[0].functionResponse?.response).toEqual({ + error: 'sku not found', + }); + }); + + it('prefers the error type the thrown error classified itself with', async () => { + const tool = new ThrowingTool( + new ClassifiedError('upstream refused', 'TOOL_ERROR'), + ); + + await runTool(tool); + + const span = toolSpan(tool); + expect(span.attributes['error.type']).toBe('TOOL_ERROR'); + expect(span.status.message).toBe('TOOL_ERROR'); + }); + + it('leaves the span of a succeeding tool clean', async () => { + const tool = new SucceedingTool(); + + await runTool(tool); + + const span = toolSpan(tool); + expect(span.status.code).not.toBe(SpanStatusCode.ERROR); + expect('error.type' in span.attributes).toBe(false); + expect(span.events.filter((e) => e.name === 'exception')).toHaveLength(0); + }); +}); diff --git a/core/test/telemetry/tracing_test.ts b/core/test/telemetry/tracing_test.ts index a150f1a63..ed4140de0 100644 --- a/core/test/telemetry/tracing_test.ts +++ b/core/test/telemetry/tracing_test.ts @@ -4,7 +4,7 @@ * SPDX-License-Identifier: Apache-2.0 */ -import {trace} from '@opentelemetry/api'; +import {SpanStatusCode, trace} from '@opentelemetry/api'; import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; import { @@ -46,6 +46,8 @@ describe('Telemetry Tracing Functions', () => { mockSpan = { setAttributes: vi.fn(), setAttribute: vi.fn(), + recordException: vi.fn(), + setStatus: vi.fn(), }; mockAgent = { @@ -185,6 +187,95 @@ describe('Telemetry Tracing Functions', () => { expect.stringContaining('not specified'), }); }); + + it('should record the exception a tool threw on the span', () => { + // Arrange + vi.mocked(trace.getActiveSpan).mockReturnValue(mockSpan); + const error = new TypeError('bad'); + + // Act + traceToolCall({tool: mockTool, args: {}, error}); + + // Assert + expect(mockSpan.recordException).toHaveBeenCalledTimes(1); + expect(mockSpan.recordException).toHaveBeenCalledWith(error); + expect(mockSpan.setAttribute).toHaveBeenCalledWith( + 'error.type', + 'TypeError', + ); + expect(mockSpan.setStatus).toHaveBeenCalledWith({ + code: SpanStatusCode.ERROR, + message: 'TypeError', + }); + }); + + it('should still set the baseline tool attributes when the tool threw', () => { + // Arrange + vi.mocked(trace.getActiveSpan).mockReturnValue(mockSpan); + const args = {param1: 'value1'}; + + // Act + traceToolCall({tool: mockTool, args, error: new TypeError('bad')}); + + // Assert + expect(mockSpan.setAttributes).toHaveBeenCalledWith({ + 'gen_ai.operation.name': 'execute_tool', + 'gen_ai.tool.description': 'A test tool', + 'gen_ai.tool.name': 'test-tool', + 'gen_ai.tool.type': 'FunctionTool', + 'gcp.vertex.agent.llm_request': '{}', + 'gcp.vertex.agent.llm_response': '{}', + 'gcp.vertex.agent.tool_call_args': expect.stringContaining('param1'), + }); + }); + + it('should leave the call id unspecified and the event id unset without a response event', () => { + // Arrange + vi.mocked(trace.getActiveSpan).mockReturnValue(mockSpan); + + // Act + traceToolCall({tool: mockTool, args: {}, error: new TypeError('bad')}); + + // Assert + expect(mockSpan.setAttributes).toHaveBeenCalledWith({ + 'gen_ai.tool.call.id': '', + 'gcp.vertex.agent.event_id': undefined, + 'gcp.vertex.agent.tool_response': + expect.stringContaining('not specified'), + }); + }); + + it('should leave the span status unset when the tool did not throw', () => { + // Arrange + vi.mocked(trace.getActiveSpan).mockReturnValue(mockSpan); + + // Act + traceToolCall({ + tool: mockTool, + args: {}, + functionResponseEvent: mockEvent, + }); + + // Assert + expect(mockSpan.recordException).not.toHaveBeenCalled(); + expect(mockSpan.setStatus).not.toHaveBeenCalled(); + expect(mockSpan.setAttribute).not.toHaveBeenCalledWith( + 'error.type', + expect.anything(), + ); + }); + + it('should record a thrown value that is not an error', () => { + // Arrange + vi.mocked(trace.getActiveSpan).mockReturnValue(mockSpan); + + // Act + traceToolCall({tool: mockTool, args: {}, error: 'boom'}); + + // Assert + expect(mockSpan.recordException).toHaveBeenCalledWith('boom'); + expect(mockSpan.setAttribute).toHaveBeenCalledWith('error.type', 'boom'); + }); }); describe('traceMergedToolCalls', () => { diff --git a/core/test/utils/error_utils_test.ts b/core/test/utils/error_utils_test.ts index 11cfd3636..c696f0ada 100644 --- a/core/test/utils/error_utils_test.ts +++ b/core/test/utils/error_utils_test.ts @@ -4,8 +4,9 @@ * SPDX-License-Identifier: Apache-2.0 */ +import {ApiError} from '@google/genai'; import {describe, expect, it} from 'vitest'; -import {formatError} from '../../src/utils/error_utils.js'; +import {formatError, resolveErrorType} from '../../src/utils/error_utils.js'; const TRUNCATION_MARKER = '... [truncated]'; const MAX_RESPONSE_BODY_LENGTH = 1000; @@ -208,3 +209,64 @@ describe('formatError', () => { expect(formatError(err)).toContain('text body'); }); }); + +/** An error that classifies itself, the way an MCP tool failure does. */ +class ClassifiedError extends Error { + constructor( + message: string, + readonly errorType: string, + ) { + super(message); + } +} + +/** An error that never assigns `this.name`, so only its class identifies it. */ +class MyToolError extends Error {} + +describe('resolveErrorType', () => { + it('prefers the error type the error classified itself with', () => { + const err = new ClassifiedError('tool failed', 'MCP_TOOL_ERROR'); + expect(resolveErrorType(err)).toBe('MCP_TOOL_ERROR'); + }); + + it('reports the HTTP status of a genai API error', () => { + const err = new ApiError({message: 'rate limited', status: 429}); + expect(resolveErrorType(err)).toBe('429'); + }); + + it('prefers a self-classified error type over the HTTP status', () => { + const err = Object.assign(new ApiError({message: 'quota', status: 429}), { + errorType: 'QUOTA_EXCEEDED', + }); + expect(resolveErrorType(err)).toBe('QUOTA_EXCEEDED'); + }); + + it('reports the class name of a standard error', () => { + expect(resolveErrorType(new TypeError('bad'))).toBe('TypeError'); + }); + + it('reports the class name of a subclass that never sets its name', () => { + expect(resolveErrorType(new MyToolError('bad'))).toBe('MyToolError'); + }); + + it('ignores a numeric status outside the HTTP range', () => { + const err = Object.assign(new MyToolError('bad'), {status: 0}); + expect(resolveErrorType(err)).toBe('MyToolError'); + }); + + it('ignores a status that is not a number', () => { + const err = Object.assign(new MyToolError('bad'), {status: '429'}); + expect(resolveErrorType(err)).toBe('MyToolError'); + }); + + it('stringifies a thrown value that is not an error', () => { + expect(resolveErrorType('boom')).toBe('boom'); + expect(resolveErrorType(42)).toBe('42'); + }); + + it('reads the error type off a plain object that carries one', () => { + expect(resolveErrorType({errorType: 'PLAIN_OBJECT_ERROR'})).toBe( + 'PLAIN_OBJECT_ERROR', + ); + }); +});