From c676490480a77829cbb5c60119ce2b27fae02078 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Tue, 28 Jul 2026 18:07:40 -0700 Subject: [PATCH 1/4] Fix: resolve the exit_loop built-in tool in YAML agent configs The conformance agent registry mapped every name in BUILTIN_TOOLS to undefined, so a YAML agent declaring 'tools: [{name: exit_loop}]' was instantiated with no tools at all. Split the list into built-ins that resolve to a real tool object (currently just exit_loop -> EXIT_LOOP, mirroring adk-python's LlmAgent._resolve_tools) and the server-side built-ins that must keep being dropped because their processLlmRequest rejects the replay harness' DummyLlm model name. --- dev/src/integration/agent_registry.ts | 27 +++++++++++++++++---- dev/test/integration/agent_registry_test.ts | 24 ++++++++++++++++-- 2 files changed, 44 insertions(+), 7 deletions(-) diff --git a/dev/src/integration/agent_registry.ts b/dev/src/integration/agent_registry.ts index 51abe4c3b..3548c0b86 100644 --- a/dev/src/integration/agent_registry.ts +++ b/dev/src/integration/agent_registry.ts @@ -7,6 +7,8 @@ import { AgentTool, BaseAgent, + BaseTool, + EXIT_LOOP, FunctionTool, LlmAgent, LoopAgent, @@ -23,8 +25,19 @@ import { } from './agent_types.js'; import {IntegrationRegistry} from './integration_registry.js'; -const BUILTIN_TOOLS = [ - 'exit_loop', +/** + * Built-in tools that a YAML config can name directly, mirroring adk-python's + * `LlmAgent._resolve_tools`, which resolves a bare built-in name to the real + * tool object. + */ +const BUILTIN_TOOLS = new Map([['exit_loop', EXIT_LOOP]]); + +/** + * Server-side built-ins that are dropped instead of resolved: they are executed + * by the Gemini backend and their `processLlmRequest` throws for the replay + * harness' `DummyLlm` model name. + */ +const SKIPPED_BUILTIN_TOOLS = [ 'google_search', 'url_context', 'google_maps_grounding', @@ -136,8 +149,12 @@ export class AgentRegistry { const tools = config.tools ?.map((toolConfig) => { - // Built in tools are skipped - if (BUILTIN_TOOLS.includes(toolConfig.name)) { + const builtinTool = BUILTIN_TOOLS.get(toolConfig.name); + if (builtinTool) { + return builtinTool; + } + + if (SKIPPED_BUILTIN_TOOLS.includes(toolConfig.name)) { return undefined; } @@ -175,7 +192,7 @@ export class AgentRegistry { return this.findToolOrThrow(toolConfig.name); }) - // remove entries for built-in tools + // remove entries for the server-side built-in tools .filter((tool) => tool !== undefined); const options = { diff --git a/dev/test/integration/agent_registry_test.ts b/dev/test/integration/agent_registry_test.ts index 26235389c..9d7106e85 100644 --- a/dev/test/integration/agent_registry_test.ts +++ b/dev/test/integration/agent_registry_test.ts @@ -6,6 +6,7 @@ import { AgentTool, + EXIT_LOOP, FunctionTool, LlmAgent, MCPToolset, @@ -257,7 +258,7 @@ describe('AgentRegistry', () => { expect(retrieved.tools[0]).toBe(tool); }); - it('should skip built-in tools', () => { + it('should resolve exit_loop to the EXIT_LOOP built-in tool', () => { const config = { name: 'builtin_agent', model: 'model', @@ -270,7 +271,26 @@ describe('AgentRegistry', () => { agentRegistry.registerAgentConfig('builtin_agent', config); const retrieved = agentRegistry.getAgent('builtin_agent') as LlmAgent; - expect(retrieved).toBeDefined(); + expect(retrieved.tools).toEqual([EXIT_LOOP]); + }); + + it('should skip server-side built-in tools', () => { + const config = { + name: 'builtin_agent', + model: 'model', + description: 'desc', + instruction: 'inst', + agentClass: 'LlmAgent', + tools: [ + {name: 'google_search'}, + {name: 'url_context'}, + {name: 'google_maps_grounding'}, + ], + } as unknown as YamlAgentConfig; + + agentRegistry.registerAgentConfig('builtin_agent', config); + const retrieved = agentRegistry.getAgent('builtin_agent') as LlmAgent; + expect(retrieved.tools.length).toBe(0); }); }); From a73f149ac0e4db5ca5ffc8e79817a6b44bfa3a62 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Tue, 28 Jul 2026 18:07:44 -0700 Subject: [PATCH 2/4] Fix: replay the exit_loop side effects in ReplayPlugin Returning the recorded tool response short-circuits callToolAsync, so a tool whose only observable effect is on EventActions never runs during a replay. The plugin already replicated that effect for transfer_to_agent; do the same for exit_loop so escalate and skipSummarization are set, which is what stops the LoopAgent and ends the LlmAgent step loop. --- dev/src/integration/replay_plugin.ts | 19 ++- dev/test/integration/replay_plugin_test.ts | 134 +++++++++++++++++++++ 2 files changed, 148 insertions(+), 5 deletions(-) create mode 100644 dev/test/integration/replay_plugin_test.ts diff --git a/dev/src/integration/replay_plugin.ts b/dev/src/integration/replay_plugin.ts index c19709d43..d5916b100 100644 --- a/dev/src/integration/replay_plugin.ts +++ b/dev/src/integration/replay_plugin.ts @@ -74,11 +74,20 @@ export class ReplayPlugin extends BasePlugin { const rec = this.recordings[index]; (rec as unknown as {_consumed: boolean})._consumed = true; - // Handle side effects for built-in tools that modify EventActions - if (toolName === 'transfer_to_agent') { - params.toolContext.actions.transferToAgent = params.toolArgs[ - 'agentName' - ] as string; + // Returning the recorded response short-circuits the real tool call, so + // built-in tools whose only observable effect is on EventActions must have + // that effect replicated here. (adk-python instead runs the real tool in + // its replay plugin before returning the recording.) + switch (toolName) { + case 'transfer_to_agent': + params.toolContext.actions.transferToAgent = params.toolArgs[ + 'agentName' + ] as string; + break; + case 'exit_loop': + params.toolContext.actions.escalate = true; + params.toolContext.actions.skipSummarization = true; + break; } // The response from a tool call is a plain object. diff --git a/dev/test/integration/replay_plugin_test.ts b/dev/test/integration/replay_plugin_test.ts new file mode 100644 index 000000000..f3d98a23a --- /dev/null +++ b/dev/test/integration/replay_plugin_test.ts @@ -0,0 +1,134 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import { + Context, + createEventActions, + EventActions, + EXIT_LOOP, + FunctionTool, +} from '@google/adk'; +import {beforeEach, describe, expect, it} from 'vitest'; +import {ReplayPlugin} from '../../src/integration/replay_plugin.js'; +import {Recording} from '../../src/integration/test_types.js'; + +const AGENT_NAME = 'refiner_agent'; + +const TRANSFER_TO_AGENT = new FunctionTool({ + name: 'transfer_to_agent', + description: 'Transfers to another agent.', + execute: async () => ({}), +}); + +const GREET = new FunctionTool({ + name: 'greet', + description: 'Greets the user.', + execute: async () => ({}), +}); + +function toolRecording( + name: string, + response: Record, +): Recording { + return { + userMessageIndex: 0, + agentName: AGENT_NAME, + toolRecording: {toolCall: {name}, toolResponse: {response}}, + }; +} + +describe('ReplayPlugin', () => { + let actions: EventActions; + let toolContext: Context; + + beforeEach(() => { + actions = createEventActions(); + toolContext = { + actions, + invocationContext: {agent: {name: AGENT_NAME}}, + } as unknown as Context; + }); + + it('replays the recorded response and escalates for exit_loop', async () => { + const plugin = new ReplayPlugin( + [toolRecording('exit_loop', {result: null})], + { + userMessageIndex: 0, + }, + ); + + const response = await plugin.beforeToolCallback({ + tool: EXIT_LOOP, + toolArgs: {}, + toolContext, + }); + + expect(response).toEqual({result: null}); + expect(actions.escalate).toBe(true); + expect(actions.skipSummarization).toBe(true); + }); + + it('replays transfer_to_agent by setting transferToAgent', async () => { + const plugin = new ReplayPlugin( + [toolRecording('transfer_to_agent', {result: null})], + {userMessageIndex: 0}, + ); + + await plugin.beforeToolCallback({ + tool: TRANSFER_TO_AGENT, + toolArgs: {agentName: 'writer_agent'}, + toolContext, + }); + + expect(actions.transferToAgent).toBe('writer_agent'); + expect(actions.escalate).toBeUndefined(); + }); + + it('replays a plain tool without touching the actions', async () => { + const plugin = new ReplayPlugin( + [toolRecording('greet', {greeting: 'hi'})], + { + userMessageIndex: 0, + }, + ); + + const response = await plugin.beforeToolCallback({ + tool: GREET, + toolArgs: {}, + toolContext, + }); + + expect(response).toEqual({greeting: 'hi'}); + expect(actions).toEqual(createEventActions()); + }); + + it('throws when no recording matches the tool call', async () => { + const plugin = new ReplayPlugin([], {userMessageIndex: 0}); + + await expect( + plugin.beforeToolCallback({tool: GREET, toolArgs: {}, toolContext}), + ).rejects.toThrow( + `No tool recording found for agent ${AGENT_NAME}, tool greet at turn 0`, + ); + }); + + it('consumes each recording only once', async () => { + const plugin = new ReplayPlugin( + [toolRecording('greet', {greeting: 'hi'})], + { + userMessageIndex: 0, + }, + ); + const call = () => + plugin.beforeToolCallback({tool: GREET, toolArgs: {}, toolContext}); + + await call(); + + await expect(call()).rejects.toThrow( + `No tool recording found for agent ${AGENT_NAME}, tool greet at turn 0`, + ); + }); +}); From a519152349bcd5c09e5c5c08ec1022ff8f76e7f3 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Tue, 28 Jul 2026 18:07:48 -0700 Subject: [PATCH 3/4] Fix: un-skip the workflow/loop_001 conformance test ExitLoopTool is implemented and now resolved from YAML configs, so the skip reason no longer holds. Add a TestRunner suite that replays an in-memory equivalent of the case (LoopAgent -> LlmAgent calling exit_loop) as executable proof, since the conformance corpus itself is not vendored in this repo. --- dev/src/integration/test_runner.ts | 1 - dev/test/integration/test_runner_test.ts | 140 +++++++++++++++++++++++ 2 files changed, 140 insertions(+), 1 deletion(-) create mode 100644 dev/test/integration/test_runner_test.ts diff --git a/dev/src/integration/test_runner.ts b/dev/src/integration/test_runner.ts index efb35b8d8..1362355ec 100644 --- a/dev/src/integration/test_runner.ts +++ b/dev/src/integration/test_runner.ts @@ -30,7 +30,6 @@ const SKIPPED_TESTS = [ name: 'tool/example_tool_001', reason: 'ExampleTool is not implemented yet.', }, - {name: 'workflow/loop_001', reason: 'ExitLoopTool is not implemented yet.'}, { name: 'core/multi_005', reason: 'Suspected broken test. Need to re-evaluate.', diff --git a/dev/test/integration/test_runner_test.ts b/dev/test/integration/test_runner_test.ts new file mode 100644 index 000000000..600e08942 --- /dev/null +++ b/dev/test/integration/test_runner_test.ts @@ -0,0 +1,140 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {createEvent, createEventActions, createSession} from '@google/adk'; +import {beforeEach, describe, expect, it} from 'vitest'; +import {AgentRegistry} from '../../src/integration/agent_registry.js'; +import {YamlAgentConfig} from '../../src/integration/agent_types.js'; +import {IntegrationRegistry} from '../../src/integration/integration_registry.js'; +import {TestRunner} from '../../src/integration/test_runner.js'; +import {TestInfo} from '../../src/integration/test_types.js'; + +const ROOT_AGENT = 'loop_root_agent'; +const SUB_AGENT = 'refiner_agent'; +const USER_MESSAGE = 'Refine the poem.'; + +/** + * Registers the in-memory equivalent of the `workflow/loop_001` conformance + * corpus: a LoopAgent whose only sub-agent calls `exit_loop`. + */ +function registerLoopAgents(registry: AgentRegistry) { + registry.registerAgentConfig('loop_test/refiner_agent', { + name: SUB_AGENT, + model: 'gemini-2.5-flash', + description: 'Refines a poem.', + instruction: 'Refine the poem, then call exit_loop.', + agentClass: 'LlmAgent', + tools: [{name: 'exit_loop'}], + } as unknown as YamlAgentConfig); + + registry.registerAgentConfig('loop_test/root_agent', { + name: ROOT_AGENT, + model: 'gemini-2.5-flash', + description: 'Loops until the refiner exits.', + instruction: '', + agentClass: 'LoopAgent', + maxIterations: '3', + isRootAgent: true, + subAgents: [{configPath: 'loop_test/refiner_agent'}], + } as unknown as YamlAgentConfig); +} + +/** + * The session the harness must reproduce: the refiner agent calls `exit_loop` + * once and the LoopAgent stops, so there is exactly one iteration. The final + * function-response event carries `escalate` / `skipSummarization`, which only + * happens when the `exit_loop` tool is resolved and its side effects replayed. + * + * The fixture keeps the shape of a recorded `generated-session.yaml`, but note + * that `filterPartFields` strips `functionCall` / `functionResponse` from every + * part before the comparison, so those payloads are documentation rather than + * assertions. + */ +function expectedSession() { + return createSession({ + id: 'expected-session', + appName: 'test-runner', + events: [ + createEvent({ + author: 'user', + content: {role: 'user', parts: [{text: USER_MESSAGE}]}, + }), + createEvent({ + author: SUB_AGENT, + content: { + role: 'model', + parts: [{functionCall: {name: 'exit_loop', args: {}}}], + }, + }), + createEvent({ + author: SUB_AGENT, + content: { + role: 'user', + parts: [ + {functionResponse: {name: 'exit_loop', response: {result: null}}}, + ], + }, + actions: { + ...createEventActions(), + escalate: true, + skipSummarization: true, + }, + }), + ], + }); +} + +function loopTestInfo(): TestInfo { + return { + name: 'workflow/loop_001', + spec: { + description: 'The refiner agent exits the loop on the first iteration.', + agent: 'loop_test', + userMessages: [{text: USER_MESSAGE}], + }, + recordings: { + recordings: [ + { + userMessageIndex: 0, + agentName: SUB_AGENT, + llmRecording: { + llmResponse: { + content: { + role: 'model', + parts: [{functionCall: {name: 'exit_loop', args: {}}}], + }, + }, + }, + }, + { + userMessageIndex: 0, + agentName: SUB_AGENT, + toolRecording: { + toolCall: {name: 'exit_loop'}, + toolResponse: {response: {result: null}}, + }, + }, + ], + }, + session: expectedSession(), + }; +} + +describe('TestRunner', () => { + let registry: AgentRegistry; + let testRunner: TestRunner; + + beforeEach(() => { + registry = new AgentRegistry(new IntegrationRegistry()); + testRunner = new TestRunner(registry); + }); + + it('replays a loop agent that exits via the exit_loop built-in tool', async () => { + registerLoopAgents(registry); + + await expect(testRunner.run(loopTestInfo(), false)).resolves.toBe(false); + }); +}); From ca75004605a96505f709edbbef7490d2de738211 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Thu, 30 Jul 2026 22:07:17 -0700 Subject: [PATCH 4/4] Fix: use a plain Record for the built-in tool table Review feedback: prefer Record over a Map. The lookup goes through Object.hasOwn so a YAML string naming an inherited member such as 'constructor' still falls through to findToolOrThrow instead of resolving to Object.prototype.constructor. Object.hasOwn is already the guard used elsewhere in this package (test_runner.ts). --- dev/src/integration/agent_registry.ts | 10 +++++----- dev/test/integration/agent_registry_test.ts | 16 ++++++++++++++++ 2 files changed, 21 insertions(+), 5 deletions(-) diff --git a/dev/src/integration/agent_registry.ts b/dev/src/integration/agent_registry.ts index 3548c0b86..8513d86a1 100644 --- a/dev/src/integration/agent_registry.ts +++ b/dev/src/integration/agent_registry.ts @@ -28,9 +28,10 @@ import {IntegrationRegistry} from './integration_registry.js'; /** * Built-in tools that a YAML config can name directly, mirroring adk-python's * `LlmAgent._resolve_tools`, which resolves a bare built-in name to the real - * tool object. + * tool object. Looked up with `Object.hasOwn` so that a YAML string naming an + * inherited member such as `constructor` does not resolve to one. */ -const BUILTIN_TOOLS = new Map([['exit_loop', EXIT_LOOP]]); +const BUILTIN_TOOLS: Record = {exit_loop: EXIT_LOOP}; /** * Server-side built-ins that are dropped instead of resolved: they are executed @@ -149,9 +150,8 @@ export class AgentRegistry { const tools = config.tools ?.map((toolConfig) => { - const builtinTool = BUILTIN_TOOLS.get(toolConfig.name); - if (builtinTool) { - return builtinTool; + if (Object.hasOwn(BUILTIN_TOOLS, toolConfig.name)) { + return BUILTIN_TOOLS[toolConfig.name]; } if (SKIPPED_BUILTIN_TOOLS.includes(toolConfig.name)) { diff --git a/dev/test/integration/agent_registry_test.ts b/dev/test/integration/agent_registry_test.ts index 9d7106e85..ddae8effa 100644 --- a/dev/test/integration/agent_registry_test.ts +++ b/dev/test/integration/agent_registry_test.ts @@ -293,4 +293,20 @@ describe('AgentRegistry', () => { expect(retrieved.tools.length).toBe(0); }); + + it('should not resolve an inherited Object member as a built-in tool', () => { + const config = { + name: 'bad_agent', + model: 'model', + description: 'desc', + instruction: 'inst', + agentClass: 'LlmAgent', + tools: [{name: 'constructor'}], + } as unknown as YamlAgentConfig; + + agentRegistry.registerAgentConfig('bad_agent', config); + expect(() => agentRegistry.getAgent('bad_agent')).toThrow( + 'Tool constructor not found in registry', + ); + }); });