From 09b17b117c3fd2e67303cb425d18c523009c86f2 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Sun, 2 Aug 2026 21:32:21 -0700 Subject: [PATCH 1/4] Fix: export the code-execution response processor from the public API The LlmAgent default (stacked base) makes CODE_EXECUTION_RESPONSE_PROCESSOR the out-of-the-box response processor, but the symbol was still unreachable from an installed @google/adk: it was absent from common.ts and core's exports map declares no deep subpath. Callers who supply their own responseProcessors list therefore could not re-add the default, and the sandbox integration test could only import it through a path that resolves via the vitest alias. Export both the singleton and its class from common.ts (index.ts and index_web.ts re-export it), document the replace-not-merge contract on LlmAgentConfig.responseProcessors, and switch the tests to the public import. --- core/src/agents/llm_agent.ts | 5 + core/src/common.ts | 4 + core/test/agents/llm_agent_test.ts | 2 +- .../code_execution_request_processor_test.ts | 8 +- .../agent_with_default_code_execution_test.ts | 156 ++++++++++++++++++ .../agent_with_sandbox_executor_test.ts | 7 +- 6 files changed, 174 insertions(+), 8 deletions(-) create mode 100644 tests/integration/agents/agent_with_default_code_execution_test.ts diff --git a/core/src/agents/llm_agent.ts b/core/src/agents/llm_agent.ts index aaab7f9c6..64bf77aab 100644 --- a/core/src/agents/llm_agent.ts +++ b/core/src/agents/llm_agent.ts @@ -303,6 +303,11 @@ export interface LlmAgentConfig extends BaseAgentConfig { /** * Processors to run after the LLM response is received. + * + * Omitting this selects the default list, which contains + * {@link CODE_EXECUTION_RESPONSE_PROCESSOR}. Supplying a list replaces the + * default entirely rather than extending it, so a caller that needs code + * execution alongside its own processors must include the default explicitly. */ responseProcessors?: BaseLlmResponseProcessor[]; diff --git a/core/src/common.ts b/core/src/common.ts index 23f628165..3eded3277 100644 --- a/core/src/common.ts +++ b/core/src/common.ts @@ -46,6 +46,10 @@ export { BaseLlmRequestProcessor, BaseLlmResponseProcessor, } from './agents/processors/base_llm_processor.js'; +export { + CODE_EXECUTION_RESPONSE_PROCESSOR, + CodeExecutionResponseProcessor, +} from './agents/processors/code_execution_request_processor.js'; export { CONTENT_REQUEST_PROCESSOR, ContentRequestProcessor, diff --git a/core/test/agents/llm_agent_test.ts b/core/test/agents/llm_agent_test.ts index aafcf9756..f22ac2035 100644 --- a/core/test/agents/llm_agent_test.ts +++ b/core/test/agents/llm_agent_test.ts @@ -13,6 +13,7 @@ import { BaseLlmResponseProcessor, BasePlugin, BaseTool, + CODE_EXECUTION_RESPONSE_PROCESSOR, CodeExecutionInput, CodeExecutionResult, CONTENT_REQUEST_PROCESSOR, @@ -36,7 +37,6 @@ import {Content, Outcome, Schema, Type} from '@google/genai'; import {beforeEach, describe, expect, it} from 'vitest'; import {z as z3} from 'zod/v3'; import {z as z4} from 'zod/v4'; -import {CODE_EXECUTION_RESPONSE_PROCESSOR} from '../../src/agents/processors/code_execution_request_processor.js'; import {ScopedArtifactService} from '../../src/artifacts/scoped_artifact_service.js'; class MockLlmConnection implements BaseLlmConnection { diff --git a/core/test/agents/processors/code_execution_request_processor_test.ts b/core/test/agents/processors/code_execution_request_processor_test.ts index cf67f3088..9b8807c38 100644 --- a/core/test/agents/processors/code_execution_request_processor_test.ts +++ b/core/test/agents/processors/code_execution_request_processor_test.ts @@ -6,6 +6,7 @@ import { BaseAgent, + CODE_EXECUTION_RESPONSE_PROCESSOR, InvocationContext, LlmAgent, LlmRequest, @@ -13,10 +14,7 @@ import { createSession, } from '@google/adk'; import {describe, expect, it} from 'vitest'; -import { - CODE_EXECUTION_REQUEST_PROCESSOR, - CodeExecutionResponseProcessor, -} from '../../../src/agents/processors/code_execution_request_processor.js'; +import {CODE_EXECUTION_REQUEST_PROCESSOR} from '../../../src/agents/processors/code_execution_request_processor.js'; import { BaseCodeExecutor, ExecuteCodeParams, @@ -128,7 +126,7 @@ describe('CodeExecutionRequestProcessor', () => { }); describe('CodeExecutionResponseProcessor', () => { - const responseProcessor = new CodeExecutionResponseProcessor(); + const responseProcessor = CODE_EXECUTION_RESPONSE_PROCESSOR; describe('early-exit paths', () => { it('yields no events for a partial response', async () => { diff --git a/tests/integration/agents/agent_with_default_code_execution_test.ts b/tests/integration/agents/agent_with_default_code_execution_test.ts new file mode 100644 index 000000000..1a23e78e4 --- /dev/null +++ b/tests/integration/agents/agent_with_default_code_execution_test.ts @@ -0,0 +1,156 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import type { + CodeExecutionInput, + CodeExecutionResult, + Event, + ExecuteCodeParams, +} from '@google/adk'; +import {BaseCodeExecutor, LlmAgent} from '@google/adk'; +import {FinishReason, Outcome} from '@google/genai'; +import {describe, expect, it} from 'vitest'; +import { + createRunner, + GeminiWithMockResponses, + RawGenerateContentResponse, +} from '../test_case_utils.js'; + +const CODE_BLOCK = '```python\nprint("hello")\n```'; + +/** + * Builds a fresh set of recorded responses for one test. + * + * The response processor truncates `content.parts` in place and + * `GeminiWithMockResponses` hands out the same `Candidate` objects it is + * constructed with, so a fixture shared between tests would be rewritten by + * whichever test runs first. + */ +function mockResponses(): RawGenerateContentResponse[] { + return [ + { + candidates: [ + { + content: { + parts: [{text: `Here is the code:\n${CODE_BLOCK}`}], + role: 'model', + }, + finishReason: FinishReason.STOP, + }, + ], + }, + { + candidates: [ + { + content: {parts: [{text: 'Execution finished.'}], role: 'model'}, + finishReason: FinishReason.STOP, + }, + ], + }, + ]; +} + +/** + * A code executor that records what it was asked to run and returns a fixed + * result, so the test exercises the real processor pipeline without depending + * on a Python interpreter being present on the CI host. + */ +class RecordingCodeExecutor extends BaseCodeExecutor { + readonly calls: CodeExecutionInput[] = []; + + async executeCode(params: ExecuteCodeParams): Promise { + this.calls.push(params.codeExecutionInput); + return {stdout: 'hello', stderr: '', outputFiles: []}; + } +} + +async function collectEvents( + run: (prompt: string) => AsyncGenerator, + prompt: string, +): Promise { + const events: Event[] = []; + for await (const event of run(prompt)) { + events.push(event); + } + return events; +} + +describe('Agent with a codeExecutor and no explicit responseProcessors', () => { + it('executes model-emitted code and yields an execution-result event', async () => { + const executor = new RecordingCodeExecutor(); + const agent = new LlmAgent({ + model: new GeminiWithMockResponses(mockResponses()), + name: 'coderAgent', + description: 'An agent that writes and runs code', + instruction: 'Write code to solve the user request.', + codeExecutor: executor, + }); + + const {run} = await createRunner(agent); + const events = await collectEvents(run, 'Print hello'); + + expect(executor.calls).toHaveLength(1); + expect(executor.calls[0].code).toBe('print("hello")'); + + const resultParts = events.flatMap( + (e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [], + ); + expect(resultParts).toHaveLength(1); + expect(resultParts[0].codeExecutionResult?.outcome).toBe( + Outcome.OUTCOME_OK, + ); + expect(resultParts[0].text).toContain('Code execution result:'); + expect(resultParts[0].text).toContain('hello'); + }); + + it('surfaces a failed execution as OUTCOME_FAILED with the stderr text', async () => { + class FailingCodeExecutor extends BaseCodeExecutor { + async executeCode( + _params: ExecuteCodeParams, + ): Promise { + return {stdout: '', stderr: 'NameError: boom', outputFiles: []}; + } + } + + const agent = new LlmAgent({ + model: new GeminiWithMockResponses(mockResponses()), + name: 'coderAgent', + description: 'An agent that writes and runs code', + instruction: 'Write code to solve the user request.', + codeExecutor: new FailingCodeExecutor(), + }); + + const {run} = await createRunner(agent); + const events = await collectEvents(run, 'Print hello'); + + const resultParts = events.flatMap( + (e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [], + ); + expect(resultParts).toHaveLength(1); + expect(resultParts[0].codeExecutionResult?.outcome).toBe( + Outcome.OUTCOME_FAILED, + ); + expect(resultParts[0].text).toContain('NameError: boom'); + }); + + it('is inert for an agent that has no codeExecutor', async () => { + const agent = new LlmAgent({ + model: new GeminiWithMockResponses(mockResponses()), + name: 'plainAgent', + description: 'An agent with no code executor', + instruction: 'Answer the user request.', + }); + + const {run} = await createRunner(agent); + const events = await collectEvents(run, 'Print hello'); + + const resultParts = events.flatMap( + (e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [], + ); + expect(resultParts).toHaveLength(0); + expect(events.at(-1)?.content?.parts?.[0]?.text).toContain(CODE_BLOCK); + }); +}); diff --git a/tests/integration/agents/agent_with_sandbox_executor_test.ts b/tests/integration/agents/agent_with_sandbox_executor_test.ts index 9ca9eb35c..fcfc5dcf1 100644 --- a/tests/integration/agents/agent_with_sandbox_executor_test.ts +++ b/tests/integration/agents/agent_with_sandbox_executor_test.ts @@ -6,8 +6,11 @@ import {Client} from '@google-cloud/vertexai'; import type {Event} from '@google/adk'; -import {AgentEngineSandboxCodeExecutor, LlmAgent} from '@google/adk'; -import {CODE_EXECUTION_RESPONSE_PROCESSOR} from '@google/adk/agents/processors/code_execution_request_processor.js'; +import { + AgentEngineSandboxCodeExecutor, + CODE_EXECUTION_RESPONSE_PROCESSOR, + LlmAgent, +} from '@google/adk'; import {FinishReason, Outcome} from '@google/genai'; import {describe, expect, it, vi} from 'vitest'; import { From 9e62a4ec15e9ab56e5fec5b2f44dcec66bd98bd3 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Sun, 2 Aug 2026 21:38:35 -0700 Subject: [PATCH 2/4] Test: assert the no-executor control on the full event text The negative control read events.at(-1)?...?.text, which is undefined when a regression truncates the turn, so the assertion failed with a type complaint instead of a readable diff. Join the text across all parts so the mutation 'remove the codeExecutor guard in runPostProcessor' reports "expected '' to contain '```python...'". --- .../agents/agent_with_default_code_execution_test.ts | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/tests/integration/agents/agent_with_default_code_execution_test.ts b/tests/integration/agents/agent_with_default_code_execution_test.ts index 1a23e78e4..a82c5952b 100644 --- a/tests/integration/agents/agent_with_default_code_execution_test.ts +++ b/tests/integration/agents/agent_with_default_code_execution_test.ts @@ -151,6 +151,13 @@ describe('Agent with a codeExecutor and no explicit responseProcessors', () => { (e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [], ); expect(resultParts).toHaveLength(0); - expect(events.at(-1)?.content?.parts?.[0]?.text).toContain(CODE_BLOCK); + + // The model turn reaches the caller untouched: nothing truncated it to the + // first code block. + const allText = events + .flatMap((e) => e.content?.parts ?? []) + .map((p) => p.text ?? '') + .join(''); + expect(allText).toContain(CODE_BLOCK); }); }); From e488637145dc8d9692b90ebbfdf84fd7a00da34a Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Sun, 2 Aug 2026 21:40:27 -0700 Subject: [PATCH 3/4] Test: hoist FailingCodeExecutor to module level --- .../agent_with_default_code_execution_test.ts | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/tests/integration/agents/agent_with_default_code_execution_test.ts b/tests/integration/agents/agent_with_default_code_execution_test.ts index a82c5952b..e49accf8f 100644 --- a/tests/integration/agents/agent_with_default_code_execution_test.ts +++ b/tests/integration/agents/agent_with_default_code_execution_test.ts @@ -67,6 +67,13 @@ class RecordingCodeExecutor extends BaseCodeExecutor { } } +/** A code executor whose run always fails, to exercise the error path. */ +class FailingCodeExecutor extends BaseCodeExecutor { + async executeCode(_params: ExecuteCodeParams): Promise { + return {stdout: '', stderr: 'NameError: boom', outputFiles: []}; + } +} + async function collectEvents( run: (prompt: string) => AsyncGenerator, prompt: string, @@ -107,14 +114,6 @@ describe('Agent with a codeExecutor and no explicit responseProcessors', () => { }); it('surfaces a failed execution as OUTCOME_FAILED with the stderr text', async () => { - class FailingCodeExecutor extends BaseCodeExecutor { - async executeCode( - _params: ExecuteCodeParams, - ): Promise { - return {stdout: '', stderr: 'NameError: boom', outputFiles: []}; - } - } - const agent = new LlmAgent({ model: new GeminiWithMockResponses(mockResponses()), name: 'coderAgent', From 8ed5d7fd3d6e1c575b6bcaa15129ac90ae2c7f6d Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Sun, 2 Aug 2026 22:08:55 -0700 Subject: [PATCH 4/4] Test: make the sandbox fixture per-test and share collectEvents The sandbox test's module-level MOCK_RESPONSES was mutable shared state: extractCodeAndTruncateContent rewrites content.parts in place and GeminiWithMockResponses passes the caller's Candidate objects straight through, so the first test permanently rewrote the second's input from a markdown code block into an executableCode part. The second test still passed, but via a different extractor branch than the one it was written to cover, and the result flipped with run order. Build the fixture from a factory so each test gets its own. Also fold the two call-site 'as unknown as Client' casts into the mock factory, which now returns the typed client alongside the execution spy, and move the duplicated event-draining loop into test_case_utils. Revert the code_execution_request_processor_test.ts import change: nothing required it, and it left one module reached through two specifiers. The public export stays pinned by llm_agent_test.ts and the sandbox test. --- .../code_execution_request_processor_test.ts | 8 +- .../agent_with_default_code_execution_test.ts | 13 +- .../agent_with_sandbox_executor_test.ts | 124 ++++++++++-------- tests/integration/test_case_utils.ts | 17 +++ 4 files changed, 89 insertions(+), 73 deletions(-) diff --git a/core/test/agents/processors/code_execution_request_processor_test.ts b/core/test/agents/processors/code_execution_request_processor_test.ts index 9b8807c38..cf67f3088 100644 --- a/core/test/agents/processors/code_execution_request_processor_test.ts +++ b/core/test/agents/processors/code_execution_request_processor_test.ts @@ -6,7 +6,6 @@ import { BaseAgent, - CODE_EXECUTION_RESPONSE_PROCESSOR, InvocationContext, LlmAgent, LlmRequest, @@ -14,7 +13,10 @@ import { createSession, } from '@google/adk'; import {describe, expect, it} from 'vitest'; -import {CODE_EXECUTION_REQUEST_PROCESSOR} from '../../../src/agents/processors/code_execution_request_processor.js'; +import { + CODE_EXECUTION_REQUEST_PROCESSOR, + CodeExecutionResponseProcessor, +} from '../../../src/agents/processors/code_execution_request_processor.js'; import { BaseCodeExecutor, ExecuteCodeParams, @@ -126,7 +128,7 @@ describe('CodeExecutionRequestProcessor', () => { }); describe('CodeExecutionResponseProcessor', () => { - const responseProcessor = CODE_EXECUTION_RESPONSE_PROCESSOR; + const responseProcessor = new CodeExecutionResponseProcessor(); describe('early-exit paths', () => { it('yields no events for a partial response', async () => { diff --git a/tests/integration/agents/agent_with_default_code_execution_test.ts b/tests/integration/agents/agent_with_default_code_execution_test.ts index e49accf8f..769d3817c 100644 --- a/tests/integration/agents/agent_with_default_code_execution_test.ts +++ b/tests/integration/agents/agent_with_default_code_execution_test.ts @@ -7,13 +7,13 @@ import type { CodeExecutionInput, CodeExecutionResult, - Event, ExecuteCodeParams, } from '@google/adk'; import {BaseCodeExecutor, LlmAgent} from '@google/adk'; import {FinishReason, Outcome} from '@google/genai'; import {describe, expect, it} from 'vitest'; import { + collectEvents, createRunner, GeminiWithMockResponses, RawGenerateContentResponse, @@ -74,17 +74,6 @@ class FailingCodeExecutor extends BaseCodeExecutor { } } -async function collectEvents( - run: (prompt: string) => AsyncGenerator, - prompt: string, -): Promise { - const events: Event[] = []; - for await (const event of run(prompt)) { - events.push(event); - } - return events; -} - describe('Agent with a codeExecutor and no explicit responseProcessors', () => { it('executes model-emitted code and yields an execution-result event', async () => { const executor = new RecordingCodeExecutor(); diff --git a/tests/integration/agents/agent_with_sandbox_executor_test.ts b/tests/integration/agents/agent_with_sandbox_executor_test.ts index fcfc5dcf1..df71325f2 100644 --- a/tests/integration/agents/agent_with_sandbox_executor_test.ts +++ b/tests/integration/agents/agent_with_sandbox_executor_test.ts @@ -5,7 +5,6 @@ */ import {Client} from '@google-cloud/vertexai'; -import type {Event} from '@google/adk'; import { AgentEngineSandboxCodeExecutor, CODE_EXECUTION_RESPONSE_PROCESSOR, @@ -14,42 +13,68 @@ import { import {FinishReason, Outcome} from '@google/genai'; import {describe, expect, it, vi} from 'vitest'; import { + collectEvents, createRunner, GeminiWithMockResponses, RawGenerateContentResponse, } from '../test_case_utils.js'; -const MOCK_RESPONSES: RawGenerateContentResponse[] = [ - { - candidates: [ - { - content: { - parts: [ - { - text: 'Here is the code to print hello:\n```python\nprint("hello")\n```', - }, - ], - role: 'model', +/** + * Builds a fresh set of recorded responses for one test. + * + * The response processor truncates `content.parts` in place and + * `GeminiWithMockResponses` hands out the same `Candidate` objects it is + * constructed with, so a fixture shared between tests would be rewritten by + * whichever test runs first. + */ +function mockResponses(): RawGenerateContentResponse[] { + return [ + { + candidates: [ + { + content: { + parts: [ + { + text: 'Here is the code to print hello:\n```python\nprint("hello")\n```', + }, + ], + role: 'model', + }, + finishReason: FinishReason.STOP, }, - finishReason: FinishReason.STOP, - }, - ], - }, - { - candidates: [ - { - content: { - parts: [{text: 'Execution was successful.'}], - role: 'model', + ], + }, + { + candidates: [ + { + content: { + parts: [{text: 'Execution was successful.'}], + role: 'model', + }, + finishReason: FinishReason.STOP, }, - finishReason: FinishReason.STOP, + ], + }, + ]; +} + +/** + * Builds a stub Vertex AI client for one test, returning it already typed as a + * `Client` alongside the code-execution spy so callers need no cast. + */ +function createMockClient() { + const executeCodeInternal = vi.fn().mockResolvedValue({ + outputs: [ + { + mimeType: 'application/json', + data: Buffer.from( + JSON.stringify({msg_out: 'hello', msg_err: ''}), + ).toString('base64'), }, ], - }, -]; + }); -function createMockClient() { - return { + const client = { agentEnginesInternal: { createInternal: vi.fn().mockResolvedValue({ name: 'operations/create-engine-op', @@ -69,31 +94,24 @@ function createMockClient() { name: 'projects/test-project/locations/us-central1/reasoningEngines/123/sandboxEnvironments/456', }, }), - executeCodeInternal: vi.fn().mockResolvedValue({ - outputs: [ - { - mimeType: 'application/json', - data: Buffer.from( - JSON.stringify({msg_out: 'hello', msg_err: ''}), - ).toString('base64'), - }, - ], - }), + executeCodeInternal, }, }, - }; + } as unknown as Client; + + return {client, executeCodeInternal}; } describe('Agent with AgentEngineSandboxCodeExecutor', () => { it('executes code generated by the agent', async () => { - const mockClient = createMockClient(); + const {client} = createMockClient(); const executor = new AgentEngineSandboxCodeExecutor({ projectId: 'test-project', - client: mockClient as unknown as Client, + client, }); - const model = new GeminiWithMockResponses(MOCK_RESPONSES); + const model = new GeminiWithMockResponses(mockResponses()); const agent = new LlmAgent({ model, name: 'coderAgent', @@ -105,10 +123,7 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => { const {run} = await createRunner(agent); - const events: Event[] = []; - for await (const event of run('Print hello')) { - events.push(event); - } + const events = await collectEvents(run, 'Print hello'); // We expect events for: // 1. User message (handled by runner) @@ -121,9 +136,7 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => { // Check if we got a code execution result event const hasExecutionResult = events.some( (e) => - e.content?.parts?.some( - (p: {text?: string}) => p.text && p.text.includes('hello'), - ) || + e.content?.parts?.some((p) => p.text && p.text.includes('hello')) || (e.content?.parts?.[0]?.inlineData?.data && Buffer.from(e.content.parts[0].inlineData.data, 'base64') .toString('utf-8') @@ -134,14 +147,14 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => { }); it('executes code with no explicit responseProcessors', async () => { - const mockClient = createMockClient(); + const {client, executeCodeInternal} = createMockClient(); const executor = new AgentEngineSandboxCodeExecutor({ projectId: 'test-project', - client: mockClient as unknown as Client, + client, }); - const model = new GeminiWithMockResponses(MOCK_RESPONSES); + const model = new GeminiWithMockResponses(mockResponses()); const agent = new LlmAgent({ model, name: 'coderAgent', @@ -152,14 +165,9 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => { const {run} = await createRunner(agent); - const events: Event[] = []; - for await (const event of run('Print hello')) { - events.push(event); - } + const events = await collectEvents(run, 'Print hello'); - expect( - mockClient.agentEnginesInternal.sandboxes.executeCodeInternal, - ).toHaveBeenCalledTimes(1); + expect(executeCodeInternal).toHaveBeenCalledTimes(1); const resultParts = events.flatMap( (e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [], diff --git a/tests/integration/test_case_utils.ts b/tests/integration/test_case_utils.ts index ad5088fc0..a5f05a8a5 100644 --- a/tests/integration/test_case_utils.ts +++ b/tests/integration/test_case_utils.ts @@ -154,6 +154,23 @@ export async function createRunner( }; } +/** + * Drains an agent run into an array. + * @param run The `run` function returned by {@link createRunner}. + * @param prompt The user prompt to send. + * @returns Every event the run yielded, in order. + */ +export async function collectEvents( + run: (prompt: string) => AsyncGenerator, + prompt: string, +): Promise { + const events: Event[] = []; + for await (const event of run(prompt)) { + events.push(event); + } + return events; +} + const ADK_EVENT_ID_REGEX = /^[a-zA-Z0-9]{8}$/; const INVOCATION_ID_REGEX = /^e-[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;