Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions core/src/agents/llm_agent.ts
Original file line number Diff line number Diff line change
Expand Up @@ -303,6 +303,11 @@ export interface LlmAgentConfig extends BaseAgentConfig {

/**
* Processors to run after the LLM response is received.
*
* Omitting this selects the default list, which contains
* {@link CODE_EXECUTION_RESPONSE_PROCESSOR}. Supplying a list replaces the
* default entirely rather than extending it, so a caller that needs code
* execution alongside its own processors must include the default explicitly.
*/
responseProcessors?: BaseLlmResponseProcessor[];

Expand Down
4 changes: 4 additions & 0 deletions core/src/common.ts
Original file line number Diff line number Diff line change
Expand Up @@ -46,6 +46,10 @@ export {
BaseLlmRequestProcessor,
BaseLlmResponseProcessor,
} from './agents/processors/base_llm_processor.js';
export {
CODE_EXECUTION_RESPONSE_PROCESSOR,
CodeExecutionResponseProcessor,
} from './agents/processors/code_execution_request_processor.js';
export {
CONTENT_REQUEST_PROCESSOR,
ContentRequestProcessor,
Expand Down
2 changes: 1 addition & 1 deletion core/test/agents/llm_agent_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@ import {
BaseLlmResponseProcessor,
BasePlugin,
BaseTool,
CODE_EXECUTION_RESPONSE_PROCESSOR,
CodeExecutionInput,
CodeExecutionResult,
CONTENT_REQUEST_PROCESSOR,
Expand All @@ -36,7 +37,6 @@ import {Content, Outcome, Schema, Type} from '@google/genai';
import {beforeEach, describe, expect, it} from 'vitest';
import {z as z3} from 'zod/v3';
import {z as z4} from 'zod/v4';
import {CODE_EXECUTION_RESPONSE_PROCESSOR} from '../../src/agents/processors/code_execution_request_processor.js';
import {ScopedArtifactService} from '../../src/artifacts/scoped_artifact_service.js';

class MockLlmConnection implements BaseLlmConnection {
Expand Down
151 changes: 151 additions & 0 deletions tests/integration/agents/agent_with_default_code_execution_test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,151 @@
/**
* @license
* Copyright 2026 Google LLC
* SPDX-License-Identifier: Apache-2.0
*/

import type {
CodeExecutionInput,
CodeExecutionResult,
ExecuteCodeParams,
} from '@google/adk';
import {BaseCodeExecutor, LlmAgent} from '@google/adk';
import {FinishReason, Outcome} from '@google/genai';
import {describe, expect, it} from 'vitest';
import {
collectEvents,
createRunner,
GeminiWithMockResponses,
RawGenerateContentResponse,
} from '../test_case_utils.js';

const CODE_BLOCK = '```python\nprint("hello")\n```';

/**
* Builds a fresh set of recorded responses for one test.
*
* The response processor truncates `content.parts` in place and
* `GeminiWithMockResponses` hands out the same `Candidate` objects it is
* constructed with, so a fixture shared between tests would be rewritten by
* whichever test runs first.
*/
function mockResponses(): RawGenerateContentResponse[] {
return [
{
candidates: [
{
content: {
parts: [{text: `Here is the code:\n${CODE_BLOCK}`}],
role: 'model',
},
finishReason: FinishReason.STOP,
},
],
},
{
candidates: [
{
content: {parts: [{text: 'Execution finished.'}], role: 'model'},
finishReason: FinishReason.STOP,
},
],
},
];
}

/**
* A code executor that records what it was asked to run and returns a fixed
* result, so the test exercises the real processor pipeline without depending
* on a Python interpreter being present on the CI host.
*/
class RecordingCodeExecutor extends BaseCodeExecutor {
readonly calls: CodeExecutionInput[] = [];

async executeCode(params: ExecuteCodeParams): Promise<CodeExecutionResult> {
this.calls.push(params.codeExecutionInput);
return {stdout: 'hello', stderr: '', outputFiles: []};
}
}

/** A code executor whose run always fails, to exercise the error path. */
class FailingCodeExecutor extends BaseCodeExecutor {
async executeCode(_params: ExecuteCodeParams): Promise<CodeExecutionResult> {
return {stdout: '', stderr: 'NameError: boom', outputFiles: []};
}
}

describe('Agent with a codeExecutor and no explicit responseProcessors', () => {
it('executes model-emitted code and yields an execution-result event', async () => {
const executor = new RecordingCodeExecutor();
const agent = new LlmAgent({
model: new GeminiWithMockResponses(mockResponses()),
name: 'coderAgent',
description: 'An agent that writes and runs code',
instruction: 'Write code to solve the user request.',
codeExecutor: executor,
});

const {run} = await createRunner(agent);
const events = await collectEvents(run, 'Print hello');

expect(executor.calls).toHaveLength(1);
expect(executor.calls[0].code).toBe('print("hello")');

const resultParts = events.flatMap(
(e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [],
);
expect(resultParts).toHaveLength(1);
expect(resultParts[0].codeExecutionResult?.outcome).toBe(
Outcome.OUTCOME_OK,
);
expect(resultParts[0].text).toContain('Code execution result:');
expect(resultParts[0].text).toContain('hello');
});

it('surfaces a failed execution as OUTCOME_FAILED with the stderr text', async () => {
const agent = new LlmAgent({
model: new GeminiWithMockResponses(mockResponses()),
name: 'coderAgent',
description: 'An agent that writes and runs code',
instruction: 'Write code to solve the user request.',
codeExecutor: new FailingCodeExecutor(),
});

const {run} = await createRunner(agent);
const events = await collectEvents(run, 'Print hello');

const resultParts = events.flatMap(
(e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [],
);
expect(resultParts).toHaveLength(1);
expect(resultParts[0].codeExecutionResult?.outcome).toBe(
Outcome.OUTCOME_FAILED,
);
expect(resultParts[0].text).toContain('NameError: boom');
});

it('is inert for an agent that has no codeExecutor', async () => {
const agent = new LlmAgent({
model: new GeminiWithMockResponses(mockResponses()),
name: 'plainAgent',
description: 'An agent with no code executor',
instruction: 'Answer the user request.',
});

const {run} = await createRunner(agent);
const events = await collectEvents(run, 'Print hello');

const resultParts = events.flatMap(
(e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [],
);
expect(resultParts).toHaveLength(0);

// The model turn reaches the caller untouched: nothing truncated it to the
// first code block.
const allText = events
.flatMap((e) => e.content?.parts ?? [])
.map((p) => p.text ?? '')
.join('');
expect(allText).toContain(CODE_BLOCK);
});
});
131 changes: 71 additions & 60 deletions tests/integration/agents/agent_with_sandbox_executor_test.ts
Original file line number Diff line number Diff line change
Expand Up @@ -5,48 +5,76 @@
*/

import {Client} from '@google-cloud/vertexai';
import type {Event} from '@google/adk';
import {AgentEngineSandboxCodeExecutor, LlmAgent} from '@google/adk';
import {CODE_EXECUTION_RESPONSE_PROCESSOR} from '@google/adk/agents/processors/code_execution_request_processor.js';
import {
AgentEngineSandboxCodeExecutor,
CODE_EXECUTION_RESPONSE_PROCESSOR,
LlmAgent,
} from '@google/adk';
import {FinishReason, Outcome} from '@google/genai';
import {describe, expect, it, vi} from 'vitest';
import {
collectEvents,
createRunner,
GeminiWithMockResponses,
RawGenerateContentResponse,
} from '../test_case_utils.js';

const MOCK_RESPONSES: RawGenerateContentResponse[] = [
{
candidates: [
{
content: {
parts: [
{
text: 'Here is the code to print hello:\n```python\nprint("hello")\n```',
},
],
role: 'model',
/**
* Builds a fresh set of recorded responses for one test.
*
* The response processor truncates `content.parts` in place and
* `GeminiWithMockResponses` hands out the same `Candidate` objects it is
* constructed with, so a fixture shared between tests would be rewritten by
* whichever test runs first.
*/
function mockResponses(): RawGenerateContentResponse[] {
return [
{
candidates: [
{
content: {
parts: [
{
text: 'Here is the code to print hello:\n```python\nprint("hello")\n```',
},
],
role: 'model',
},
finishReason: FinishReason.STOP,
},
finishReason: FinishReason.STOP,
},
],
},
{
candidates: [
{
content: {
parts: [{text: 'Execution was successful.'}],
role: 'model',
],
},
{
candidates: [
{
content: {
parts: [{text: 'Execution was successful.'}],
role: 'model',
},
finishReason: FinishReason.STOP,
},
finishReason: FinishReason.STOP,
],
},
];
}

/**
* Builds a stub Vertex AI client for one test, returning it already typed as a
* `Client` alongside the code-execution spy so callers need no cast.
*/
function createMockClient() {
const executeCodeInternal = vi.fn().mockResolvedValue({
outputs: [
{
mimeType: 'application/json',
data: Buffer.from(
JSON.stringify({msg_out: 'hello', msg_err: ''}),
).toString('base64'),
},
],
},
];
});

function createMockClient() {
return {
const client = {
agentEnginesInternal: {
createInternal: vi.fn().mockResolvedValue({
name: 'operations/create-engine-op',
Expand All @@ -66,31 +94,24 @@ function createMockClient() {
name: 'projects/test-project/locations/us-central1/reasoningEngines/123/sandboxEnvironments/456',
},
}),
executeCodeInternal: vi.fn().mockResolvedValue({
outputs: [
{
mimeType: 'application/json',
data: Buffer.from(
JSON.stringify({msg_out: 'hello', msg_err: ''}),
).toString('base64'),
},
],
}),
executeCodeInternal,
},
},
};
} as unknown as Client;

return {client, executeCodeInternal};
}

describe('Agent with AgentEngineSandboxCodeExecutor', () => {
it('executes code generated by the agent', async () => {
const mockClient = createMockClient();
const {client} = createMockClient();

const executor = new AgentEngineSandboxCodeExecutor({
projectId: 'test-project',
client: mockClient as unknown as Client,
client,
});

const model = new GeminiWithMockResponses(MOCK_RESPONSES);
const model = new GeminiWithMockResponses(mockResponses());
const agent = new LlmAgent({
model,
name: 'coderAgent',
Expand All @@ -102,10 +123,7 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => {

const {run} = await createRunner(agent);

const events: Event[] = [];
for await (const event of run('Print hello')) {
events.push(event);
}
const events = await collectEvents(run, 'Print hello');

// We expect events for:
// 1. User message (handled by runner)
Expand All @@ -118,9 +136,7 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => {
// Check if we got a code execution result event
const hasExecutionResult = events.some(
(e) =>
e.content?.parts?.some(
(p: {text?: string}) => p.text && p.text.includes('hello'),
) ||
e.content?.parts?.some((p) => p.text && p.text.includes('hello')) ||
(e.content?.parts?.[0]?.inlineData?.data &&
Buffer.from(e.content.parts[0].inlineData.data, 'base64')
.toString('utf-8')
Expand All @@ -131,14 +147,14 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => {
});

it('executes code with no explicit responseProcessors', async () => {
const mockClient = createMockClient();
const {client, executeCodeInternal} = createMockClient();

const executor = new AgentEngineSandboxCodeExecutor({
projectId: 'test-project',
client: mockClient as unknown as Client,
client,
});

const model = new GeminiWithMockResponses(MOCK_RESPONSES);
const model = new GeminiWithMockResponses(mockResponses());
const agent = new LlmAgent({
model,
name: 'coderAgent',
Expand All @@ -149,14 +165,9 @@ describe('Agent with AgentEngineSandboxCodeExecutor', () => {

const {run} = await createRunner(agent);

const events: Event[] = [];
for await (const event of run('Print hello')) {
events.push(event);
}
const events = await collectEvents(run, 'Print hello');

expect(
mockClient.agentEnginesInternal.sandboxes.executeCodeInternal,
).toHaveBeenCalledTimes(1);
expect(executeCodeInternal).toHaveBeenCalledTimes(1);

const resultParts = events.flatMap(
(e) => e.content?.parts?.filter((p) => p.codeExecutionResult) ?? [],
Expand Down
Loading