diff --git a/dev/src/cli/cli.ts b/dev/src/cli/cli.ts index da6db54ab..09db67ca7 100644 --- a/dev/src/cli/cli.ts +++ b/dev/src/cli/cli.ts @@ -537,11 +537,25 @@ export function createProgram(): Command { ) .option('--force', 'Force run skipped tests.') .action(async (options: Record) => { - runIntegrationTests({ - agentsDir: options['agents_dir'], - testsDir: options['tests_dir'], - forceRunAll: getBoolean(options['force']), - }); + try { + const failedCount = await runIntegrationTests({ + agentsDir: options['agents_dir'], + testsDir: options['tests_dir'], + forceRunAll: getBoolean(options['force']), + }); + + if (failedCount > 0) { + // Not `process.exit()`: the run has just written its summary, and + // exiting immediately discards stdout writes still pending on a pipe. + process.exitCode = 1; + } + } catch (error) { + logger.error( + 'Error running conformance tests:', + (error as Error).message, + ); + process.exitCode = 1; + } }); return program; diff --git a/dev/src/integration/run_integration_tests.ts b/dev/src/integration/run_integration_tests.ts index 9cc353758..5bb91fae1 100644 --- a/dev/src/integration/run_integration_tests.ts +++ b/dev/src/integration/run_integration_tests.ts @@ -11,6 +11,12 @@ import {AgentRegistry} from './agent_registry.js'; import {IntegrationRegistry} from './integration_registry.js'; import {TestRunner} from './test_runner.js'; +/** + * Runs every conformance test found under `testsDir` against the agents found + * under `agentsDir`. + * + * @returns the number of tests that failed. + */ export async function runIntegrationTests({ agentsDir, testsDir, @@ -19,7 +25,7 @@ export async function runIntegrationTests({ agentsDir: string; testsDir: string; forceRunAll: boolean; -}) { +}): Promise { console.log(`Loading agents from ${agentsDir}`); const agentConfigs = await batchLoadYamlAgentConfig(agentsDir); console.log(agentConfigs.size, 'agents found'); @@ -75,4 +81,6 @@ export async function runIntegrationTests({ console.log('Skipped tests:', skippedTests.join(', ')); console.log('Failed tests:', failedTests.join(', ')); console.log('\n'); + + return failedTests.length; } diff --git a/dev/test/cli/cli_test.ts b/dev/test/cli/cli_test.ts index f0f940e99..68c9d4b76 100644 --- a/dev/test/cli/cli_test.ts +++ b/dev/test/cli/cli_test.ts @@ -11,7 +11,9 @@ import {createAgent} from '../../src/cli/cli_create.js'; import {runAgent} from '../../src/cli/cli_run.js'; import {deployToAgentEngine} from '../../src/cli/deploy/cli_deploy_agent_engine.js'; import {deployToCloudRun} from '../../src/cli/deploy/cli_deploy_cloud_run.js'; +import {runIntegrationTests} from '../../src/integration/run_integration_tests.js'; import {AdkApiServer} from '../../src/server/adk_api_server.js'; +import {AdkLogger} from '../../src/utils/logger.js'; vi.mock('../../src/server/adk_api_server', () => { return { @@ -37,6 +39,12 @@ vi.mock('../../src/cli/cli_run', () => ({ runAgent: vi.fn(), })); +vi.mock('../../src/integration/run_integration_tests', () => ({ + // The factory-argument fake survives the suite's `vi.restoreAllMocks()`; + // an implementation installed with `mockResolvedValue` would not. + runIntegrationTests: vi.fn(async () => 0), +})); + vi.mock('../../src/version', () => ({ version: '1.0.0-test', })); @@ -51,15 +59,19 @@ vi.mock('@google/adk', async (importOriginal) => { describe('CLI Entrypoint', () => { let program: ReturnType; + let originalExitCode: typeof process.exitCode; beforeEach(() => { vi.clearAllMocks(); program = createProgram(); program.exitOverride(); + originalExitCode = process.exitCode; + process.exitCode = 0; }); afterEach(() => { vi.restoreAllMocks(); + process.exitCode = originalExitCode; }); const parse = async (args: string[]) => { @@ -431,4 +443,79 @@ describe('CLI Entrypoint', () => { }); }); }); + + describe('command: integration conformance', () => { + it('should pass parsed options to runIntegrationTests', async () => { + await parse([ + 'integration', + 'conformance', + '--agents_dir', + '/a', + '--tests_dir', + '/t', + '--force', + ]); + + expect(runIntegrationTests).toHaveBeenCalledWith({ + agentsDir: '/a', + testsDir: '/t', + forceRunAll: true, + }); + }); + + it('should wait for the conformance run before the action resolves', async () => { + let finishRun!: (failed: number) => void; + vi.mocked(runIntegrationTests).mockReturnValue( + new Promise((resolve) => { + finishRun = resolve; + }), + ); + + let settled = false; + const parsed = parse(['integration', 'conformance']).then(() => { + settled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + + expect(settled).toBe(false); + + finishRun(0); + await parsed; + + expect(settled).toBe(true); + }); + + it('should exit with status 1 when a conformance test failed', async () => { + vi.mocked(runIntegrationTests).mockResolvedValue(2); + + await parse(['integration', 'conformance']); + + expect(process.exitCode).toBe(1); + }); + + it('should leave the exit code alone when every test passed', async () => { + vi.mocked(runIntegrationTests).mockResolvedValue(0); + + await parse(['integration', 'conformance']); + + expect(process.exitCode).toBe(0); + }); + + it('should exit with status 1 when the conformance run throws', async () => { + const errorSpy = vi + .spyOn(AdkLogger.prototype, 'error') + .mockImplementation(() => {}); + vi.mocked(runIntegrationTests).mockRejectedValue(new Error('boom')); + + await expect( + parse(['integration', 'conformance']), + ).resolves.toBeUndefined(); + + expect(errorSpy).toHaveBeenCalledWith( + 'Error running conformance tests:', + 'boom', + ); + expect(process.exitCode).toBe(1); + }); + }); }); diff --git a/dev/test/integration/run_integration_tests_test.ts b/dev/test/integration/run_integration_tests_test.ts new file mode 100644 index 000000000..c964afc07 --- /dev/null +++ b/dev/test/integration/run_integration_tests_test.ts @@ -0,0 +1,77 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; +import {batchLoadYamlTestDefs} from '../../src/conformance/yaml_test_loader.js'; +import {runIntegrationTests} from '../../src/integration/run_integration_tests.js'; +import {TestRunner} from '../../src/integration/test_runner.js'; +import {TestInfo} from '../../src/integration/test_types.js'; + +vi.mock('../../src/conformance/yaml_agent_loader.js', () => ({ + batchLoadYamlAgentConfig: vi.fn(async () => new Map()), +})); + +vi.mock('../../src/conformance/yaml_test_loader.js', () => ({ + batchLoadYamlTestDefs: vi.fn(async () => new Map()), +})); + +function testInfo(name: string): TestInfo { + return { + name, + spec: {description: `spec for ${name}`, agent: 'agent'}, + session: { + id: `session-${name}`, + appName: 'app', + userId: 'user', + state: {}, + events: [], + lastUpdateTime: 0, + }, + recordings: {recordings: []}, + }; +} + +function testDefs(...names: string[]): Map { + return new Map(names.map((name) => [name, testInfo(name)])); +} + +const OPTIONS = {agentsDir: '/agents', testsDir: '/tests', forceRunAll: false}; + +describe('runIntegrationTests', () => { + beforeEach(() => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns 0 when there are no tests', async () => { + vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs()); + + await expect(runIntegrationTests(OPTIONS)).resolves.toBe(0); + }); + + it('returns the number of failed tests', async () => { + vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs('a', 'b')); + vi.spyOn(TestRunner.prototype, 'run').mockRejectedValue( + new Error('failed'), + ); + + await expect(runIntegrationTests(OPTIONS)).resolves.toBe(2); + }); + + it('does not count passed or skipped tests as failures', async () => { + vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs('a', 'b', 'c')); + vi.spyOn(TestRunner.prototype, 'run') + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true) + .mockRejectedValueOnce(new Error('failed')); + + await expect(runIntegrationTests(OPTIONS)).resolves.toBe(1); + }); +});