From 22fc04812928ace866a6baa70a05d00335437b33 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Sat, 8 Aug 2026 19:25:22 -0700 Subject: [PATCH 1/3] fix(dev): return the conformance failure count from runIntegrationTests The CLI needs one fact from a conformance run: did anything fail? The function accumulated the counts and threw them away, so no caller could tell a clean run from a failed one. --- dev/src/integration/run_integration_tests.ts | 10 ++- .../run_integration_tests_result_test.ts | 80 +++++++++++++++++++ 2 files changed, 89 insertions(+), 1 deletion(-) create mode 100644 dev/test/integration/run_integration_tests_result_test.ts diff --git a/dev/src/integration/run_integration_tests.ts b/dev/src/integration/run_integration_tests.ts index 9cc353758..5bb91fae1 100644 --- a/dev/src/integration/run_integration_tests.ts +++ b/dev/src/integration/run_integration_tests.ts @@ -11,6 +11,12 @@ import {AgentRegistry} from './agent_registry.js'; import {IntegrationRegistry} from './integration_registry.js'; import {TestRunner} from './test_runner.js'; +/** + * Runs every conformance test found under `testsDir` against the agents found + * under `agentsDir`. + * + * @returns the number of tests that failed. + */ export async function runIntegrationTests({ agentsDir, testsDir, @@ -19,7 +25,7 @@ export async function runIntegrationTests({ agentsDir: string; testsDir: string; forceRunAll: boolean; -}) { +}): Promise { console.log(`Loading agents from ${agentsDir}`); const agentConfigs = await batchLoadYamlAgentConfig(agentsDir); console.log(agentConfigs.size, 'agents found'); @@ -75,4 +81,6 @@ export async function runIntegrationTests({ console.log('Skipped tests:', skippedTests.join(', ')); console.log('Failed tests:', failedTests.join(', ')); console.log('\n'); + + return failedTests.length; } diff --git a/dev/test/integration/run_integration_tests_result_test.ts b/dev/test/integration/run_integration_tests_result_test.ts new file mode 100644 index 000000000..6401f5f77 --- /dev/null +++ b/dev/test/integration/run_integration_tests_result_test.ts @@ -0,0 +1,80 @@ +/** + * @license + * Copyright 2026 Google LLC + * SPDX-License-Identifier: Apache-2.0 + */ + +// Covers the failure count `runIntegrationTests` returns. The filename avoids +// `run_integration_tests_test.ts`, which a concurrent change adds. + +import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; +import {batchLoadYamlTestDefs} from '../../src/conformance/yaml_test_loader.js'; +import {runIntegrationTests} from '../../src/integration/run_integration_tests.js'; +import {TestRunner} from '../../src/integration/test_runner.js'; +import {TestInfo} from '../../src/integration/test_types.js'; + +vi.mock('../../src/conformance/yaml_agent_loader.js', () => ({ + batchLoadYamlAgentConfig: vi.fn(async () => new Map()), +})); + +vi.mock('../../src/conformance/yaml_test_loader.js', () => ({ + batchLoadYamlTestDefs: vi.fn(async () => new Map()), +})); + +function testInfo(name: string): TestInfo { + return { + name, + spec: {description: `spec for ${name}`, agent: 'agent'}, + session: { + id: `session-${name}`, + appName: 'app', + userId: 'user', + state: {}, + events: [], + lastUpdateTime: 0, + }, + recordings: {recordings: []}, + }; +} + +function testDefs(...names: string[]): Map { + return new Map(names.map((name) => [name, testInfo(name)])); +} + +const OPTIONS = {agentsDir: '/agents', testsDir: '/tests', forceRunAll: false}; + +describe('runIntegrationTests', () => { + beforeEach(() => { + vi.spyOn(console, 'log').mockImplementation(() => {}); + vi.spyOn(console, 'error').mockImplementation(() => {}); + }); + + afterEach(() => { + vi.restoreAllMocks(); + }); + + it('returns 0 when there are no tests', async () => { + vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs()); + + await expect(runIntegrationTests(OPTIONS)).resolves.toBe(0); + }); + + it('returns the number of failed tests', async () => { + vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs('a', 'b')); + vi.spyOn(TestRunner.prototype, 'run').mockRejectedValue( + new Error('failed'), + ); + + await expect(runIntegrationTests(OPTIONS)).resolves.toBe(2); + }); + + it('does not count passed or skipped tests as failures', async () => { + vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs('a', 'b', 'c')); + vi.spyOn(TestRunner.prototype, 'run') + .mockResolvedValueOnce(false) + .mockResolvedValueOnce(true) + .mockRejectedValueOnce(new Error('failed')); + + await expect(runIntegrationTests(OPTIONS)).resolves.toBe(1); + }); +}); From 947a623439349e31a1e7e52ab1f304e1acff36d2 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Sat, 8 Aug 2026 19:25:29 -0700 Subject: [PATCH 2/3] fix(cli): await the conformance run and exit non-zero when a test fails The action dropped the runIntegrationTests promise, so the command exited 0 after a run in which every test failed, and a thrown error surfaced as an unhandled rejection. It now awaits the run, reports an error through the CLI logger, and sets process.exitCode. --- dev/src/cli/cli.ts | 24 ++++++++--- dev/test/cli/cli_test.ts | 87 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 106 insertions(+), 5 deletions(-) diff --git a/dev/src/cli/cli.ts b/dev/src/cli/cli.ts index da6db54ab..09db67ca7 100644 --- a/dev/src/cli/cli.ts +++ b/dev/src/cli/cli.ts @@ -537,11 +537,25 @@ export function createProgram(): Command { ) .option('--force', 'Force run skipped tests.') .action(async (options: Record) => { - runIntegrationTests({ - agentsDir: options['agents_dir'], - testsDir: options['tests_dir'], - forceRunAll: getBoolean(options['force']), - }); + try { + const failedCount = await runIntegrationTests({ + agentsDir: options['agents_dir'], + testsDir: options['tests_dir'], + forceRunAll: getBoolean(options['force']), + }); + + if (failedCount > 0) { + // Not `process.exit()`: the run has just written its summary, and + // exiting immediately discards stdout writes still pending on a pipe. + process.exitCode = 1; + } + } catch (error) { + logger.error( + 'Error running conformance tests:', + (error as Error).message, + ); + process.exitCode = 1; + } }); return program; diff --git a/dev/test/cli/cli_test.ts b/dev/test/cli/cli_test.ts index f0f940e99..68c9d4b76 100644 --- a/dev/test/cli/cli_test.ts +++ b/dev/test/cli/cli_test.ts @@ -11,7 +11,9 @@ import {createAgent} from '../../src/cli/cli_create.js'; import {runAgent} from '../../src/cli/cli_run.js'; import {deployToAgentEngine} from '../../src/cli/deploy/cli_deploy_agent_engine.js'; import {deployToCloudRun} from '../../src/cli/deploy/cli_deploy_cloud_run.js'; +import {runIntegrationTests} from '../../src/integration/run_integration_tests.js'; import {AdkApiServer} from '../../src/server/adk_api_server.js'; +import {AdkLogger} from '../../src/utils/logger.js'; vi.mock('../../src/server/adk_api_server', () => { return { @@ -37,6 +39,12 @@ vi.mock('../../src/cli/cli_run', () => ({ runAgent: vi.fn(), })); +vi.mock('../../src/integration/run_integration_tests', () => ({ + // The factory-argument fake survives the suite's `vi.restoreAllMocks()`; + // an implementation installed with `mockResolvedValue` would not. + runIntegrationTests: vi.fn(async () => 0), +})); + vi.mock('../../src/version', () => ({ version: '1.0.0-test', })); @@ -51,15 +59,19 @@ vi.mock('@google/adk', async (importOriginal) => { describe('CLI Entrypoint', () => { let program: ReturnType; + let originalExitCode: typeof process.exitCode; beforeEach(() => { vi.clearAllMocks(); program = createProgram(); program.exitOverride(); + originalExitCode = process.exitCode; + process.exitCode = 0; }); afterEach(() => { vi.restoreAllMocks(); + process.exitCode = originalExitCode; }); const parse = async (args: string[]) => { @@ -431,4 +443,79 @@ describe('CLI Entrypoint', () => { }); }); }); + + describe('command: integration conformance', () => { + it('should pass parsed options to runIntegrationTests', async () => { + await parse([ + 'integration', + 'conformance', + '--agents_dir', + '/a', + '--tests_dir', + '/t', + '--force', + ]); + + expect(runIntegrationTests).toHaveBeenCalledWith({ + agentsDir: '/a', + testsDir: '/t', + forceRunAll: true, + }); + }); + + it('should wait for the conformance run before the action resolves', async () => { + let finishRun!: (failed: number) => void; + vi.mocked(runIntegrationTests).mockReturnValue( + new Promise((resolve) => { + finishRun = resolve; + }), + ); + + let settled = false; + const parsed = parse(['integration', 'conformance']).then(() => { + settled = true; + }); + await new Promise((resolve) => setImmediate(resolve)); + + expect(settled).toBe(false); + + finishRun(0); + await parsed; + + expect(settled).toBe(true); + }); + + it('should exit with status 1 when a conformance test failed', async () => { + vi.mocked(runIntegrationTests).mockResolvedValue(2); + + await parse(['integration', 'conformance']); + + expect(process.exitCode).toBe(1); + }); + + it('should leave the exit code alone when every test passed', async () => { + vi.mocked(runIntegrationTests).mockResolvedValue(0); + + await parse(['integration', 'conformance']); + + expect(process.exitCode).toBe(0); + }); + + it('should exit with status 1 when the conformance run throws', async () => { + const errorSpy = vi + .spyOn(AdkLogger.prototype, 'error') + .mockImplementation(() => {}); + vi.mocked(runIntegrationTests).mockRejectedValue(new Error('boom')); + + await expect( + parse(['integration', 'conformance']), + ).resolves.toBeUndefined(); + + expect(errorSpy).toHaveBeenCalledWith( + 'Error running conformance tests:', + 'boom', + ); + expect(process.exitCode).toBe(1); + }); + }); }); From c3005a6a215be627bd02d64962f82d1c0b943058 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Sat, 8 Aug 2026 19:51:16 -0700 Subject: [PATCH 3/3] test(dev): name the conformance result test after its module Sibling suites in this directory are named _test.ts. --- ...tion_tests_result_test.ts => run_integration_tests_test.ts} | 3 --- 1 file changed, 3 deletions(-) rename dev/test/integration/{run_integration_tests_result_test.ts => run_integration_tests_test.ts} (94%) diff --git a/dev/test/integration/run_integration_tests_result_test.ts b/dev/test/integration/run_integration_tests_test.ts similarity index 94% rename from dev/test/integration/run_integration_tests_result_test.ts rename to dev/test/integration/run_integration_tests_test.ts index 6401f5f77..c964afc07 100644 --- a/dev/test/integration/run_integration_tests_result_test.ts +++ b/dev/test/integration/run_integration_tests_test.ts @@ -4,9 +4,6 @@ * SPDX-License-Identifier: Apache-2.0 */ -// Covers the failure count `runIntegrationTests` returns. The filename avoids -// `run_integration_tests_test.ts`, which a concurrent change adds. - import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest'; import {batchLoadYamlTestDefs} from '../../src/conformance/yaml_test_loader.js'; import {runIntegrationTests} from '../../src/integration/run_integration_tests.js';