Fix: await the conformance run and exit non-zero when a conformance test fails

dev/src/cli/cli.ts

@@ -537,11 +537,25 @@ export function createProgram(): Command {
     )
     .option('--force', 'Force run skipped tests.')
     .action(async (options: Record<string, string>) => {
-      runIntegrationTests({
-        agentsDir: options['agents_dir'],
-        testsDir: options['tests_dir'],
-        forceRunAll: getBoolean(options['force']),
-      });
+      try {
+        const failedCount = await runIntegrationTests({
+          agentsDir: options['agents_dir'],
+          testsDir: options['tests_dir'],
+          forceRunAll: getBoolean(options['force']),
+        });
+
+        if (failedCount > 0) {
+          // Not `process.exit()`: the run has just written its summary, and
+          // exiting immediately discards stdout writes still pending on a pipe.
+          process.exitCode = 1;
+        }
+      } catch (error) {
+        logger.error(
+          'Error running conformance tests:',
+          (error as Error).message,
+        );
+        process.exitCode = 1;
+      }
     });
 
   return program;

dev/src/integration/run_integration_tests.ts

@@ -11,6 +11,12 @@ import {AgentRegistry} from './agent_registry.js';
 import {IntegrationRegistry} from './integration_registry.js';
 import {TestRunner} from './test_runner.js';
 
+/**
+ * Runs every conformance test found under `testsDir` against the agents found
+ * under `agentsDir`.
+ *
+ * @returns the number of tests that failed.
+ */
 export async function runIntegrationTests({
   agentsDir,
   testsDir,
@@ -19,7 +25,7 @@ export async function runIntegrationTests({
   agentsDir: string;
   testsDir: string;
   forceRunAll: boolean;
-}) {
+}): Promise<number> {
   console.log(`Loading agents from ${agentsDir}`);
   const agentConfigs = await batchLoadYamlAgentConfig(agentsDir);
   console.log(agentConfigs.size, 'agents found');
@@ -75,4 +81,6 @@ export async function runIntegrationTests({
   console.log('Skipped tests:', skippedTests.join(', '));
   console.log('Failed tests:', failedTests.join(', '));
   console.log('\n');
+
+  return failedTests.length;
 }

dev/test/cli/cli_test.ts

@@ -11,7 +11,9 @@ import {createAgent} from '../../src/cli/cli_create.js';
 import {runAgent} from '../../src/cli/cli_run.js';
 import {deployToAgentEngine} from '../../src/cli/deploy/cli_deploy_agent_engine.js';
 import {deployToCloudRun} from '../../src/cli/deploy/cli_deploy_cloud_run.js';
+import {runIntegrationTests} from '../../src/integration/run_integration_tests.js';
 import {AdkApiServer} from '../../src/server/adk_api_server.js';
+import {AdkLogger} from '../../src/utils/logger.js';
 
 vi.mock('../../src/server/adk_api_server', () => {
   return {
@@ -37,6 +39,12 @@ vi.mock('../../src/cli/cli_run', () => ({
   runAgent: vi.fn(),
 }));
 
+vi.mock('../../src/integration/run_integration_tests', () => ({
+  // The factory-argument fake survives the suite's `vi.restoreAllMocks()`;
+  // an implementation installed with `mockResolvedValue` would not.
+  runIntegrationTests: vi.fn(async () => 0),
+}));
+
 vi.mock('../../src/version', () => ({
   version: '1.0.0-test',
 }));
@@ -51,15 +59,19 @@ vi.mock('@google/adk', async (importOriginal) => {
 
 describe('CLI Entrypoint', () => {
   let program: ReturnType<typeof createProgram>;
+  let originalExitCode: typeof process.exitCode;
 
   beforeEach(() => {
     vi.clearAllMocks();
     program = createProgram();
     program.exitOverride();
+    originalExitCode = process.exitCode;
+    process.exitCode = 0;
   });
 
   afterEach(() => {
     vi.restoreAllMocks();
+    process.exitCode = originalExitCode;
   });
 
   const parse = async (args: string[]) => {
@@ -431,4 +443,79 @@ describe('CLI Entrypoint', () => {
       });
     });
   });
+
+  describe('command: integration conformance', () => {
+    it('should pass parsed options to runIntegrationTests', async () => {
+      await parse([
+        'integration',
+        'conformance',
+        '--agents_dir',
+        '/a',
+        '--tests_dir',
+        '/t',
+        '--force',
+      ]);
+
+      expect(runIntegrationTests).toHaveBeenCalledWith({
+        agentsDir: '/a',
+        testsDir: '/t',
+        forceRunAll: true,
+      });
+    });
+
+    it('should wait for the conformance run before the action resolves', async () => {
+      let finishRun!: (failed: number) => void;
+      vi.mocked(runIntegrationTests).mockReturnValue(
+        new Promise<number>((resolve) => {
+          finishRun = resolve;
+        }),
+      );
+
+      let settled = false;
+      const parsed = parse(['integration', 'conformance']).then(() => {
+        settled = true;
+      });
+      await new Promise((resolve) => setImmediate(resolve));
+
+      expect(settled).toBe(false);
+
+      finishRun(0);
+      await parsed;
+
+      expect(settled).toBe(true);
+    });
+
+    it('should exit with status 1 when a conformance test failed', async () => {
+      vi.mocked(runIntegrationTests).mockResolvedValue(2);
+
+      await parse(['integration', 'conformance']);
+
+      expect(process.exitCode).toBe(1);
+    });
+
+    it('should leave the exit code alone when every test passed', async () => {
+      vi.mocked(runIntegrationTests).mockResolvedValue(0);
+
+      await parse(['integration', 'conformance']);
+
+      expect(process.exitCode).toBe(0);
+    });
+
+    it('should exit with status 1 when the conformance run throws', async () => {
+      const errorSpy = vi
+        .spyOn(AdkLogger.prototype, 'error')
+        .mockImplementation(() => {});
+      vi.mocked(runIntegrationTests).mockRejectedValue(new Error('boom'));
+
+      await expect(
+        parse(['integration', 'conformance']),
+      ).resolves.toBeUndefined();
+
+      expect(errorSpy).toHaveBeenCalledWith(
+        'Error running conformance tests:',
+        'boom',
+      );
+      expect(process.exitCode).toBe(1);
+    });
+  });
 });

dev/test/integration/run_integration_tests_test.ts

@@ -0,0 +1,77 @@
+/**
+ * @license
+ * Copyright 2026 Google LLC
+ * SPDX-License-Identifier: Apache-2.0
+ */
+
+import {afterEach, beforeEach, describe, expect, it, vi} from 'vitest';
+import {batchLoadYamlTestDefs} from '../../src/conformance/yaml_test_loader.js';
+import {runIntegrationTests} from '../../src/integration/run_integration_tests.js';
+import {TestRunner} from '../../src/integration/test_runner.js';
+import {TestInfo} from '../../src/integration/test_types.js';
+
+vi.mock('../../src/conformance/yaml_agent_loader.js', () => ({
+  batchLoadYamlAgentConfig: vi.fn(async () => new Map()),
+}));
+
+vi.mock('../../src/conformance/yaml_test_loader.js', () => ({
+  batchLoadYamlTestDefs: vi.fn(async () => new Map()),
+}));
+
+function testInfo(name: string): TestInfo {
+  return {
+    name,
+    spec: {description: `spec for ${name}`, agent: 'agent'},
+    session: {
+      id: `session-${name}`,
+      appName: 'app',
+      userId: 'user',
+      state: {},
+      events: [],
+      lastUpdateTime: 0,
+    },
+    recordings: {recordings: []},
+  };
+}
+
+function testDefs(...names: string[]): Map<string, TestInfo> {
+  return new Map(names.map((name) => [name, testInfo(name)]));
+}
+
+const OPTIONS = {agentsDir: '/agents', testsDir: '/tests', forceRunAll: false};
+
+describe('runIntegrationTests', () => {
+  beforeEach(() => {
+    vi.spyOn(console, 'log').mockImplementation(() => {});
+    vi.spyOn(console, 'error').mockImplementation(() => {});
+  });
+
+  afterEach(() => {
+    vi.restoreAllMocks();
+  });
+
+  it('returns 0 when there are no tests', async () => {
+    vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs());
+
+    await expect(runIntegrationTests(OPTIONS)).resolves.toBe(0);
+  });
+
+  it('returns the number of failed tests', async () => {
+    vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs('a', 'b'));
+    vi.spyOn(TestRunner.prototype, 'run').mockRejectedValue(
+      new Error('failed'),
+    );
+
+    await expect(runIntegrationTests(OPTIONS)).resolves.toBe(2);
+  });
+
+  it('does not count passed or skipped tests as failures', async () => {
+    vi.mocked(batchLoadYamlTestDefs).mockResolvedValue(testDefs('a', 'b', 'c'));
+    vi.spyOn(TestRunner.prototype, 'run')
+      .mockResolvedValueOnce(false)
+      .mockResolvedValueOnce(true)
+      .mockRejectedValueOnce(new Error('failed'));
+
+    await expect(runIntegrationTests(OPTIONS)).resolves.toBe(1);
+  });
+});