diff --git a/tests/integration/a2a/basic/a2a_agent_test.ts b/tests/integration/a2a/basic/a2a_agent_test.ts index 5e0f82c69..bc59e564c 100644 --- a/tests/integration/a2a/basic/a2a_agent_test.ts +++ b/tests/integration/a2a/basic/a2a_agent_test.ts @@ -22,7 +22,7 @@ describe('A2A: Remote Agent Basic', () => { startFailureTimeout: 60000, }); await server.start(); - }, 60000); + }); afterAll(async () => { await server.stop(); diff --git a/tests/integration/a2a/input_required/input_required_test.ts b/tests/integration/a2a/input_required/input_required_test.ts index 33231d200..48d8c5879 100644 --- a/tests/integration/a2a/input_required/input_required_test.ts +++ b/tests/integration/a2a/input_required/input_required_test.ts @@ -10,7 +10,9 @@ import * as path from 'node:path'; import {afterAll, beforeAll, describe, expect, it} from 'vitest'; import {AdkTsApiServer} from '../../test_api_server.js'; -const TEST_TIMEOUT = 60000; +// The server's own start watchdog, not a vitest budget. It stays below the +// project hookTimeout so the server's captured-stdout diagnostic wins. +const SERVER_START_TIMEOUT_MS = 60000; describe('A2A: RemoteAgent InputRequired', () => { let server: AdkTsApiServer; @@ -19,10 +21,10 @@ describe('A2A: RemoteAgent InputRequired', () => { server = new AdkTsApiServer({ agentsDir: path.join(__dirname, 'test_agents'), a2a: true, - startFailureTimeout: TEST_TIMEOUT, + startFailureTimeout: SERVER_START_TIMEOUT_MS, }); await server.start(); - }, TEST_TIMEOUT); + }); afterAll(async () => { await server.stop(); diff --git a/tests/integration/a2a/stream/stream_test.ts b/tests/integration/a2a/stream/stream_test.ts index 5926bbbf2..e17c4a591 100644 --- a/tests/integration/a2a/stream/stream_test.ts +++ b/tests/integration/a2a/stream/stream_test.ts @@ -10,7 +10,9 @@ import * as path from 'node:path'; import {afterAll, beforeAll, describe, expect, it} from 'vitest'; import {AdkTsApiServer} from '../../test_api_server.js'; -const TEST_TIMEOUT = 60000; +// The server's own start watchdog, not a vitest budget. It stays below the +// project hookTimeout so the server's captured-stdout diagnostic wins. +const SERVER_START_TIMEOUT_MS = 60000; describe('A2A: RemoteAgent Streaming', () => { let server: AdkTsApiServer; @@ -19,10 +21,10 @@ describe('A2A: RemoteAgent Streaming', () => { server = new AdkTsApiServer({ agentsDir: path.join(__dirname, 'test_agents'), a2a: true, - startFailureTimeout: TEST_TIMEOUT, + startFailureTimeout: SERVER_START_TIMEOUT_MS, }); await server.start(); - }, TEST_TIMEOUT); + }); afterAll(async () => { await server.stop(); diff --git a/tests/integration/adk_web/webui_test.ts b/tests/integration/adk_web/webui_test.ts index 8e525cef1..d85c04428 100644 --- a/tests/integration/adk_web/webui_test.ts +++ b/tests/integration/adk_web/webui_test.ts @@ -34,6 +34,9 @@ describe('WebUI Integration Test', () => { let server: AdkApiServer | AdkCliApiServer; let url: string; + // No local timeout: the hook inherits vitest.config.ts's hookTimeout, + // which must stay above AdkTsApiServer's 60s start watchdog so the + // server's captured-stdout diagnostic wins (test_api_server.ts). beforeAll(async () => { server = new serverClass({ agentsDir: path.resolve(__dirname, './agent'), @@ -42,7 +45,7 @@ describe('WebUI Integration Test', () => { }); await server.start(); url = server.url; - }, 20000); + }); afterAll(async () => { if (server) { @@ -82,4 +85,4 @@ describe('WebUI Integration Test', () => { }); }, ); -}, 20000); +}); diff --git a/tests/integration/agent_loader/agent_dirname_test.ts b/tests/integration/agent_loader/agent_dirname_test.ts index 1e71a3a1d..fafe428ea 100644 --- a/tests/integration/agent_loader/agent_dirname_test.ts +++ b/tests/integration/agent_loader/agent_dirname_test.ts @@ -13,7 +13,6 @@ import {sendInput} from '../test_case_utils.js'; const execAsync = promisify(exec); const dirname = process.cwd(); -const TEST_EXECUTION_TIMEOUT = 40000; describe.each(['__dirname', '__filename', 'import_meta_url'])( 'Agent with %s', @@ -26,25 +25,21 @@ describe.each(['__dirname', '__filename', 'import_meta_url'])( beforeAll(async () => { await execAsync('npm install', {cwd: projectPath}); - }, TEST_EXECUTION_TIMEOUT); + }); - it( - 'should run agent and load params from file nearby via package.json script', - async () => { - const childProcess = spawn('npm', ['run', 'start'], { - cwd: projectPath, - shell: true, - }); + it('should run agent and load params from file nearby via package.json script', async () => { + const childProcess = spawn('npm', ['run', 'start'], { + cwd: projectPath, + shell: true, + }); - let response = await sendInput(childProcess, 'Tell me a joke.\n'); + let response = await sendInput(childProcess, 'Tell me a joke.\n'); - expect(response.toString()).toContain("I'm stubby model response!"); + expect(response.toString()).toContain("I'm stubby model response!"); - response = await sendInput(childProcess, 'exit\n'); - expect(response.toString()).toContain(''); - }, - TEST_EXECUTION_TIMEOUT, - ); + response = await sendInput(childProcess, 'exit\n'); + expect(response.toString()).toContain(''); + }); afterAll(async () => { await fs @@ -56,6 +51,6 @@ describe.each(['__dirname', '__filename', 'import_meta_url'])( await fs .unlink(path.join(projectPath, 'package-lock.json')) .catch(() => {}); - }, TEST_EXECUTION_TIMEOUT); + }); }, ); diff --git a/tests/integration/app_loader/app_loader_test.ts b/tests/integration/app_loader/app_loader_test.ts index 4367307b8..bf10cc059 100644 --- a/tests/integration/app_loader/app_loader_test.ts +++ b/tests/integration/app_loader/app_loader_test.ts @@ -15,6 +15,8 @@ import {sendInput} from '../test_case_utils.js'; const execAsync = promisify(exec); const dirname = process.cwd(); +// Assertion budget only: the install/teardown hooks inherit vitest.config.ts's +// hookTimeout so a slow cold install is not reported as a test failure. const TEST_EXECUTION_TIMEOUT = 40000; describe('App loader CLI integration', () => { @@ -29,7 +31,7 @@ describe('App loader CLI integration', () => { beforeAll(async () => { await execAsync('npm install', {cwd: projectPath}); - }, TEST_EXECUTION_TIMEOUT); + }); it( 'should run app via package.json start script and get responses', @@ -62,7 +64,7 @@ describe('App loader CLI integration', () => { await fs .unlink(path.join(projectPath, 'package-lock.json')) .catch(() => {}); - }, TEST_EXECUTION_TIMEOUT); + }); }, ); }); @@ -77,7 +79,7 @@ describe('AgentLoader discovery and loading integration', () => { beforeAll(async () => { await execAsync('npm install', {cwd: projectPath}); loader = new AgentLoader(projectPath); - }, TEST_EXECUTION_TIMEOUT); + }); it( 'should discover apps vs agents across directories and standalone files', @@ -138,5 +140,5 @@ describe('AgentLoader discovery and loading integration', () => { await fs .unlink(path.join(projectPath, 'package-lock.json')) .catch(() => {}); - }, TEST_EXECUTION_TIMEOUT); + }); }); diff --git a/tests/integration/build_setup/build_setup_test.ts b/tests/integration/build_setup/build_setup_test.ts index 880dee469..6da732bcd 100644 --- a/tests/integration/build_setup/build_setup_test.ts +++ b/tests/integration/build_setup/build_setup_test.ts @@ -12,17 +12,11 @@ import {getResponse, sendInput} from '../test_case_utils.js'; const execAsync = promisify(exec); const dirname = process.cwd(); +// Below the 60s project testTimeout on purpose: the install and the build run +// in the hook, so each test only spawns an installed fixture and reads its +// output. A hang there should surface in 20s. const TEST_EXECUTION_TIMEOUT = 20000; -// These hooks run `npm install` (plus `npm run build` for ts_* setups) and the -// recursive node_modules teardown, overrunning vitest's default 10s hookTimeout -// and causing flaky "Hook timed out in 10000ms" failures. All twelve hook runs -// take ~16s combined on ubuntu-latest, but a cold, network-bound install has -// been measured at ~70s, so 120s covers the worst case. Don't raise -// TEST_EXECUTION_TIMEOUT for hook flakes; that stays the 20s per-test budget. -// Trade-off: a stuck hook now takes this long to surface. -const HOOK_TIMEOUT = 120000; - describe('Build setup', () => { describe.each([ 'js_commonjs', @@ -52,7 +46,7 @@ describe('Build setup', () => { expect(buildResult.stderr).toBe(''); expect(buildResult.stdout).toContain('\nBuild complete'); } - }, HOOK_TIMEOUT); + }); it( 'should build and run agent successfully', @@ -128,6 +122,6 @@ describe('Build setup', () => { .rm(`${projectPath}/dist`, {recursive: true, force: true}) .catch(() => {}); } - }, HOOK_TIMEOUT); + }); }); }); diff --git a/tests/integration/skills/script_js/agent_test.ts b/tests/integration/skills/script_js/agent_test.ts index b17df2c86..cd311aa18 100644 --- a/tests/integration/skills/script_js/agent_test.ts +++ b/tests/integration/skills/script_js/agent_test.ts @@ -12,7 +12,6 @@ import {normalizeLineEndings, sendInput} from '../../test_case_utils.js'; const execAsync = promisify(exec); const dirname = process.cwd(); const PROJECT_PATH = `${dirname}/tests/integration/skills/script_js`; -const TEST_EXECUTION_TIMEOUT = 60000; /** * This integration test verifies that an agent equipped with script execution skills @@ -31,66 +30,62 @@ const TEST_EXECUTION_TIMEOUT = 60000; describe('Agent with skills that generates JS script and runs it locally', () => { beforeAll(async () => { await execAsync('npm install', {cwd: PROJECT_PATH}); - }, TEST_EXECUTION_TIMEOUT); + }); - it( - 'should run agent with skills successfully', - async () => { - const childProcess = spawn('npm', ['run', 'start'], { - cwd: PROJECT_PATH, - shell: true, - }); + it('should run agent with skills successfully', async () => { + const childProcess = spawn('npm', ['run', 'start'], { + cwd: PROJECT_PATH, + shell: true, + }); - let response = await sendInput( - childProcess, - 'Let`s create algorithmic art.\n', - ); - expect(response.toString()).toContain( - 'I have created an original algorithmic art piece titled **"Ephemeral Entanglement"**.\n\nFollowing the generative art movement philosophy, I\'ve generated three files for you:\n\n1. **`ephemeral_entanglement.md`**: The algorithmic philosophy detailing the conceptual foundation of this piece. It explores the delicate dance between deterministic forces and stochastic drift, visualizing unseen connections in a dynamic system.\n2. **`index.html`**: The interactive viewer for the generative art. It includes a user interface to adjust parameters like particle count, connection radius, and noise scale, allowing you to explore the algorithm\'s emergent behavior.\n3. **`sketch.js`**: The meticulously crafted p5.js algorithm that brings the philosophy to life. It uses layered Perlin noise to drive a flow field, guiding particles that form ephemeral, glowing bonds when they come into proximity. \n\nYou can view the art by opening the `index.html` file in your web browser. Let the algorithmic dance begin!', - ); + let response = await sendInput( + childProcess, + 'Let`s create algorithmic art.\n', + ); + expect(response.toString()).toContain( + 'I have created an original algorithmic art piece titled **"Ephemeral Entanglement"**.\n\nFollowing the generative art movement philosophy, I\'ve generated three files for you:\n\n1. **`ephemeral_entanglement.md`**: The algorithmic philosophy detailing the conceptual foundation of this piece. It explores the delicate dance between deterministic forces and stochastic drift, visualizing unseen connections in a dynamic system.\n2. **`index.html`**: The interactive viewer for the generative art. It includes a user interface to adjust parameters like particle count, connection radius, and noise scale, allowing you to explore the algorithm\'s emergent behavior.\n3. **`sketch.js`**: The meticulously crafted p5.js algorithm that brings the philosophy to life. It uses layered Perlin noise to drive a flow field, guiding particles that form ephemeral, glowing bonds when they come into proximity. \n\nYou can view the art by opening the `index.html` file in your web browser. Let the algorithmic dance begin!', + ); - response = await sendInput(childProcess, 'exit\n'); - expect(response.toString()).toContain(''); + response = await sendInput(childProcess, 'exit\n'); + expect(response.toString()).toContain(''); - // verify that files were created and have the expected content - const resultMdFile = await fs.readFile( - `${PROJECT_PATH}/ephemeral_entanglement.md`, - 'utf-8', - ); - const resultScriptFile = await fs.readFile( - `${PROJECT_PATH}/sketch.js`, - 'utf-8', - ); - const resultHtmlFile = await fs.readFile( - `${PROJECT_PATH}/index.html`, - 'utf-8', - ); + // verify that files were created and have the expected content + const resultMdFile = await fs.readFile( + `${PROJECT_PATH}/ephemeral_entanglement.md`, + 'utf-8', + ); + const resultScriptFile = await fs.readFile( + `${PROJECT_PATH}/sketch.js`, + 'utf-8', + ); + const resultHtmlFile = await fs.readFile( + `${PROJECT_PATH}/index.html`, + 'utf-8', + ); - const expectedMdFile = await fs.readFile( - `${PROJECT_PATH}/expected/ephemeral_entanglement.md`, - 'utf-8', - ); - const expectedScriptFile = await fs.readFile( - `${PROJECT_PATH}/expected/sketch.js`, - 'utf-8', - ); - const expectedHtmlFile = await fs.readFile( - `${PROJECT_PATH}/expected/index.html`, - 'utf-8', - ); + const expectedMdFile = await fs.readFile( + `${PROJECT_PATH}/expected/ephemeral_entanglement.md`, + 'utf-8', + ); + const expectedScriptFile = await fs.readFile( + `${PROJECT_PATH}/expected/sketch.js`, + 'utf-8', + ); + const expectedHtmlFile = await fs.readFile( + `${PROJECT_PATH}/expected/index.html`, + 'utf-8', + ); - expect((normalizeLineEndings(resultMdFile) as string).trim()).toEqual( - (normalizeLineEndings(expectedMdFile) as string).trim(), - ); - expect((normalizeLineEndings(resultScriptFile) as string).trim()).toEqual( - (normalizeLineEndings(expectedScriptFile) as string).trim(), - ); - expect((normalizeLineEndings(resultHtmlFile) as string).trim()).toEqual( - (normalizeLineEndings(expectedHtmlFile) as string).trim(), - ); - }, - TEST_EXECUTION_TIMEOUT, - ); + expect((normalizeLineEndings(resultMdFile) as string).trim()).toEqual( + (normalizeLineEndings(expectedMdFile) as string).trim(), + ); + expect((normalizeLineEndings(resultScriptFile) as string).trim()).toEqual( + (normalizeLineEndings(expectedScriptFile) as string).trim(), + ); + expect((normalizeLineEndings(resultHtmlFile) as string).trim()).toEqual( + (normalizeLineEndings(expectedHtmlFile) as string).trim(), + ); + }); afterAll(async () => { // delete generated files diff --git a/vitest.config.ts b/vitest.config.ts index 1b7eaee63..d936ecf34 100644 --- a/vitest.config.ts +++ b/vitest.config.ts @@ -10,7 +10,12 @@ import {defineConfig} from 'vitest/config'; /** * Hook budget (ms) for the `integration` project: install-heavy `beforeAll` * hooks run `npm install` (and sometimes `npm run build`) per fixture, which - * exceeds Vitest's 10s default on a slow or loaded machine. + * exceeds Vitest's 10s default on a slow or loaded machine. The twelve + * `build_setup` hook runs take ~16s combined on ubuntu-latest, but a cold, + * network-bound install has been measured at ~70s, so 120s covers the worst + * case. It also stays above the 60s start watchdog in + * `tests/integration/test_api_server.ts`, so a server that fails to start + * reports its own captured stdout instead of a bare "Hook timed out". */ const INTEGRATION_HOOK_TIMEOUT_MS = 120000;