From d43b56f751b84f0a81fdc4363dcf67d08d3a395a Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Fri, 7 Aug 2026 02:34:18 -0700 Subject: [PATCH 1/2] Fix: sweep the script_js fixture outputs by name shape The skill write path de-duplicates against existing files: it appends `_2`, `_3`, ... before the extension when the target name is taken. A teardown keyed on the three exact names therefore leaves every variant behind, and those variants are not gitignored. Sweep the fixture directory by name shape instead, in `beforeAll` and in `afterAll`. Sweeping first also stops the content assertions comparing a previous run's output, which is what kept the leak invisible. --- .../skills/script_js/agent_test.ts | 52 ++++++++++++++++--- 1 file changed, 46 insertions(+), 6 deletions(-) diff --git a/tests/integration/skills/script_js/agent_test.ts b/tests/integration/skills/script_js/agent_test.ts index b17df2c86..a99a1d537 100644 --- a/tests/integration/skills/script_js/agent_test.ts +++ b/tests/integration/skills/script_js/agent_test.ts @@ -5,6 +5,7 @@ */ import {exec, spawn} from 'node:child_process'; import * as fs from 'node:fs/promises'; +import * as path from 'node:path'; import {promisify} from 'node:util'; import {afterAll, beforeAll, describe, expect, it} from 'vitest'; import {normalizeLineEndings, sendInput} from '../../test_case_utils.js'; @@ -14,6 +15,35 @@ const dirname = process.cwd(); const PROJECT_PATH = `${dirname}/tests/integration/skills/script_js`; const TEST_EXECUTION_TIMEOUT = 60000; +/** + * Files the skill script materializes into the fixture directory. The write + * path de-duplicates against existing files by appending `_2`, `_3`, ... + * before the extension (materializeFiles in core/src/utils/file_utils.ts), so + * a teardown keyed on these exact names leaves the variants behind. + */ +const GENERATED_FILE_NAMES = [ + 'ephemeral_entanglement.md', + 'index.html', + 'sketch.js', +]; + +/** True for a generated output name or one of its `_` de-duplicated variants. */ +function isGeneratedOutput(entry: string): boolean { + const ext = path.extname(entry); + const base = path.basename(entry, ext); + return GENERATED_FILE_NAMES.includes(`${base.replace(/_\d+$/, '')}${ext}`); +} + +/** Removes every generated output in the fixture directory, variants included. */ +async function removeGeneratedOutputs(): Promise { + const entries = await fs.readdir(PROJECT_PATH); + await Promise.all( + entries + .filter(isGeneratedOutput) + .map((entry) => fs.rm(path.join(PROJECT_PATH, entry), {force: true})), + ); +} + /** * This integration test verifies that an agent equipped with script execution skills * can successfully process a request to create algorithmic art. @@ -30,6 +60,11 @@ const TEST_EXECUTION_TIMEOUT = 60000; */ describe('Agent with skills that generates JS script and runs it locally', () => { beforeAll(async () => { + // A previous run's outputs must not survive into this one: the skill's + // write path de-duplicates against existing files, so the agent would + // write `_2.` and the comparisons below would read the stale + // file instead of this run's output. + await removeGeneratedOutputs(); await execAsync('npm install', {cwd: PROJECT_PATH}); }, TEST_EXECUTION_TIMEOUT); @@ -88,17 +123,22 @@ describe('Agent with skills that generates JS script and runs it locally', () => expect((normalizeLineEndings(resultHtmlFile) as string).trim()).toEqual( (normalizeLineEndings(expectedHtmlFile) as string).trim(), ); + + // The run must have written the three names verbatim. A `_` variant + // here means the fixture directory was dirty and the comparisons above + // read a previous run's output. + const generated = (await fs.readdir(PROJECT_PATH)).filter( + isGeneratedOutput, + ); + expect(generated.sort()).toEqual([...GENERATED_FILE_NAMES].sort()); }, TEST_EXECUTION_TIMEOUT, ); afterAll(async () => { - // delete generated files - await fs - .rm(`${PROJECT_PATH}/ephemeral_entanglement.md`, {force: true}) - .catch(() => {}); - await fs.rm(`${PROJECT_PATH}/index.html`, {force: true}).catch(() => {}); - await fs.rm(`${PROJECT_PATH}/sketch.js`, {force: true}).catch(() => {}); + // Removed by name shape, not by exact name: a run that started dirty + // produces `_2.`, which an exact-name teardown leaves behind. + await removeGeneratedOutputs().catch(() => {}); await fs .rm(`${PROJECT_PATH}/node_modules`, {recursive: true, force: true}) From d18f58f5c5e811d1751a5bc161ff9093322a3849 Mon Sep 17 00:00:00 2001 From: Amaad Martin Date: Fri, 7 Aug 2026 03:18:46 -0700 Subject: [PATCH 2/2] Chore: keep the de-duplication rationale in one place --- tests/integration/skills/script_js/agent_test.ts | 12 +++--------- 1 file changed, 3 insertions(+), 9 deletions(-) diff --git a/tests/integration/skills/script_js/agent_test.ts b/tests/integration/skills/script_js/agent_test.ts index a99a1d537..5b4884275 100644 --- a/tests/integration/skills/script_js/agent_test.ts +++ b/tests/integration/skills/script_js/agent_test.ts @@ -60,10 +60,7 @@ async function removeGeneratedOutputs(): Promise { */ describe('Agent with skills that generates JS script and runs it locally', () => { beforeAll(async () => { - // A previous run's outputs must not survive into this one: the skill's - // write path de-duplicates against existing files, so the agent would - // write `_2.` and the comparisons below would read the stale - // file instead of this run's output. + // Start from a clean fixture dir; see GENERATED_FILE_NAMES. await removeGeneratedOutputs(); await execAsync('npm install', {cwd: PROJECT_PATH}); }, TEST_EXECUTION_TIMEOUT); @@ -124,9 +121,7 @@ describe('Agent with skills that generates JS script and runs it locally', () => (normalizeLineEndings(expectedHtmlFile) as string).trim(), ); - // The run must have written the three names verbatim. A `_` variant - // here means the fixture directory was dirty and the comparisons above - // read a previous run's output. + // Fail loudly if the run produced a `_` variant. const generated = (await fs.readdir(PROJECT_PATH)).filter( isGeneratedOutput, ); @@ -136,8 +131,7 @@ describe('Agent with skills that generates JS script and runs it locally', () => ); afterAll(async () => { - // Removed by name shape, not by exact name: a run that started dirty - // produces `_2.`, which an exact-name teardown leaves behind. + // By name shape, not exact name: variants must go too. await removeGeneratedOutputs().catch(() => {}); await fs