From 30412f498eedca8a1e8aa5cb4d4a583fc98f3ed0 Mon Sep 17 00:00:00 2001 From: Yannic Charlon <52761674+JustYannicc@users.noreply.github.com> Date: Fri, 3 Jul 2026 21:16:59 +0200 Subject: [PATCH] perf: optimize file search delete batches --- .../benchmark-file-search-delete-batch.mjs | 130 ++++++++++++ scripts/test-file-search-delete-batch.mjs | 190 ++++++++++++++++++ src/main/file-search-index.ts | 73 ++++++- 3 files changed, 386 insertions(+), 7 deletions(-) create mode 100644 scripts/benchmark-file-search-delete-batch.mjs create mode 100644 scripts/test-file-search-delete-batch.mjs diff --git a/scripts/benchmark-file-search-delete-batch.mjs b/scripts/benchmark-file-search-delete-batch.mjs new file mode 100644 index 00000000..d1c8072c --- /dev/null +++ b/scripts/benchmark-file-search-delete-batch.mjs @@ -0,0 +1,130 @@ +#!/usr/bin/env node + +import { createRequire } from 'node:module'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import { performance } from 'node:perf_hooks'; +import vm from 'node:vm'; + +const require = createRequire(import.meta.url); +const ts = require('typescript'); + +function loadFileSearchIndexInternals() { + const filePath = path.resolve('src/main/file-search-index.ts'); + const source = `${fs.readFileSync(filePath, 'utf8')} + +exports.__test = { + indexEntry, + tombstoneDeletedPaths, + makeSnapshot() { + return { + entries: [], + prefixToEntryIds: new Map(), + pathToEntryId: new Map(), + builtAt: Date.now(), + }; + }, +}; +`; + + const transpiled = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2022, + esModuleInterop: true, + importsNotUsedAsValues: ts.ImportsNotUsedAsValues.Remove, + }, + fileName: filePath, + }); + + const module = { exports: {} }; + const sandbox = { + module, + exports: module.exports, + require, + console, + Date, + Map, + Promise, + Set, + clearInterval, + clearTimeout, + process, + setInterval, + setTimeout, + }; + vm.runInNewContext(transpiled.outputText, sandbox, { filename: filePath }); + return module.exports.__test; +} + +function addEntry(api, snapshot, filePath, isDirectory = false) { + api.indexEntry(snapshot, { + path: filePath, + name: path.basename(filePath), + parentPath: path.dirname(filePath), + isDirectory, + }); +} + +function buildScenario(api) { + const homeDir = '/tmp/supercmd-file-index-bench'; + const snapshot = api.makeSnapshot(); + const deletedRoot = path.join(homeDir, 'deleted-root'); + const survivorRoot = path.join(homeDir, 'survivors'); + const deletePaths = [deletedRoot]; + + addEntry(api, snapshot, deletedRoot, true); + for (let group = 0; group < 40; group += 1) { + const groupDir = path.join(deletedRoot, `group-${group}`); + addEntry(api, snapshot, groupDir, true); + deletePaths.push(groupDir); + for (let item = 0; item < 100; item += 1) { + const nestedDir = path.join(groupDir, `nested-${item}`); + addEntry(api, snapshot, nestedDir, true); + addEntry(api, snapshot, path.join(nestedDir, `deleted-${group}-${item}.txt`)); + deletePaths.push(nestedDir); + } + } + + addEntry(api, snapshot, survivorRoot, true); + for (let dir = 0; dir < 120; dir += 1) { + const dirPath = path.join(survivorRoot, `dir-${dir}`); + addEntry(api, snapshot, dirPath, true); + for (let file = 0; file < 500; file += 1) { + addEntry(api, snapshot, path.join(dirPath, `survivor-${dir}-${file}.txt`)); + } + } + + return { + deletePaths, + deletedRoot, + expectedDeletedCount: 1 + (40 * 100) + 40 + (40 * 100), + snapshot, + survivorPath: path.join(survivorRoot, 'dir-119', 'survivor-119-499.txt'), + }; +} + +const api = loadFileSearchIndexInternals(); +const scenario = buildScenario(api); +const started = performance.now(); +api.tombstoneDeletedPaths(scenario.snapshot, scenario.deletePaths); +const elapsedMs = performance.now() - started; + +let deletedCount = 0; +for (const entry of scenario.snapshot.entries) { + if (entry.deleted) deletedCount += 1; +} + +const survivorId = scenario.snapshot.pathToEntryId.get(scenario.survivorPath); +assert.equal(deletedCount, scenario.expectedDeletedCount); +assert.notEqual(survivorId, undefined); +assert.equal(scenario.snapshot.entries[survivorId].deleted, undefined); + +console.log(JSON.stringify({ + benchmark: 'file-search-delete-batch', + entries: scenario.snapshot.entries.length, + deletePaths: scenario.deletePaths.length, + deletedCount, + elapsedMs: Number(elapsedMs.toFixed(2)), +}, null, 2)); diff --git a/scripts/test-file-search-delete-batch.mjs b/scripts/test-file-search-delete-batch.mjs new file mode 100644 index 00000000..684d3001 --- /dev/null +++ b/scripts/test-file-search-delete-batch.mjs @@ -0,0 +1,190 @@ +#!/usr/bin/env node + +import { createRequire } from 'node:module'; +import assert from 'node:assert/strict'; +import fs from 'node:fs'; +import path from 'node:path'; +import test from 'node:test'; +import vm from 'node:vm'; + +const require = createRequire(import.meta.url); +const ts = require('typescript'); + +function loadFileSearchIndexInternals() { + const filePath = path.resolve('src/main/file-search-index.ts'); + const source = `${fs.readFileSync(filePath, 'utf8')} + +exports.__test = { + collapseNestedDeletedPaths, + indexEntry, + searchIndexedFiles, + tombstoneDeletedPaths, + makeSnapshot() { + return { + entries: [], + prefixToEntryIds: new Map(), + pathToEntryId: new Map(), + builtAt: Date.now(), + }; + }, + setActiveIndex(snapshot, homeDir) { + activeIndex = snapshot; + configuredHomeDir = homeDir; + includeRoots = [homeDir]; + includeProtectedHomeRoots = true; + lastBuildStartedAt = 0; + lastIndexError = null; + }, +}; +`; + + const transpiled = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2022, + esModuleInterop: true, + importsNotUsedAsValues: ts.ImportsNotUsedAsValues.Remove, + }, + fileName: filePath, + }); + + const module = { exports: {} }; + const sandbox = { + module, + exports: module.exports, + require, + console, + Date, + Map, + Promise, + Set, + clearInterval, + clearTimeout, + process, + setInterval, + setTimeout, + }; + vm.runInNewContext(transpiled.outputText, sandbox, { filename: filePath }); + return module.exports.__test; +} + +const api = loadFileSearchIndexInternals(); +const homeDir = '/tmp/supercmd-file-search-delete-tests'; + +function addEntry(snapshot, filePath, isDirectory = false) { + api.indexEntry(snapshot, { + path: filePath, + name: path.basename(filePath), + parentPath: path.dirname(filePath), + isDirectory, + }); +} + +function entryForPath(snapshot, filePath) { + const id = snapshot.pathToEntryId.get(filePath); + assert.notEqual(id, undefined, `${filePath} should be indexed`); + return snapshot.entries[id]; +} + +function isDeleted(snapshot, filePath) { + return Boolean(entryForPath(snapshot, filePath).deleted); +} + +test('file delete tombstones only the exact indexed file', () => { + const snapshot = api.makeSnapshot(); + const deletedFile = path.join(homeDir, 'notes', 'alpha.txt'); + const prefixSibling = path.join(homeDir, 'notes', 'alpha.txt.backup'); + const survivor = path.join(homeDir, 'notes', 'beta.txt'); + + addEntry(snapshot, path.dirname(deletedFile), true); + addEntry(snapshot, deletedFile); + addEntry(snapshot, prefixSibling); + addEntry(snapshot, survivor); + + api.tombstoneDeletedPaths(snapshot, [deletedFile]); + + assert.equal(isDeleted(snapshot, deletedFile), true); + assert.equal(isDeleted(snapshot, prefixSibling), false); + assert.equal(isDeleted(snapshot, survivor), false); + + api.indexEntry(snapshot, { + path: deletedFile, + name: path.basename(deletedFile), + parentPath: path.dirname(deletedFile), + isDirectory: false, + }); + assert.equal(isDeleted(snapshot, deletedFile), false); +}); + +test('directory delete tombstones the directory and all indexed descendants', () => { + const snapshot = api.makeSnapshot(); + const deletedDir = path.join(homeDir, 'project'); + const childDir = path.join(deletedDir, 'src'); + const childFile = path.join(childDir, 'index.ts'); + const survivor = path.join(homeDir, 'other', 'index.ts'); + + addEntry(snapshot, deletedDir, true); + addEntry(snapshot, childDir, true); + addEntry(snapshot, childFile); + addEntry(snapshot, path.dirname(survivor), true); + addEntry(snapshot, survivor); + + api.tombstoneDeletedPaths(snapshot, [deletedDir]); + + assert.equal(isDeleted(snapshot, deletedDir), true); + assert.equal(isDeleted(snapshot, childDir), true); + assert.equal(isDeleted(snapshot, childFile), true); + assert.equal(isDeleted(snapshot, survivor), false); +}); + +test('nested directory deletes collapse to the outermost deleted root', () => { + const deletedRoot = path.join(homeDir, 'nested-root'); + const nestedDir = path.join(deletedRoot, 'child'); + const nestedFile = path.join(nestedDir, 'deep.txt'); + + assert.deepEqual( + Array.from(api.collapseNestedDeletedPaths([nestedFile, nestedDir, deletedRoot])), + [deletedRoot] + ); + + const snapshot = api.makeSnapshot(); + const similarSibling = path.join(homeDir, 'nested-root-copy', 'deep.txt'); + addEntry(snapshot, deletedRoot, true); + addEntry(snapshot, nestedDir, true); + addEntry(snapshot, nestedFile); + addEntry(snapshot, path.dirname(similarSibling), true); + addEntry(snapshot, similarSibling); + + api.tombstoneDeletedPaths(snapshot, [nestedFile, nestedDir, deletedRoot]); + + assert.equal(isDeleted(snapshot, deletedRoot), true); + assert.equal(isDeleted(snapshot, nestedDir), true); + assert.equal(isDeleted(snapshot, nestedFile), true); + assert.equal(isDeleted(snapshot, similarSibling), false); +}); + +test('non-matching delete paths do not tombstone prefix-like neighbors', async () => { + const snapshot = api.makeSnapshot(); + const prefixLikeDelete = path.join(homeDir, 'docs'); + const survivorDir = path.join(homeDir, 'docs-old'); + const survivor = path.join(survivorDir, 'report.txt'); + const searchable = path.join(homeDir, 'searchable', 'needle-survivor.txt'); + + addEntry(snapshot, survivorDir, true); + addEntry(snapshot, survivor); + addEntry(snapshot, path.dirname(searchable), true); + addEntry(snapshot, searchable); + + api.tombstoneDeletedPaths(snapshot, [ + prefixLikeDelete, + path.join(homeDir, 'missing', 'child'), + ]); + + assert.equal(isDeleted(snapshot, survivor), false); + assert.equal(isDeleted(snapshot, searchable), false); + + api.setActiveIndex(snapshot, homeDir); + const results = await api.searchIndexedFiles('needle-survivor', { limit: 1 }); + assert.equal(results.length, 1); + assert.equal(results[0].path, searchable); +}); diff --git a/src/main/file-search-index.ts b/src/main/file-search-index.ts index 9065be73..6db6ea2e 100644 --- a/src/main/file-search-index.ts +++ b/src/main/file-search-index.ts @@ -796,13 +796,75 @@ async function applyWatchEventBatch(paths: string[]): Promise { } } +function hasDeletedPathAncestor(candidatePath: string, deletedPathSet: Set): boolean { + let currentPath = path.dirname(candidatePath); + while (currentPath && currentPath !== candidatePath) { + if (deletedPathSet.has(currentPath)) return true; + const parentPath = path.dirname(currentPath); + if (parentPath === currentPath) break; + currentPath = parentPath; + } + return false; +} + +function normalizeDeletedPath(candidatePath: string): string | null { + const rawPath = String(candidatePath || ''); + if (!rawPath) return null; + return path.resolve(rawPath); +} + +function collapseNestedDeletedPaths(deletePaths: string[]): string[] { + const uniquePaths = new Set(); + for (const deletePath of deletePaths) { + const normalizedPath = normalizeDeletedPath(deletePath); + if (normalizedPath) uniquePaths.add(normalizedPath); + } + + const sortedPaths = [...uniquePaths].sort((a, b) => { + if (a.length !== b.length) return a.length - b.length; + return a.localeCompare(b); + }); + const collapsedPaths: string[] = []; + const collapsedPathSet = new Set(); + + for (const deletePath of sortedPaths) { + if (hasDeletedPathAncestor(deletePath, collapsedPathSet)) { + continue; + } + collapsedPaths.push(deletePath); + collapsedPathSet.add(deletePath); + } + + return collapsedPaths; +} + +function getDescendantPathPrefix(rootPath: string): string { + return rootPath.endsWith(path.sep) ? rootPath : `${rootPath}${path.sep}`; +} + function tombstoneDeletedPaths(snapshot: IndexSnapshot, deletePaths: string[]): void { + const collapsedDeletePaths = collapseNestedDeletedPaths(deletePaths); + if (collapsedDeletePaths.length === 0) return; + const directIds = new Set(); - for (const deletedPath of deletePaths) { + for (const deletedPath of collapsedDeletePaths) { const id = snapshot.pathToEntryId.get(deletedPath); if (id !== undefined) directIds.add(id); } - const prefixes = deletePaths.map((p) => p + path.sep); + + if (collapsedDeletePaths.length === 1) { + const descendantPrefix = getDescendantPathPrefix(collapsedDeletePaths[0]); + for (let i = 0; i < snapshot.entries.length; i += 1) { + const entry = snapshot.entries[i]; + if (entry.deleted) continue; + if (directIds.has(i) || entry.path.startsWith(descendantPrefix)) { + entry.deleted = true; + } + } + return; + } + + const deletedPathSet = new Set(collapsedDeletePaths); for (let i = 0; i < snapshot.entries.length; i += 1) { const entry = snapshot.entries[i]; @@ -811,11 +873,8 @@ function tombstoneDeletedPaths(snapshot: IndexSnapshot, deletePaths: string[]): entry.deleted = true; continue; } - for (const prefix of prefixes) { - if (entry.path.startsWith(prefix)) { - entry.deleted = true; - break; - } + if (hasDeletedPathAncestor(entry.path, deletedPathSet)) { + entry.deleted = true; } } }