diff --git a/scripts/test-file-search-index.mjs b/scripts/test-file-search-index.mjs new file mode 100644 index 00000000..6b2bdb9d --- /dev/null +++ b/scripts/test-file-search-index.mjs @@ -0,0 +1,248 @@ +#!/usr/bin/env node + +import assert from 'assert/strict'; +import fs from 'fs'; +import os from 'os'; +import path from 'path'; +import vm from 'vm'; +import { createRequire } from 'module'; +import { performance } from 'perf_hooks'; + +const require = createRequire(import.meta.url); +const ts = require('typescript'); + +const moduleCache = new Map(); + +function loadTsModule(filePath) { + const resolvedPath = path.resolve(filePath); + if (moduleCache.has(resolvedPath)) return moduleCache.get(resolvedPath).exports; + + const source = fs.readFileSync(resolvedPath, 'utf8'); + const transpiled = ts.transpileModule(source, { + compilerOptions: { + module: ts.ModuleKind.CommonJS, + target: ts.ScriptTarget.ES2022, + esModuleInterop: true, + importsNotUsedAsValues: ts.ImportsNotUsedAsValues.Remove, + }, + fileName: resolvedPath, + }); + + const module = { exports: {} }; + moduleCache.set(resolvedPath, module); + const localRequire = (request) => { + if (request.startsWith('.')) { + const candidate = path.resolve(path.dirname(resolvedPath), request); + for (const suffix of ['', '.ts', '.tsx', '.js', '.jsx', '/index.ts', '/index.tsx']) { + const nextPath = `${candidate}${suffix}`; + if (fs.existsSync(nextPath) && fs.statSync(nextPath).isFile()) { + if (nextPath.endsWith('.ts') || nextPath.endsWith('.tsx')) return loadTsModule(nextPath); + return require(nextPath); + } + } + } + return require(request); + }; + const sandbox = { + module, + exports: module.exports, + require: localRequire, + console, + URL, + Date, + Math, + String, + Number, + Boolean, + Set, + Map, + WeakMap, + Object, + Array, + RegExp, + Promise, + process, + Buffer, + setTimeout, + clearTimeout, + setInterval, + clearInterval, + }; + sandbox.global = sandbox; + sandbox.globalThis = sandbox; + vm.runInNewContext(transpiled.outputText, sandbox, { filename: resolvedPath }); + return module.exports; +} + +function writeFile(filePath, contents = '') { + fs.mkdirSync(path.dirname(filePath), { recursive: true }); + fs.writeFileSync(filePath, contents); +} + +function makeTempHome(label) { + const tmpRoot = fs.realpathSync(os.tmpdir()); + return fs.mkdtempSync(path.join(tmpRoot, `supercmd-file-search-${label}-`)); +} + +function removeTempHome(homeDir) { + try { + fs.rmSync(homeDir, { recursive: true, force: true }); + } catch { + // Best-effort cleanup for temp fixtures. + } +} + +async function withIndexedHome(homeDir, fn) { + moduleCache.delete(path.resolve('src/main/file-search-index.ts')); + const fileSearch = loadTsModule('src/main/file-search-index.ts'); + fileSearch.startFileSearchIndexing({ + homeDir, + refreshIntervalMs: 30_000, + includeProtectedHomeRoots: true, + }); + await fileSearch.rebuildFileSearchIndex('test'); + try { + await fn(fileSearch); + } finally { + fileSearch.stopFileSearchIndexing(); + } +} + +function resultPaths(results) { + return results.map((result) => result.path); +} + +async function test(name, fn) { + try { + await fn(); + console.log(`✓ ${name}`); + } catch (error) { + console.error(`✗ ${name}`); + throw error; + } +} + +await test('path-like queries match absolute, tilde, and relative paths', async () => { + const homeDir = makeTempHome('correctness'); + try { + const srcDir = path.join(homeDir, 'Projects', 'app-042', 'src'); + const exactFile = path.join(srcDir, 'Report Final.txt'); + writeFile(exactFile, 'report'); + writeFile(path.join(srcDir, 'Report Notes.md'), 'notes'); + writeFile(path.join(homeDir, 'Projects', 'app-042', 'README.md'), 'readme'); + writeFile(path.join(homeDir, 'Archive', 'Reports', 'src', 'Q2 Plan.txt'), 'plan'); + + await withIndexedHome(homeDir, async ({ searchIndexedFiles }) => { + const absolute = await searchIndexedFiles(srcDir, { limit: 8 }); + assert.equal(absolute[0]?.path, srcDir); + assert.ok(resultPaths(absolute).includes(exactFile)); + + const tilde = await searchIndexedFiles('~/Projects/app-042/src', { limit: 8 }); + assert.equal(tilde[0]?.path, srcDir); + assert.ok(resultPaths(tilde).includes(exactFile)); + + const relative = await searchIndexedFiles('Projects/app-042/src', { limit: 8 }); + assert.equal(relative[0]?.path, srcDir); + assert.ok(resultPaths(relative).includes(exactFile)); + + const exact = await searchIndexedFiles('~/Projects/app-042/src/Report Final.txt', { limit: 4 }); + assert.equal(exact[0]?.path, exactFile); + assert.equal(exact[0]?.matchKind, 'path'); + }); + } finally { + removeTempHome(homeDir); + } +}); + +await test('path-like fallback preserves mid-token slash matches', async () => { + const homeDir = makeTempHome('fallback'); + try { + const fallbackFile = path.join(homeDir, 'Archive', 'Reports', 'src', 'Q2 Plan.txt'); + writeFile(fallbackFile, 'plan'); + writeFile(path.join(homeDir, 'Archive', 'Exports', 'src', 'Other.txt'), 'other'); + + await withIndexedHome(homeDir, async ({ searchIndexedFiles }) => { + const midToken = await searchIndexedFiles('ports/src', { limit: 8 }); + assert.ok(resultPaths(midToken).includes(fallbackFile)); + + const trailingSlash = await searchIndexedFiles('ports/', { limit: 8 }); + assert.ok(resultPaths(trailingSlash).includes(fallbackFile)); + + const noMatch = await searchIndexedFiles('~/Archive/Missing/src', { limit: 8 }); + assert.equal(noMatch.length, 0); + }); + } finally { + removeTempHome(homeDir); + } +}); + +function populateSyntheticTree(homeDir, targetEntries) { + const projectsDir = path.join(homeDir, 'Projects'); + const filesPerApp = 48; + const appCount = Math.max(1, Math.ceil(targetEntries / (filesPerApp + 2))); + + for (let appIndex = 0; appIndex < appCount; appIndex += 1) { + const appName = `app-${String(appIndex).padStart(4, '0')}`; + const srcDir = path.join(projectsDir, appName, 'src'); + const docsDir = path.join(projectsDir, appName, 'docs'); + fs.mkdirSync(srcDir, { recursive: true }); + fs.mkdirSync(docsDir, { recursive: true }); + for (let fileIndex = 0; fileIndex < filesPerApp; fileIndex += 1) { + const bucket = fileIndex % 2 === 0 ? srcDir : docsDir; + const name = `module-${String(fileIndex).padStart(3, '0')}-${appName}.ts`; + fs.writeFileSync(path.join(bucket, name), ''); + } + } + + return { + appCount, + filesPerApp, + indexedEntryEstimate: appCount * (filesPerApp + 3) + 1, + }; +} + +async function runPerformanceHarness() { + const targetEntries = Number(process.env.FILE_SEARCH_PERF_ENTRIES || 60000); + const iterations = Number(process.env.FILE_SEARCH_PERF_ITERATIONS || 24); + const homeDir = makeTempHome('perf'); + try { + const fixture = populateSyntheticTree(homeDir, targetEntries); + await withIndexedHome(homeDir, async ({ getFileSearchIndexStatus, searchIndexedFiles }) => { + const status = getFileSearchIndexStatus(); + const queryAppIds = [7, 42, 137, Math.floor(fixture.appCount / 2), fixture.appCount - 3] + .filter((value, index, values) => value >= 0 && value < fixture.appCount && values.indexOf(value) === index) + .map((value) => String(value).padStart(4, '0')); + const queries = queryAppIds.flatMap((id) => [ + path.join(homeDir, 'Projects', `app-${id}`, 'src'), + `~/Projects/app-${id}/src`, + `Projects/app-${id}/src`, + ]); + + const startedAt = performance.now(); + let totalResults = 0; + for (let iteration = 0; iteration < iterations; iteration += 1) { + for (const query of queries) { + const results = await searchIndexedFiles(query, { limit: 1 }); + totalResults += results.length; + } + } + const elapsedMs = performance.now() - startedAt; + const queryCount = iterations * queries.length; + const metric = { + entries: status.indexedEntryCount, + estimatedEntries: fixture.indexedEntryEstimate, + queries: queryCount, + elapsedMs: Number(elapsedMs.toFixed(2)), + avgMsPerQuery: Number((elapsedMs / queryCount).toFixed(3)), + totalResults, + }; + console.log(`FILE_SEARCH_PERF ${JSON.stringify(metric)}`); + }); + } finally { + removeTempHome(homeDir); + } +} + +if (process.env.SUPERCMD_FILE_SEARCH_PERF === '1') { + await runPerformanceHarness(); +} diff --git a/src/main/file-search-index.ts b/src/main/file-search-index.ts index 9065be73..4162b57a 100644 --- a/src/main/file-search-index.ts +++ b/src/main/file-search-index.ts @@ -41,6 +41,7 @@ type IndexedEntry = { parentPath: string; normalizedName: string; normalizedPath: string; + normalizedTildePath: string; compactName: string; tokens: string[]; pathTokens: string[]; @@ -132,6 +133,7 @@ const PROTECTED_TOP_LEVEL_SET = new Set( FILE_SEARCH_INDEX_PROTECTED_HOME_TOP_LEVEL_DIRECTORIES.map((name) => name.toLowerCase()) ); const EXCLUDED_FILE_EXTENSIONS = new Set(['.tmp', '.temp', '.log', '.cache', '.crdownload', '.download']); +const PATH_CANDIDATE_TERM_REGEX = /[a-z0-9]/; let activeIndex: IndexSnapshot | null = null; let rebuildPromise: Promise | null = null; @@ -263,12 +265,13 @@ function addPrefixIndexValue(prefixToEntryIds: Map, key: strin function indexEntry( snapshot: IndexSnapshot, - entry: Omit + entry: Omit ): void { const normalizedName = normalizeSearchText(entry.name); if (!normalizedName) return; const normalizedPath = normalizePathSearchText(entry.path); if (!normalizedPath) return; + const normalizedTildePath = normalizePathSearchText(asTildePath(entry.path, configuredHomeDir)); const existingId = snapshot.pathToEntryId.get(entry.path); if (existingId !== undefined) { @@ -292,6 +295,7 @@ function indexEntry( ...entry, normalizedName, normalizedPath, + normalizedTildePath, compactName, tokens, pathTokens, @@ -589,6 +593,62 @@ function resolveCandidateIds(snapshot: IndexSnapshot, terms: string[]): number[] return intersectCandidates(indexedLists); } +function getPathLikeBoundaryTerms(needle: string): string[] { + const normalizedNeedle = normalizePathSearchText(needle); + if (!normalizedNeedle) return []; + + const terms: string[] = []; + let termStart = -1; + + const addTerm = (endIndex: number) => { + if (termStart <= 0) return; + const previousChar = normalizedNeedle[termStart - 1] || ''; + if (PATH_CANDIDATE_TERM_REGEX.test(previousChar)) return; + const term = normalizedNeedle.slice(termStart, endIndex); + if (term.length >= 2) terms.push(term); + }; + + for (let index = 0; index <= normalizedNeedle.length; index += 1) { + const char = normalizedNeedle[index] || ''; + if (PATH_CANDIDATE_TERM_REGEX.test(char)) { + if (termStart < 0) termStart = index; + continue; + } + + if (termStart >= 0) { + addTerm(index); + termStart = -1; + } + } + + return [...new Set(terms)]; +} + +function resolvePathLikeCandidateIds( + snapshot: IndexSnapshot, + rawNeedle: string, + expandedNeedle: string, + trimmedQuery: string +): number[] | null { + const isHomeTildeQuery = trimmedQuery === '~' || trimmedQuery.startsWith('~/') || trimmedQuery.startsWith('~\\'); + const terms = getPathLikeBoundaryTerms(rawNeedle); + if (terms.length === 0 && isHomeTildeQuery) { + terms.push(...getPathLikeBoundaryTerms(expandedNeedle)); + } + if (terms.length === 0) return null; + + let smallestBucket: number[] | null = null; + for (const term of terms) { + const key = term.slice(0, Math.min(MAX_PREFIX_LENGTH, term.length)); + const matches = snapshot.prefixToEntryIds.get(key); + if (!matches || matches.length === 0) return []; + if (!smallestBucket || matches.length < smallestBucket.length) { + smallestBucket = matches; + } + } + return smallestBucket ? [...smallestBucket] : null; +} + function resolveHomeDir(inputHomeDir?: string): string { const candidate = String(inputHomeDir || '').trim(); if (candidate) return path.resolve(candidate); @@ -907,13 +967,16 @@ export async function searchIndexedFiles( const expandedNeedle = trimmedQuery.startsWith('~') && configuredHomeDir ? normalizePathSearchText(`${configuredHomeDir}${trimmedQuery.slice(1)}`) : rawNeedle; + const candidateIds = resolvePathLikeCandidateIds(snapshot, rawNeedle, expandedNeedle, trimmedQuery); + const candidateEntries = candidateIds === null + ? snapshot.entries + : candidateIds.map((entryId) => snapshot.entries[entryId]).filter(Boolean); const scored: Array<{ entry: IndexedEntry; score: number }> = []; - for (const entry of snapshot.entries) { + for (const entry of candidateEntries) { if (entry.deleted) continue; const pathIndex = entry.normalizedPath.indexOf(expandedNeedle); - const tildePath = normalizePathSearchText(asTildePath(entry.path, configuredHomeDir)); - const tildeIndex = tildePath.indexOf(rawNeedle); + const tildeIndex = entry.normalizedTildePath.indexOf(rawNeedle); const matchIndex = pathIndex >= 0 ? pathIndex : tildeIndex; if (matchIndex < 0) continue; @@ -1059,6 +1122,7 @@ export async function searchIndexedFiles( parentPath: path.dirname(candidatePath), normalizedName, normalizedPath: normalizePathSearchText(candidatePath), + normalizedTildePath: normalizePathSearchText(asTildePath(candidatePath, configuredHomeDir)), compactName: normalizedName.replace(/\s+/g, ''), tokens: tokenizeSearchText(candidateName), pathTokens: tokenizeSearchText(candidatePath),