perf(file-search): speed up path-like queries

scripts/test-file-search-index.mjs

@@ -0,0 +1,248 @@
+#!/usr/bin/env node
+
+import assert from 'assert/strict';
+import fs from 'fs';
+import os from 'os';
+import path from 'path';
+import vm from 'vm';
+import { createRequire } from 'module';
+import { performance } from 'perf_hooks';
+
+const require = createRequire(import.meta.url);
+const ts = require('typescript');
+
+const moduleCache = new Map();
+
+function loadTsModule(filePath) {
+  const resolvedPath = path.resolve(filePath);
+  if (moduleCache.has(resolvedPath)) return moduleCache.get(resolvedPath).exports;
+
+  const source = fs.readFileSync(resolvedPath, 'utf8');
+  const transpiled = ts.transpileModule(source, {
+    compilerOptions: {
+      module: ts.ModuleKind.CommonJS,
+      target: ts.ScriptTarget.ES2022,
+      esModuleInterop: true,
+      importsNotUsedAsValues: ts.ImportsNotUsedAsValues.Remove,
+    },
+    fileName: resolvedPath,
+  });
+
+  const module = { exports: {} };
+  moduleCache.set(resolvedPath, module);
+  const localRequire = (request) => {
+    if (request.startsWith('.')) {
+      const candidate = path.resolve(path.dirname(resolvedPath), request);
+      for (const suffix of ['', '.ts', '.tsx', '.js', '.jsx', '/index.ts', '/index.tsx']) {
+        const nextPath = `${candidate}${suffix}`;
+        if (fs.existsSync(nextPath) && fs.statSync(nextPath).isFile()) {
+          if (nextPath.endsWith('.ts') || nextPath.endsWith('.tsx')) return loadTsModule(nextPath);
+          return require(nextPath);
+        }
+      }
+    }
+    return require(request);
+  };
+  const sandbox = {
+    module,
+    exports: module.exports,
+    require: localRequire,
+    console,
+    URL,
+    Date,
+    Math,
+    String,
+    Number,
+    Boolean,
+    Set,
+    Map,
+    WeakMap,
+    Object,
+    Array,
+    RegExp,
+    Promise,
+    process,
+    Buffer,
+    setTimeout,
+    clearTimeout,
+    setInterval,
+    clearInterval,
+  };
+  sandbox.global = sandbox;
+  sandbox.globalThis = sandbox;
+  vm.runInNewContext(transpiled.outputText, sandbox, { filename: resolvedPath });
+  return module.exports;
+}
+
+function writeFile(filePath, contents = '') {
+  fs.mkdirSync(path.dirname(filePath), { recursive: true });
+  fs.writeFileSync(filePath, contents);
+}
+
+function makeTempHome(label) {
+  const tmpRoot = fs.realpathSync(os.tmpdir());
+  return fs.mkdtempSync(path.join(tmpRoot, `supercmd-file-search-${label}-`));
+}
+
+function removeTempHome(homeDir) {
+  try {
+    fs.rmSync(homeDir, { recursive: true, force: true });
+  } catch {
+    // Best-effort cleanup for temp fixtures.
+  }
+}
+
+async function withIndexedHome(homeDir, fn) {
+  moduleCache.delete(path.resolve('src/main/file-search-index.ts'));
+  const fileSearch = loadTsModule('src/main/file-search-index.ts');
+  fileSearch.startFileSearchIndexing({
+    homeDir,
+    refreshIntervalMs: 30_000,
+    includeProtectedHomeRoots: true,
+  });
+  await fileSearch.rebuildFileSearchIndex('test');
+  try {
+    await fn(fileSearch);
+  } finally {
+    fileSearch.stopFileSearchIndexing();
+  }
+}
+
+function resultPaths(results) {
+  return results.map((result) => result.path);
+}
+
+async function test(name, fn) {
+  try {
+    await fn();
+    console.log(`✓ ${name}`);
+  } catch (error) {
+    console.error(`✗ ${name}`);
+    throw error;
+  }
+}
+
+await test('path-like queries match absolute, tilde, and relative paths', async () => {
+  const homeDir = makeTempHome('correctness');
+  try {
+    const srcDir = path.join(homeDir, 'Projects', 'app-042', 'src');
+    const exactFile = path.join(srcDir, 'Report Final.txt');
+    writeFile(exactFile, 'report');
+    writeFile(path.join(srcDir, 'Report Notes.md'), 'notes');
+    writeFile(path.join(homeDir, 'Projects', 'app-042', 'README.md'), 'readme');
+    writeFile(path.join(homeDir, 'Archive', 'Reports', 'src', 'Q2 Plan.txt'), 'plan');
+
+    await withIndexedHome(homeDir, async ({ searchIndexedFiles }) => {
+      const absolute = await searchIndexedFiles(srcDir, { limit: 8 });
+      assert.equal(absolute[0]?.path, srcDir);
+      assert.ok(resultPaths(absolute).includes(exactFile));
+
+      const tilde = await searchIndexedFiles('~/Projects/app-042/src', { limit: 8 });
+      assert.equal(tilde[0]?.path, srcDir);
+      assert.ok(resultPaths(tilde).includes(exactFile));
+
+      const relative = await searchIndexedFiles('Projects/app-042/src', { limit: 8 });
+      assert.equal(relative[0]?.path, srcDir);
+      assert.ok(resultPaths(relative).includes(exactFile));
+
+      const exact = await searchIndexedFiles('~/Projects/app-042/src/Report Final.txt', { limit: 4 });
+      assert.equal(exact[0]?.path, exactFile);
+      assert.equal(exact[0]?.matchKind, 'path');
+    });
+  } finally {
+    removeTempHome(homeDir);
+  }
+});
+
+await test('path-like fallback preserves mid-token slash matches', async () => {
+  const homeDir = makeTempHome('fallback');
+  try {
+    const fallbackFile = path.join(homeDir, 'Archive', 'Reports', 'src', 'Q2 Plan.txt');
+    writeFile(fallbackFile, 'plan');
+    writeFile(path.join(homeDir, 'Archive', 'Exports', 'src', 'Other.txt'), 'other');
+
+    await withIndexedHome(homeDir, async ({ searchIndexedFiles }) => {
+      const midToken = await searchIndexedFiles('ports/src', { limit: 8 });
+      assert.ok(resultPaths(midToken).includes(fallbackFile));
+
+      const trailingSlash = await searchIndexedFiles('ports/', { limit: 8 });
+      assert.ok(resultPaths(trailingSlash).includes(fallbackFile));
+
+      const noMatch = await searchIndexedFiles('~/Archive/Missing/src', { limit: 8 });
+      assert.equal(noMatch.length, 0);
+    });
+  } finally {
+    removeTempHome(homeDir);
+  }
+});
+
+function populateSyntheticTree(homeDir, targetEntries) {
+  const projectsDir = path.join(homeDir, 'Projects');
+  const filesPerApp = 48;
+  const appCount = Math.max(1, Math.ceil(targetEntries / (filesPerApp + 2)));
+
+  for (let appIndex = 0; appIndex < appCount; appIndex += 1) {
+    const appName = `app-${String(appIndex).padStart(4, '0')}`;
+    const srcDir = path.join(projectsDir, appName, 'src');
+    const docsDir = path.join(projectsDir, appName, 'docs');
+    fs.mkdirSync(srcDir, { recursive: true });
+    fs.mkdirSync(docsDir, { recursive: true });
+    for (let fileIndex = 0; fileIndex < filesPerApp; fileIndex += 1) {
+      const bucket = fileIndex % 2 === 0 ? srcDir : docsDir;
+      const name = `module-${String(fileIndex).padStart(3, '0')}-${appName}.ts`;
+      fs.writeFileSync(path.join(bucket, name), '');
+    }
+  }
+
+  return {
+    appCount,
+    filesPerApp,
+    indexedEntryEstimate: appCount * (filesPerApp + 3) + 1,
+  };
+}
+
+async function runPerformanceHarness() {
+  const targetEntries = Number(process.env.FILE_SEARCH_PERF_ENTRIES || 60000);
+  const iterations = Number(process.env.FILE_SEARCH_PERF_ITERATIONS || 24);
+  const homeDir = makeTempHome('perf');
+  try {
+    const fixture = populateSyntheticTree(homeDir, targetEntries);
+    await withIndexedHome(homeDir, async ({ getFileSearchIndexStatus, searchIndexedFiles }) => {
+      const status = getFileSearchIndexStatus();
+      const queryAppIds = [7, 42, 137, Math.floor(fixture.appCount / 2), fixture.appCount - 3]
+        .filter((value, index, values) => value >= 0 && value < fixture.appCount && values.indexOf(value) === index)
+        .map((value) => String(value).padStart(4, '0'));
+      const queries = queryAppIds.flatMap((id) => [
+        path.join(homeDir, 'Projects', `app-${id}`, 'src'),
+        `~/Projects/app-${id}/src`,
+        `Projects/app-${id}/src`,
+      ]);
+
+      const startedAt = performance.now();
+      let totalResults = 0;
+      for (let iteration = 0; iteration < iterations; iteration += 1) {
+        for (const query of queries) {
+          const results = await searchIndexedFiles(query, { limit: 1 });
+          totalResults += results.length;
+        }
+      }
+      const elapsedMs = performance.now() - startedAt;
+      const queryCount = iterations * queries.length;
+      const metric = {
+        entries: status.indexedEntryCount,
+        estimatedEntries: fixture.indexedEntryEstimate,
+        queries: queryCount,
+        elapsedMs: Number(elapsedMs.toFixed(2)),
+        avgMsPerQuery: Number((elapsedMs / queryCount).toFixed(3)),
+        totalResults,
+      };
+      console.log(`FILE_SEARCH_PERF ${JSON.stringify(metric)}`);
+    });
+  } finally {
+    removeTempHome(homeDir);
+  }
+}
+
+if (process.env.SUPERCMD_FILE_SEARCH_PERF === '1') {
+  await runPerformanceHarness();
+}

src/main/file-search-index.ts

@@ -41,6 +41,7 @@ type IndexedEntry = {
   parentPath: string;
   normalizedName: string;
   normalizedPath: string;
+  normalizedTildePath: string;
   compactName: string;
   tokens: string[];
   pathTokens: string[];
@@ -132,6 +133,7 @@ const PROTECTED_TOP_LEVEL_SET = new Set(
   FILE_SEARCH_INDEX_PROTECTED_HOME_TOP_LEVEL_DIRECTORIES.map((name) => name.toLowerCase())
 );
 const EXCLUDED_FILE_EXTENSIONS = new Set(['.tmp', '.temp', '.log', '.cache', '.crdownload', '.download']);
+const PATH_CANDIDATE_TERM_REGEX = /[a-z0-9]/;
 
 let activeIndex: IndexSnapshot | null = null;
 let rebuildPromise: Promise<void> | null = null;
@@ -263,12 +265,13 @@ function addPrefixIndexValue(prefixToEntryIds: Map<string, number[]>, key: strin
 
 function indexEntry(
   snapshot: IndexSnapshot,
-  entry: Omit<IndexedEntry, 'normalizedName' | 'normalizedPath' | 'compactName' | 'tokens' | 'pathTokens' | 'deleted'>
+  entry: Omit<IndexedEntry, 'normalizedName' | 'normalizedPath' | 'normalizedTildePath' | 'compactName' | 'tokens' | 'pathTokens' | 'deleted'>
 ): void {
   const normalizedName = normalizeSearchText(entry.name);
   if (!normalizedName) return;
   const normalizedPath = normalizePathSearchText(entry.path);
   if (!normalizedPath) return;
+  const normalizedTildePath = normalizePathSearchText(asTildePath(entry.path, configuredHomeDir));
 
   const existingId = snapshot.pathToEntryId.get(entry.path);
   if (existingId !== undefined) {
@@ -292,6 +295,7 @@ function indexEntry(
     ...entry,
     normalizedName,
     normalizedPath,
+    normalizedTildePath,
     compactName,
     tokens,
     pathTokens,
@@ -589,6 +593,62 @@ function resolveCandidateIds(snapshot: IndexSnapshot, terms: string[]): number[]
   return intersectCandidates(indexedLists);
 }
 
+function getPathLikeBoundaryTerms(needle: string): string[] {
+  const normalizedNeedle = normalizePathSearchText(needle);
+  if (!normalizedNeedle) return [];
+
+  const terms: string[] = [];
+  let termStart = -1;
+
+  const addTerm = (endIndex: number) => {
+    if (termStart <= 0) return;
+    const previousChar = normalizedNeedle[termStart - 1] || '';
+    if (PATH_CANDIDATE_TERM_REGEX.test(previousChar)) return;
+    const term = normalizedNeedle.slice(termStart, endIndex);
+    if (term.length >= 2) terms.push(term);
+  };
+
+  for (let index = 0; index <= normalizedNeedle.length; index += 1) {
+    const char = normalizedNeedle[index] || '';
+    if (PATH_CANDIDATE_TERM_REGEX.test(char)) {
+      if (termStart < 0) termStart = index;
+      continue;
+    }
+
+    if (termStart >= 0) {
+      addTerm(index);
+      termStart = -1;
+    }
+  }
+
+  return [...new Set(terms)];
+}
+
+function resolvePathLikeCandidateIds(
+  snapshot: IndexSnapshot,
+  rawNeedle: string,
+  expandedNeedle: string,
+  trimmedQuery: string
+): number[] | null {
+  const isHomeTildeQuery = trimmedQuery === '~' || trimmedQuery.startsWith('~/') || trimmedQuery.startsWith('~\\');
+  const terms = getPathLikeBoundaryTerms(rawNeedle);
+  if (terms.length === 0 && isHomeTildeQuery) {
+    terms.push(...getPathLikeBoundaryTerms(expandedNeedle));
+  }
+  if (terms.length === 0) return null;
+
+  let smallestBucket: number[] | null = null;
+  for (const term of terms) {
+    const key = term.slice(0, Math.min(MAX_PREFIX_LENGTH, term.length));
+    const matches = snapshot.prefixToEntryIds.get(key);
+    if (!matches || matches.length === 0) return [];
+    if (!smallestBucket || matches.length < smallestBucket.length) {
+      smallestBucket = matches;
+    }
+  }
+  return smallestBucket ? [...smallestBucket] : null;
+}
+
 function resolveHomeDir(inputHomeDir?: string): string {
   const candidate = String(inputHomeDir || '').trim();
   if (candidate) return path.resolve(candidate);
@@ -907,13 +967,16 @@ export async function searchIndexedFiles(
         const expandedNeedle = trimmedQuery.startsWith('~') && configuredHomeDir
           ? normalizePathSearchText(`${configuredHomeDir}${trimmedQuery.slice(1)}`)
           : rawNeedle;
+        const candidateIds = resolvePathLikeCandidateIds(snapshot, rawNeedle, expandedNeedle, trimmedQuery);
+        const candidateEntries = candidateIds === null
+          ? snapshot.entries
+          : candidateIds.map((entryId) => snapshot.entries[entryId]).filter(Boolean);
 
         const scored: Array<{ entry: IndexedEntry; score: number }> = [];
-        for (const entry of snapshot.entries) {
+        for (const entry of candidateEntries) {
           if (entry.deleted) continue;
           const pathIndex = entry.normalizedPath.indexOf(expandedNeedle);
-          const tildePath = normalizePathSearchText(asTildePath(entry.path, configuredHomeDir));
-          const tildeIndex = tildePath.indexOf(rawNeedle);
+          const tildeIndex = entry.normalizedTildePath.indexOf(rawNeedle);
           const matchIndex = pathIndex >= 0 ? pathIndex : tildeIndex;
           if (matchIndex < 0) continue;
 
@@ -1059,6 +1122,7 @@ export async function searchIndexedFiles(
         parentPath: path.dirname(candidatePath),
         normalizedName,
         normalizedPath: normalizePathSearchText(candidatePath),
+        normalizedTildePath: normalizePathSearchText(asTildePath(candidatePath, configuredHomeDir)),
         compactName: normalizedName.replace(/\s+/g, ''),
         tokens: tokenizeSearchText(candidateName),
         pathTokens: tokenizeSearchText(candidatePath),