From 9aabfe2e064a83f86e69d43a0b3f13571dc47e76 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Fri, 24 Apr 2026 14:17:06 +0200 Subject: [PATCH 01/12] refactor(indexer-core): extract shared scaffolding (Phases 1-4) Introduce @statewalker/indexer-core, a workspace-internal package that consolidates engine-agnostic code previously forked across backends. See openspec/changes/indexer-core-consolidation/ for full design/specs. Phase 1 - Scaffold: * New @statewalker/indexer-core (private:true, not published) * Add as workspace:* dep in the 5 backends Phase 2 - Extract pure helpers: * compositeKey, matchesPrefix, sanitizePrefix, validateDimensionality * toAsyncIterable (replaces 10 unsafe `as AsyncIterable<>` casts in deleteDocuments implementations) * Persistence byte helpers (toBytes, singleChunk, readEntryBytes) * resolveDocId DEFERRED to Phase 6 (absorbed into SqlRetrieverBase) Phase 3 - Unify hybrid merge: * One mergeByRRF/mergeByWeights/mergeHybrid in indexer-core * Three byte-identical hybrid-search.ts files deleted * Preserves composite-key semantics (path + blockId); not routed through indexer-api's reciprocalRankFusion, which keys by blockId alone and applies a top-rank bonus (deferred as separate change) Phase 4 - Unify composite index: * One createCompositeIndex in indexer-core (~225 LOC) replaces three ~275-LOC composites (MemIndex / DuckDbIndex / PGLiteIndex) * Old classes become thin factory functions (~20-40 LOC each) passing getSize closure + optional onDeleteIndex hook * Fixes O(N^2) getDocumentsBlocks bug: removes trackedBlocks map and persistence-replay dance; enumeration now unions sub-indexes once and de-dupes by composite key * Regression test in indexer-tests document-paths.suite (500 blocks must complete in <3s; O(N^2) would take 2+ seconds) Net: -1232 LOC working tree. No public API change: every @statewalker/indexer-api symbol remains exported and behaviourally identical. Tests: 575/575 passing across 6 packages. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/indexer-core/README.md | 17 + packages/indexer-core/package.json | 44 +++ packages/indexer-core/src/async.ts | 9 + packages/indexer-core/src/composite-key.ts | 5 + .../src/create-composite-index.ts | 263 +++++++++++++++ packages/indexer-core/src/index.ts | 11 + .../src/merge.ts} | 19 +- packages/indexer-core/src/path-prefix.ts | 5 + .../indexer-core/src/persistence-bytes.ts | 28 ++ packages/indexer-core/src/sanitize-prefix.ts | 3 + .../src/validate-dimensionality.ts | 12 + packages/indexer-core/tsconfig.json | 27 ++ packages/indexer-duckdb/package.json | 1 + .../src/duckdb-full-text-index.ts | 3 +- packages/indexer-duckdb/src/duckdb-index.ts | 302 +++--------------- packages/indexer-duckdb/src/duckdb-indexer.ts | 11 +- .../indexer-duckdb/src/duckdb-vector-index.ts | 15 +- packages/indexer-duckdb/src/hybrid-search.ts | 136 -------- packages/indexer-mem-flexsearch/package.json | 1 + .../src/flexsearch-full-text-index.ts | 11 +- .../src/flexsearch-indexer.ts | 54 +--- packages/indexer-mem-minisearch/package.json | 1 + .../src/minisearch-full-text-index.ts | 11 +- .../src/minisearch-indexer.ts | 52 +-- packages/indexer-mem/package.json | 1 + packages/indexer-mem/src/index.ts | 2 +- packages/indexer-mem/src/mem-index.ts | 286 +---------------- packages/indexer-mem/src/mem-vector-index.ts | 28 +- packages/indexer-pglite/package.json | 1 + packages/indexer-pglite/src/hybrid-search.ts | 136 -------- .../src/pglite-full-text-index.ts | 3 +- packages/indexer-pglite/src/pglite-index.ts | 300 +++-------------- packages/indexer-pglite/src/pglite-indexer.ts | 11 +- .../indexer-pglite/src/pglite-vector-index.ts | 15 +- .../src/suites/document-paths.suite.ts | 23 ++ 35 files changed, 594 insertions(+), 1253 deletions(-) create mode 100644 packages/indexer-core/README.md create mode 100644 packages/indexer-core/package.json create mode 100644 packages/indexer-core/src/async.ts create mode 100644 packages/indexer-core/src/composite-key.ts create mode 100644 packages/indexer-core/src/create-composite-index.ts create mode 100644 packages/indexer-core/src/index.ts rename packages/{indexer-mem/src/hybrid-search.ts => indexer-core/src/merge.ts} (90%) create mode 100644 packages/indexer-core/src/path-prefix.ts create mode 100644 packages/indexer-core/src/persistence-bytes.ts create mode 100644 packages/indexer-core/src/sanitize-prefix.ts create mode 100644 packages/indexer-core/src/validate-dimensionality.ts create mode 100644 packages/indexer-core/tsconfig.json delete mode 100644 packages/indexer-duckdb/src/hybrid-search.ts delete mode 100644 packages/indexer-pglite/src/hybrid-search.ts diff --git a/packages/indexer-core/README.md b/packages/indexer-core/README.md new file mode 100644 index 0000000..4b5fc85 --- /dev/null +++ b/packages/indexer-core/README.md @@ -0,0 +1,17 @@ +# @statewalker/indexer-core + +Workspace-internal scaffolding shared by the `@statewalker/indexer-*` backends. + +**Not published to npm.** This package is consumed via `workspace:*` by sibling backend packages only (`indexer-mem`, `indexer-mem-flexsearch`, `indexer-mem-minisearch`, `indexer-duckdb`, `indexer-pglite`). + +## Purpose + +Holds engine-agnostic code that would otherwise be forked across every backend: + +- Shared pure helpers (`compositeKey`, `matchesPrefix`, `sanitizePrefix`, `validateDimensionality`, `toAsyncIterable`, persistence byte helpers). +- Unified hybrid merge (`mergeHybrid`) delegating RRF to `@statewalker/indexer-api`'s `reciprocalRankFusion`. +- One composite-index factory (`createCompositeIndex`) replacing `MemIndex` / `DuckDbIndex` / `PGLiteIndex`. +- Two generic `Indexer` factory builders: `createPersistenceBackedIndexer` (mem) and `createSqlBackedIndexer` (SQL). +- A `SqlRetrieverBase` + `SqlDialect` pair holding shared SQL CRUD; dialects override only search SQL + DDL + embedding binding. + +See [openspec change `indexer-core-consolidation`](../../../../openspec/changes/indexer-core-consolidation/) for the design and migration plan. diff --git a/packages/indexer-core/package.json b/packages/indexer-core/package.json new file mode 100644 index 0000000..7ac29ff --- /dev/null +++ b/packages/indexer-core/package.json @@ -0,0 +1,44 @@ +{ + "name": "@statewalker/indexer-core", + "version": "0.1.0", + "private": true, + "type": "module", + "description": "Workspace-internal scaffolding shared by @statewalker/indexer-* backends. Not published.", + "homepage": "https://github.com/statewalker/statewalker-indexer", + "author": { + "name": "Mikhail Kotelnikov", + "email": "mikhail.kotelnikov@gmail.com" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git" + }, + "exports": { + ".": "./src/index.ts" + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "tsdown", + "dev": "tsdown --watch", + "test": "vitest run --passWithNoTests", + "test:watch": "vitest", + "typecheck": "tsc --noEmit", + "clean": "rimraf dist", + "lint": "biome check --write .", + "format": "biome format --write ." + }, + "dependencies": { + "@statewalker/indexer-api": "workspace:*" + }, + "devDependencies": { + "rimraf": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "sideEffects": false +} diff --git a/packages/indexer-core/src/async.ts b/packages/indexer-core/src/async.ts new file mode 100644 index 0000000..8802e3b --- /dev/null +++ b/packages/indexer-core/src/async.ts @@ -0,0 +1,9 @@ +export async function* toAsyncIterable( + source: Iterable | AsyncIterable, +): AsyncGenerator { + if (Symbol.asyncIterator in source) { + yield* source as AsyncIterable; + } else { + yield* source as Iterable; + } +} diff --git a/packages/indexer-core/src/composite-key.ts b/packages/indexer-core/src/composite-key.ts new file mode 100644 index 0000000..0cf66ec --- /dev/null +++ b/packages/indexer-core/src/composite-key.ts @@ -0,0 +1,5 @@ +import type { DocumentPath } from "@statewalker/indexer-api"; + +export function compositeKey(path: DocumentPath, blockId: string): string { + return `${path}\0${blockId}`; +} diff --git a/packages/indexer-core/src/create-composite-index.ts b/packages/indexer-core/src/create-composite-index.ts new file mode 100644 index 0000000..fa60bc2 --- /dev/null +++ b/packages/indexer-core/src/create-composite-index.ts @@ -0,0 +1,263 @@ +import type { + BlockReference, + DocumentPath, + EmbeddingIndex, + FullTextIndex, + HybridSearchParams, + HybridSearchResult, + Index, + IndexedBlock, + Metadata, + PathSelector, +} from "@statewalker/indexer-api"; +import { compositeKey } from "./composite-key.js"; +import { mergeByRRF, mergeByWeights } from "./merge.js"; +import { toAsyncIterable } from "./async.js"; + +export interface CompositeIndexOptions { + name: string; + fts: FullTextIndex | null; + vec: EmbeddingIndex | null; + metadata?: Metadata; + /** Engine-specific count implementation. Defaults to a sub-index union for in-memory backends. */ + getSize?: (pathPrefix?: DocumentPath) => Promise; + /** Engine-specific cleanup invoked by `deleteIndex()` AFTER sub-indexes are deleted. SQL backends pass a closure here to `DROP TABLE` the shared docs table. */ + onDeleteIndex?: () => Promise; +} + +/** + * Engine-agnostic composite `Index` that delegates FTS and vector retrieval to sub-indexes and merges results + * via RRF or weighted linear blend. Replaces `MemIndex` / `DuckDbIndex` / `PGLiteIndex`. + */ +export function createCompositeIndex(opts: CompositeIndexOptions): Index { + const { name, fts, vec, metadata, onDeleteIndex } = opts; + let closed = false; + + const ensureOpen = (): void => { + if (closed) throw new Error(`Index "${name}" is closed`); + }; + + const defaultGetSize = async (pathPrefix?: DocumentPath): Promise => { + const seen = new Set(); + if (fts) { + for await (const ref of fts.getDocumentBlocksRefs(pathPrefix)) { + seen.add(compositeKey(ref.path, ref.blockId)); + } + } + if (vec) { + for await (const ref of vec.getDocumentBlocksRefs(pathPrefix)) { + seen.add(compositeKey(ref.path, ref.blockId)); + } + } + return seen.size; + }; + + const getSize = opts.getSize ?? defaultGetSize; + + return { + name, + metadata, + + async *search(params: HybridSearchParams): AsyncGenerator { + ensureOpen(); + const { queries, embeddings, topK, weights, paths } = params; + + const hasQueries = queries && queries.length > 0 && fts !== null; + const hasEmbeddings = embeddings && embeddings.length > 0 && vec !== null; + + if (!hasQueries && !hasEmbeddings) return; + + const ftsResults = []; + if (hasQueries) { + for await (const r of fts.search({ queries, topK, paths })) { + ftsResults.push(r); + } + } + + const vecResults = []; + if (hasEmbeddings) { + for await (const r of vec.search({ embeddings, topK, paths })) { + vecResults.push(r); + } + } + + let merged: HybridSearchResult[]; + if (ftsResults.length > 0 && vecResults.length > 0) { + merged = weights + ? mergeByWeights(ftsResults, vecResults, weights, topK) + : mergeByRRF(ftsResults, vecResults, topK); + } else if (ftsResults.length > 0) { + merged = ftsResults.map((r) => ({ + path: r.path, + blockId: r.blockId, + score: r.score, + fts: r, + embedding: null, + })); + } else { + merged = vecResults.map((r) => ({ + path: r.path, + blockId: r.blockId, + score: r.score, + fts: null, + embedding: r, + })); + } + + for (const r of merged.slice(0, topK)) yield r; + }, + + async addDocument(blocks: IndexedBlock[]): Promise { + ensureOpen(); + const ftsBlocks = []; + const vecBlocks = []; + + for (const block of blocks) { + if (block.content !== undefined && fts !== null) { + ftsBlocks.push({ + path: block.path, + blockId: block.blockId, + content: block.content, + metadata: block.metadata, + }); + } + if (block.embedding !== undefined && vec !== null) { + vecBlocks.push({ + path: block.path, + blockId: block.blockId, + embedding: block.embedding, + metadata: block.metadata, + }); + } + } + + if (ftsBlocks.length > 0) await fts?.addDocument(ftsBlocks); + if (vecBlocks.length > 0) await vec?.addDocument(vecBlocks); + }, + + async addDocuments( + blocks: Iterable | AsyncIterable, + ): Promise { + ensureOpen(); + for await (const batch of blocks) { + await this.addDocument(batch); + } + }, + + async deleteDocuments( + pathSelectors: PathSelector[] | AsyncIterable, + ): Promise { + ensureOpen(); + const selectors: PathSelector[] = []; + for await (const sel of toAsyncIterable(pathSelectors)) { + selectors.push(sel); + } + if (fts !== null) await fts.deleteDocuments(selectors); + if (vec !== null) await vec.deleteDocuments(selectors); + }, + + async getSize(pathPrefix?: DocumentPath): Promise { + ensureOpen(); + return getSize(pathPrefix); + }, + + async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const paths = new Set(); + if (fts) { + for await (const p of fts.getDocumentPaths(pathPrefix)) paths.add(p); + } + if (vec) { + for await (const p of vec.getDocumentPaths(pathPrefix)) paths.add(p); + } + for (const p of paths) yield p as DocumentPath; + }, + + async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const seen = new Set(); + if (fts) { + for await (const ref of fts.getDocumentBlocksRefs(pathPrefix)) { + const key = compositeKey(ref.path, ref.blockId); + if (!seen.has(key)) { + seen.add(key); + yield ref; + } + } + } + if (vec) { + for await (const ref of vec.getDocumentBlocksRefs(pathPrefix)) { + const key = compositeKey(ref.path, ref.blockId); + if (!seen.has(key)) { + seen.add(key); + yield ref; + } + } + } + }, + + async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const blockMap = new Map(); + + if (fts) { + for await (const b of fts.getDocumentsBlocks(pathPrefix)) { + blockMap.set(compositeKey(b.path, b.blockId), { + path: b.path, + blockId: b.blockId, + content: b.content, + metadata: b.metadata, + }); + } + } + if (vec) { + for await (const b of vec.getDocumentsBlocks(pathPrefix)) { + const key = compositeKey(b.path, b.blockId); + const existing = blockMap.get(key); + if (existing) { + existing.embedding = b.embedding; + if (!existing.metadata) existing.metadata = b.metadata; + } else { + blockMap.set(key, { + path: b.path, + blockId: b.blockId, + embedding: b.embedding, + metadata: b.metadata, + }); + } + } + } + + for (const block of blockMap.values()) yield block; + }, + + getFullTextIndex(): FullTextIndex | null { + return fts; + }, + + getVectorIndex(): EmbeddingIndex | null { + return vec; + }, + + async close(_options?: { force?: boolean }): Promise { + if (closed) return; + closed = true; + if (fts !== null) await fts.close(); + if (vec !== null) await vec.close(); + }, + + async flush(): Promise { + ensureOpen(); + if (fts !== null) await fts.flush(); + if (vec !== null) await vec.flush(); + }, + + async deleteIndex(): Promise { + ensureOpen(); + if (fts !== null) await fts.deleteIndex(); + if (vec !== null) await vec.deleteIndex(); + if (onDeleteIndex) await onDeleteIndex(); + closed = true; + }, + }; +} diff --git a/packages/indexer-core/src/index.ts b/packages/indexer-core/src/index.ts new file mode 100644 index 0000000..1ac7f10 --- /dev/null +++ b/packages/indexer-core/src/index.ts @@ -0,0 +1,11 @@ +// @statewalker/indexer-core — workspace-internal scaffolding shared by @statewalker/indexer-* backends. +// Not published to npm. Consumed via workspace:* by sibling backend packages only. + +export { toAsyncIterable } from "./async.js"; +export { compositeKey } from "./composite-key.js"; +export { createCompositeIndex, type CompositeIndexOptions } from "./create-composite-index.js"; +export { mergeByRRF, mergeByWeights, mergeHybrid } from "./merge.js"; +export { matchesPrefix } from "./path-prefix.js"; +export { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; +export { sanitizePrefix } from "./sanitize-prefix.js"; +export { validateDimensionality } from "./validate-dimensionality.js"; diff --git a/packages/indexer-mem/src/hybrid-search.ts b/packages/indexer-core/src/merge.ts similarity index 90% rename from packages/indexer-mem/src/hybrid-search.ts rename to packages/indexer-core/src/merge.ts index b952840..2674ae1 100644 --- a/packages/indexer-mem/src/hybrid-search.ts +++ b/packages/indexer-core/src/merge.ts @@ -3,11 +3,9 @@ import type { EmbeddingSearchResult, FullTextSearchResult, HybridSearchResult, + HybridWeights, } from "@statewalker/indexer-api"; - -function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; -} +import { compositeKey } from "./composite-key.js"; export function mergeByRRF( ftsResults: FullTextSearchResult[], @@ -57,7 +55,7 @@ export function mergeByRRF( export function mergeByWeights( ftsResults: FullTextSearchResult[], vecResults: EmbeddingSearchResult[], - weights: { fts: number; embedding: number }, + weights: HybridWeights, topK: number, ): HybridSearchResult[] { const normalize = (results: Array<{ score: number }>): Map => { @@ -138,3 +136,14 @@ export function mergeByWeights( results.sort((a, b) => b.score - a.score); return results.slice(0, topK); } + +export function mergeHybrid( + ftsResults: FullTextSearchResult[], + vecResults: EmbeddingSearchResult[], + topK: number, + weights?: HybridWeights, +): HybridSearchResult[] { + return weights + ? mergeByWeights(ftsResults, vecResults, weights, topK) + : mergeByRRF(ftsResults, vecResults, topK); +} diff --git a/packages/indexer-core/src/path-prefix.ts b/packages/indexer-core/src/path-prefix.ts new file mode 100644 index 0000000..7ee5518 --- /dev/null +++ b/packages/indexer-core/src/path-prefix.ts @@ -0,0 +1,5 @@ +import type { DocumentPath } from "@statewalker/indexer-api"; + +export function matchesPrefix(path: DocumentPath, prefix: DocumentPath): boolean { + return path.startsWith(prefix); +} diff --git a/packages/indexer-core/src/persistence-bytes.ts b/packages/indexer-core/src/persistence-bytes.ts new file mode 100644 index 0000000..6063e67 --- /dev/null +++ b/packages/indexer-core/src/persistence-bytes.ts @@ -0,0 +1,28 @@ +import type { PersistenceEntry } from "@statewalker/indexer-api"; + +export function toBytes(str: string): Uint8Array { + return new TextEncoder().encode(str); +} + +export function singleChunk(data: Uint8Array): AsyncIterable { + return { + async *[Symbol.asyncIterator]() { + yield data; + }, + }; +} + +export async function readEntryBytes(entry: PersistenceEntry): Promise { + const chunks: Uint8Array[] = []; + for await (const chunk of entry.content) { + chunks.push(chunk); + } + const totalLength = chunks.reduce((sum, c) => sum + c.length, 0); + const result = new Uint8Array(totalLength); + let offset = 0; + for (const chunk of chunks) { + result.set(chunk, offset); + offset += chunk.length; + } + return result; +} diff --git a/packages/indexer-core/src/sanitize-prefix.ts b/packages/indexer-core/src/sanitize-prefix.ts new file mode 100644 index 0000000..71a6c4e --- /dev/null +++ b/packages/indexer-core/src/sanitize-prefix.ts @@ -0,0 +1,3 @@ +export function sanitizePrefix(name: string): string { + return name.replace(/[^a-zA-Z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`); +} diff --git a/packages/indexer-core/src/validate-dimensionality.ts b/packages/indexer-core/src/validate-dimensionality.ts new file mode 100644 index 0000000..004a9a3 --- /dev/null +++ b/packages/indexer-core/src/validate-dimensionality.ts @@ -0,0 +1,12 @@ +import type { EmbeddingIndexInfo } from "@statewalker/indexer-api"; + +export function validateDimensionality( + info: Pick, + embedding: Float32Array, +): void { + if (embedding.length !== info.dimensionality) { + throw new Error( + `Expected dimensionality ${info.dimensionality}, got ${embedding.length}`, + ); + } +} diff --git a/packages/indexer-core/tsconfig.json b/packages/indexer-core/tsconfig.json new file mode 100644 index 0000000..6dbcc68 --- /dev/null +++ b/packages/indexer-core/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "Preserve", + "moduleResolution": "Bundler", + "lib": ["ESNext"], + "strict": true, + "skipLibCheck": true, + "verbatimModuleSyntax": true, + "resolveJsonModule": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "resolvePackageJsonExports": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "noEmit": true + }, + "include": ["./src", "./tests"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/indexer-duckdb/package.json b/packages/indexer-duckdb/package.json index b0a0eff..93b97ea 100644 --- a/packages/indexer-duckdb/package.json +++ b/packages/indexer-duckdb/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@statewalker/indexer-api": "workspace:*", + "@statewalker/indexer-core": "workspace:*", "@statewalker/db-api": "catalog:" }, "devDependencies": { diff --git a/packages/indexer-duckdb/src/duckdb-full-text-index.ts b/packages/indexer-duckdb/src/duckdb-full-text-index.ts index 1588e63..6ed2d4d 100644 --- a/packages/indexer-duckdb/src/duckdb-full-text-index.ts +++ b/packages/indexer-duckdb/src/duckdb-full-text-index.ts @@ -10,6 +10,7 @@ import type { Metadata, PathSelector, } from "@statewalker/indexer-api"; +import { toAsyncIterable } from "@statewalker/indexer-core"; export class DuckDbFullTextIndex implements FullTextIndex { private readonly db: Db; @@ -156,7 +157,7 @@ export class DuckDbFullTextIndex implements FullTextIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); - for await (const sel of pathSelectors as AsyncIterable) { + for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { await this.db.query( `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, diff --git a/packages/indexer-duckdb/src/duckdb-index.ts b/packages/indexer-duckdb/src/duckdb-index.ts index 7c9ded2..4a21ff6 100644 --- a/packages/indexer-duckdb/src/duckdb-index.ts +++ b/packages/indexer-duckdb/src/duckdb-index.ts @@ -1,271 +1,41 @@ import type { Db } from "@statewalker/db-api"; -import type { - BlockReference, - DocumentPath, - EmbeddingIndex, - FullTextIndex, - HybridSearchParams, - HybridSearchResult, - Index, - IndexedBlock, - Metadata, - PathSelector, -} from "@statewalker/indexer-api"; +import type { DocumentPath, Index, Metadata } from "@statewalker/indexer-api"; +import { createCompositeIndex } from "@statewalker/indexer-core"; import type { DuckDbFullTextIndex } from "./duckdb-full-text-index.js"; import type { DuckDbVectorIndex } from "./duckdb-vector-index.js"; -import { mergeByRRF, mergeByWeights } from "./hybrid-search.js"; -export class DuckDbIndex implements Index { - readonly name: string; - readonly metadata?: Metadata; - private readonly db: Db; - private readonly docsTable: string; - private readonly fts: DuckDbFullTextIndex | null; - private readonly vec: DuckDbVectorIndex | null; - private closed = false; - - constructor( - name: string, - db: Db, - docsTable: string, - fts: DuckDbFullTextIndex | null, - vec: DuckDbVectorIndex | null, - metadata?: Metadata, - ) { - this.name = name; - this.db = db; - this.docsTable = docsTable; - this.fts = fts; - this.vec = vec; - this.metadata = metadata; - } - - private ensureOpen(): void { - if (this.closed) { - throw new Error(`Index "${this.name}" is closed`); - } - } - - async *search(params: HybridSearchParams): AsyncGenerator { - this.ensureOpen(); - const { queries, embeddings, topK, weights, paths } = params; - - const hasQueries = queries && queries.length > 0 && this.fts !== null; - const hasEmbeddings = embeddings && embeddings.length > 0 && this.vec !== null; - - if (!hasQueries && !hasEmbeddings) return; - - const ftsResults = []; - if (hasQueries) { - for await (const r of this.fts.search({ queries, topK, paths })) { - ftsResults.push(r); - } - } - - const vecResults = []; - if (hasEmbeddings) { - for await (const r of this.vec.search({ embeddings, topK, paths })) { - vecResults.push(r); - } - } - - let merged: HybridSearchResult[]; - if (ftsResults.length > 0 && vecResults.length > 0) { - merged = weights - ? mergeByWeights(ftsResults, vecResults, weights, topK) - : mergeByRRF(ftsResults, vecResults, topK); - } else if (ftsResults.length > 0) { - merged = ftsResults.map((r) => ({ - path: r.path, - blockId: r.blockId, - score: r.score, - fts: r, - embedding: null, - })); - } else { - merged = vecResults.map((r) => ({ - path: r.path, - blockId: r.blockId, - score: r.score, - fts: null, - embedding: r, - })); - } - - for (const r of merged.slice(0, topK)) { - yield r; - } - } - - async addDocument(blocks: IndexedBlock[]): Promise { - this.ensureOpen(); - const ftsBlocks = []; - const vecBlocks = []; - - for (const block of blocks) { - if (block.content !== undefined && this.fts !== null) { - ftsBlocks.push({ - path: block.path, - blockId: block.blockId, - content: block.content, - metadata: block.metadata, - }); - } - if (block.embedding !== undefined && this.vec !== null) { - vecBlocks.push({ - path: block.path, - blockId: block.blockId, - embedding: block.embedding, - metadata: block.metadata, - }); - } - } - - if (ftsBlocks.length > 0) await this.fts?.addDocument(ftsBlocks); - if (vecBlocks.length > 0) await this.vec?.addDocument(vecBlocks); - } - - async addDocuments( - blocks: Iterable | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const batch of blocks) { - await this.addDocument(batch); - } - } - - async deleteDocuments( - pathSelectors: PathSelector[] | AsyncIterable, - ): Promise { - this.ensureOpen(); - const selectors: PathSelector[] = []; - for await (const sel of pathSelectors as AsyncIterable) { - selectors.push(sel); - } - if (this.fts !== null) await this.fts.deleteDocuments(selectors); - if (this.vec !== null) await this.vec.deleteDocuments(selectors); - } - - async getSize(pathPrefix?: DocumentPath): Promise { - this.ensureOpen(); - // Count unique (path, blockId) pairs across both sub-indexes - const hasFts = this.fts !== null; - const hasVec = this.vec !== null; - - if (hasFts && hasVec) { - const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const ftsTable = (this.fts as DuckDbFullTextIndex).tableName; - const vecTable = (this.vec as DuckDbVectorIndex).tableName; - const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${ftsTable} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vecTable} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id${pathClause})`; - const rows = await this.db.query<{ cnt: number | bigint }>(sql, params); - return Number(rows[0]?.cnt ?? 0); - } - - if (hasFts) return this.fts.getSize(pathPrefix); - if (hasVec) return this.vec.getSize(pathPrefix); - return 0; - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const paths = new Set(); - if (this.fts) { - for await (const p of this.fts.getDocumentPaths(pathPrefix)) { - paths.add(p); - } - } - if (this.vec) { - for await (const p of this.vec.getDocumentPaths(pathPrefix)) { - paths.add(p); - } - } - for (const p of paths) { - yield p as DocumentPath; - } - } - - async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const seen = new Set(); - if (this.fts) { - for await (const ref of this.fts.getDocumentBlocksRefs(pathPrefix)) { - const key = `${ref.path}\0${ref.blockId}`; - seen.add(key); - yield ref; - } - } - if (this.vec) { - for await (const ref of this.vec.getDocumentBlocksRefs(pathPrefix)) { - const key = `${ref.path}\0${ref.blockId}`; - if (!seen.has(key)) { - yield ref; - } - } - } - } - - async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - // Merge blocks from both sub-indexes - const blockMap = new Map(); - - if (this.fts) { - for await (const b of this.fts.getDocumentsBlocks(pathPrefix)) { - const key = `${b.path}\0${b.blockId}`; - blockMap.set(key, { - path: b.path, - blockId: b.blockId, - content: b.content, - metadata: b.metadata, - }); - } - } - if (this.vec) { - for await (const b of this.vec.getDocumentsBlocks(pathPrefix)) { - const key = `${b.path}\0${b.blockId}`; - const existing = blockMap.get(key); - if (existing) { - existing.embedding = b.embedding; - } else { - blockMap.set(key, { - path: b.path, - blockId: b.blockId, - embedding: b.embedding, - }); - } - } - } - - for (const block of blockMap.values()) { - yield block; - } - } - - getFullTextIndex(): FullTextIndex | null { - return this.fts; - } - - getVectorIndex(): EmbeddingIndex | null { - return this.vec; - } - - async close(_options?: { force?: boolean }): Promise { - if (this.closed) return; - this.closed = true; - if (this.fts !== null) await this.fts.close(); - if (this.vec !== null) await this.vec.close(); - } - - async flush(): Promise { - this.ensureOpen(); - } - - async deleteIndex(): Promise { - this.ensureOpen(); - if (this.fts !== null) await this.fts.deleteIndex(); - if (this.vec !== null) await this.vec.deleteIndex(); - await this.db.exec(`DROP TABLE IF EXISTS ${this.docsTable}`); - this.closed = true; - } +/** + * DuckDB-backed composite `Index`. + * + * @deprecated Use `createCompositeIndex` from `@statewalker/indexer-core` directly. Kept as a thin factory for one transitional release. + */ +export function DuckDbIndex( + name: string, + db: Db, + docsTable: string, + fts: DuckDbFullTextIndex | null, + vec: DuckDbVectorIndex | null, + metadata?: Metadata, +): Index { + return createCompositeIndex({ + name, + fts, + vec, + metadata, + getSize: async (pathPrefix?: DocumentPath): Promise => { + if (fts !== null && vec !== null) { + const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause})`; + const rows = await db.query<{ cnt: number | bigint }>(sql, params); + return Number(rows[0]?.cnt ?? 0); + } + if (fts !== null) return fts.getSize(pathPrefix); + if (vec !== null) return vec.getSize(pathPrefix); + return 0; + }, + onDeleteIndex: async () => { + await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); + }, + }); } diff --git a/packages/indexer-duckdb/src/duckdb-indexer.ts b/packages/indexer-duckdb/src/duckdb-indexer.ts index df1ad0c..92c664c 100644 --- a/packages/indexer-duckdb/src/duckdb-indexer.ts +++ b/packages/indexer-duckdb/src/duckdb-indexer.ts @@ -1,5 +1,6 @@ import type { Db } from "@statewalker/db-api"; import type { CreateIndexParams, Index, Indexer, IndexInfo } from "@statewalker/indexer-api"; +import { sanitizePrefix } from "@statewalker/indexer-core"; import { DuckDbFullTextIndex } from "./duckdb-full-text-index.js"; import { DuckDbIndex } from "./duckdb-index.js"; import { DuckDbVectorIndex } from "./duckdb-vector-index.js"; @@ -8,13 +9,9 @@ export interface DuckDbIndexerOptions { db: Db; } -function sanitizePrefix(name: string): string { - return name.replace(/[^a-zA-Z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`); -} - export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promise { const { db } = options; - const indexes = new Map(); + const indexes = new Map(); const manifest = new Map(); let closed = false; @@ -112,7 +109,7 @@ export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promis config, ]); - const index = new DuckDbIndex(name, db, docsTable, fts, vec); + const index = DuckDbIndex(name, db, docsTable, fts, vec); indexes.set(name, index); manifest.set(name, { name }); @@ -162,7 +159,7 @@ export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promis }) : null; - const index = new DuckDbIndex(name, db, docsTable, fts, vec); + const index = DuckDbIndex(name, db, docsTable, fts, vec); indexes.set(name, index); return index; }, diff --git a/packages/indexer-duckdb/src/duckdb-vector-index.ts b/packages/indexer-duckdb/src/duckdb-vector-index.ts index f01c59c..18f0363 100644 --- a/packages/indexer-duckdb/src/duckdb-vector-index.ts +++ b/packages/indexer-duckdb/src/duckdb-vector-index.ts @@ -9,6 +9,7 @@ import type { EmbeddingSearchResult, PathSelector, } from "@statewalker/indexer-api"; +import { toAsyncIterable, validateDimensionality } from "@statewalker/indexer-core"; export class DuckDbVectorIndex implements EmbeddingIndex { private readonly db: Db; @@ -42,14 +43,6 @@ export class DuckDbVectorIndex implements EmbeddingIndex { } } - private validateDimensionality(embedding: Float32Array): void { - if (embedding.length !== this.info.dimensionality) { - throw new Error( - `Expected dimensionality ${this.info.dimensionality}, got ${embedding.length}`, - ); - } - } - private embeddingToSql(embedding: Float32Array): string { return `[${Array.from(embedding).join(",")}]`; } @@ -81,7 +74,7 @@ export class DuckDbVectorIndex implements EmbeddingIndex { const dim = this.info.dimensionality; for (const queryEmb of embeddings) { - this.validateDimensionality(queryEmb); + validateDimensionality(this.info,queryEmb); const vecLiteral = this.embeddingToSql(queryEmb); let pathClause = ""; @@ -128,7 +121,7 @@ export class DuckDbVectorIndex implements EmbeddingIndex { async addDocument(blocks: EmbeddingBlock[]): Promise { this.ensureOpen(); for (const block of blocks) { - this.validateDimensionality(block.embedding); + validateDimensionality(this.info,block.embedding); const docId = await this.resolveDocId(block.path); const dim = this.info.dimensionality; const vecLiteral = this.embeddingToSql(block.embedding); @@ -157,7 +150,7 @@ export class DuckDbVectorIndex implements EmbeddingIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); - for await (const sel of pathSelectors as AsyncIterable) { + for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { await this.db.query( `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, diff --git a/packages/indexer-duckdb/src/hybrid-search.ts b/packages/indexer-duckdb/src/hybrid-search.ts deleted file mode 100644 index ed5ac42..0000000 --- a/packages/indexer-duckdb/src/hybrid-search.ts +++ /dev/null @@ -1,136 +0,0 @@ -import type { - DocumentPath, - EmbeddingSearchResult, - FullTextSearchResult, - HybridSearchResult, -} from "@statewalker/indexer-api"; - -function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; -} - -export function mergeByRRF( - ftsResults: FullTextSearchResult[], - vecResults: EmbeddingSearchResult[], - topK: number, - k = 60, -): HybridSearchResult[] { - const scores = new Map(); - const ftsMap = new Map(); - const vecMap = new Map(); - const pathMap = new Map(); - const blockIdMap = new Map(); - - for (let i = 0; i < ftsResults.length; i++) { - const r = ftsResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - scores.set(key, (scores.get(key) ?? 0) + 1 / (k + i + 1)); - ftsMap.set(key, r); - pathMap.set(key, r.path); - blockIdMap.set(key, r.blockId); - } - for (let i = 0; i < vecResults.length; i++) { - const r = vecResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - scores.set(key, (scores.get(key) ?? 0) + 1 / (k + i + 1)); - vecMap.set(key, r); - if (!pathMap.has(key)) pathMap.set(key, r.path); - if (!blockIdMap.has(key)) blockIdMap.set(key, r.blockId); - } - - const results: HybridSearchResult[] = []; - for (const [key, score] of scores) { - results.push({ - path: pathMap.get(key) as DocumentPath, - blockId: blockIdMap.get(key) as string, - score, - fts: ftsMap.get(key) ?? null, - embedding: vecMap.get(key) ?? null, - }); - } - results.sort((a, b) => b.score - a.score); - return results.slice(0, topK); -} - -export function mergeByWeights( - ftsResults: FullTextSearchResult[], - vecResults: EmbeddingSearchResult[], - weights: { fts: number; embedding: number }, - topK: number, -): HybridSearchResult[] { - const normalize = (results: Array<{ score: number }>): Map => { - const map = new Map(); - if (results.length === 0) return map; - let min = Number.POSITIVE_INFINITY; - let max = Number.NEGATIVE_INFINITY; - for (const r of results) { - if (r.score < min) min = r.score; - if (r.score > max) max = r.score; - } - const range = max - min; - for (let i = 0; i < results.length; i++) { - const r = results[i]; - if (!r) continue; - map.set(i, range === 0 ? 1 : (r.score - min) / range); - } - return map; - }; - - const ftsNorm = normalize(ftsResults); - const vecNorm = normalize(vecResults); - - const allKeys = new Map< - string, - { - path: DocumentPath; - blockId: string; - fts: FullTextSearchResult | null; - embedding: EmbeddingSearchResult | null; - } - >(); - const ftsScoreMap = new Map(); - const vecScoreMap = new Map(); - - for (let i = 0; i < ftsResults.length; i++) { - const r = ftsResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - if (!allKeys.has(key)) { - allKeys.set(key, { - path: r.path, - blockId: r.blockId, - fts: r, - embedding: null, - }); - } - ftsScoreMap.set(key, ftsNorm.get(i) ?? 0); - } - for (let i = 0; i < vecResults.length; i++) { - const r = vecResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - const existing = allKeys.get(key); - if (existing) { - existing.embedding = r; - } else { - allKeys.set(key, { - path: r.path, - blockId: r.blockId, - fts: null, - embedding: r, - }); - } - vecScoreMap.set(key, vecNorm.get(i) ?? 0); - } - - const results: HybridSearchResult[] = []; - for (const [key, entry] of allKeys) { - const ftsScore = (ftsScoreMap.get(key) ?? 0) * weights.fts; - const vecScore = (vecScoreMap.get(key) ?? 0) * weights.embedding; - results.push({ ...entry, score: ftsScore + vecScore }); - } - results.sort((a, b) => b.score - a.score); - return results.slice(0, topK); -} diff --git a/packages/indexer-mem-flexsearch/package.json b/packages/indexer-mem-flexsearch/package.json index 5fa7f96..94cc949 100644 --- a/packages/indexer-mem-flexsearch/package.json +++ b/packages/indexer-mem-flexsearch/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@statewalker/indexer-api": "workspace:*", + "@statewalker/indexer-core": "workspace:*", "@statewalker/indexer-mem": "workspace:*", "flexsearch": "catalog:" }, diff --git a/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts b/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts index 71695b0..243028f 100644 --- a/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts +++ b/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts @@ -9,6 +9,7 @@ import type { Metadata, PathSelector, } from "@statewalker/indexer-api"; +import { compositeKey, matchesPrefix, toAsyncIterable } from "@statewalker/indexer-core"; import FlexSearch from "flexsearch"; interface StoredBlock { @@ -18,14 +19,6 @@ interface StoredBlock { metadata?: Metadata; } -function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; -} - -function matchesPrefix(path: DocumentPath, prefix: DocumentPath): boolean { - return path.startsWith(prefix); -} - export class FlexSearchFullTextIndex implements FullTextIndex { private readonly info: FullTextIndexInfo; private flexIndex: FlexSearch.Index; @@ -180,7 +173,7 @@ export class FlexSearchFullTextIndex implements FullTextIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); - for await (const sel of pathSelectors as AsyncIterable) { + for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { const key = compositeKey(sel.path, sel.blockId); this.removeByKey(key); diff --git a/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts b/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts index 909411c..426fff8 100644 --- a/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts +++ b/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts @@ -6,6 +6,7 @@ import type { IndexInfo, PersistenceEntry, } from "@statewalker/indexer-api"; +import { readEntryBytes, singleChunk, toBytes } from "@statewalker/indexer-core"; import { MemIndex, MemVectorIndex } from "@statewalker/indexer-mem"; import { FlexSearchFullTextIndex } from "./flexsearch-full-text-index.js"; @@ -23,35 +24,8 @@ interface StoredIndexConfig { }; } -function toBytes(str: string): Uint8Array { - return new TextEncoder().encode(str); -} - -function singleChunk(data: Uint8Array): AsyncIterable { - return { - async *[Symbol.asyncIterator]() { - yield data; - }, - }; -} - -async function readEntryBytes(entry: PersistenceEntry): Promise { - const chunks: Uint8Array[] = []; - for await (const chunk of entry.content) { - chunks.push(chunk); - } - const totalLength = chunks.reduce((sum, c) => sum + c.length, 0); - const result = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - result.set(chunk, offset); - offset += chunk.length; - } - return result; -} - export function createFlexSearchIndexer(options?: FlexSearchIndexerOptions): Indexer { - const indexes = new Map(); + const indexes = new Map(); const configs = new Map(); const manifest = new Map(); let closed = false; @@ -111,27 +85,7 @@ export function createFlexSearchIndexer(options?: FlexSearchIndexerOptions): Ind } } - const index = new MemIndex(name, fts, vec); - - // Restore block tracking by iterating sub-indexes. - // addDocument with no content/embedding just populates the tracking map - // without re-adding to sub-indexes. - const seen = new Set(); - if (fts) { - for await (const ref of fts.getDocumentBlocksRefs()) { - seen.add(`${ref.path}\0${ref.blockId}`); - await index.addDocument([{ path: ref.path, blockId: ref.blockId }]); - } - } - if (vec) { - for await (const ref of vec.getDocumentBlocksRefs()) { - const key = `${ref.path}\0${ref.blockId}`; - if (!seen.has(key)) { - await index.addDocument([{ path: ref.path, blockId: ref.blockId }]); - } - } - } - + const index = MemIndex(name, fts, vec); indexes.set(name, index); manifest.set(name, { name }); } @@ -233,7 +187,7 @@ export function createFlexSearchIndexer(options?: FlexSearchIndexerOptions): Ind }) : null; - const index = new MemIndex(name, fts, vec); + const index = MemIndex(name, fts, vec); indexes.set(name, index); manifest.set(name, { name }); configs.set(name, { name, fulltext, vector }); diff --git a/packages/indexer-mem-minisearch/package.json b/packages/indexer-mem-minisearch/package.json index eb367bf..6b5fa0a 100644 --- a/packages/indexer-mem-minisearch/package.json +++ b/packages/indexer-mem-minisearch/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@statewalker/indexer-api": "workspace:*", + "@statewalker/indexer-core": "workspace:*", "@statewalker/indexer-mem": "workspace:*", "minisearch": "catalog:" }, diff --git a/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts b/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts index 1d0d34e..77743ff 100644 --- a/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts +++ b/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts @@ -9,6 +9,7 @@ import type { Metadata, PathSelector, } from "@statewalker/indexer-api"; +import { compositeKey, matchesPrefix, toAsyncIterable } from "@statewalker/indexer-core"; import MiniSearch from "minisearch"; interface StoredBlock { @@ -18,14 +19,6 @@ interface StoredBlock { metadata?: Metadata; } -function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; -} - -function matchesPrefix(path: DocumentPath, prefix: DocumentPath): boolean { - return path.startsWith(prefix); -} - export class MiniSearchFullTextIndex implements FullTextIndex { private readonly info: FullTextIndexInfo; private miniSearch: MiniSearch; @@ -157,7 +150,7 @@ export class MiniSearchFullTextIndex implements FullTextIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); - for await (const sel of pathSelectors as AsyncIterable) { + for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { const key = compositeKey(sel.path, sel.blockId); this.removeByKey(key); diff --git a/packages/indexer-mem-minisearch/src/minisearch-indexer.ts b/packages/indexer-mem-minisearch/src/minisearch-indexer.ts index de42e45..d1afcc1 100644 --- a/packages/indexer-mem-minisearch/src/minisearch-indexer.ts +++ b/packages/indexer-mem-minisearch/src/minisearch-indexer.ts @@ -6,6 +6,7 @@ import type { IndexInfo, PersistenceEntry, } from "@statewalker/indexer-api"; +import { readEntryBytes, singleChunk, toBytes } from "@statewalker/indexer-core"; import { MemIndex, MemVectorIndex } from "@statewalker/indexer-mem"; import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js"; @@ -23,35 +24,8 @@ interface StoredIndexConfig { }; } -function toBytes(str: string): Uint8Array { - return new TextEncoder().encode(str); -} - -function singleChunk(data: Uint8Array): AsyncIterable { - return { - async *[Symbol.asyncIterator]() { - yield data; - }, - }; -} - -async function readEntryBytes(entry: PersistenceEntry): Promise { - const chunks: Uint8Array[] = []; - for await (const chunk of entry.content) { - chunks.push(chunk); - } - const totalLength = chunks.reduce((sum, c) => sum + c.length, 0); - const result = new Uint8Array(totalLength); - let offset = 0; - for (const chunk of chunks) { - result.set(chunk, offset); - offset += chunk.length; - } - return result; -} - export function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer { - const indexes = new Map(); + const indexes = new Map(); const configs = new Map(); const manifest = new Map(); let closed = false; @@ -111,25 +85,7 @@ export function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Ind } } - const index = new MemIndex(name, fts, vec); - - // Restore block tracking by iterating sub-indexes - const seen = new Set(); - if (fts) { - for await (const ref of fts.getDocumentBlocksRefs()) { - seen.add(`${ref.path}\0${ref.blockId}`); - await index.addDocument([{ path: ref.path, blockId: ref.blockId }]); - } - } - if (vec) { - for await (const ref of vec.getDocumentBlocksRefs()) { - const key = `${ref.path}\0${ref.blockId}`; - if (!seen.has(key)) { - await index.addDocument([{ path: ref.path, blockId: ref.blockId }]); - } - } - } - + const index = MemIndex(name, fts, vec); indexes.set(name, index); manifest.set(name, { name }); } @@ -231,7 +187,7 @@ export function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Ind }) : null; - const index = new MemIndex(name, fts, vec); + const index = MemIndex(name, fts, vec); indexes.set(name, index); manifest.set(name, { name }); configs.set(name, { name, fulltext, vector }); diff --git a/packages/indexer-mem/package.json b/packages/indexer-mem/package.json index a44be2b..a136681 100644 --- a/packages/indexer-mem/package.json +++ b/packages/indexer-mem/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@statewalker/indexer-api": "workspace:*", + "@statewalker/indexer-core": "workspace:*", "@uwdata/flechette": "catalog:" }, "devDependencies": { diff --git a/packages/indexer-mem/src/index.ts b/packages/indexer-mem/src/index.ts index 77aad8a..2f48cf1 100644 --- a/packages/indexer-mem/src/index.ts +++ b/packages/indexer-mem/src/index.ts @@ -1,4 +1,4 @@ -export { mergeByRRF, mergeByWeights } from "./hybrid-search.js"; +export { mergeByRRF, mergeByWeights } from "@statewalker/indexer-core"; export { MemIndex } from "./mem-index.js"; export { MemVectorIndex } from "./mem-vector-index.js"; export { bruteForceSearch, cosineSimilarity } from "./vector-search.js"; diff --git a/packages/indexer-mem/src/mem-index.ts b/packages/indexer-mem/src/mem-index.ts index c47e1ad..d649785 100644 --- a/packages/indexer-mem/src/mem-index.ts +++ b/packages/indexer-mem/src/mem-index.ts @@ -1,279 +1,21 @@ import type { - BlockReference, - DocumentPath, EmbeddingIndex, FullTextIndex, - HybridSearchParams, - HybridSearchResult, Index, - IndexedBlock, Metadata, - PathSelector, } from "@statewalker/indexer-api"; -import { mergeByRRF, mergeByWeights } from "./hybrid-search.js"; - -function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; -} - -function matchesPrefix(path: DocumentPath, prefix: DocumentPath): boolean { - return path.startsWith(prefix); -} - -export class MemIndex implements Index { - readonly name: string; - readonly metadata?: Metadata; - private readonly fts: FullTextIndex | null; - private readonly vec: EmbeddingIndex | null; - private readonly trackedBlocks = new Map(); - private closed = false; - - constructor( - name: string, - fts: FullTextIndex | null, - vec: EmbeddingIndex | null, - metadata?: Metadata, - ) { - this.name = name; - this.fts = fts; - this.vec = vec; - this.metadata = metadata; - } - - private ensureOpen(): void { - if (this.closed) { - throw new Error(`Index "${this.name}" is closed`); - } - } - - async *search(params: HybridSearchParams): AsyncGenerator { - this.ensureOpen(); - const { queries, embeddings, topK, weights, paths } = params; - - const hasQueries = queries && queries.length > 0 && this.fts !== null; - const hasEmbeddings = embeddings && embeddings.length > 0 && this.vec !== null; - - if (!hasQueries && !hasEmbeddings) return; - - // Collect FTS results - const ftsResults = []; - if (hasQueries) { - for await (const r of this.fts.search({ - queries, - topK, - paths, - })) { - ftsResults.push(r); - } - } - - // Collect embedding results - const vecResults = []; - if (hasEmbeddings) { - for await (const r of this.vec.search({ - embeddings, - topK, - paths, - })) { - vecResults.push(r); - } - } - - // Merge - let merged: HybridSearchResult[]; - if (ftsResults.length > 0 && vecResults.length > 0) { - merged = weights - ? mergeByWeights(ftsResults, vecResults, weights, topK) - : mergeByRRF(ftsResults, vecResults, topK); - } else if (ftsResults.length > 0) { - merged = ftsResults.map((r) => ({ - path: r.path, - blockId: r.blockId, - score: r.score, - fts: r, - embedding: null, - })); - } else { - merged = vecResults.map((r) => ({ - path: r.path, - blockId: r.blockId, - score: r.score, - fts: null, - embedding: r, - })); - } - - for (const r of merged.slice(0, topK)) { - yield r; - } - } - - async addDocument(blocks: IndexedBlock[]): Promise { - this.ensureOpen(); - const ftsBlocks = []; - const vecBlocks = []; - - for (const block of blocks) { - if (block.content !== undefined && this.fts !== null) { - ftsBlocks.push({ - path: block.path, - blockId: block.blockId, - content: block.content, - metadata: block.metadata, - }); - } - if (block.embedding !== undefined && this.vec !== null) { - vecBlocks.push({ - path: block.path, - blockId: block.blockId, - embedding: block.embedding, - metadata: block.metadata, - }); - } - const key = compositeKey(block.path, block.blockId); - this.trackedBlocks.set(key, { - path: block.path, - blockId: block.blockId, - }); - } - - if (ftsBlocks.length > 0) await this.fts?.addDocument(ftsBlocks); - if (vecBlocks.length > 0) await this.vec?.addDocument(vecBlocks); - } - - async addDocuments( - blocks: Iterable | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const batch of blocks) { - await this.addDocument(batch); - } - } - - async deleteDocuments( - pathSelectors: PathSelector[] | AsyncIterable, - ): Promise { - this.ensureOpen(); - const selectors: PathSelector[] = []; - for await (const sel of pathSelectors as AsyncIterable) { - selectors.push(sel); - } - - // Determine which tracked blocks match - const toDelete: PathSelector[] = []; - for (const sel of selectors) { - if (sel.blockId !== undefined) { - const key = compositeKey(sel.path, sel.blockId); - if (this.trackedBlocks.has(key)) { - this.trackedBlocks.delete(key); - toDelete.push(sel); - } - } else { - for (const [key, entry] of this.trackedBlocks) { - if (matchesPrefix(entry.path, sel.path)) { - this.trackedBlocks.delete(key); - toDelete.push({ path: entry.path, blockId: entry.blockId }); - } - } - } - } - - if (toDelete.length > 0) { - if (this.fts !== null) await this.fts.deleteDocuments(toDelete); - if (this.vec !== null) await this.vec.deleteDocuments(toDelete); - } - } - - async getSize(pathPrefix?: DocumentPath): Promise { - this.ensureOpen(); - if (pathPrefix === undefined) return this.trackedBlocks.size; - let count = 0; - for (const entry of this.trackedBlocks.values()) { - if (matchesPrefix(entry.path, pathPrefix)) count++; - } - return count; - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const paths = new Set(); - for (const entry of this.trackedBlocks.values()) { - if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) { - paths.add(entry.path); - } - } - for (const p of paths) yield p; - } - - async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - for (const entry of this.trackedBlocks.values()) { - if (pathPrefix === undefined || matchesPrefix(entry.path, pathPrefix)) { - yield { path: entry.path, blockId: entry.blockId }; - } - } - } - - async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - for (const entry of this.trackedBlocks.values()) { - if (pathPrefix !== undefined && !matchesPrefix(entry.path, pathPrefix)) { - continue; - } - const block: IndexedBlock = { - path: entry.path, - blockId: entry.blockId, - }; - // Retrieve content from FTS sub-index if available - if (this.fts !== null) { - for await (const ftsBlock of this.fts.getDocumentsBlocks()) { - if (ftsBlock.path === entry.path && ftsBlock.blockId === entry.blockId) { - block.content = ftsBlock.content; - block.metadata = ftsBlock.metadata; - break; - } - } - } - // Retrieve embedding from vec sub-index if available - if (this.vec !== null) { - for await (const vecBlock of this.vec.getDocumentsBlocks()) { - if (vecBlock.path === entry.path && vecBlock.blockId === entry.blockId) { - block.embedding = vecBlock.embedding; - if (!block.metadata) block.metadata = vecBlock.metadata; - break; - } - } - } - yield block; - } - } - - getFullTextIndex(): FullTextIndex | null { - return this.fts; - } - - getVectorIndex(): EmbeddingIndex | null { - return this.vec; - } - - async close(_options?: { force?: boolean }): Promise { - if (this.closed) return; - this.closed = true; - if (this.fts !== null) await this.fts.close(); - if (this.vec !== null) await this.vec.close(); - } - - async flush(): Promise { - this.ensureOpen(); - if (this.fts !== null) await this.fts.flush(); - if (this.vec !== null) await this.vec.flush(); - } - - async deleteIndex(): Promise { - this.ensureOpen(); - if (this.fts !== null) await this.fts.deleteIndex(); - if (this.vec !== null) await this.vec.deleteIndex(); - this.trackedBlocks.clear(); - this.closed = true; - } +import { createCompositeIndex } from "@statewalker/indexer-core"; + +/** + * In-memory composite `Index`. + * + * @deprecated Use `createCompositeIndex` from `@statewalker/indexer-core` directly. Kept as a thin re-export for one transitional release. + */ +export function MemIndex( + name: string, + fts: FullTextIndex | null, + vec: EmbeddingIndex | null, + metadata?: Metadata, +): Index { + return createCompositeIndex({ name, fts, vec, metadata }); } diff --git a/packages/indexer-mem/src/mem-vector-index.ts b/packages/indexer-mem/src/mem-vector-index.ts index 96196cc..4b054dc 100644 --- a/packages/indexer-mem/src/mem-vector-index.ts +++ b/packages/indexer-mem/src/mem-vector-index.ts @@ -9,6 +9,12 @@ import type { Metadata, PathSelector, } from "@statewalker/indexer-api"; +import { + compositeKey, + matchesPrefix, + toAsyncIterable, + validateDimensionality, +} from "@statewalker/indexer-core"; import { fixedSizeList, float32, @@ -26,14 +32,6 @@ interface StoredEntry { metadata?: Metadata; } -function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; -} - -function matchesPrefix(path: DocumentPath, prefix: DocumentPath): boolean { - return path.startsWith(prefix); -} - export class MemVectorIndex implements EmbeddingIndex { private readonly info: EmbeddingIndexInfo; private readonly entries = new Map(); @@ -49,14 +47,6 @@ export class MemVectorIndex implements EmbeddingIndex { } } - private validateDimensionality(embedding: Float32Array): void { - if (embedding.length !== this.info.dimensionality) { - throw new Error( - `Expected dimensionality ${this.info.dimensionality}, got ${embedding.length}`, - ); - } - } - private *filteredEntries(pathPrefixes?: DocumentPath[]): Iterable { if (!pathPrefixes || pathPrefixes.length === 0) { yield* this.entries.values(); @@ -84,7 +74,7 @@ export class MemVectorIndex implements EmbeddingIndex { const bestScores = new Map(); for (const queryEmb of embeddings) { - this.validateDimensionality(queryEmb); + validateDimensionality(this.info,queryEmb); const filtered = [...this.filteredEntries(paths)]; const results = bruteForceSearch(queryEmb, filtered, topK); for (const r of results) { @@ -105,7 +95,7 @@ export class MemVectorIndex implements EmbeddingIndex { async addDocument(blocks: EmbeddingBlock[]): Promise { this.ensureOpen(); for (const block of blocks) { - this.validateDimensionality(block.embedding); + validateDimensionality(this.info,block.embedding); const key = compositeKey(block.path, block.blockId); this.entries.set(key, { path: block.path, @@ -129,7 +119,7 @@ export class MemVectorIndex implements EmbeddingIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); - for await (const sel of pathSelectors as AsyncIterable) { + for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { this.entries.delete(compositeKey(sel.path, sel.blockId)); } else { diff --git a/packages/indexer-pglite/package.json b/packages/indexer-pglite/package.json index 4f86046..b92ed22 100644 --- a/packages/indexer-pglite/package.json +++ b/packages/indexer-pglite/package.json @@ -33,6 +33,7 @@ }, "dependencies": { "@statewalker/indexer-api": "workspace:*", + "@statewalker/indexer-core": "workspace:*", "@electric-sql/pglite": "catalog:" }, "devDependencies": { diff --git a/packages/indexer-pglite/src/hybrid-search.ts b/packages/indexer-pglite/src/hybrid-search.ts deleted file mode 100644 index ed5ac42..0000000 --- a/packages/indexer-pglite/src/hybrid-search.ts +++ /dev/null @@ -1,136 +0,0 @@ -import type { - DocumentPath, - EmbeddingSearchResult, - FullTextSearchResult, - HybridSearchResult, -} from "@statewalker/indexer-api"; - -function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; -} - -export function mergeByRRF( - ftsResults: FullTextSearchResult[], - vecResults: EmbeddingSearchResult[], - topK: number, - k = 60, -): HybridSearchResult[] { - const scores = new Map(); - const ftsMap = new Map(); - const vecMap = new Map(); - const pathMap = new Map(); - const blockIdMap = new Map(); - - for (let i = 0; i < ftsResults.length; i++) { - const r = ftsResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - scores.set(key, (scores.get(key) ?? 0) + 1 / (k + i + 1)); - ftsMap.set(key, r); - pathMap.set(key, r.path); - blockIdMap.set(key, r.blockId); - } - for (let i = 0; i < vecResults.length; i++) { - const r = vecResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - scores.set(key, (scores.get(key) ?? 0) + 1 / (k + i + 1)); - vecMap.set(key, r); - if (!pathMap.has(key)) pathMap.set(key, r.path); - if (!blockIdMap.has(key)) blockIdMap.set(key, r.blockId); - } - - const results: HybridSearchResult[] = []; - for (const [key, score] of scores) { - results.push({ - path: pathMap.get(key) as DocumentPath, - blockId: blockIdMap.get(key) as string, - score, - fts: ftsMap.get(key) ?? null, - embedding: vecMap.get(key) ?? null, - }); - } - results.sort((a, b) => b.score - a.score); - return results.slice(0, topK); -} - -export function mergeByWeights( - ftsResults: FullTextSearchResult[], - vecResults: EmbeddingSearchResult[], - weights: { fts: number; embedding: number }, - topK: number, -): HybridSearchResult[] { - const normalize = (results: Array<{ score: number }>): Map => { - const map = new Map(); - if (results.length === 0) return map; - let min = Number.POSITIVE_INFINITY; - let max = Number.NEGATIVE_INFINITY; - for (const r of results) { - if (r.score < min) min = r.score; - if (r.score > max) max = r.score; - } - const range = max - min; - for (let i = 0; i < results.length; i++) { - const r = results[i]; - if (!r) continue; - map.set(i, range === 0 ? 1 : (r.score - min) / range); - } - return map; - }; - - const ftsNorm = normalize(ftsResults); - const vecNorm = normalize(vecResults); - - const allKeys = new Map< - string, - { - path: DocumentPath; - blockId: string; - fts: FullTextSearchResult | null; - embedding: EmbeddingSearchResult | null; - } - >(); - const ftsScoreMap = new Map(); - const vecScoreMap = new Map(); - - for (let i = 0; i < ftsResults.length; i++) { - const r = ftsResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - if (!allKeys.has(key)) { - allKeys.set(key, { - path: r.path, - blockId: r.blockId, - fts: r, - embedding: null, - }); - } - ftsScoreMap.set(key, ftsNorm.get(i) ?? 0); - } - for (let i = 0; i < vecResults.length; i++) { - const r = vecResults[i]; - if (!r) continue; - const key = compositeKey(r.path, r.blockId); - const existing = allKeys.get(key); - if (existing) { - existing.embedding = r; - } else { - allKeys.set(key, { - path: r.path, - blockId: r.blockId, - fts: null, - embedding: r, - }); - } - vecScoreMap.set(key, vecNorm.get(i) ?? 0); - } - - const results: HybridSearchResult[] = []; - for (const [key, entry] of allKeys) { - const ftsScore = (ftsScoreMap.get(key) ?? 0) * weights.fts; - const vecScore = (vecScoreMap.get(key) ?? 0) * weights.embedding; - results.push({ ...entry, score: ftsScore + vecScore }); - } - results.sort((a, b) => b.score - a.score); - return results.slice(0, topK); -} diff --git a/packages/indexer-pglite/src/pglite-full-text-index.ts b/packages/indexer-pglite/src/pglite-full-text-index.ts index 9793e23..ae9f728 100644 --- a/packages/indexer-pglite/src/pglite-full-text-index.ts +++ b/packages/indexer-pglite/src/pglite-full-text-index.ts @@ -10,6 +10,7 @@ import type { Metadata, PathSelector, } from "@statewalker/indexer-api"; +import { toAsyncIterable } from "@statewalker/indexer-core"; const LANGUAGE_MAP: Record = { en: "english", @@ -179,7 +180,7 @@ export class PGLiteFullTextIndex implements FullTextIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); - for await (const sel of pathSelectors as AsyncIterable) { + for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { await this.db.query( `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, diff --git a/packages/indexer-pglite/src/pglite-index.ts b/packages/indexer-pglite/src/pglite-index.ts index cb12d6c..3132a8f 100644 --- a/packages/indexer-pglite/src/pglite-index.ts +++ b/packages/indexer-pglite/src/pglite-index.ts @@ -1,269 +1,41 @@ import type { PGlite } from "@electric-sql/pglite"; -import type { - BlockReference, - DocumentPath, - EmbeddingIndex, - FullTextIndex, - HybridSearchParams, - HybridSearchResult, - Index, - IndexedBlock, - Metadata, - PathSelector, -} from "@statewalker/indexer-api"; -import { mergeByRRF, mergeByWeights } from "./hybrid-search.js"; +import type { DocumentPath, Index, Metadata } from "@statewalker/indexer-api"; +import { createCompositeIndex } from "@statewalker/indexer-core"; import type { PGLiteFullTextIndex } from "./pglite-full-text-index.js"; import type { PGLiteVectorIndex } from "./pglite-vector-index.js"; -export class PGLiteIndex implements Index { - readonly name: string; - readonly metadata?: Metadata; - private readonly db: PGlite; - private readonly docsTable: string; - private readonly fts: PGLiteFullTextIndex | null; - private readonly vec: PGLiteVectorIndex | null; - private closed = false; - - constructor( - name: string, - db: PGlite, - docsTable: string, - fts: PGLiteFullTextIndex | null, - vec: PGLiteVectorIndex | null, - metadata?: Metadata, - ) { - this.name = name; - this.db = db; - this.docsTable = docsTable; - this.fts = fts; - this.vec = vec; - this.metadata = metadata; - } - - private ensureOpen(): void { - if (this.closed) { - throw new Error(`Index "${this.name}" is closed`); - } - } - - async *search(params: HybridSearchParams): AsyncGenerator { - this.ensureOpen(); - const { queries, embeddings, topK, weights, paths } = params; - - const hasQueries = queries && queries.length > 0 && this.fts !== null; - const hasEmbeddings = embeddings && embeddings.length > 0 && this.vec !== null; - - if (!hasQueries && !hasEmbeddings) return; - - const ftsResults = []; - if (hasQueries) { - for await (const r of this.fts.search({ queries, topK, paths })) { - ftsResults.push(r); - } - } - - const vecResults = []; - if (hasEmbeddings) { - for await (const r of this.vec.search({ embeddings, topK, paths })) { - vecResults.push(r); - } - } - - let merged: HybridSearchResult[]; - if (ftsResults.length > 0 && vecResults.length > 0) { - merged = weights - ? mergeByWeights(ftsResults, vecResults, weights, topK) - : mergeByRRF(ftsResults, vecResults, topK); - } else if (ftsResults.length > 0) { - merged = ftsResults.map((r) => ({ - path: r.path, - blockId: r.blockId, - score: r.score, - fts: r, - embedding: null, - })); - } else { - merged = vecResults.map((r) => ({ - path: r.path, - blockId: r.blockId, - score: r.score, - fts: null, - embedding: r, - })); - } - - for (const r of merged.slice(0, topK)) { - yield r; - } - } - - async addDocument(blocks: IndexedBlock[]): Promise { - this.ensureOpen(); - const ftsBlocks = []; - const vecBlocks = []; - - for (const block of blocks) { - if (block.content !== undefined && this.fts !== null) { - ftsBlocks.push({ - path: block.path, - blockId: block.blockId, - content: block.content, - metadata: block.metadata, - }); - } - if (block.embedding !== undefined && this.vec !== null) { - vecBlocks.push({ - path: block.path, - blockId: block.blockId, - embedding: block.embedding, - metadata: block.metadata, - }); - } - } - - if (ftsBlocks.length > 0) await this.fts?.addDocument(ftsBlocks); - if (vecBlocks.length > 0) await this.vec?.addDocument(vecBlocks); - } - - async addDocuments( - blocks: Iterable | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const batch of blocks) { - await this.addDocument(batch); - } - } - - async deleteDocuments( - pathSelectors: PathSelector[] | AsyncIterable, - ): Promise { - this.ensureOpen(); - const selectors: PathSelector[] = []; - for await (const sel of pathSelectors as AsyncIterable) { - selectors.push(sel); - } - if (this.fts !== null) await this.fts.deleteDocuments(selectors); - if (this.vec !== null) await this.vec.deleteDocuments(selectors); - } - - async getSize(pathPrefix?: DocumentPath): Promise { - this.ensureOpen(); - const hasFts = this.fts !== null; - const hasVec = this.vec !== null; - - if (hasFts && hasVec) { - const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const ftsTable = (this.fts as PGLiteFullTextIndex).tableName; - const vecTable = (this.vec as PGLiteVectorIndex).tableName; - const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${ftsTable} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vecTable} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id${pathClause}) AS combined`; - const { rows } = await this.db.query<{ cnt: number | bigint }>(sql, params); - return Number(rows[0]?.cnt ?? 0); - } - - if (hasFts) return this.fts.getSize(pathPrefix); - if (hasVec) return this.vec.getSize(pathPrefix); - return 0; - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const paths = new Set(); - if (this.fts) { - for await (const p of this.fts.getDocumentPaths(pathPrefix)) { - paths.add(p); - } - } - if (this.vec) { - for await (const p of this.vec.getDocumentPaths(pathPrefix)) { - paths.add(p); - } - } - for (const p of paths) { - yield p as DocumentPath; - } - } - - async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const seen = new Set(); - if (this.fts) { - for await (const ref of this.fts.getDocumentBlocksRefs(pathPrefix)) { - const key = `${ref.path}\0${ref.blockId}`; - seen.add(key); - yield ref; - } - } - if (this.vec) { - for await (const ref of this.vec.getDocumentBlocksRefs(pathPrefix)) { - const key = `${ref.path}\0${ref.blockId}`; - if (!seen.has(key)) { - yield ref; - } - } - } - } - - async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const blockMap = new Map(); - - if (this.fts) { - for await (const b of this.fts.getDocumentsBlocks(pathPrefix)) { - const key = `${b.path}\0${b.blockId}`; - blockMap.set(key, { - path: b.path, - blockId: b.blockId, - content: b.content, - metadata: b.metadata, - }); - } - } - if (this.vec) { - for await (const b of this.vec.getDocumentsBlocks(pathPrefix)) { - const key = `${b.path}\0${b.blockId}`; - const existing = blockMap.get(key); - if (existing) { - existing.embedding = b.embedding; - } else { - blockMap.set(key, { - path: b.path, - blockId: b.blockId, - embedding: b.embedding, - }); - } - } - } - - for (const block of blockMap.values()) { - yield block; - } - } - - getFullTextIndex(): FullTextIndex | null { - return this.fts; - } - - getVectorIndex(): EmbeddingIndex | null { - return this.vec; - } - - async close(_options?: { force?: boolean }): Promise { - if (this.closed) return; - this.closed = true; - if (this.fts !== null) await this.fts.close(); - if (this.vec !== null) await this.vec.close(); - } - - async flush(): Promise { - this.ensureOpen(); - } - - async deleteIndex(): Promise { - this.ensureOpen(); - if (this.fts !== null) await this.fts.deleteIndex(); - if (this.vec !== null) await this.vec.deleteIndex(); - await this.db.exec(`DROP TABLE IF EXISTS ${this.docsTable}`); - this.closed = true; - } +/** + * PGlite-backed composite `Index`. + * + * @deprecated Use `createCompositeIndex` from `@statewalker/indexer-core` directly. Kept as a thin factory for one transitional release. + */ +export function PGLiteIndex( + name: string, + db: PGlite, + docsTable: string, + fts: PGLiteFullTextIndex | null, + vec: PGLiteVectorIndex | null, + metadata?: Metadata, +): Index { + return createCompositeIndex({ + name, + fts, + vec, + metadata, + getSize: async (pathPrefix?: DocumentPath): Promise => { + if (fts !== null && vec !== null) { + const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause}) AS combined`; + const { rows } = await db.query<{ cnt: number | bigint }>(sql, params); + return Number(rows[0]?.cnt ?? 0); + } + if (fts !== null) return fts.getSize(pathPrefix); + if (vec !== null) return vec.getSize(pathPrefix); + return 0; + }, + onDeleteIndex: async () => { + await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); + }, + }); } diff --git a/packages/indexer-pglite/src/pglite-indexer.ts b/packages/indexer-pglite/src/pglite-indexer.ts index 6ddb36e..47c3c3e 100644 --- a/packages/indexer-pglite/src/pglite-indexer.ts +++ b/packages/indexer-pglite/src/pglite-indexer.ts @@ -1,6 +1,7 @@ import { PGlite } from "@electric-sql/pglite"; import { vector } from "@electric-sql/pglite/vector"; import type { CreateIndexParams, Index, Indexer, IndexInfo } from "@statewalker/indexer-api"; +import { sanitizePrefix } from "@statewalker/indexer-core"; import { PGLiteFullTextIndex } from "./pglite-full-text-index.js"; import { PGLiteIndex } from "./pglite-index.js"; import { PGLiteVectorIndex } from "./pglite-vector-index.js"; @@ -9,14 +10,10 @@ export interface PGLiteIndexerOptions { db?: PGlite; } -function sanitizePrefix(name: string): string { - return name.replace(/[^a-zA-Z0-9]/g, (ch) => `_${ch.charCodeAt(0)}_`); -} - export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promise { const ownsDb = !options?.db; const db = options?.db ?? (await PGlite.create({ extensions: { vector } })); - const indexes = new Map(); + const indexes = new Map(); const manifest = new Map(); let closed = false; @@ -104,7 +101,7 @@ export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promi config, ]); - const index = new PGLiteIndex(name, db, docsTable, fts, vec); + const index = PGLiteIndex(name, db, docsTable, fts, vec); indexes.set(name, index); manifest.set(name, { name }); @@ -153,7 +150,7 @@ export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promi }) : null; - const index = new PGLiteIndex(name, db, docsTable, fts, vec); + const index = PGLiteIndex(name, db, docsTable, fts, vec); indexes.set(name, index); return index; }, diff --git a/packages/indexer-pglite/src/pglite-vector-index.ts b/packages/indexer-pglite/src/pglite-vector-index.ts index c47a4be..d4f068c 100644 --- a/packages/indexer-pglite/src/pglite-vector-index.ts +++ b/packages/indexer-pglite/src/pglite-vector-index.ts @@ -9,6 +9,7 @@ import type { EmbeddingSearchResult, PathSelector, } from "@statewalker/indexer-api"; +import { toAsyncIterable, validateDimensionality } from "@statewalker/indexer-core"; export class PGLiteVectorIndex implements EmbeddingIndex { private readonly db: PGlite; @@ -42,14 +43,6 @@ export class PGLiteVectorIndex implements EmbeddingIndex { } } - private validateDimensionality(embedding: Float32Array): void { - if (embedding.length !== this.info.dimensionality) { - throw new Error( - `Expected dimensionality ${this.info.dimensionality}, got ${embedding.length}`, - ); - } - } - private embeddingToSql(embedding: Float32Array): string { return `[${Array.from(embedding).join(",")}]`; } @@ -81,7 +74,7 @@ export class PGLiteVectorIndex implements EmbeddingIndex { const dim = this.info.dimensionality; for (const queryEmb of embeddings) { - this.validateDimensionality(queryEmb); + validateDimensionality(this.info,queryEmb); const vecLiteral = this.embeddingToSql(queryEmb); let pathClause = ""; @@ -128,7 +121,7 @@ export class PGLiteVectorIndex implements EmbeddingIndex { async addDocument(blocks: EmbeddingBlock[]): Promise { this.ensureOpen(); for (const block of blocks) { - this.validateDimensionality(block.embedding); + validateDimensionality(this.info,block.embedding); const docId = await this.resolveDocId(block.path); const dim = this.info.dimensionality; const vecLiteral = this.embeddingToSql(block.embedding); @@ -157,7 +150,7 @@ export class PGLiteVectorIndex implements EmbeddingIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); - for await (const sel of pathSelectors as AsyncIterable) { + for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { await this.db.query( `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, diff --git a/packages/indexer-tests/src/suites/document-paths.suite.ts b/packages/indexer-tests/src/suites/document-paths.suite.ts index aae175c..254151c 100644 --- a/packages/indexer-tests/src/suites/document-paths.suite.ts +++ b/packages/indexer-tests/src/suites/document-paths.suite.ts @@ -228,6 +228,29 @@ export function runDocumentPathsSuite(getIndexer: () => Indexer): void { expect(blocks[0]?.content).toBe("hello world"); }); + it("getDocumentsBlocks scales linearly with block count (regression: O(N²) bug)", async () => { + const indexer = getIndexer(); + const index = await indexer.createIndex({ + name: "test", + fulltext: { language: "en" }, + }); + const N = 500; + const batch = []; + for (let i = 0; i < N; i++) { + batch.push({ path: `/docs/p${i}`, blockId: `b${i}`, content: `content-${i}` }); + } + await index.addDocument(batch); + + const start = Date.now(); + const blocks = await collect(index.getDocumentsBlocks()); + const duration = Date.now() - start; + + expect(blocks).toHaveLength(N); + // O(N) should be well under 1s for 500 blocks; O(N²) would take 2+ seconds + // Use a generous 3s ceiling as a sharp signal against quadratic regressions. + expect(duration).toBeLessThan(3000); + }); + // --- Path prefix filtering on sub-indexes --- it("FTS sub-index respects path prefix in search", async () => { From b7de0e1740da64c0e269aad245e4f61d171596c1 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Fri, 24 Apr 2026 14:30:14 +0200 Subject: [PATCH 02/12] refactor(indexer-core): collapse mem + SQL factory scaffolding (Phase 5) Phase 5 of indexer-core-consolidation: extract the persistence-backed Indexer factory pattern into @statewalker/indexer-core, and remove all transitional compatibility wrappers. New: createPersistenceBackedIndexer 6 hooks (createFts, serializeFts, deserializeFts, createVec, serializeVec, deserializeVec) + optional persistence. Generic over concrete sub-index types so backends keep their instance shapes. Preserves the wire format byte-for-byte: __manifest__ / /__config__ / /fts / /vec. Callers collapse: * flexsearch-indexer.ts: 243 -> 20 LOC * minisearch-indexer.ts: 243 -> 20 LOC Compatibility wrappers removed (no transitional layer kept): * mem-index.ts (MemIndex function) deleted * duckdb-index.ts (DuckDbIndex function) deleted * pglite-index.ts (PGLiteIndex function) deleted * SQL factories now call createCompositeIndex directly via a local buildIndex closure shared between createIndex and getIndex Barrel cleanup (indexer-mem): * Only MemVectorIndex remains exported; dead re-exports of MemIndex, mergeByRRF, mergeByWeights, bruteForceSearch, cosineSimilarity dropped (no external consumers) * Stale README rewritten to describe actual exports (closes audit task 10.2) Net: -494 LOC. No public API change. Tests: 575/575 passing across 6 packages. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/create-persistence-backed-indexer.ts | 262 ++++++++++++++++++ packages/indexer-core/src/index.ts | 6 +- packages/indexer-duckdb/src/duckdb-index.ts | 41 --- packages/indexer-duckdb/src/duckdb-indexer.ts | 68 +++-- .../src/flexsearch-indexer.ts | 247 +---------------- .../src/minisearch-indexer.ts | 247 +---------------- packages/indexer-mem/README.md | 39 ++- packages/indexer-mem/src/index.ts | 3 - packages/indexer-mem/src/mem-index.ts | 21 -- packages/indexer-pglite/src/pglite-index.ts | 41 --- packages/indexer-pglite/src/pglite-indexer.ts | 67 +++-- 11 files changed, 405 insertions(+), 637 deletions(-) create mode 100644 packages/indexer-core/src/create-persistence-backed-indexer.ts delete mode 100644 packages/indexer-duckdb/src/duckdb-index.ts delete mode 100644 packages/indexer-mem/src/mem-index.ts delete mode 100644 packages/indexer-pglite/src/pglite-index.ts diff --git a/packages/indexer-core/src/create-persistence-backed-indexer.ts b/packages/indexer-core/src/create-persistence-backed-indexer.ts new file mode 100644 index 0000000..fbd9bd0 --- /dev/null +++ b/packages/indexer-core/src/create-persistence-backed-indexer.ts @@ -0,0 +1,262 @@ +import type { + CreateIndexParams, + EmbeddingIndex, + EmbeddingIndexInfo, + FullTextIndex, + FullTextIndexInfo, + Index, + Indexer, + IndexerPersistence, + IndexInfo, + PersistenceEntry, +} from "@statewalker/indexer-api"; +import { createCompositeIndex } from "./create-composite-index.js"; +import { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; + +/** Stored per-index config for the persistence-backed wire format. */ +interface StoredIndexConfig { + name: string; + fulltext?: FullTextIndexInfo; + vector?: EmbeddingIndexInfo; +} + +export interface PersistenceBackedIndexerOptions< + F extends FullTextIndex, + V extends EmbeddingIndex, +> { + /** Build a fresh FTS sub-index from its info. */ + createFts(info: FullTextIndexInfo): F; + /** Serialize an FTS sub-index to a JSON/text payload. Sync or async. */ + serializeFts(fts: F): string | Promise; + /** Build an FTS sub-index from a serialized payload + its info. */ + deserializeFts(info: FullTextIndexInfo, data: string): F; + + /** Build a fresh vector sub-index from its info. */ + createVec(info: EmbeddingIndexInfo): V; + /** Serialize a vector sub-index to a binary payload. Sync or async. */ + serializeVec(vec: V): Uint8Array | Promise; + /** Build a vector sub-index from a serialized binary payload + its info. */ + deserializeVec(info: EmbeddingIndexInfo, data: Uint8Array): V; + + /** Optional persistence port. When absent, the indexer runs in pure in-memory mode. */ + persistence?: IndexerPersistence; +} + +/** + * Generic persistence-backed `Indexer` factory for in-memory backends. + * + * Wire format (preserved byte-for-byte from the pre-refactor flexsearch/minisearch factories): + * - `__manifest__` : JSON array of index names + * - `${name}/__config__` : JSON of StoredIndexConfig for each index + * - `${name}/fts` : UTF-8 bytes of serialized FTS (text payload) + * - `${name}/vec` : binary bytes of serialized vector sub-index + * + * Replaces the ~287-LOC `createFlexSearchIndexer` / `createMiniSearchIndexer` factories. + */ +export function createPersistenceBackedIndexer( + opts: PersistenceBackedIndexerOptions, +): Indexer { + const indexes = new Map(); + const ftsInstances = new Map(); + const vecInstances = new Map(); + const configs = new Map(); + const manifest = new Map(); + let closed = false; + let initialized = false; + let initPromise: Promise | null = null; + + const persistence = opts.persistence; + + function ensureOpen(): void { + if (closed) throw new Error("Indexer is closed"); + } + + async function loadFromPersistence(): Promise { + if (!persistence || initialized) return; + initialized = true; + + const textEntries = new Map(); + const binaryEntries = new Map(); + for await (const entry of persistence.load()) { + const bytes = await readEntryBytes(entry); + binaryEntries.set(entry.name, bytes); + textEntries.set(entry.name, new TextDecoder().decode(bytes)); + } + + const manifestJson = textEntries.get("__manifest__"); + if (!manifestJson) return; + + const indexNames = JSON.parse(manifestJson) as string[]; + + for (const name of indexNames) { + const configJson = textEntries.get(`${name}/__config__`); + if (!configJson) continue; + + const config = JSON.parse(configJson) as StoredIndexConfig; + configs.set(name, config); + + let fts: F | null = null; + let vec: V | null = null; + + if (config.fulltext) { + const ftsJson = textEntries.get(`${name}/fts`); + fts = ftsJson + ? opts.deserializeFts(config.fulltext, ftsJson) + : opts.createFts(config.fulltext); + ftsInstances.set(name, fts); + } + + if (config.vector) { + const vecBytes = binaryEntries.get(`${name}/vec`); + vec = vecBytes + ? opts.deserializeVec(config.vector, vecBytes) + : opts.createVec(config.vector); + vecInstances.set(name, vec); + } + + const index = createCompositeIndex({ name, fts, vec }); + indexes.set(name, index); + manifest.set(name, { name }); + } + } + + async function saveToPersistence(): Promise { + if (!persistence) return; + + async function* generateEntries(): AsyncIterable { + const names = [...indexes.keys()]; + yield { + name: "__manifest__", + content: singleChunk(toBytes(JSON.stringify(names))), + }; + + for (const indexName of names) { + const config = configs.get(indexName); + if (config) { + yield { + name: `${indexName}/__config__`, + content: singleChunk(toBytes(JSON.stringify(config))), + }; + } + + const fts = ftsInstances.get(indexName); + if (fts) { + const data = await opts.serializeFts(fts); + yield { + name: `${indexName}/fts`, + content: singleChunk(toBytes(data)), + }; + } + + const vec = vecInstances.get(indexName); + if (vec) { + const bytes = await opts.serializeVec(vec); + yield { + name: `${indexName}/vec`, + content: singleChunk(bytes), + }; + } + } + } + + await persistence.save(generateEntries()); + } + + async function ensureInitialized(): Promise { + if (initialized) return; + if (!initPromise) initPromise = loadFromPersistence(); + await initPromise; + } + + return { + async getIndexNames(): Promise { + ensureOpen(); + await ensureInitialized(); + return [...manifest.values()]; + }, + + async createIndex(params: CreateIndexParams): Promise { + ensureOpen(); + await ensureInitialized(); + const { name, fulltext, vector, overwrite } = params; + + if (!fulltext && !vector) { + throw new Error("At least one of fulltext or vector must be provided"); + } + + if (indexes.has(name)) { + if (overwrite) { + const old = indexes.get(name); + await old?.close(); + indexes.delete(name); + ftsInstances.delete(name); + vecInstances.delete(name); + manifest.delete(name); + configs.delete(name); + } else { + throw new Error(`Index "${name}" already exists`); + } + } + + const fts = fulltext ? opts.createFts(fulltext) : null; + if (fts) ftsInstances.set(name, fts); + + const vec = vector ? opts.createVec(vector) : null; + if (vec) vecInstances.set(name, vec); + + const index = createCompositeIndex({ name, fts, vec }); + indexes.set(name, index); + manifest.set(name, { name }); + configs.set(name, { name, fulltext, vector }); + + return index; + }, + + async getIndex(name: string): Promise { + ensureOpen(); + await ensureInitialized(); + return indexes.get(name) ?? null; + }, + + async hasIndex(name: string): Promise { + ensureOpen(); + await ensureInitialized(); + return indexes.has(name); + }, + + async deleteIndex(name: string): Promise { + ensureOpen(); + await ensureInitialized(); + const index = indexes.get(name); + if (index) { + await index.close(); + indexes.delete(name); + ftsInstances.delete(name); + vecInstances.delete(name); + manifest.delete(name); + configs.delete(name); + } + }, + + async flush(): Promise { + ensureOpen(); + await ensureInitialized(); + await saveToPersistence(); + }, + + async close(): Promise { + if (closed) return; + await ensureInitialized(); + await saveToPersistence(); + closed = true; + for (const index of indexes.values()) { + await index.close(); + } + indexes.clear(); + ftsInstances.clear(); + vecInstances.clear(); + manifest.clear(); + configs.clear(); + }, + }; +} diff --git a/packages/indexer-core/src/index.ts b/packages/indexer-core/src/index.ts index 1ac7f10..8118813 100644 --- a/packages/indexer-core/src/index.ts +++ b/packages/indexer-core/src/index.ts @@ -3,7 +3,11 @@ export { toAsyncIterable } from "./async.js"; export { compositeKey } from "./composite-key.js"; -export { createCompositeIndex, type CompositeIndexOptions } from "./create-composite-index.js"; +export { type CompositeIndexOptions, createCompositeIndex } from "./create-composite-index.js"; +export { + createPersistenceBackedIndexer, + type PersistenceBackedIndexerOptions, +} from "./create-persistence-backed-indexer.js"; export { mergeByRRF, mergeByWeights, mergeHybrid } from "./merge.js"; export { matchesPrefix } from "./path-prefix.js"; export { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; diff --git a/packages/indexer-duckdb/src/duckdb-index.ts b/packages/indexer-duckdb/src/duckdb-index.ts deleted file mode 100644 index 4a21ff6..0000000 --- a/packages/indexer-duckdb/src/duckdb-index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { Db } from "@statewalker/db-api"; -import type { DocumentPath, Index, Metadata } from "@statewalker/indexer-api"; -import { createCompositeIndex } from "@statewalker/indexer-core"; -import type { DuckDbFullTextIndex } from "./duckdb-full-text-index.js"; -import type { DuckDbVectorIndex } from "./duckdb-vector-index.js"; - -/** - * DuckDB-backed composite `Index`. - * - * @deprecated Use `createCompositeIndex` from `@statewalker/indexer-core` directly. Kept as a thin factory for one transitional release. - */ -export function DuckDbIndex( - name: string, - db: Db, - docsTable: string, - fts: DuckDbFullTextIndex | null, - vec: DuckDbVectorIndex | null, - metadata?: Metadata, -): Index { - return createCompositeIndex({ - name, - fts, - vec, - metadata, - getSize: async (pathPrefix?: DocumentPath): Promise => { - if (fts !== null && vec !== null) { - const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause})`; - const rows = await db.query<{ cnt: number | bigint }>(sql, params); - return Number(rows[0]?.cnt ?? 0); - } - if (fts !== null) return fts.getSize(pathPrefix); - if (vec !== null) return vec.getSize(pathPrefix); - return 0; - }, - onDeleteIndex: async () => { - await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); - }, - }); -} diff --git a/packages/indexer-duckdb/src/duckdb-indexer.ts b/packages/indexer-duckdb/src/duckdb-indexer.ts index 92c664c..4c109ff 100644 --- a/packages/indexer-duckdb/src/duckdb-indexer.ts +++ b/packages/indexer-duckdb/src/duckdb-indexer.ts @@ -1,8 +1,13 @@ import type { Db } from "@statewalker/db-api"; -import type { CreateIndexParams, Index, Indexer, IndexInfo } from "@statewalker/indexer-api"; -import { sanitizePrefix } from "@statewalker/indexer-core"; +import type { + CreateIndexParams, + DocumentPath, + Index, + Indexer, + IndexInfo, +} from "@statewalker/indexer-api"; +import { createCompositeIndex, sanitizePrefix } from "@statewalker/indexer-core"; import { DuckDbFullTextIndex } from "./duckdb-full-text-index.js"; -import { DuckDbIndex } from "./duckdb-index.js"; import { DuckDbVectorIndex } from "./duckdb-vector-index.js"; export interface DuckDbIndexerOptions { @@ -30,9 +35,11 @@ export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promis } function ensureOpen(): void { - if (closed) { - throw new Error("Indexer is closed"); - } + if (closed) throw new Error("Indexer is closed"); + } + + async function ensureDocsSequence(prefix: string): Promise { + await db.exec(`CREATE SEQUENCE IF NOT EXISTS idx_${prefix}_docs_seq START 1`); } async function createDocsTable(prefix: string): Promise { @@ -43,9 +50,32 @@ export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promis return docsTable; } - async function ensureDocsSequence(prefix: string): Promise { - const seqName = `idx_${prefix}_docs_seq`; - await db.exec(`CREATE SEQUENCE IF NOT EXISTS ${seqName} START 1`); + function buildIndex( + name: string, + docsTable: string, + fts: DuckDbFullTextIndex | null, + vec: DuckDbVectorIndex | null, + ): Index { + return createCompositeIndex({ + name, + fts, + vec, + getSize: async (pathPrefix?: DocumentPath): Promise => { + if (fts !== null && vec !== null) { + const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause})`; + const rows = await db.query<{ cnt: number | bigint }>(sql, params); + return Number(rows[0]?.cnt ?? 0); + } + if (fts !== null) return fts.getSize(pathPrefix); + if (vec !== null) return vec.getSize(pathPrefix); + return 0; + }, + onDeleteIndex: async () => { + await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); + }, + }); } const indexer: Indexer = { @@ -103,27 +133,21 @@ export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promis if (fts) await fts.init(); if (vec) await vec.init(); - const config = JSON.stringify({ fulltext, vector }); await db.query("INSERT INTO __indexer_manifest (name, config) VALUES ($1, $2)", [ name, - config, + JSON.stringify({ fulltext, vector }), ]); - const index = DuckDbIndex(name, db, docsTable, fts, vec); + const index = buildIndex(name, docsTable, fts, vec); indexes.set(name, index); manifest.set(name, { name }); - return index; }, async getIndex(name: string): Promise { ensureOpen(); - if (indexes.has(name)) { - return indexes.get(name) ?? null; - } - if (!manifest.has(name)) { - return null; - } + if (indexes.has(name)) return indexes.get(name) ?? null; + if (!manifest.has(name)) return null; const rows = await db.query<{ config: string }>( "SELECT config FROM __indexer_manifest WHERE name = $1", @@ -159,7 +183,7 @@ export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promis }) : null; - const index = DuckDbIndex(name, db, docsTable, fts, vec); + const index = buildIndex(name, docsTable, fts, vec); indexes.set(name, index); return index; }, @@ -194,9 +218,7 @@ export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promis async close(): Promise { if (closed) return; closed = true; - for (const index of indexes.values()) { - await index.close(); - } + for (const index of indexes.values()) await index.close(); indexes.clear(); manifest.clear(); }, diff --git a/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts b/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts index 426fff8..f872204 100644 --- a/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts +++ b/packages/indexer-mem-flexsearch/src/flexsearch-indexer.ts @@ -1,243 +1,20 @@ -import type { - CreateIndexParams, - Index, - Indexer, - IndexerPersistence, - IndexInfo, - PersistenceEntry, -} from "@statewalker/indexer-api"; -import { readEntryBytes, singleChunk, toBytes } from "@statewalker/indexer-core"; -import { MemIndex, MemVectorIndex } from "@statewalker/indexer-mem"; +import type { Indexer, IndexerPersistence } from "@statewalker/indexer-api"; +import { createPersistenceBackedIndexer } from "@statewalker/indexer-core"; +import { MemVectorIndex } from "@statewalker/indexer-mem"; import { FlexSearchFullTextIndex } from "./flexsearch-full-text-index.js"; export interface FlexSearchIndexerOptions { persistence?: IndexerPersistence; } -interface StoredIndexConfig { - name: string; - fulltext?: { language: string; metadata?: Record }; - vector?: { - dimensionality: number; - model: string; - metadata?: Record; - }; -} - export function createFlexSearchIndexer(options?: FlexSearchIndexerOptions): Indexer { - const indexes = new Map(); - const configs = new Map(); - const manifest = new Map(); - let closed = false; - let initialized = false; - let initPromise: Promise | null = null; - - const persistence = options?.persistence; - - function ensureOpen(): void { - if (closed) { - throw new Error("Indexer is closed"); - } - } - - async function loadFromPersistence(): Promise { - if (!persistence || initialized) return; - initialized = true; - - const textEntries = new Map(); - const binaryEntries = new Map(); - for await (const entry of persistence.load()) { - const bytes = await readEntryBytes(entry); - binaryEntries.set(entry.name, bytes); - textEntries.set(entry.name, new TextDecoder().decode(bytes)); - } - - const manifestJson = textEntries.get("__manifest__"); - if (!manifestJson) return; - - const indexNames = JSON.parse(manifestJson) as string[]; - - for (const name of indexNames) { - const configJson = textEntries.get(`${name}/__config__`); - if (!configJson) continue; - - const config = JSON.parse(configJson) as StoredIndexConfig; - configs.set(name, config); - - let fts: FlexSearchFullTextIndex | null = null; - let vec: MemVectorIndex | null = null; - - if (config.fulltext) { - const ftsJson = textEntries.get(`${name}/fts`); - if (ftsJson) { - fts = FlexSearchFullTextIndex.deserialize(config.fulltext, ftsJson); - } else { - fts = new FlexSearchFullTextIndex(config.fulltext); - } - } - - if (config.vector) { - const vecBytes = binaryEntries.get(`${name}/vec`); - if (vecBytes) { - vec = MemVectorIndex.deserializeFromArrow(config.vector, vecBytes); - } else { - vec = new MemVectorIndex(config.vector); - } - } - - const index = MemIndex(name, fts, vec); - indexes.set(name, index); - manifest.set(name, { name }); - } - } - - async function saveToPersistence(): Promise { - if (!persistence) return; - - async function* generateEntries(): AsyncIterable { - const names = [...indexes.keys()]; - yield { - name: "__manifest__", - content: singleChunk(toBytes(JSON.stringify(names))), - }; - - for (const indexName of names) { - const config = configs.get(indexName); - if (config) { - yield { - name: `${indexName}/__config__`, - content: singleChunk(toBytes(JSON.stringify(config))), - }; - } - - const index = indexes.get(indexName); - if (!index) continue; - - const fts = index.getFullTextIndex(); - if (fts && fts instanceof FlexSearchFullTextIndex) { - const data = await fts.serialize(); - yield { - name: `${indexName}/fts`, - content: singleChunk(toBytes(data)), - }; - } - - const vec = index.getVectorIndex(); - if (vec && vec instanceof MemVectorIndex) { - const arrowBytes = vec.serializeToArrow(); - yield { - name: `${indexName}/vec`, - content: singleChunk(arrowBytes), - }; - } - } - } - - await persistence.save(generateEntries()); - } - - async function ensureInitialized(): Promise { - if (initialized) return; - if (!initPromise) { - initPromise = loadFromPersistence(); - } - await initPromise; - } - - const indexer: Indexer = { - async getIndexNames(): Promise { - ensureOpen(); - await ensureInitialized(); - return [...manifest.values()]; - }, - - async createIndex(params: CreateIndexParams): Promise { - ensureOpen(); - await ensureInitialized(); - const { name, fulltext, vector, overwrite } = params; - - if (!fulltext && !vector) { - throw new Error("At least one of fulltext or vector must be provided"); - } - - if (indexes.has(name)) { - if (overwrite) { - const old = indexes.get(name); - await old?.close(); - indexes.delete(name); - manifest.delete(name); - configs.delete(name); - } else { - throw new Error(`Index "${name}" already exists`); - } - } - - const fts = fulltext - ? new FlexSearchFullTextIndex({ - language: fulltext.language, - metadata: fulltext.metadata, - }) - : null; - - const vec = vector - ? new MemVectorIndex({ - dimensionality: vector.dimensionality, - model: vector.model, - metadata: vector.metadata, - }) - : null; - - const index = MemIndex(name, fts, vec); - indexes.set(name, index); - manifest.set(name, { name }); - configs.set(name, { name, fulltext, vector }); - - return index; - }, - - async getIndex(name: string): Promise { - ensureOpen(); - await ensureInitialized(); - return indexes.get(name) ?? null; - }, - - async hasIndex(name: string): Promise { - ensureOpen(); - await ensureInitialized(); - return indexes.has(name); - }, - - async deleteIndex(name: string): Promise { - ensureOpen(); - await ensureInitialized(); - const index = indexes.get(name); - if (index) { - await index.close(); - indexes.delete(name); - manifest.delete(name); - configs.delete(name); - } - }, - - async flush(): Promise { - ensureOpen(); - await ensureInitialized(); - await saveToPersistence(); - }, - - async close(): Promise { - if (closed) return; - await ensureInitialized(); - await saveToPersistence(); - closed = true; - for (const index of indexes.values()) { - await index.close(); - } - indexes.clear(); - manifest.clear(); - configs.clear(); - }, - }; - - return indexer; + return createPersistenceBackedIndexer({ + persistence: options?.persistence, + createFts: (info) => new FlexSearchFullTextIndex(info), + serializeFts: (fts) => fts.serialize(), + deserializeFts: (info, data) => FlexSearchFullTextIndex.deserialize(info, data), + createVec: (info) => new MemVectorIndex(info), + serializeVec: (vec) => vec.serializeToArrow(), + deserializeVec: (info, data) => MemVectorIndex.deserializeFromArrow(info, data), + }); } diff --git a/packages/indexer-mem-minisearch/src/minisearch-indexer.ts b/packages/indexer-mem-minisearch/src/minisearch-indexer.ts index d1afcc1..db7a276 100644 --- a/packages/indexer-mem-minisearch/src/minisearch-indexer.ts +++ b/packages/indexer-mem-minisearch/src/minisearch-indexer.ts @@ -1,243 +1,20 @@ -import type { - CreateIndexParams, - Index, - Indexer, - IndexerPersistence, - IndexInfo, - PersistenceEntry, -} from "@statewalker/indexer-api"; -import { readEntryBytes, singleChunk, toBytes } from "@statewalker/indexer-core"; -import { MemIndex, MemVectorIndex } from "@statewalker/indexer-mem"; +import type { Indexer, IndexerPersistence } from "@statewalker/indexer-api"; +import { createPersistenceBackedIndexer } from "@statewalker/indexer-core"; +import { MemVectorIndex } from "@statewalker/indexer-mem"; import { MiniSearchFullTextIndex } from "./minisearch-full-text-index.js"; export interface MiniSearchIndexerOptions { persistence?: IndexerPersistence; } -interface StoredIndexConfig { - name: string; - fulltext?: { language: string; metadata?: Record }; - vector?: { - dimensionality: number; - model: string; - metadata?: Record; - }; -} - export function createMiniSearchIndexer(options?: MiniSearchIndexerOptions): Indexer { - const indexes = new Map(); - const configs = new Map(); - const manifest = new Map(); - let closed = false; - let initialized = false; - let initPromise: Promise | null = null; - - const persistence = options?.persistence; - - function ensureOpen(): void { - if (closed) { - throw new Error("Indexer is closed"); - } - } - - async function loadFromPersistence(): Promise { - if (!persistence || initialized) return; - initialized = true; - - const textEntries = new Map(); - const binaryEntries = new Map(); - for await (const entry of persistence.load()) { - const bytes = await readEntryBytes(entry); - binaryEntries.set(entry.name, bytes); - textEntries.set(entry.name, new TextDecoder().decode(bytes)); - } - - const manifestJson = textEntries.get("__manifest__"); - if (!manifestJson) return; - - const indexNames = JSON.parse(manifestJson) as string[]; - - for (const name of indexNames) { - const configJson = textEntries.get(`${name}/__config__`); - if (!configJson) continue; - - const config = JSON.parse(configJson) as StoredIndexConfig; - configs.set(name, config); - - let fts: MiniSearchFullTextIndex | null = null; - let vec: MemVectorIndex | null = null; - - if (config.fulltext) { - const ftsJson = textEntries.get(`${name}/fts`); - if (ftsJson) { - fts = MiniSearchFullTextIndex.deserialize(config.fulltext, ftsJson); - } else { - fts = new MiniSearchFullTextIndex(config.fulltext); - } - } - - if (config.vector) { - const vecBytes = binaryEntries.get(`${name}/vec`); - if (vecBytes) { - vec = MemVectorIndex.deserializeFromArrow(config.vector, vecBytes); - } else { - vec = new MemVectorIndex(config.vector); - } - } - - const index = MemIndex(name, fts, vec); - indexes.set(name, index); - manifest.set(name, { name }); - } - } - - async function saveToPersistence(): Promise { - if (!persistence) return; - - async function* generateEntries(): AsyncIterable { - const names = [...indexes.keys()]; - yield { - name: "__manifest__", - content: singleChunk(toBytes(JSON.stringify(names))), - }; - - for (const indexName of names) { - const config = configs.get(indexName); - if (config) { - yield { - name: `${indexName}/__config__`, - content: singleChunk(toBytes(JSON.stringify(config))), - }; - } - - const index = indexes.get(indexName); - if (!index) continue; - - const fts = index.getFullTextIndex(); - if (fts && fts instanceof MiniSearchFullTextIndex) { - const data = fts.serialize(); - yield { - name: `${indexName}/fts`, - content: singleChunk(toBytes(data)), - }; - } - - const vec = index.getVectorIndex(); - if (vec && vec instanceof MemVectorIndex) { - const arrowBytes = vec.serializeToArrow(); - yield { - name: `${indexName}/vec`, - content: singleChunk(arrowBytes), - }; - } - } - } - - await persistence.save(generateEntries()); - } - - async function ensureInitialized(): Promise { - if (initialized) return; - if (!initPromise) { - initPromise = loadFromPersistence(); - } - await initPromise; - } - - const indexer: Indexer = { - async getIndexNames(): Promise { - ensureOpen(); - await ensureInitialized(); - return [...manifest.values()]; - }, - - async createIndex(params: CreateIndexParams): Promise { - ensureOpen(); - await ensureInitialized(); - const { name, fulltext, vector, overwrite } = params; - - if (!fulltext && !vector) { - throw new Error("At least one of fulltext or vector must be provided"); - } - - if (indexes.has(name)) { - if (overwrite) { - const old = indexes.get(name); - await old?.close(); - indexes.delete(name); - manifest.delete(name); - configs.delete(name); - } else { - throw new Error(`Index "${name}" already exists`); - } - } - - const fts = fulltext - ? new MiniSearchFullTextIndex({ - language: fulltext.language, - metadata: fulltext.metadata, - }) - : null; - - const vec = vector - ? new MemVectorIndex({ - dimensionality: vector.dimensionality, - model: vector.model, - metadata: vector.metadata, - }) - : null; - - const index = MemIndex(name, fts, vec); - indexes.set(name, index); - manifest.set(name, { name }); - configs.set(name, { name, fulltext, vector }); - - return index; - }, - - async getIndex(name: string): Promise { - ensureOpen(); - await ensureInitialized(); - return indexes.get(name) ?? null; - }, - - async hasIndex(name: string): Promise { - ensureOpen(); - await ensureInitialized(); - return indexes.has(name); - }, - - async deleteIndex(name: string): Promise { - ensureOpen(); - await ensureInitialized(); - const index = indexes.get(name); - if (index) { - await index.close(); - indexes.delete(name); - manifest.delete(name); - configs.delete(name); - } - }, - - async flush(): Promise { - ensureOpen(); - await ensureInitialized(); - await saveToPersistence(); - }, - - async close(): Promise { - if (closed) return; - await ensureInitialized(); - await saveToPersistence(); - closed = true; - for (const index of indexes.values()) { - await index.close(); - } - indexes.clear(); - manifest.clear(); - configs.clear(); - }, - }; - - return indexer; + return createPersistenceBackedIndexer({ + persistence: options?.persistence, + createFts: (info) => new MiniSearchFullTextIndex(info), + serializeFts: (fts) => fts.serialize(), + deserializeFts: (info, data) => MiniSearchFullTextIndex.deserialize(info, data), + createVec: (info) => new MemVectorIndex(info), + serializeVec: (vec) => vec.serializeToArrow(), + deserializeVec: (info, data) => MemVectorIndex.deserializeFromArrow(info, data), + }); } diff --git a/packages/indexer-mem/README.md b/packages/indexer-mem/README.md index bca4e01..a6f38cf 100644 --- a/packages/indexer-mem/README.md +++ b/packages/indexer-mem/README.md @@ -1,26 +1,37 @@ # @statewalker/indexer-mem -In-memory base implementation of `@statewalker/indexer-api`. Scaffold reused by `indexer-mem-flexsearch` and `indexer-mem-minisearch`. - -## Installation - -```sh -pnpm add @statewalker/indexer-mem -``` +In-memory vector sub-index (`MemVectorIndex`) implementing the `EmbeddingIndex` contract from `@statewalker/indexer-api`. Consumed by the FTS-backed mem indexers (`indexer-mem-flexsearch`, `indexer-mem-minisearch`), which combine it with a full-text sub-index via `@statewalker/indexer-core`'s composite-index factory. ## Usage -```ts -import { createMemIndex } from "@statewalker/indexer-mem"; +This package is a piece of scaffolding — consumers should reach for the concrete indexer packages: -const idx = createMemIndex(); -await idx.add({ id: "1", text: "hello" }); +```ts +import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch"; +// or +import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch"; + +const indexer = createFlexSearchIndexer(); +const index = await indexer.createIndex({ + name: "docs", + fulltext: { language: "en" }, + vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" }, +}); + +await index.addDocument([ + { path: "/docs/a", blockId: "1", content: "hello world", embedding: queryEmbedding }, +]); + +for await (const hit of index.search({ queries: ["hello"], embeddings: [queryEmbedding], topK: 10 })) { + console.log(hit); +} ``` -## API +## Exports -- `createMemIndex(options)` — lazy in-memory index. +- `MemVectorIndex` — in-memory vector sub-index with brute-force cosine search and Arrow IPC serialization (used as the vector side of both mem-flexsearch and mem-minisearch). ## Related -- `@statewalker/indexer-mem-flexsearch` / `@statewalker/indexer-mem-minisearch` — drop-in FTS backends. +- `@statewalker/indexer-api` — public types and contract +- `@statewalker/indexer-mem-flexsearch` / `@statewalker/indexer-mem-minisearch` — end-user factories combining `MemVectorIndex` with a full-text backend diff --git a/packages/indexer-mem/src/index.ts b/packages/indexer-mem/src/index.ts index 2f48cf1..1eace72 100644 --- a/packages/indexer-mem/src/index.ts +++ b/packages/indexer-mem/src/index.ts @@ -1,4 +1 @@ -export { mergeByRRF, mergeByWeights } from "@statewalker/indexer-core"; -export { MemIndex } from "./mem-index.js"; export { MemVectorIndex } from "./mem-vector-index.js"; -export { bruteForceSearch, cosineSimilarity } from "./vector-search.js"; diff --git a/packages/indexer-mem/src/mem-index.ts b/packages/indexer-mem/src/mem-index.ts deleted file mode 100644 index d649785..0000000 --- a/packages/indexer-mem/src/mem-index.ts +++ /dev/null @@ -1,21 +0,0 @@ -import type { - EmbeddingIndex, - FullTextIndex, - Index, - Metadata, -} from "@statewalker/indexer-api"; -import { createCompositeIndex } from "@statewalker/indexer-core"; - -/** - * In-memory composite `Index`. - * - * @deprecated Use `createCompositeIndex` from `@statewalker/indexer-core` directly. Kept as a thin re-export for one transitional release. - */ -export function MemIndex( - name: string, - fts: FullTextIndex | null, - vec: EmbeddingIndex | null, - metadata?: Metadata, -): Index { - return createCompositeIndex({ name, fts, vec, metadata }); -} diff --git a/packages/indexer-pglite/src/pglite-index.ts b/packages/indexer-pglite/src/pglite-index.ts deleted file mode 100644 index 3132a8f..0000000 --- a/packages/indexer-pglite/src/pglite-index.ts +++ /dev/null @@ -1,41 +0,0 @@ -import type { PGlite } from "@electric-sql/pglite"; -import type { DocumentPath, Index, Metadata } from "@statewalker/indexer-api"; -import { createCompositeIndex } from "@statewalker/indexer-core"; -import type { PGLiteFullTextIndex } from "./pglite-full-text-index.js"; -import type { PGLiteVectorIndex } from "./pglite-vector-index.js"; - -/** - * PGlite-backed composite `Index`. - * - * @deprecated Use `createCompositeIndex` from `@statewalker/indexer-core` directly. Kept as a thin factory for one transitional release. - */ -export function PGLiteIndex( - name: string, - db: PGlite, - docsTable: string, - fts: PGLiteFullTextIndex | null, - vec: PGLiteVectorIndex | null, - metadata?: Metadata, -): Index { - return createCompositeIndex({ - name, - fts, - vec, - metadata, - getSize: async (pathPrefix?: DocumentPath): Promise => { - if (fts !== null && vec !== null) { - const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause}) AS combined`; - const { rows } = await db.query<{ cnt: number | bigint }>(sql, params); - return Number(rows[0]?.cnt ?? 0); - } - if (fts !== null) return fts.getSize(pathPrefix); - if (vec !== null) return vec.getSize(pathPrefix); - return 0; - }, - onDeleteIndex: async () => { - await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); - }, - }); -} diff --git a/packages/indexer-pglite/src/pglite-indexer.ts b/packages/indexer-pglite/src/pglite-indexer.ts index 47c3c3e..1a5e9c8 100644 --- a/packages/indexer-pglite/src/pglite-indexer.ts +++ b/packages/indexer-pglite/src/pglite-indexer.ts @@ -1,9 +1,14 @@ import { PGlite } from "@electric-sql/pglite"; import { vector } from "@electric-sql/pglite/vector"; -import type { CreateIndexParams, Index, Indexer, IndexInfo } from "@statewalker/indexer-api"; -import { sanitizePrefix } from "@statewalker/indexer-core"; +import type { + CreateIndexParams, + DocumentPath, + Index, + Indexer, + IndexInfo, +} from "@statewalker/indexer-api"; +import { createCompositeIndex, sanitizePrefix } from "@statewalker/indexer-core"; import { PGLiteFullTextIndex } from "./pglite-full-text-index.js"; -import { PGLiteIndex } from "./pglite-index.js"; import { PGLiteVectorIndex } from "./pglite-vector-index.js"; export interface PGLiteIndexerOptions { @@ -31,9 +36,7 @@ export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promi } function ensureOpen(): void { - if (closed) { - throw new Error("Indexer is closed"); - } + if (closed) throw new Error("Indexer is closed"); } async function createDocsTable(prefix: string): Promise { @@ -44,6 +47,34 @@ export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promi return docsTable; } + function buildIndex( + name: string, + docsTable: string, + fts: PGLiteFullTextIndex | null, + vec: PGLiteVectorIndex | null, + ): Index { + return createCompositeIndex({ + name, + fts, + vec, + getSize: async (pathPrefix?: DocumentPath): Promise => { + if (fts !== null && vec !== null) { + const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause}) AS combined`; + const { rows } = await db.query<{ cnt: number | bigint }>(sql, params); + return Number(rows[0]?.cnt ?? 0); + } + if (fts !== null) return fts.getSize(pathPrefix); + if (vec !== null) return vec.getSize(pathPrefix); + return 0; + }, + onDeleteIndex: async () => { + await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); + }, + }); + } + const indexer: Indexer = { async getIndexNames(): Promise { ensureOpen(); @@ -95,27 +126,21 @@ export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promi if (fts) await fts.init(); if (vec) await vec.init(); - const config = JSON.stringify({ fulltext, vector }); await db.query("INSERT INTO __indexer_manifest (name, config) VALUES ($1, $2)", [ name, - config, + JSON.stringify({ fulltext, vector }), ]); - const index = PGLiteIndex(name, db, docsTable, fts, vec); + const index = buildIndex(name, docsTable, fts, vec); indexes.set(name, index); manifest.set(name, { name }); - return index; }, async getIndex(name: string): Promise { ensureOpen(); - if (indexes.has(name)) { - return indexes.get(name) ?? null; - } - if (!manifest.has(name)) { - return null; - } + if (indexes.has(name)) return indexes.get(name) ?? null; + if (!manifest.has(name)) return null; const result = await db.query<{ config: string }>( "SELECT config FROM __indexer_manifest WHERE name = $1", @@ -150,7 +175,7 @@ export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promi }) : null; - const index = PGLiteIndex(name, db, docsTable, fts, vec); + const index = buildIndex(name, docsTable, fts, vec); indexes.set(name, index); return index; }, @@ -184,14 +209,10 @@ export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promi async close(): Promise { if (closed) return; closed = true; - for (const index of indexes.values()) { - await index.close(); - } + for (const index of indexes.values()) await index.close(); indexes.clear(); manifest.clear(); - if (ownsDb) { - await db.close(); - } + if (ownsDb) await db.close(); }, }; From f011656e054da4bbc8f7c520c425f866fc08421a Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Fri, 24 Apr 2026 14:47:35 +0200 Subject: [PATCH 03/12] refactor(indexer-core): SqlRetrieverBase + SQL-backed Indexer factory (Phases 6-7) Extract the shared SQL scaffolding from indexer-duckdb and indexer-pglite into indexer-core. Both backends now differ only in their dialect objects and a minimal SqlDb adapter. New in @statewalker/indexer-core: * sql-db.ts (8 LOC) - SqlDb: minimal normalised async SQL client shape * create-sql-fts-retriever.ts (237 LOC) - shared FullTextIndex CRUD + resolveDocId + path-filter SQL, driven by a SqlFtsDialect * create-sql-vector-retriever.ts (267 LOC) - shared EmbeddingIndex CRUD + HNSW DDL + embedding bind/cast/decode, driven by a SqlVectorDialect * create-sql-backed-indexer.ts (~200 LOC) - shared Indexer shell: __indexer_manifest table, extension init, per-index DDL/drop, composite assembly with dialect-specific unionAliasSuffix New in each backend: * dialect.ts (DuckDB: 146 LOC, PGlite: 161 LOC) - the pieces that genuinely differ: DDL, search SQL, embedding-literal strategy, embedding decode, extension init. Exports an aggregated duckdbDialect / pgliteDialect for createSqlBackedIndexer. * wrapDbAsSqlDb() adapter normalising each driver's result shape (db-api returns bare array; PGlite returns { rows }). Sub-index / factory shrink: * duckdb-full-text-index.ts: 252 -> 24 LOC * duckdb-vector-index.ts: 243 -> 24 LOC * duckdb-indexer.ts: 228 -> 15 LOC * pglite-full-text-index.ts: 275 -> 24 LOC * pglite-vector-index.ts: 245 -> 24 LOC * pglite-indexer.ts: 220 -> 24 LOC Absorbs deferred task 2.4 (resolveDocId extraction) which was held for Phase 6 because the two backends' driver query methods return results in different shapes - now normalised via SqlDb. LIKE-based DuckDB FTS stays in the dialect for now; Phase 8 replaces it with BM25 via the official `fts` community extension. Net: -1426 LOC from backend files, -494 LOC effective across the workspace. No public API change. Tests: 575/575 passing. Co-Authored-By: Claude Opus 4.7 (1M context) --- .../src/create-sql-backed-indexer.ts | 243 +++++++++++++++ .../src/create-sql-fts-retriever.ts | 237 ++++++++++++++ .../src/create-sql-vector-retriever.ts | 267 ++++++++++++++++ packages/indexer-core/src/index.ts | 16 + packages/indexer-core/src/sql-db.ts | 8 + packages/indexer-duckdb/src/dialect.ts | 146 +++++++++ .../src/duckdb-full-text-index.ts | 268 ++-------------- packages/indexer-duckdb/src/duckdb-indexer.ts | 229 +------------- .../indexer-duckdb/src/duckdb-vector-index.ts | 259 ++-------------- packages/indexer-pglite/src/dialect.ts | 161 ++++++++++ .../src/pglite-full-text-index.ts | 289 ++---------------- packages/indexer-pglite/src/pglite-indexer.ts | 218 +------------ .../indexer-pglite/src/pglite-vector-index.ts | 261 ++-------------- 13 files changed, 1176 insertions(+), 1426 deletions(-) create mode 100644 packages/indexer-core/src/create-sql-backed-indexer.ts create mode 100644 packages/indexer-core/src/create-sql-fts-retriever.ts create mode 100644 packages/indexer-core/src/create-sql-vector-retriever.ts create mode 100644 packages/indexer-core/src/sql-db.ts create mode 100644 packages/indexer-duckdb/src/dialect.ts create mode 100644 packages/indexer-pglite/src/dialect.ts diff --git a/packages/indexer-core/src/create-sql-backed-indexer.ts b/packages/indexer-core/src/create-sql-backed-indexer.ts new file mode 100644 index 0000000..417f631 --- /dev/null +++ b/packages/indexer-core/src/create-sql-backed-indexer.ts @@ -0,0 +1,243 @@ +import type { + CreateIndexParams, + DocumentPath, + EmbeddingIndex, + EmbeddingIndexInfo, + FullTextIndex, + FullTextIndexInfo, + Index, + Indexer, + IndexInfo, +} from "@statewalker/indexer-api"; +import { createCompositeIndex } from "./create-composite-index.js"; +import type { SqlFtsDialect } from "./create-sql-fts-retriever.js"; +import { createSqlFtsRetriever } from "./create-sql-fts-retriever.js"; +import type { SqlVectorDialect } from "./create-sql-vector-retriever.js"; +import { createSqlVectorRetriever } from "./create-sql-vector-retriever.js"; +import { sanitizePrefix } from "./sanitize-prefix.js"; +import type { SqlDb } from "./sql-db.js"; + +type FtsWithTable = FullTextIndex & { readonly tableName: string; init(): Promise }; +type VecWithTable = EmbeddingIndex & { readonly tableName: string; init(): Promise }; + +/** Per-backend dialect aggregating the pieces that genuinely differ between SQL backends. */ +export interface SqlBackedDialect { + /** SQL strings to run once during init, after the manifest table is created. */ + extensionInit: string[]; + /** DDL for the per-index docs table. Returns one or more statements. */ + docsTableDdl(prefix: string): string[]; + /** Optional per-index cleanup SQL (e.g. dropping auxiliary sequences). Runs after the docs table is dropped. */ + extraCleanup?(prefix: string): string[]; + /** Suffix appended to the inner UNION subquery in the composite `getSize` SQL. PGlite requires ` AS combined`; DuckDB leaves it empty. */ + unionAliasSuffix: string; + + fts: SqlFtsDialect; + vec: SqlVectorDialect; +} + +export interface SqlBackedIndexerOptions { + db: SqlDb; + dialect: SqlBackedDialect; + /** Optional finaliser invoked by `indexer.close()` — backends that own their `db` can close it here. */ + onClose?(): Promise; +} + +interface StoredConfig { + fulltext?: FullTextIndexInfo; + vector?: EmbeddingIndexInfo; +} + +/** + * Generic SQL-backed `Indexer` factory. Carries the manifest table, index-lifecycle SQL, and + * composite-assembly shared by every SQL backend; defers all dialect-specific SQL to `opts.dialect`. + * + * Replaces the ~200 LOC factories in `indexer-duckdb` and `indexer-pglite`. + */ +export async function createSqlBackedIndexer(opts: SqlBackedIndexerOptions): Promise { + const { db, dialect, onClose } = opts; + const indexes = new Map(); + const manifest = new Map(); + let closed = false; + + for (const stmt of dialect.extensionInit) await db.exec(stmt); + + await db.exec( + "CREATE TABLE IF NOT EXISTS __indexer_manifest (name TEXT PRIMARY KEY, config TEXT NOT NULL)", + ); + + const existing = await db.query<{ name: string; config: string }>( + "SELECT name, config FROM __indexer_manifest", + ); + for (const entry of existing) manifest.set(entry.name, { name: entry.name }); + + function ensureOpen(): void { + if (closed) throw new Error("Indexer is closed"); + } + + async function createDocsTable(prefix: string): Promise { + for (const stmt of dialect.docsTableDdl(prefix)) await db.exec(stmt); + return `idx_${prefix}_docs`; + } + + async function dropIndexTables(prefix: string): Promise { + await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_fts`); + await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_vec`); + await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_docs`); + if (dialect.extraCleanup) { + for (const stmt of dialect.extraCleanup(prefix)) await db.exec(stmt); + } + } + + function buildIndex( + name: string, + docsTable: string, + fts: FtsWithTable | null, + vec: VecWithTable | null, + ): Index { + return createCompositeIndex({ + name, + fts, + vec, + getSize: async (pathPrefix?: DocumentPath): Promise => { + if (fts !== null && vec !== null) { + const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause})${dialect.unionAliasSuffix}`; + const rows = await db.query<{ cnt: number | bigint }>(sql, params); + return Number(rows[0]?.cnt ?? 0); + } + if (fts !== null) return fts.getSize(pathPrefix); + if (vec !== null) return vec.getSize(pathPrefix); + return 0; + }, + onDeleteIndex: async () => { + await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); + }, + }); + } + + return { + async getIndexNames(): Promise { + ensureOpen(); + return [...manifest.values()]; + }, + + async createIndex(params: CreateIndexParams): Promise { + ensureOpen(); + const { name, fulltext, vector, overwrite } = params; + if (!fulltext && !vector) { + throw new Error("At least one of fulltext or vector must be provided"); + } + + if (indexes.has(name) || manifest.has(name)) { + if (overwrite) { + const old = indexes.get(name); + if (old) await old.close(); + indexes.delete(name); + manifest.delete(name); + const prefix = sanitizePrefix(name); + await dropIndexTables(prefix); + await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); + } else { + throw new Error(`Index "${name}" already exists`); + } + } + + const prefix = sanitizePrefix(name); + const docsTable = await createDocsTable(prefix); + + const fts = fulltext + ? createSqlFtsRetriever({ db, prefix, docsTable, info: fulltext, dialect: dialect.fts }) + : null; + const vec = vector + ? createSqlVectorRetriever({ db, prefix, docsTable, info: vector, dialect: dialect.vec }) + : null; + + if (fts) await fts.init(); + if (vec) await vec.init(); + + await db.query("INSERT INTO __indexer_manifest (name, config) VALUES ($1, $2)", [ + name, + JSON.stringify({ fulltext, vector }), + ]); + + const index = buildIndex(name, docsTable, fts, vec); + indexes.set(name, index); + manifest.set(name, { name }); + return index; + }, + + async getIndex(name: string): Promise { + ensureOpen(); + if (indexes.has(name)) return indexes.get(name) ?? null; + if (!manifest.has(name)) return null; + + const rows = await db.query<{ config: string }>( + "SELECT config FROM __indexer_manifest WHERE name = $1", + [name], + ); + if (rows.length === 0) return null; + + const config = JSON.parse(rows[0]?.config ?? "{}") as StoredConfig; + + const prefix = sanitizePrefix(name); + const docsTable = await createDocsTable(prefix); + + const fts = config.fulltext + ? createSqlFtsRetriever({ + db, + prefix, + docsTable, + info: config.fulltext, + dialect: dialect.fts, + }) + : null; + const vec = config.vector + ? createSqlVectorRetriever({ + db, + prefix, + docsTable, + info: config.vector, + dialect: dialect.vec, + }) + : null; + + const index = buildIndex(name, docsTable, fts, vec); + indexes.set(name, index); + return index; + }, + + async hasIndex(name: string): Promise { + ensureOpen(); + return manifest.has(name); + }, + + async deleteIndex(name: string): Promise { + ensureOpen(); + const index = indexes.get(name); + if (index) { + await index.close(); + indexes.delete(name); + } + if (manifest.has(name)) { + const prefix = sanitizePrefix(name); + await dropIndexTables(prefix); + await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); + manifest.delete(name); + } + }, + + async flush(): Promise { + ensureOpen(); + }, + + async close(): Promise { + if (closed) return; + closed = true; + for (const index of indexes.values()) await index.close(); + indexes.clear(); + manifest.clear(); + if (onClose) await onClose(); + }, + }; +} diff --git a/packages/indexer-core/src/create-sql-fts-retriever.ts b/packages/indexer-core/src/create-sql-fts-retriever.ts new file mode 100644 index 0000000..2bc4324 --- /dev/null +++ b/packages/indexer-core/src/create-sql-fts-retriever.ts @@ -0,0 +1,237 @@ +import type { + BlockReference, + DocumentPath, + FullTextBlock, + FullTextIndex, + FullTextIndexInfo, + FullTextSearchParams, + FullTextSearchResult, + Metadata, + PathSelector, +} from "@statewalker/indexer-api"; +import { toAsyncIterable } from "./async.js"; +import { compositeKey } from "./composite-key.js"; +import type { SqlDb } from "./sql-db.js"; + +/** + * Per-dialect SQL hooks for a full-text sub-index. + * + * Only the table DDL and the search SQL differ between SQL backends; everything else (CRUD, path filters, + * doc-id resolution, enumeration) is shared in {@link createSqlFtsRetriever}. + */ +export interface SqlFtsDialect { + /** + * DDL statements to create the FTS table and any auxiliary indexes. Runs once per sub-index on `init()`. + * Each returned string is `exec()`'d in order. + */ + createTableDdl(opts: { tableName: string; info: FullTextIndexInfo }): string[]; + + /** + * Execute a lexical search for a single query string. The base iterates `queries[]` and merges by best score. + */ + search(opts: { + db: SqlDb; + tableName: string; + docsTable: string; + info: FullTextIndexInfo; + query: string; + paths: DocumentPath[] | undefined; + topK: number; + }): Promise>; +} + +export interface SqlFtsRetrieverOptions { + db: SqlDb; + prefix: string; + docsTable: string; + info: FullTextIndexInfo; + dialect: SqlFtsDialect; +} + +/** + * SQL-backed `FullTextIndex` implementation shared by every SQL backend. Per-dialect differences are + * supplied via {@link SqlFtsDialect}. Callers typically expose this as `XxxFullTextIndex` in their barrel. + */ +export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextIndex & { + readonly tableName: string; + init(): Promise; +} { + const { db, prefix, docsTable, info, dialect } = opts; + const tableName = `idx_${prefix}_fts`; + let closed = false; + + const ensureOpen = (): void => { + if (closed) throw new Error("FullTextIndex is closed"); + }; + + const resolveDocId = async (path: DocumentPath): Promise => { + await db.query( + `INSERT INTO ${docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, + [path], + ); + const rows = await db.query<{ doc_id: number }>( + `SELECT doc_id FROM ${docsTable} WHERE path = $1`, + [path], + ); + return rows[0]?.doc_id ?? -1; + }; + + return { + tableName, + + async init(): Promise { + for (const stmt of dialect.createTableDdl({ tableName, info })) { + await db.exec(stmt); + } + }, + + async getIndexInfo(): Promise { + ensureOpen(); + return { ...info }; + }, + + async *search(params: FullTextSearchParams): AsyncGenerator { + ensureOpen(); + const { queries, topK, paths } = params; + if (!queries || queries.length === 0) return; + + const bestScores = new Map(); + for (const query of queries) { + const rows = await dialect.search({ db, tableName, docsTable, info, query, paths, topK }); + for (const row of rows) { + const key = compositeKey(row.path, row.blockId); + const existing = bestScores.get(key); + if (!existing || row.score > existing.score) { + bestScores.set(key, { + path: row.path, + blockId: row.blockId, + snippet: row.content, + score: row.score, + }); + } + } + } + + const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score); + for (const r of sorted.slice(0, topK)) yield r; + }, + + async addDocument(blocks: FullTextBlock[]): Promise { + ensureOpen(); + for (const block of blocks) { + const docId = await resolveDocId(block.path); + const metaJson = block.metadata ? JSON.stringify(block.metadata) : null; + await db.query(`DELETE FROM ${tableName} WHERE doc_id = $1 AND block_id = $2`, [ + docId, + block.blockId, + ]); + await db.query( + `INSERT INTO ${tableName} (doc_id, block_id, content, metadata) VALUES ($1, $2, $3, $4)`, + [docId, block.blockId, block.content, metaJson], + ); + } + }, + + async addDocuments( + blocks: Iterable | AsyncIterable, + ): Promise { + ensureOpen(); + for await (const batch of blocks) await this.addDocument(batch); + }, + + async deleteDocuments( + pathSelectors: PathSelector[] | AsyncIterable, + ): Promise { + ensureOpen(); + for await (const sel of toAsyncIterable(pathSelectors)) { + if (sel.blockId !== undefined) { + await db.query( + `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE path = $1) AND block_id = $2`, + [sel.path, sel.blockId], + ); + } else { + await db.query( + `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE path LIKE $1 || '%')`, + [sel.path], + ); + } + } + }, + + async getSize(pathPrefix?: DocumentPath): Promise { + ensureOpen(); + if (pathPrefix !== undefined) { + const rows = await db.query<{ cnt: number | bigint }>( + `SELECT COUNT(*) AS cnt FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, + [pathPrefix], + ); + return Number(rows[0]?.cnt ?? 0); + } + const rows = await db.query<{ cnt: number | bigint }>( + `SELECT COUNT(*) AS cnt FROM ${tableName}`, + ); + return Number(rows[0]?.cnt ?? 0); + }, + + async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const sql = + pathPrefix !== undefined + ? `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` + : `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const rows = await db.query<{ path: string }>(sql, params); + for (const row of rows) yield row.path as DocumentPath; + }, + + async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const sql = + pathPrefix !== undefined + ? `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` + : `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const rows = await db.query<{ path: string; block_id: string }>(sql, params); + for (const row of rows) { + yield { path: row.path as DocumentPath, blockId: row.block_id }; + } + }, + + async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const sql = + pathPrefix !== undefined + ? `SELECT d.path, b.block_id, b.content, b.metadata FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` + : `SELECT d.path, b.block_id, b.content, b.metadata FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const rows = await db.query<{ + path: string; + block_id: string; + content: string; + metadata: string | null; + }>(sql, params); + for (const row of rows) { + yield { + path: row.path as DocumentPath, + blockId: row.block_id, + content: row.content, + metadata: row.metadata ? (JSON.parse(row.metadata) as Metadata) : undefined, + }; + } + }, + + async close(_options?: { force?: boolean }): Promise { + closed = true; + }, + + async flush(): Promise { + ensureOpen(); + }, + + async deleteIndex(): Promise { + ensureOpen(); + closed = true; + await db.exec(`DROP TABLE IF EXISTS ${tableName}`); + }, + }; +} diff --git a/packages/indexer-core/src/create-sql-vector-retriever.ts b/packages/indexer-core/src/create-sql-vector-retriever.ts new file mode 100644 index 0000000..9f1f84d --- /dev/null +++ b/packages/indexer-core/src/create-sql-vector-retriever.ts @@ -0,0 +1,267 @@ +import type { + BlockReference, + DocumentPath, + EmbeddingBlock, + EmbeddingIndex, + EmbeddingIndexInfo, + EmbeddingSearchParams, + EmbeddingSearchResult, + PathSelector, +} from "@statewalker/indexer-api"; +import { toAsyncIterable } from "./async.js"; +import { compositeKey } from "./composite-key.js"; +import type { SqlDb } from "./sql-db.js"; +import { validateDimensionality } from "./validate-dimensionality.js"; + +/** + * Per-dialect SQL hooks for a vector sub-index. + * + * Only the table/HNSW DDL, the search SQL, embedding binding strategy, and the embedding-row decode + * differ between SQL backends; everything else (CRUD, path filters, doc-id resolution, enumeration) + * is shared in {@link createSqlVectorRetriever}. + */ +export interface SqlVectorDialect { + /** + * DDL statements to create the vector table and its HNSW index. Runs once per sub-index on `init()`. + */ + createTableDdl(opts: { + tableName: string; + indexName: string; + info: EmbeddingIndexInfo; + }): string[]; + + /** Driver-native binding for an embedding value used in INSERT / SELECT parameter slots. */ + bindEmbedding(embedding: Float32Array): unknown; + + /** + * The SQL cast suffix for the embedding param; e.g. `::FLOAT[1536]` (DuckDB) or `::vector(1536)` (PGlite). + * Appended to the embedding parameter placeholder in INSERT and search SQL. + */ + embeddingCastSuffix(dim: number): string; + + /** Execute a cosine/ANN search for a single query vector. The base iterates embeddings and merges by best score. */ + search(opts: { + db: SqlDb; + tableName: string; + docsTable: string; + queryEmbedding: Float32Array; + paths: DocumentPath[] | undefined; + topK: number; + info: EmbeddingIndexInfo; + bindEmbedding(embedding: Float32Array): unknown; + embeddingCastSuffix(dim: number): string; + }): Promise>; + + /** Driver-specific decode of a raw embedding column value from a SELECT row into a Float32Array. */ + decodeEmbedding(raw: unknown): Float32Array; +} + +export interface SqlVectorRetrieverOptions { + db: SqlDb; + prefix: string; + docsTable: string; + info: EmbeddingIndexInfo; + dialect: SqlVectorDialect; +} + +/** + * SQL-backed `EmbeddingIndex` implementation shared by every SQL backend. Per-dialect differences are + * supplied via {@link SqlVectorDialect}. Callers typically expose this as `XxxVectorIndex` in their barrel. + */ +export function createSqlVectorRetriever(opts: SqlVectorRetrieverOptions): EmbeddingIndex & { + readonly tableName: string; + init(): Promise; +} { + const { db, prefix, docsTable, info, dialect } = opts; + const tableName = `idx_${prefix}_vec`; + const indexName = `idx_${prefix}_vec_hnsw`; + const dim = info.dimensionality; + let closed = false; + + const ensureOpen = (): void => { + if (closed) throw new Error("EmbeddingIndex is closed"); + }; + + const resolveDocId = async (path: DocumentPath): Promise => { + await db.query( + `INSERT INTO ${docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, + [path], + ); + const rows = await db.query<{ doc_id: number }>( + `SELECT doc_id FROM ${docsTable} WHERE path = $1`, + [path], + ); + return rows[0]?.doc_id ?? -1; + }; + + return { + tableName, + + async init(): Promise { + for (const stmt of dialect.createTableDdl({ tableName, indexName, info })) { + await db.exec(stmt); + } + }, + + async getIndexInfo(): Promise { + ensureOpen(); + return { ...info }; + }, + + async *search(params: EmbeddingSearchParams): AsyncGenerator { + ensureOpen(); + const { embeddings, topK, paths } = params; + if (!embeddings || embeddings.length === 0) return; + + const bestScores = new Map(); + + for (const queryEmb of embeddings) { + validateDimensionality(info, queryEmb); + const rows = await dialect.search({ + db, + tableName, + docsTable, + queryEmbedding: queryEmb, + paths, + topK, + info, + bindEmbedding: dialect.bindEmbedding, + embeddingCastSuffix: dialect.embeddingCastSuffix, + }); + for (const row of rows) { + const key = compositeKey(row.path, row.blockId); + const existing = bestScores.get(key); + if (!existing || row.score > existing.score) { + bestScores.set(key, { + path: row.path, + blockId: row.blockId, + score: row.score, + }); + } + } + } + + const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score); + for (const r of sorted.slice(0, topK)) yield r; + }, + + async addDocument(blocks: EmbeddingBlock[]): Promise { + ensureOpen(); + for (const block of blocks) { + validateDimensionality(info, block.embedding); + const docId = await resolveDocId(block.path); + const bound = dialect.bindEmbedding(block.embedding); + const cast = dialect.embeddingCastSuffix(dim); + + await db.query(`DELETE FROM ${tableName} WHERE doc_id = $1 AND block_id = $2`, [ + docId, + block.blockId, + ]); + await db.query( + `INSERT INTO ${tableName} (doc_id, block_id, embedding) VALUES ($1, $2, $3${cast})`, + [docId, block.blockId, bound], + ); + } + }, + + async addDocuments( + blocks: Iterable | AsyncIterable, + ): Promise { + ensureOpen(); + for await (const batch of blocks) await this.addDocument(batch); + }, + + async deleteDocuments( + pathSelectors: PathSelector[] | AsyncIterable, + ): Promise { + ensureOpen(); + for await (const sel of toAsyncIterable(pathSelectors)) { + if (sel.blockId !== undefined) { + await db.query( + `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE path = $1) AND block_id = $2`, + [sel.path, sel.blockId], + ); + } else { + await db.query( + `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE path LIKE $1 || '%')`, + [sel.path], + ); + } + } + }, + + async getSize(pathPrefix?: DocumentPath): Promise { + ensureOpen(); + if (pathPrefix !== undefined) { + const rows = await db.query<{ cnt: number | bigint }>( + `SELECT COUNT(*) AS cnt FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, + [pathPrefix], + ); + return Number(rows[0]?.cnt ?? 0); + } + const rows = await db.query<{ cnt: number | bigint }>( + `SELECT COUNT(*) AS cnt FROM ${tableName}`, + ); + return Number(rows[0]?.cnt ?? 0); + }, + + async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const sql = + pathPrefix !== undefined + ? `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` + : `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const rows = await db.query<{ path: string }>(sql, params); + for (const row of rows) yield row.path as DocumentPath; + }, + + async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const sql = + pathPrefix !== undefined + ? `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` + : `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const rows = await db.query<{ path: string; block_id: string }>(sql, params); + for (const row of rows) { + yield { path: row.path as DocumentPath, blockId: row.block_id }; + } + }, + + async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { + ensureOpen(); + const sql = + pathPrefix !== undefined + ? `SELECT d.path, b.block_id, b.embedding FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` + : `SELECT d.path, b.block_id, b.embedding FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = pathPrefix !== undefined ? [pathPrefix] : []; + const rows = await db.query<{ + path: string; + block_id: string; + embedding: unknown; + }>(sql, params); + for (const row of rows) { + yield { + path: row.path as DocumentPath, + blockId: row.block_id, + embedding: dialect.decodeEmbedding(row.embedding), + }; + } + }, + + async close(_options?: { force?: boolean }): Promise { + closed = true; + }, + + async flush(): Promise { + ensureOpen(); + }, + + async deleteIndex(): Promise { + ensureOpen(); + closed = true; + await db.exec(`DROP TABLE IF EXISTS ${tableName}`); + }, + }; +} diff --git a/packages/indexer-core/src/index.ts b/packages/indexer-core/src/index.ts index 8118813..87b9cd4 100644 --- a/packages/indexer-core/src/index.ts +++ b/packages/indexer-core/src/index.ts @@ -8,8 +8,24 @@ export { createPersistenceBackedIndexer, type PersistenceBackedIndexerOptions, } from "./create-persistence-backed-indexer.js"; +export { + createSqlBackedIndexer, + type SqlBackedDialect, + type SqlBackedIndexerOptions, +} from "./create-sql-backed-indexer.js"; +export { + createSqlFtsRetriever, + type SqlFtsDialect, + type SqlFtsRetrieverOptions, +} from "./create-sql-fts-retriever.js"; +export { + createSqlVectorRetriever, + type SqlVectorDialect, + type SqlVectorRetrieverOptions, +} from "./create-sql-vector-retriever.js"; export { mergeByRRF, mergeByWeights, mergeHybrid } from "./merge.js"; export { matchesPrefix } from "./path-prefix.js"; export { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; export { sanitizePrefix } from "./sanitize-prefix.js"; +export type { SqlDb } from "./sql-db.js"; export { validateDimensionality } from "./validate-dimensionality.js"; diff --git a/packages/indexer-core/src/sql-db.ts b/packages/indexer-core/src/sql-db.ts new file mode 100644 index 0000000..425195f --- /dev/null +++ b/packages/indexer-core/src/sql-db.ts @@ -0,0 +1,8 @@ +/** + * Minimal normalised async SQL client used by indexer-core's SQL retrievers. + * Each backend wraps its native driver (e.g. `@statewalker/db-api` `Db` or `PGlite`) to satisfy this shape. + */ +export interface SqlDb { + exec(sql: string): Promise; + query>(sql: string, params?: unknown[]): Promise; +} diff --git a/packages/indexer-duckdb/src/dialect.ts b/packages/indexer-duckdb/src/dialect.ts new file mode 100644 index 0000000..87b6812 --- /dev/null +++ b/packages/indexer-duckdb/src/dialect.ts @@ -0,0 +1,146 @@ +import type { Db } from "@statewalker/db-api"; +import type { + SqlBackedDialect, + SqlDb, + SqlFtsDialect, + SqlVectorDialect, +} from "@statewalker/indexer-core"; + +/** Adapt `@statewalker/db-api`'s `Db` to `@statewalker/indexer-core`'s minimal `SqlDb`. */ +export function wrapDbAsSqlDb(db: Db): SqlDb { + return { + exec: (sql) => db.exec(sql), + query: (sql: string, params?: unknown[]) => db.query(sql, params ?? []), + }; +} + +function embeddingToLiteral(embedding: Float32Array): string { + return `[${Array.from(embedding).join(",")}]`; +} + +/** + * DuckDB FTS dialect. + * + * Current implementation: LIKE-based scanning (word-count + rank-decay). Phase 8 replaces this with + * the official `fts` community extension (BM25 via `PRAGMA create_fts_index` + `match_bm25`). + */ +export const duckdbFtsDialect: SqlFtsDialect = { + createTableDdl({ tableName }) { + return [ + `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, metadata TEXT, PRIMARY KEY (doc_id, block_id))`, + ]; + }, + + async search({ db, tableName, docsTable, query, paths, topK }) { + const words = query + .toLowerCase() + .split(/\s+/) + .filter((w) => w.length > 0); + if (words.length === 0) return []; + + const likeParams = words.map((w) => `%${w}%`); + const conditions = words.map((_, i) => `LOWER(b.content) LIKE $${i + 1}`); + const scoreExpr = words + .map((_, i) => `CASE WHEN LOWER(b.content) LIKE $${i + 1} THEN 1 ELSE 0 END`) + .join(" + "); + + const allParams: unknown[] = [...likeParams]; + + let pathClause = ""; + if (paths && paths.length > 0) { + const pathOffset = allParams.length + 1; + pathClause = ` AND (${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")})`; + allParams.push(...(paths as string[])); + } + + const topKParam = `$${allParams.length + 1}`; + allParams.push(topK); + + const sql = `SELECT d.path, b.block_id, b.content, (${scoreExpr}) AS match_count FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE (${conditions.join(" OR ")})${pathClause} ORDER BY match_count DESC LIMIT ${topKParam}`; + + const rows = await db.query<{ + path: string; + block_id: string; + content: string; + match_count: number; + }>(sql, allParams); + + return rows.map((row, rank) => ({ + path: row.path as import("@statewalker/indexer-api").DocumentPath, + blockId: row.block_id, + content: row.content, + score: (row.match_count / words.length) * (1 - rank / (rows.length + 1)), + })); + }, +}; + +/** + * DuckDB vector dialect. + * + * Uses the `vss` extension's HNSW index with `array_cosine_distance` for cosine ANN search. + * Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound + * driver-native arrays for correctness and ingest speed. + */ +export const duckdbVectorDialect: SqlVectorDialect = { + createTableDdl({ tableName, indexName, info }) { + const dim = info.dimensionality; + return [ + `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding FLOAT[${dim}] NOT NULL, PRIMARY KEY (doc_id, block_id))`, + `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING HNSW (embedding)`, + ]; + }, + + bindEmbedding: embeddingToLiteral, + embeddingCastSuffix: (dim) => `::FLOAT[${dim}]`, + + async search({ db, tableName, docsTable, queryEmbedding, paths, topK, info, bindEmbedding, embeddingCastSuffix }) { + const vecLiteral = bindEmbedding(queryEmbedding); + const dim = info.dimensionality; + + const allParams: unknown[] = [vecLiteral]; + let pathClause = ""; + if (paths && paths.length > 0) { + const pathOffset = allParams.length + 1; + pathClause = `WHERE ${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")} `; + allParams.push(...(paths as string[])); + } + + const topKParam = `$${allParams.length + 1}`; + allParams.push(topK); + + const rows = await db.query<{ path: string; block_id: string; dist: number }>( + `SELECT d.path, b.block_id, array_cosine_distance(b.embedding, $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`, + allParams, + ); + + return rows.map((row) => ({ + path: row.path as import("@statewalker/indexer-api").DocumentPath, + blockId: row.block_id, + score: 1 - row.dist, + })); + }, + + decodeEmbedding(raw) { + return new Float32Array(raw as number[]); + }, +}; + +/** Aggregate DuckDB dialect for `createSqlBackedIndexer`. */ +export const duckdbDialect: SqlBackedDialect = { + extensionInit: [ + "INSTALL vss; LOAD vss;", + "SET hnsw_enable_experimental_persistence = true;", + ], + docsTableDdl(prefix) { + return [ + `CREATE SEQUENCE IF NOT EXISTS idx_${prefix}_docs_seq START 1`, + `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id INTEGER PRIMARY KEY DEFAULT(nextval('idx_${prefix}_docs_seq')), path TEXT NOT NULL UNIQUE)`, + ]; + }, + extraCleanup(prefix) { + return [`DROP SEQUENCE IF EXISTS idx_${prefix}_docs_seq`]; + }, + unionAliasSuffix: "", + fts: duckdbFtsDialect, + vec: duckdbVectorDialect, +}; diff --git a/packages/indexer-duckdb/src/duckdb-full-text-index.ts b/packages/indexer-duckdb/src/duckdb-full-text-index.ts index 6ed2d4d..daddb56 100644 --- a/packages/indexer-duckdb/src/duckdb-full-text-index.ts +++ b/packages/indexer-duckdb/src/duckdb-full-text-index.ts @@ -1,252 +1,24 @@ import type { Db } from "@statewalker/db-api"; -import type { - BlockReference, - DocumentPath, - FullTextBlock, - FullTextIndex, - FullTextIndexInfo, - FullTextSearchParams, - FullTextSearchResult, - Metadata, - PathSelector, -} from "@statewalker/indexer-api"; -import { toAsyncIterable } from "@statewalker/indexer-core"; +import type { FullTextIndex, FullTextIndexInfo } from "@statewalker/indexer-api"; +import { createSqlFtsRetriever } from "@statewalker/indexer-core"; +import { duckdbFtsDialect, wrapDbAsSqlDb } from "./dialect.js"; -export class DuckDbFullTextIndex implements FullTextIndex { - private readonly db: Db; +export type DuckDbFullTextIndex = FullTextIndex & { readonly tableName: string; - private readonly docsTable: string; - private readonly info: FullTextIndexInfo; - private closed = false; - - constructor(db: Db, prefix: string, docsTable: string, info: FullTextIndexInfo) { - this.db = db; - this.tableName = `idx_${prefix}_fts`; - this.docsTable = docsTable; - this.info = info; - } - - async init(): Promise { - await this.db.exec( - `CREATE TABLE IF NOT EXISTS ${this.tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, metadata TEXT, PRIMARY KEY (doc_id, block_id))`, - ); - } - - private ensureOpen(): void { - if (this.closed) { - throw new Error("FullTextIndex is closed"); - } - } - - private async resolveDocId(path: DocumentPath): Promise { - await this.db.query( - `INSERT INTO ${this.docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, - [path], - ); - const rows = await this.db.query<{ doc_id: number }>( - `SELECT doc_id FROM ${this.docsTable} WHERE path = $1`, - [path], - ); - return rows[0]?.doc_id ?? -1; - } - - private pathFilterClause( - paths: DocumentPath[] | undefined, - paramOffset: number, - ): { sql: string; params: string[] } { - if (!paths || paths.length === 0) return { sql: "", params: [] }; - const conditions = paths.map((_, i) => `d.path LIKE $${paramOffset + i} || '%'`); - return { - sql: ` AND (${conditions.join(" OR ")})`, - params: paths as string[], - }; - } - - async getIndexInfo(): Promise { - this.ensureOpen(); - return { ...this.info }; - } - - async *search(params: FullTextSearchParams): AsyncGenerator { - this.ensureOpen(); - const { queries, topK, paths } = params; - - if (!queries || queries.length === 0) return; - - const bestScores = new Map(); - - for (const query of queries) { - const words = query - .toLowerCase() - .split(/\s+/) - .filter((w) => w.length > 0); - if (words.length === 0) continue; - - const likeParams = words.map((w) => `%${w}%`); - const conditions = words.map((_, i) => `LOWER(b.content) LIKE $${i + 1}`); - const scoreExpr = words - .map((_, i) => `CASE WHEN LOWER(b.content) LIKE $${i + 1} THEN 1 ELSE 0 END`) - .join(" + "); - - const allParams: (string | number)[] = [...likeParams]; - - const pathFilter = this.pathFilterClause(paths, allParams.length + 1); - allParams.push(...pathFilter.params); - - const topKParam = `$${allParams.length + 1}`; - allParams.push(topK); - - const sql = `SELECT d.path, b.block_id, b.content, (${scoreExpr}) AS match_count FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE (${conditions.join(" OR ")})${pathFilter.sql} ORDER BY match_count DESC LIMIT ${topKParam}`; - - const rows = await this.db.query<{ - path: string; - block_id: string; - content: string; - match_count: number; - }>(sql, allParams); - - for (let rank = 0; rank < rows.length; rank++) { - const row = rows[rank]; - if (!row) continue; - const key = `${row.path}\0${row.block_id}`; - const score = (row.match_count / words.length) * (1 - rank / (rows.length + 1)); - const existing = bestScores.get(key); - if (!existing || score > existing.score) { - bestScores.set(key, { - path: row.path as DocumentPath, - blockId: row.block_id, - snippet: row.content, - score, - }); - } - } - } - - const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score); - for (const r of sorted.slice(0, topK)) { - yield r; - } - } - - async addDocument(blocks: FullTextBlock[]): Promise { - this.ensureOpen(); - for (const block of blocks) { - const docId = await this.resolveDocId(block.path); - const metaJson = block.metadata ? JSON.stringify(block.metadata) : null; - await this.db.query(`DELETE FROM ${this.tableName} WHERE doc_id = $1 AND block_id = $2`, [ - docId, - block.blockId, - ]); - await this.db.query( - `INSERT INTO ${this.tableName} (doc_id, block_id, content, metadata) VALUES ($1, $2, $3, $4)`, - [docId, block.blockId, block.content, metaJson], - ); - } - } - - async addDocuments( - blocks: Iterable | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const batch of blocks) { - await this.addDocument(batch); - } - } - - async deleteDocuments( - pathSelectors: PathSelector[] | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const sel of toAsyncIterable(pathSelectors)) { - if (sel.blockId !== undefined) { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, - [sel.path, sel.blockId], - ); - } else { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path LIKE $1 || '%')`, - [sel.path], - ); - } - } - } - - async getSize(pathPrefix?: DocumentPath): Promise { - this.ensureOpen(); - if (pathPrefix !== undefined) { - const rows = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, - [pathPrefix], - ); - return Number(rows[0]?.cnt ?? 0); - } - const rows = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName}`, - ); - return Number(rows[0]?.cnt ?? 0); - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await this.db.query<{ path: string }>(sql, params); - for (const row of rows) { - yield row.path as DocumentPath; - } - } - - async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await this.db.query<{ path: string; block_id: string }>(sql, params); - for (const row of rows) { - yield { path: row.path as DocumentPath, blockId: row.block_id }; - } - } - - async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id, b.content, b.metadata FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id, b.content, b.metadata FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await this.db.query<{ - path: string; - block_id: string; - content: string; - metadata: string | null; - }>(sql, params); - for (const row of rows) { - yield { - path: row.path as DocumentPath, - blockId: row.block_id, - content: row.content, - metadata: row.metadata ? (JSON.parse(row.metadata) as Metadata) : undefined, - }; - } - } - - async close(_options?: { force?: boolean }): Promise { - this.closed = true; - } - - async flush(): Promise { - this.ensureOpen(); - } - - async deleteIndex(): Promise { - this.ensureOpen(); - this.closed = true; - await this.db.exec(`DROP TABLE IF EXISTS ${this.tableName}`); - } + init(): Promise; +}; + +export function createDuckDbFullTextIndex( + db: Db, + prefix: string, + docsTable: string, + info: FullTextIndexInfo, +): DuckDbFullTextIndex { + return createSqlFtsRetriever({ + db: wrapDbAsSqlDb(db), + prefix, + docsTable, + info, + dialect: duckdbFtsDialect, + }); } diff --git a/packages/indexer-duckdb/src/duckdb-indexer.ts b/packages/indexer-duckdb/src/duckdb-indexer.ts index 4c109ff..9136ba5 100644 --- a/packages/indexer-duckdb/src/duckdb-indexer.ts +++ b/packages/indexer-duckdb/src/duckdb-indexer.ts @@ -1,228 +1,15 @@ import type { Db } from "@statewalker/db-api"; -import type { - CreateIndexParams, - DocumentPath, - Index, - Indexer, - IndexInfo, -} from "@statewalker/indexer-api"; -import { createCompositeIndex, sanitizePrefix } from "@statewalker/indexer-core"; -import { DuckDbFullTextIndex } from "./duckdb-full-text-index.js"; -import { DuckDbVectorIndex } from "./duckdb-vector-index.js"; +import type { Indexer } from "@statewalker/indexer-api"; +import { createSqlBackedIndexer } from "@statewalker/indexer-core"; +import { duckdbDialect, wrapDbAsSqlDb } from "./dialect.js"; export interface DuckDbIndexerOptions { db: Db; } -export async function createDuckDbIndexer(options: DuckDbIndexerOptions): Promise { - const { db } = options; - const indexes = new Map(); - const manifest = new Map(); - let closed = false; - - await db.exec("INSTALL vss; LOAD vss;"); - await db.exec("SET hnsw_enable_experimental_persistence = true;"); - - await db.exec( - "CREATE TABLE IF NOT EXISTS __indexer_manifest (name TEXT PRIMARY KEY, config TEXT NOT NULL)", - ); - - const existingEntries = await db.query<{ name: string; config: string }>( - "SELECT name, config FROM __indexer_manifest", - ); - for (const entry of existingEntries) { - manifest.set(entry.name, { name: entry.name }); - } - - function ensureOpen(): void { - if (closed) throw new Error("Indexer is closed"); - } - - async function ensureDocsSequence(prefix: string): Promise { - await db.exec(`CREATE SEQUENCE IF NOT EXISTS idx_${prefix}_docs_seq START 1`); - } - - async function createDocsTable(prefix: string): Promise { - const docsTable = `idx_${prefix}_docs`; - await db.exec( - `CREATE TABLE IF NOT EXISTS ${docsTable} (doc_id INTEGER PRIMARY KEY DEFAULT(nextval('${docsTable}_seq')), path TEXT NOT NULL UNIQUE)`, - ); - return docsTable; - } - - function buildIndex( - name: string, - docsTable: string, - fts: DuckDbFullTextIndex | null, - vec: DuckDbVectorIndex | null, - ): Index { - return createCompositeIndex({ - name, - fts, - vec, - getSize: async (pathPrefix?: DocumentPath): Promise => { - if (fts !== null && vec !== null) { - const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause})`; - const rows = await db.query<{ cnt: number | bigint }>(sql, params); - return Number(rows[0]?.cnt ?? 0); - } - if (fts !== null) return fts.getSize(pathPrefix); - if (vec !== null) return vec.getSize(pathPrefix); - return 0; - }, - onDeleteIndex: async () => { - await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); - }, - }); - } - - const indexer: Indexer = { - async getIndexNames(): Promise { - ensureOpen(); - return [...manifest.values()]; - }, - - async createIndex(params: CreateIndexParams): Promise { - ensureOpen(); - const { name, fulltext, vector, overwrite } = params; - - if (!fulltext && !vector) { - throw new Error("At least one of fulltext or vector must be provided"); - } - - if (indexes.has(name) || manifest.has(name)) { - if (overwrite) { - const old = indexes.get(name); - if (old) await old.close(); - indexes.delete(name); - manifest.delete(name); - const prefix = sanitizePrefix(name); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_fts`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_vec`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_docs`); - await db.exec(`DROP SEQUENCE IF EXISTS idx_${prefix}_docs_seq`); - await db.exec( - `DELETE FROM __indexer_manifest WHERE name = '${name.replace(/'/g, "''")}'`, - ); - } else { - throw new Error(`Index "${name}" already exists`); - } - } - - const prefix = sanitizePrefix(name); - await ensureDocsSequence(prefix); - const docsTable = await createDocsTable(prefix); - - const fts = fulltext - ? new DuckDbFullTextIndex(db, prefix, docsTable, { - language: fulltext.language, - metadata: fulltext.metadata, - }) - : null; - - const vec = vector - ? new DuckDbVectorIndex(db, prefix, docsTable, { - dimensionality: vector.dimensionality, - model: vector.model, - metadata: vector.metadata, - }) - : null; - - if (fts) await fts.init(); - if (vec) await vec.init(); - - await db.query("INSERT INTO __indexer_manifest (name, config) VALUES ($1, $2)", [ - name, - JSON.stringify({ fulltext, vector }), - ]); - - const index = buildIndex(name, docsTable, fts, vec); - indexes.set(name, index); - manifest.set(name, { name }); - return index; - }, - - async getIndex(name: string): Promise { - ensureOpen(); - if (indexes.has(name)) return indexes.get(name) ?? null; - if (!manifest.has(name)) return null; - - const rows = await db.query<{ config: string }>( - "SELECT config FROM __indexer_manifest WHERE name = $1", - [name], - ); - if (rows.length === 0) return null; - - const config = JSON.parse(rows[0]?.config ?? "{}") as { - fulltext?: { language: string; metadata?: Record }; - vector?: { - dimensionality: number; - model: string; - metadata?: Record; - }; - }; - - const prefix = sanitizePrefix(name); - await ensureDocsSequence(prefix); - const docsTable = await createDocsTable(prefix); - - const fts = config.fulltext - ? new DuckDbFullTextIndex(db, prefix, docsTable, { - language: config.fulltext.language, - metadata: config.fulltext.metadata, - }) - : null; - - const vec = config.vector - ? new DuckDbVectorIndex(db, prefix, docsTable, { - dimensionality: config.vector.dimensionality, - model: config.vector.model, - metadata: config.vector.metadata, - }) - : null; - - const index = buildIndex(name, docsTable, fts, vec); - indexes.set(name, index); - return index; - }, - - async hasIndex(name: string): Promise { - ensureOpen(); - return manifest.has(name); - }, - - async deleteIndex(name: string): Promise { - ensureOpen(); - const index = indexes.get(name); - if (index) { - await index.close(); - indexes.delete(name); - } - if (manifest.has(name)) { - const prefix = sanitizePrefix(name); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_fts`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_vec`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_docs`); - await db.exec(`DROP SEQUENCE IF EXISTS idx_${prefix}_docs_seq`); - await db.exec(`DELETE FROM __indexer_manifest WHERE name = '${name.replace(/'/g, "''")}'`); - manifest.delete(name); - } - }, - - async flush(): Promise { - ensureOpen(); - }, - - async close(): Promise { - if (closed) return; - closed = true; - for (const index of indexes.values()) await index.close(); - indexes.clear(); - manifest.clear(); - }, - }; - - return indexer; +export function createDuckDbIndexer(options: DuckDbIndexerOptions): Promise { + return createSqlBackedIndexer({ + db: wrapDbAsSqlDb(options.db), + dialect: duckdbDialect, + }); } diff --git a/packages/indexer-duckdb/src/duckdb-vector-index.ts b/packages/indexer-duckdb/src/duckdb-vector-index.ts index 18f0363..8608f43 100644 --- a/packages/indexer-duckdb/src/duckdb-vector-index.ts +++ b/packages/indexer-duckdb/src/duckdb-vector-index.ts @@ -1,243 +1,24 @@ import type { Db } from "@statewalker/db-api"; -import type { - BlockReference, - DocumentPath, - EmbeddingBlock, - EmbeddingIndex, - EmbeddingIndexInfo, - EmbeddingSearchParams, - EmbeddingSearchResult, - PathSelector, -} from "@statewalker/indexer-api"; -import { toAsyncIterable, validateDimensionality } from "@statewalker/indexer-core"; +import type { EmbeddingIndex, EmbeddingIndexInfo } from "@statewalker/indexer-api"; +import { createSqlVectorRetriever } from "@statewalker/indexer-core"; +import { duckdbVectorDialect, wrapDbAsSqlDb } from "./dialect.js"; -export class DuckDbVectorIndex implements EmbeddingIndex { - private readonly db: Db; +export type DuckDbVectorIndex = EmbeddingIndex & { readonly tableName: string; - private readonly indexName: string; - private readonly docsTable: string; - private readonly info: EmbeddingIndexInfo; - private closed = false; - - constructor(db: Db, prefix: string, docsTable: string, info: EmbeddingIndexInfo) { - this.db = db; - this.tableName = `idx_${prefix}_vec`; - this.indexName = `idx_${prefix}_vec_hnsw`; - this.docsTable = docsTable; - this.info = info; - } - - async init(): Promise { - const dim = this.info.dimensionality; - await this.db.exec( - `CREATE TABLE IF NOT EXISTS ${this.tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding FLOAT[${dim}] NOT NULL, PRIMARY KEY (doc_id, block_id))`, - ); - await this.db.exec( - `CREATE INDEX IF NOT EXISTS ${this.indexName} ON ${this.tableName} USING HNSW (embedding)`, - ); - } - - private ensureOpen(): void { - if (this.closed) { - throw new Error("EmbeddingIndex is closed"); - } - } - - private embeddingToSql(embedding: Float32Array): string { - return `[${Array.from(embedding).join(",")}]`; - } - - private async resolveDocId(path: DocumentPath): Promise { - await this.db.query( - `INSERT INTO ${this.docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, - [path], - ); - const rows = await this.db.query<{ doc_id: number }>( - `SELECT doc_id FROM ${this.docsTable} WHERE path = $1`, - [path], - ); - return rows[0]?.doc_id ?? -1; - } - - async getIndexInfo(): Promise { - this.ensureOpen(); - return { ...this.info }; - } - - async *search(params: EmbeddingSearchParams): AsyncGenerator { - this.ensureOpen(); - const { embeddings, topK, paths } = params; - - if (!embeddings || embeddings.length === 0) return; - - const bestScores = new Map(); - const dim = this.info.dimensionality; - - for (const queryEmb of embeddings) { - validateDimensionality(this.info,queryEmb); - const vecLiteral = this.embeddingToSql(queryEmb); - - let pathClause = ""; - const queryParams: (string | number)[] = [vecLiteral]; - - if (paths && paths.length > 0) { - const pathConditions = paths.map((_, i) => `d.path LIKE $${i + 2} || '%'`); - pathClause = `WHERE ${pathConditions.join(" OR ")} `; - queryParams.push(...(paths as string[])); - } - - const topKParam = `$${queryParams.length + 1}`; - queryParams.push(topK); - - const rows = await this.db.query<{ - path: string; - block_id: string; - dist: number; - }>( - `SELECT d.path, b.block_id, array_cosine_distance(b.embedding, $1::FLOAT[${dim}]) AS dist FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`, - queryParams, - ); - - for (const row of rows) { - const key = `${row.path}\0${row.block_id}`; - const score = 1 - row.dist; - const existing = bestScores.get(key); - if (!existing || score > existing.score) { - bestScores.set(key, { - path: row.path as DocumentPath, - blockId: row.block_id, - score, - }); - } - } - } - - const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score); - for (const r of sorted.slice(0, topK)) { - yield r; - } - } - - async addDocument(blocks: EmbeddingBlock[]): Promise { - this.ensureOpen(); - for (const block of blocks) { - validateDimensionality(this.info,block.embedding); - const docId = await this.resolveDocId(block.path); - const dim = this.info.dimensionality; - const vecLiteral = this.embeddingToSql(block.embedding); - - await this.db.query(`DELETE FROM ${this.tableName} WHERE doc_id = $1 AND block_id = $2`, [ - docId, - block.blockId, - ]); - await this.db.query( - `INSERT INTO ${this.tableName} (doc_id, block_id, embedding) VALUES ($1, $2, $3::FLOAT[${dim}])`, - [docId, block.blockId, vecLiteral], - ); - } - } - - async addDocuments( - blocks: Iterable | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const batch of blocks) { - await this.addDocument(batch); - } - } - - async deleteDocuments( - pathSelectors: PathSelector[] | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const sel of toAsyncIterable(pathSelectors)) { - if (sel.blockId !== undefined) { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, - [sel.path, sel.blockId], - ); - } else { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path LIKE $1 || '%')`, - [sel.path], - ); - } - } - } - - async getSize(pathPrefix?: DocumentPath): Promise { - this.ensureOpen(); - if (pathPrefix !== undefined) { - const rows = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, - [pathPrefix], - ); - return Number(rows[0]?.cnt ?? 0); - } - const rows = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName}`, - ); - return Number(rows[0]?.cnt ?? 0); - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await this.db.query<{ path: string }>(sql, params); - for (const row of rows) { - yield row.path as DocumentPath; - } - } - - async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await this.db.query<{ path: string; block_id: string }>(sql, params); - for (const row of rows) { - yield { path: row.path as DocumentPath, blockId: row.block_id }; - } - } - - async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id, b.embedding FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id, b.embedding FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await this.db.query<{ - path: string; - block_id: string; - embedding: number[]; - }>(sql, params); - for (const row of rows) { - yield { - path: row.path as DocumentPath, - blockId: row.block_id, - embedding: new Float32Array(row.embedding), - }; - } - } - - async close(_options?: { force?: boolean }): Promise { - this.closed = true; - } - - async flush(): Promise { - this.ensureOpen(); - } - - async deleteIndex(): Promise { - this.ensureOpen(); - this.closed = true; - await this.db.exec(`DROP TABLE IF EXISTS ${this.tableName}`); - } + init(): Promise; +}; + +export function createDuckDbVectorIndex( + db: Db, + prefix: string, + docsTable: string, + info: EmbeddingIndexInfo, +): DuckDbVectorIndex { + return createSqlVectorRetriever({ + db: wrapDbAsSqlDb(db), + prefix, + docsTable, + info, + dialect: duckdbVectorDialect, + }); } diff --git a/packages/indexer-pglite/src/dialect.ts b/packages/indexer-pglite/src/dialect.ts new file mode 100644 index 0000000..a95630e --- /dev/null +++ b/packages/indexer-pglite/src/dialect.ts @@ -0,0 +1,161 @@ +import type { PGlite } from "@electric-sql/pglite"; +import type { + SqlBackedDialect, + SqlDb, + SqlFtsDialect, + SqlVectorDialect, +} from "@statewalker/indexer-core"; + +/** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */ +export function wrapDbAsSqlDb(db: PGlite): SqlDb { + return { + exec: (sql) => db.exec(sql).then(() => undefined), + async query(sql: string, params?: unknown[]): Promise { + const { rows } = await db.query(sql, params ?? []); + return rows; + }, + }; +} + +const LANGUAGE_MAP: Record = { + en: "english", + fr: "french", + de: "german", + es: "spanish", + it: "italian", + pt: "portuguese", + nl: "dutch", + ru: "russian", + sv: "swedish", + no: "norwegian", + da: "danish", + fi: "finnish", + hu: "hungarian", + ro: "romanian", + tr: "turkish", + simple: "simple", +}; + +function resolvePgLanguage(lang: string): string { + return LANGUAGE_MAP[lang] ?? lang; +} + +function embeddingToLiteral(embedding: Float32Array): string { + return `[${Array.from(embedding).join(",")}]`; +} + +/** + * PGlite FTS dialect — uses native PostgreSQL TSVECTOR with a GIN index, `ts_rank_cd` scoring, + * and `to_tsquery` with a language-specific stemmer. + */ +export const pgliteFtsDialect: SqlFtsDialect = { + createTableDdl({ tableName, info }) { + const pgLang = resolvePgLanguage(info.language); + return [ + `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`, + `CREATE INDEX IF NOT EXISTS ${tableName}_tsv_idx ON ${tableName} USING GIN (content_tsv)`, + ]; + }, + + async search({ db, tableName, docsTable, info, query, paths, topK }) { + const pgLang = resolvePgLanguage(info.language); + const validWords = query + .toLowerCase() + .split(/\s+/) + .map((w) => w.replace(/[^a-zA-Z0-9]/g, "")) + .filter((w) => w.length > 0); + if (validWords.length === 0) return []; + + const orTerms = validWords.join(" | "); + + const allParams: unknown[] = [orTerms]; + let pathClause = ""; + if (paths && paths.length > 0) { + const pathOffset = allParams.length + 1; + pathClause = ` AND (${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")})`; + allParams.push(...(paths as string[])); + } + + const topKParam = `$${allParams.length + 1}`; + allParams.push(topK); + + const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${pgLang}', $1)) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${pgLang}', $1)${pathClause} ORDER BY score DESC LIMIT ${topKParam}`; + + const rows = await db.query<{ + path: string; + block_id: string; + content: string; + score: number; + }>(sql, allParams); + + return rows.map((row) => ({ + path: row.path as import("@statewalker/indexer-api").DocumentPath, + blockId: row.block_id, + content: row.content, + score: row.score, + })); + }, +}; + +/** + * PGlite vector dialect — uses pgvector's HNSW index with the `<=>` cosine-distance operator. + * Embeddings are currently bound as stringified array literals — Phase 9 switches to parameter-bound + * driver-native arrays. + */ +export const pgliteVectorDialect: SqlVectorDialect = { + createTableDdl({ tableName, indexName, info }) { + const dim = info.dimensionality; + return [ + `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`, + `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING hnsw (embedding vector_cosine_ops)`, + ]; + }, + + bindEmbedding: embeddingToLiteral, + embeddingCastSuffix: (dim) => `::vector(${dim})`, + + async search({ db, tableName, docsTable, queryEmbedding, paths, topK, info, bindEmbedding, embeddingCastSuffix }) { + const vecLiteral = bindEmbedding(queryEmbedding); + const dim = info.dimensionality; + + const allParams: unknown[] = [vecLiteral]; + let pathClause = ""; + if (paths && paths.length > 0) { + const pathOffset = allParams.length + 1; + pathClause = `WHERE ${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")} `; + allParams.push(...(paths as string[])); + } + + const topKParam = `$${allParams.length + 1}`; + allParams.push(topK); + + const rows = await db.query<{ path: string; block_id: string; dist: number }>( + `SELECT d.path, b.block_id, (b.embedding <=> $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`, + allParams, + ); + + return rows.map((row) => ({ + path: row.path as import("@statewalker/indexer-api").DocumentPath, + blockId: row.block_id, + score: 1 - row.dist, + })); + }, + + decodeEmbedding(raw) { + // PGlite returns vector as a string "[1,2,3]". + return new Float32Array(JSON.parse(raw as string) as number[]); + }, +}; + +/** Aggregate PGlite dialect for `createSqlBackedIndexer`. */ +export const pgliteDialect: SqlBackedDialect = { + extensionInit: ["CREATE EXTENSION IF NOT EXISTS vector"], + docsTableDdl(prefix) { + return [ + `CREATE TABLE IF NOT EXISTS idx_${prefix}_docs (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`, + ]; + }, + unionAliasSuffix: " AS combined", + fts: pgliteFtsDialect, + vec: pgliteVectorDialect, +}; diff --git a/packages/indexer-pglite/src/pglite-full-text-index.ts b/packages/indexer-pglite/src/pglite-full-text-index.ts index ae9f728..a5db48f 100644 --- a/packages/indexer-pglite/src/pglite-full-text-index.ts +++ b/packages/indexer-pglite/src/pglite-full-text-index.ts @@ -1,275 +1,24 @@ import type { PGlite } from "@electric-sql/pglite"; -import type { - BlockReference, - DocumentPath, - FullTextBlock, - FullTextIndex, - FullTextIndexInfo, - FullTextSearchParams, - FullTextSearchResult, - Metadata, - PathSelector, -} from "@statewalker/indexer-api"; -import { toAsyncIterable } from "@statewalker/indexer-core"; +import type { FullTextIndex, FullTextIndexInfo } from "@statewalker/indexer-api"; +import { createSqlFtsRetriever } from "@statewalker/indexer-core"; +import { pgliteFtsDialect, wrapDbAsSqlDb } from "./dialect.js"; -const LANGUAGE_MAP: Record = { - en: "english", - fr: "french", - de: "german", - es: "spanish", - it: "italian", - pt: "portuguese", - nl: "dutch", - ru: "russian", - sv: "swedish", - no: "norwegian", - da: "danish", - fi: "finnish", - hu: "hungarian", - ro: "romanian", - tr: "turkish", - simple: "simple", -}; - -function resolvePgLanguage(lang: string): string { - return LANGUAGE_MAP[lang] ?? lang; -} - -export class PGLiteFullTextIndex implements FullTextIndex { - private readonly db: PGlite; +export type PGLiteFullTextIndex = FullTextIndex & { readonly tableName: string; - private readonly docsTable: string; - private readonly info: FullTextIndexInfo; - private readonly pgLang: string; - private closed = false; - - constructor(db: PGlite, prefix: string, docsTable: string, info: FullTextIndexInfo) { - this.db = db; - this.tableName = `idx_${prefix}_fts`; - this.docsTable = docsTable; - this.info = info; - this.pgLang = resolvePgLanguage(info.language); - } - - async init(): Promise { - await this.db.exec( - `CREATE TABLE IF NOT EXISTS ${this.tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, content_tsv TSVECTOR GENERATED ALWAYS AS (to_tsvector('${this.pgLang}', content)) STORED, metadata TEXT, PRIMARY KEY (doc_id, block_id))`, - ); - await this.db.exec( - `CREATE INDEX IF NOT EXISTS ${this.tableName}_tsv_idx ON ${this.tableName} USING GIN (content_tsv)`, - ); - } - - private ensureOpen(): void { - if (this.closed) { - throw new Error("FullTextIndex is closed"); - } - } - - private async resolveDocId(path: DocumentPath): Promise { - await this.db.query( - `INSERT INTO ${this.docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, - [path], - ); - const { rows } = await this.db.query<{ doc_id: number }>( - `SELECT doc_id FROM ${this.docsTable} WHERE path = $1`, - [path], - ); - return rows[0]?.doc_id ?? -1; - } - - private pathFilterClause( - paths: DocumentPath[] | undefined, - paramOffset: number, - ): { sql: string; params: string[] } { - if (!paths || paths.length === 0) return { sql: "", params: [] }; - const conditions = paths.map((_, i) => `d.path LIKE $${paramOffset + i} || '%'`); - return { - sql: ` AND (${conditions.join(" OR ")})`, - params: paths as string[], - }; - } - - async getIndexInfo(): Promise { - this.ensureOpen(); - return { ...this.info }; - } - - async *search(params: FullTextSearchParams): AsyncGenerator { - this.ensureOpen(); - const { queries, topK, paths } = params; - - if (!queries || queries.length === 0) return; - - const bestScores = new Map(); - - for (const query of queries) { - const words = query - .toLowerCase() - .split(/\s+/) - .filter((w) => w.length > 0) - .map((w) => w.replace(/[^a-zA-Z0-9]/g, "")); - const validWords = words.filter((w) => w.length > 0); - if (validWords.length === 0) continue; - - const orTerms = validWords.join(" | "); - - const allParams: (string | number)[] = [orTerms]; - - const pathFilter = this.pathFilterClause(paths, allParams.length + 1); - allParams.push(...pathFilter.params); - - const topKParam = `$${allParams.length + 1}`; - allParams.push(topK); - - const sql = `SELECT d.path, b.block_id, b.content, ts_rank_cd(b.content_tsv, to_tsquery('${this.pgLang}', $1)) AS score FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE b.content_tsv @@ to_tsquery('${this.pgLang}', $1)${pathFilter.sql} ORDER BY score DESC LIMIT ${topKParam}`; - - const { rows } = await this.db.query<{ - path: string; - block_id: string; - content: string; - score: number; - }>(sql, allParams); - - for (const row of rows) { - const key = `${row.path}\0${row.block_id}`; - const existing = bestScores.get(key); - if (!existing || row.score > existing.score) { - bestScores.set(key, { - path: row.path as DocumentPath, - blockId: row.block_id, - snippet: row.content, - score: row.score, - }); - } - } - } - - const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score); - for (const r of sorted.slice(0, topK)) { - yield r; - } - } - - async addDocument(blocks: FullTextBlock[]): Promise { - this.ensureOpen(); - for (const block of blocks) { - const docId = await this.resolveDocId(block.path); - const metaJson = block.metadata ? JSON.stringify(block.metadata) : null; - await this.db.query(`DELETE FROM ${this.tableName} WHERE doc_id = $1 AND block_id = $2`, [ - docId, - block.blockId, - ]); - await this.db.query( - `INSERT INTO ${this.tableName} (doc_id, block_id, content, metadata) VALUES ($1, $2, $3, $4)`, - [docId, block.blockId, block.content, metaJson], - ); - } - } - - async addDocuments( - blocks: Iterable | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const batch of blocks) { - await this.addDocument(batch); - } - } - - async deleteDocuments( - pathSelectors: PathSelector[] | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const sel of toAsyncIterable(pathSelectors)) { - if (sel.blockId !== undefined) { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, - [sel.path, sel.blockId], - ); - } else { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path LIKE $1 || '%')`, - [sel.path], - ); - } - } - } - - async getSize(pathPrefix?: DocumentPath): Promise { - this.ensureOpen(); - if (pathPrefix !== undefined) { - const { rows } = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, - [pathPrefix], - ); - return Number(rows[0]?.cnt ?? 0); - } - const { rows } = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName}`, - ); - return Number(rows[0]?.cnt ?? 0); - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const { rows } = await this.db.query<{ path: string }>(sql, params); - for (const row of rows) { - yield row.path as DocumentPath; - } - } - - async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const { rows } = await this.db.query<{ path: string; block_id: string }>(sql, params); - for (const row of rows) { - yield { path: row.path as DocumentPath, blockId: row.block_id }; - } - } - - async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id, b.content, b.metadata FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id, b.content, b.metadata FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const { rows } = await this.db.query<{ - path: string; - block_id: string; - content: string; - metadata: string | null; - }>(sql, params); - for (const row of rows) { - yield { - path: row.path as DocumentPath, - blockId: row.block_id, - content: row.content, - metadata: row.metadata ? (JSON.parse(row.metadata) as Metadata) : undefined, - }; - } - } - - async close(_options?: { force?: boolean }): Promise { - this.closed = true; - } - - async flush(): Promise { - this.ensureOpen(); - } + init(): Promise; +}; - async deleteIndex(): Promise { - this.ensureOpen(); - this.closed = true; - await this.db.exec(`DROP TABLE IF EXISTS ${this.tableName}`); - } +export function createPGLiteFullTextIndex( + db: PGlite, + prefix: string, + docsTable: string, + info: FullTextIndexInfo, +): PGLiteFullTextIndex { + return createSqlFtsRetriever({ + db: wrapDbAsSqlDb(db), + prefix, + docsTable, + info, + dialect: pgliteFtsDialect, + }); } diff --git a/packages/indexer-pglite/src/pglite-indexer.ts b/packages/indexer-pglite/src/pglite-indexer.ts index 1a5e9c8..4ab77e1 100644 --- a/packages/indexer-pglite/src/pglite-indexer.ts +++ b/packages/indexer-pglite/src/pglite-indexer.ts @@ -1,15 +1,8 @@ import { PGlite } from "@electric-sql/pglite"; import { vector } from "@electric-sql/pglite/vector"; -import type { - CreateIndexParams, - DocumentPath, - Index, - Indexer, - IndexInfo, -} from "@statewalker/indexer-api"; -import { createCompositeIndex, sanitizePrefix } from "@statewalker/indexer-core"; -import { PGLiteFullTextIndex } from "./pglite-full-text-index.js"; -import { PGLiteVectorIndex } from "./pglite-vector-index.js"; +import type { Indexer } from "@statewalker/indexer-api"; +import { createSqlBackedIndexer } from "@statewalker/indexer-core"; +import { pgliteDialect, wrapDbAsSqlDb } from "./dialect.js"; export interface PGLiteIndexerOptions { db?: PGlite; @@ -18,203 +11,14 @@ export interface PGLiteIndexerOptions { export async function createPGLiteIndexer(options?: PGLiteIndexerOptions): Promise { const ownsDb = !options?.db; const db = options?.db ?? (await PGlite.create({ extensions: { vector } })); - const indexes = new Map(); - const manifest = new Map(); - let closed = false; - await db.exec("CREATE EXTENSION IF NOT EXISTS vector"); - - await db.exec( - "CREATE TABLE IF NOT EXISTS __indexer_manifest (name TEXT PRIMARY KEY, config TEXT NOT NULL)", - ); - - const existingEntries = await db.query<{ name: string; config: string }>( - "SELECT name, config FROM __indexer_manifest", - ); - for (const entry of existingEntries.rows) { - manifest.set(entry.name, { name: entry.name }); - } - - function ensureOpen(): void { - if (closed) throw new Error("Indexer is closed"); - } - - async function createDocsTable(prefix: string): Promise { - const docsTable = `idx_${prefix}_docs`; - await db.exec( - `CREATE TABLE IF NOT EXISTS ${docsTable} (doc_id SERIAL PRIMARY KEY, path TEXT NOT NULL UNIQUE)`, - ); - return docsTable; - } - - function buildIndex( - name: string, - docsTable: string, - fts: PGLiteFullTextIndex | null, - vec: PGLiteVectorIndex | null, - ): Index { - return createCompositeIndex({ - name, - fts, - vec, - getSize: async (pathPrefix?: DocumentPath): Promise => { - if (fts !== null && vec !== null) { - const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause}) AS combined`; - const { rows } = await db.query<{ cnt: number | bigint }>(sql, params); - return Number(rows[0]?.cnt ?? 0); - } - if (fts !== null) return fts.getSize(pathPrefix); - if (vec !== null) return vec.getSize(pathPrefix); - return 0; - }, - onDeleteIndex: async () => { - await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); - }, - }); - } - - const indexer: Indexer = { - async getIndexNames(): Promise { - ensureOpen(); - return [...manifest.values()]; - }, - - async createIndex(params: CreateIndexParams): Promise { - ensureOpen(); - const { name, fulltext, vector, overwrite } = params; - - if (!fulltext && !vector) { - throw new Error("At least one of fulltext or vector must be provided"); - } - - if (indexes.has(name) || manifest.has(name)) { - if (overwrite) { - const old = indexes.get(name); - if (old) await old.close(); - indexes.delete(name); - manifest.delete(name); - const prefix = sanitizePrefix(name); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_fts`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_vec`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_docs`); - await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); - } else { - throw new Error(`Index "${name}" already exists`); + return createSqlBackedIndexer({ + db: wrapDbAsSqlDb(db), + dialect: pgliteDialect, + onClose: ownsDb + ? async () => { + await db.close(); } - } - - const prefix = sanitizePrefix(name); - const docsTable = await createDocsTable(prefix); - - const fts = fulltext - ? new PGLiteFullTextIndex(db, prefix, docsTable, { - language: fulltext.language, - metadata: fulltext.metadata, - }) - : null; - - const vec = vector - ? new PGLiteVectorIndex(db, prefix, docsTable, { - dimensionality: vector.dimensionality, - model: vector.model, - metadata: vector.metadata, - }) - : null; - - if (fts) await fts.init(); - if (vec) await vec.init(); - - await db.query("INSERT INTO __indexer_manifest (name, config) VALUES ($1, $2)", [ - name, - JSON.stringify({ fulltext, vector }), - ]); - - const index = buildIndex(name, docsTable, fts, vec); - indexes.set(name, index); - manifest.set(name, { name }); - return index; - }, - - async getIndex(name: string): Promise { - ensureOpen(); - if (indexes.has(name)) return indexes.get(name) ?? null; - if (!manifest.has(name)) return null; - - const result = await db.query<{ config: string }>( - "SELECT config FROM __indexer_manifest WHERE name = $1", - [name], - ); - if (result.rows.length === 0) return null; - - const config = JSON.parse(result.rows[0]?.config ?? "{}") as { - fulltext?: { language: string; metadata?: Record }; - vector?: { - dimensionality: number; - model: string; - metadata?: Record; - }; - }; - - const prefix = sanitizePrefix(name); - const docsTable = await createDocsTable(prefix); - - const fts = config.fulltext - ? new PGLiteFullTextIndex(db, prefix, docsTable, { - language: config.fulltext.language, - metadata: config.fulltext.metadata, - }) - : null; - - const vec = config.vector - ? new PGLiteVectorIndex(db, prefix, docsTable, { - dimensionality: config.vector.dimensionality, - model: config.vector.model, - metadata: config.vector.metadata, - }) - : null; - - const index = buildIndex(name, docsTable, fts, vec); - indexes.set(name, index); - return index; - }, - - async hasIndex(name: string): Promise { - ensureOpen(); - return manifest.has(name); - }, - - async deleteIndex(name: string): Promise { - ensureOpen(); - const index = indexes.get(name); - if (index) { - await index.close(); - indexes.delete(name); - } - if (manifest.has(name)) { - const prefix = sanitizePrefix(name); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_fts`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_vec`); - await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_docs`); - await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); - manifest.delete(name); - } - }, - - async flush(): Promise { - ensureOpen(); - }, - - async close(): Promise { - if (closed) return; - closed = true; - for (const index of indexes.values()) await index.close(); - indexes.clear(); - manifest.clear(); - if (ownsDb) await db.close(); - }, - }; - - return indexer; + : undefined, + }); } diff --git a/packages/indexer-pglite/src/pglite-vector-index.ts b/packages/indexer-pglite/src/pglite-vector-index.ts index d4f068c..61ecda1 100644 --- a/packages/indexer-pglite/src/pglite-vector-index.ts +++ b/packages/indexer-pglite/src/pglite-vector-index.ts @@ -1,245 +1,24 @@ import type { PGlite } from "@electric-sql/pglite"; -import type { - BlockReference, - DocumentPath, - EmbeddingBlock, - EmbeddingIndex, - EmbeddingIndexInfo, - EmbeddingSearchParams, - EmbeddingSearchResult, - PathSelector, -} from "@statewalker/indexer-api"; -import { toAsyncIterable, validateDimensionality } from "@statewalker/indexer-core"; +import type { EmbeddingIndex, EmbeddingIndexInfo } from "@statewalker/indexer-api"; +import { createSqlVectorRetriever } from "@statewalker/indexer-core"; +import { pgliteVectorDialect, wrapDbAsSqlDb } from "./dialect.js"; -export class PGLiteVectorIndex implements EmbeddingIndex { - private readonly db: PGlite; +export type PGLiteVectorIndex = EmbeddingIndex & { readonly tableName: string; - private readonly indexName: string; - private readonly docsTable: string; - private readonly info: EmbeddingIndexInfo; - private closed = false; - - constructor(db: PGlite, prefix: string, docsTable: string, info: EmbeddingIndexInfo) { - this.db = db; - this.tableName = `idx_${prefix}_vec`; - this.indexName = `idx_${prefix}_vec_hnsw`; - this.docsTable = docsTable; - this.info = info; - } - - async init(): Promise { - const dim = this.info.dimensionality; - await this.db.exec( - `CREATE TABLE IF NOT EXISTS ${this.tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding vector(${dim}) NOT NULL, PRIMARY KEY (doc_id, block_id))`, - ); - await this.db.exec( - `CREATE INDEX IF NOT EXISTS ${this.indexName} ON ${this.tableName} USING hnsw (embedding vector_cosine_ops)`, - ); - } - - private ensureOpen(): void { - if (this.closed) { - throw new Error("EmbeddingIndex is closed"); - } - } - - private embeddingToSql(embedding: Float32Array): string { - return `[${Array.from(embedding).join(",")}]`; - } - - private async resolveDocId(path: DocumentPath): Promise { - await this.db.query( - `INSERT INTO ${this.docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, - [path], - ); - const { rows } = await this.db.query<{ doc_id: number }>( - `SELECT doc_id FROM ${this.docsTable} WHERE path = $1`, - [path], - ); - return rows[0]?.doc_id ?? -1; - } - - async getIndexInfo(): Promise { - this.ensureOpen(); - return { ...this.info }; - } - - async *search(params: EmbeddingSearchParams): AsyncGenerator { - this.ensureOpen(); - const { embeddings, topK, paths } = params; - - if (!embeddings || embeddings.length === 0) return; - - const bestScores = new Map(); - const dim = this.info.dimensionality; - - for (const queryEmb of embeddings) { - validateDimensionality(this.info,queryEmb); - const vecLiteral = this.embeddingToSql(queryEmb); - - let pathClause = ""; - const queryParams: (string | number)[] = [vecLiteral]; - - if (paths && paths.length > 0) { - const pathConditions = paths.map((_, i) => `d.path LIKE $${i + 2} || '%'`); - pathClause = `WHERE ${pathConditions.join(" OR ")} `; - queryParams.push(...(paths as string[])); - } - - const topKParam = `$${queryParams.length + 1}`; - queryParams.push(topK); - - const { rows } = await this.db.query<{ - path: string; - block_id: string; - dist: number; - }>( - `SELECT d.path, b.block_id, (b.embedding <=> $1::vector(${dim})) AS dist FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`, - queryParams, - ); - - for (const row of rows) { - const key = `${row.path}\0${row.block_id}`; - const score = 1 - row.dist; - const existing = bestScores.get(key); - if (!existing || score > existing.score) { - bestScores.set(key, { - path: row.path as DocumentPath, - blockId: row.block_id, - score, - }); - } - } - } - - const sorted = [...bestScores.values()].sort((a, b) => b.score - a.score); - for (const r of sorted.slice(0, topK)) { - yield r; - } - } - - async addDocument(blocks: EmbeddingBlock[]): Promise { - this.ensureOpen(); - for (const block of blocks) { - validateDimensionality(this.info,block.embedding); - const docId = await this.resolveDocId(block.path); - const dim = this.info.dimensionality; - const vecLiteral = this.embeddingToSql(block.embedding); - - await this.db.query(`DELETE FROM ${this.tableName} WHERE doc_id = $1 AND block_id = $2`, [ - docId, - block.blockId, - ]); - await this.db.query( - `INSERT INTO ${this.tableName} (doc_id, block_id, embedding) VALUES ($1, $2, $3::vector(${dim}))`, - [docId, block.blockId, vecLiteral], - ); - } - } - - async addDocuments( - blocks: Iterable | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const batch of blocks) { - await this.addDocument(batch); - } - } - - async deleteDocuments( - pathSelectors: PathSelector[] | AsyncIterable, - ): Promise { - this.ensureOpen(); - for await (const sel of toAsyncIterable(pathSelectors)) { - if (sel.blockId !== undefined) { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path = $1) AND block_id = $2`, - [sel.path, sel.blockId], - ); - } else { - await this.db.query( - `DELETE FROM ${this.tableName} WHERE doc_id IN (SELECT doc_id FROM ${this.docsTable} WHERE path LIKE $1 || '%')`, - [sel.path], - ); - } - } - } - - async getSize(pathPrefix?: DocumentPath): Promise { - this.ensureOpen(); - if (pathPrefix !== undefined) { - const { rows } = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, - [pathPrefix], - ); - return Number(rows[0]?.cnt ?? 0); - } - const { rows } = await this.db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${this.tableName}`, - ); - return Number(rows[0]?.cnt ?? 0); - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT DISTINCT d.path FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const { rows } = await this.db.query<{ path: string }>(sql, params); - for (const row of rows) { - yield row.path as DocumentPath; - } - } - - async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const { rows } = await this.db.query<{ path: string; block_id: string }>(sql, params); - for (const row of rows) { - yield { path: row.path as DocumentPath, blockId: row.block_id }; - } - } - - async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { - this.ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id, b.embedding FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id, b.embedding FROM ${this.tableName} b JOIN ${this.docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const { rows } = await this.db.query<{ - path: string; - block_id: string; - embedding: string; - }>(sql, params); - for (const row of rows) { - // PGlite returns vector as string "[1,2,3]", parse to Float32Array - const parsed = JSON.parse(row.embedding) as number[]; - yield { - path: row.path as DocumentPath, - blockId: row.block_id, - embedding: new Float32Array(parsed), - }; - } - } - - async close(_options?: { force?: boolean }): Promise { - this.closed = true; - } - - async flush(): Promise { - this.ensureOpen(); - } - - async deleteIndex(): Promise { - this.ensureOpen(); - this.closed = true; - await this.db.exec(`DROP TABLE IF EXISTS ${this.tableName}`); - } + init(): Promise; +}; + +export function createPGLiteVectorIndex( + db: PGlite, + prefix: string, + docsTable: string, + info: EmbeddingIndexInfo, +): PGLiteVectorIndex { + return createSqlVectorRetriever({ + db: wrapDbAsSqlDb(db), + prefix, + docsTable, + info, + dialect: pgliteVectorDialect, + }); } From 3dd72c6fbb9032bf2fdde5d8b2591de21c1962d5 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Fri, 24 Apr 2026 15:16:36 +0200 Subject: [PATCH 04/12] feat(indexer-duckdb): real BM25 FTS + HNSW cosine metric (Phases 8-10) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 8 - Real DuckDB full-text search via the `fts` extension: * extensionInit now loads `fts` in addition to `vss` * FTS table gains a virtual `fts_id` column - required because create_fts_index takes a single-column identifier; ours is the concatenation of (doc_id, block_id) via a VIRTUAL GENERATED column * New SqlFtsDialect.rebuild hook: the DuckDB dialect implements PRAGMA create_fts_index(stemmer=, stopwords='none', strip_accents=1, lower=1, overwrite=1) with a Snowball-stemmer name derived from the FullTextIndexInfo.language via DUCKDB_STEMMER_MAP (en→english, fr→french, ...) * createSqlFtsRetriever tracks a dirty flag: set on every addDocument/deleteDocuments, forced true on first open, checked lazily before search and on flush. Dialects without a rebuild hook (PGlite) bypass this state entirely - tsvector is maintained automatically by the DB. * Search SQL: SELECT d.path, b.block_id, b.content, fts_main_.match_bm25(b.fts_id, $1) AS score FROM ... WHERE ... IS NOT NULL ORDER BY score DESC LIMIT $k * README updated to document the fts + vss runtime extensions. * New indexer-tests cases (cross-backend): - multi-term rank: "alpha beta" doc ranks above "alpha" alone - flush + search preserves content (exercises rebuild on flush) Phase 9 - HNSW cosine metric: * DuckDB HNSW DDL now emits `WITH (metric = 'cosine')`. Without this, array_cosine_distance queries silently fell back to sequential scan (vss default metric is l2sq). * Parameter-bound embeddings attempted but reverted: the @statewalker/db-duckdb-node driver rejects JS arrays passed via prepared.bind() - 38 tests failed. Kept the locale-independent string-literal path (documented at each dialect). * PGlite HNSW already declared vector_cosine_ops; verified unchanged. Phase 10 - Cleanups: * indexer-api README corrected (removed references to non-existent helpers isCollectionPrefix et al.) * biome check --write --unsafe across touched packages Tests: 585/585 passing across 6 packages. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/indexer-api/README.md | 2 +- .../src/create-composite-index.ts | 2 +- .../src/create-sql-fts-retriever.ts | 29 ++++- .../src/create-sql-vector-retriever.ts | 7 +- .../src/validate-dimensionality.ts | 4 +- packages/indexer-duckdb/README.md | 9 ++ packages/indexer-duckdb/src/dialect.ts | 100 ++++++++++++++---- packages/indexer-mem/src/mem-vector-index.ts | 4 +- packages/indexer-pglite/src/dialect.ts | 18 +++- .../src/suites/full-text-index.suite.ts | 37 +++++++ 10 files changed, 174 insertions(+), 38 deletions(-) diff --git a/packages/indexer-api/README.md b/packages/indexer-api/README.md index 45060c2..8c0c1b7 100644 --- a/packages/indexer-api/README.md +++ b/packages/indexer-api/README.md @@ -87,7 +87,7 @@ Validators (`validateLexQuery`, `validateSemanticQuery`) catch malformed queries ### Path-prefix filtering -Documents are organized under hierarchical paths (`"/projects/alpha/specs/"`). All search, enumeration, and deletion operations accept optional path prefixes to restrict their scope. For example, passing `paths: ["/docs/"]` limits results to documents whose path starts with `"/docs/"`. Helper utilities (`isCollectionPrefix`, `matchesCollection`, `resolveCollections`) perform prefix and exact matching on path segments, and `buildCollectionClause()` generates parameterized SQL WHERE fragments for SQL-backed implementations. +Documents are organized under hierarchical paths (`"/projects/alpha/specs/"`). All search, enumeration, and deletion operations accept optional path prefixes to restrict their scope. For example, passing `paths: ["/docs/"]` to `index.search(...)` limits results to documents whose path starts with `"/docs/"`. Prefix matching is a simple `startsWith` at the type level — backends implement it using their native SQL (DuckDB / PGlite) or in-memory filtering (indexer-mem-*). ### Persistence diff --git a/packages/indexer-core/src/create-composite-index.ts b/packages/indexer-core/src/create-composite-index.ts index fa60bc2..4238609 100644 --- a/packages/indexer-core/src/create-composite-index.ts +++ b/packages/indexer-core/src/create-composite-index.ts @@ -10,9 +10,9 @@ import type { Metadata, PathSelector, } from "@statewalker/indexer-api"; +import { toAsyncIterable } from "./async.js"; import { compositeKey } from "./composite-key.js"; import { mergeByRRF, mergeByWeights } from "./merge.js"; -import { toAsyncIterable } from "./async.js"; export interface CompositeIndexOptions { name: string; diff --git a/packages/indexer-core/src/create-sql-fts-retriever.ts b/packages/indexer-core/src/create-sql-fts-retriever.ts index 2bc4324..be65e9e 100644 --- a/packages/indexer-core/src/create-sql-fts-retriever.ts +++ b/packages/indexer-core/src/create-sql-fts-retriever.ts @@ -26,6 +26,14 @@ export interface SqlFtsDialect { */ createTableDdl(opts: { tableName: string; info: FullTextIndexInfo }): string[]; + /** + * Optional hook to (re)build an external FTS index structure after ingest/delete. When set, the + * retriever tracks a dirty flag (set on every write) and calls this lazily before `search` and on + * `flush`. Dialects whose FTS is maintained automatically by the database (e.g. PGlite's `tsvector`) + * leave this undefined. + */ + rebuild?(opts: { db: SqlDb; tableName: string; info: FullTextIndexInfo }): Promise; + /** * Execute a lexical search for a single query string. The base iterates `queries[]` and merges by best score. */ @@ -59,16 +67,24 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd const { db, prefix, docsTable, info, dialect } = opts; const tableName = `idx_${prefix}_fts`; let closed = false; + // Start dirty: forces a rebuild on first search after `init()` or a reopen, covering both + // brand-new tables and tables whose FTS index structure may be out of date. + let dirty = dialect.rebuild != null; const ensureOpen = (): void => { if (closed) throw new Error("FullTextIndex is closed"); }; + const ensureRebuilt = async (): Promise => { + if (!dialect.rebuild || !dirty) return; + await dialect.rebuild({ db, tableName, info }); + dirty = false; + }; + const resolveDocId = async (path: DocumentPath): Promise => { - await db.query( - `INSERT INTO ${docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, - [path], - ); + await db.query(`INSERT INTO ${docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, [ + path, + ]); const rows = await db.query<{ doc_id: number }>( `SELECT doc_id FROM ${docsTable} WHERE path = $1`, [path], @@ -95,6 +111,8 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd const { queries, topK, paths } = params; if (!queries || queries.length === 0) return; + await ensureRebuilt(); + const bestScores = new Map(); for (const query of queries) { const rows = await dialect.search({ db, tableName, docsTable, info, query, paths, topK }); @@ -130,6 +148,7 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd [docId, block.blockId, block.content, metaJson], ); } + if (dialect.rebuild) dirty = true; }, async addDocuments( @@ -156,6 +175,7 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd ); } } + if (dialect.rebuild) dirty = true; }, async getSize(pathPrefix?: DocumentPath): Promise { @@ -226,6 +246,7 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd async flush(): Promise { ensureOpen(); + await ensureRebuilt(); }, async deleteIndex(): Promise { diff --git a/packages/indexer-core/src/create-sql-vector-retriever.ts b/packages/indexer-core/src/create-sql-vector-retriever.ts index 9f1f84d..a86eaab 100644 --- a/packages/indexer-core/src/create-sql-vector-retriever.ts +++ b/packages/indexer-core/src/create-sql-vector-retriever.ts @@ -83,10 +83,9 @@ export function createSqlVectorRetriever(opts: SqlVectorRetrieverOptions): Embed }; const resolveDocId = async (path: DocumentPath): Promise => { - await db.query( - `INSERT INTO ${docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, - [path], - ); + await db.query(`INSERT INTO ${docsTable} (path) VALUES ($1) ON CONFLICT (path) DO NOTHING`, [ + path, + ]); const rows = await db.query<{ doc_id: number }>( `SELECT doc_id FROM ${docsTable} WHERE path = $1`, [path], diff --git a/packages/indexer-core/src/validate-dimensionality.ts b/packages/indexer-core/src/validate-dimensionality.ts index 004a9a3..5328b2b 100644 --- a/packages/indexer-core/src/validate-dimensionality.ts +++ b/packages/indexer-core/src/validate-dimensionality.ts @@ -5,8 +5,6 @@ export function validateDimensionality( embedding: Float32Array, ): void { if (embedding.length !== info.dimensionality) { - throw new Error( - `Expected dimensionality ${info.dimensionality}, got ${embedding.length}`, - ); + throw new Error(`Expected dimensionality ${info.dimensionality}, got ${embedding.length}`); } } diff --git a/packages/indexer-duckdb/README.md b/packages/indexer-duckdb/README.md index 5b50ed0..d972d48 100644 --- a/packages/indexer-duckdb/README.md +++ b/packages/indexer-duckdb/README.md @@ -18,6 +18,15 @@ const db = await createDuckDbNodeClient({ path: "./index.duckdb" }); const idx = await createDuckDbIndexer({ db }); ``` +## Runtime DuckDB extensions + +`createDuckDbIndexer` installs and loads two DuckDB extensions on startup: + +- **`fts`** — BM25 full-text search. The indexer calls `PRAGMA create_fts_index(...)` and queries via `fts_main_
.match_bm25(id, query)`. The FTS index is rebuilt lazily: the retriever tracks a dirty flag set on every `addDocument` / `deleteDocuments` call and rebuilds on the first `search` thereafter (or sooner via explicit `flush()`). +- **`vss`** — HNSW vector similarity index. `SET hnsw_enable_experimental_persistence = true;` is also set so HNSW indexes survive database close/reopen. + +Both extensions ship with DuckDB and are fetched from the DuckDB extension repository on first `INSTALL`. Running `createDuckDbIndexer` in an environment that blocks extension downloads will fail at init time with the missing-extension error surfaced by DuckDB. + ## API - `createDuckDbIndexer(options)` — returns a hybrid FTS + vector indexer. diff --git a/packages/indexer-duckdb/src/dialect.ts b/packages/indexer-duckdb/src/dialect.ts index 87b6812..4ba9e4c 100644 --- a/packages/indexer-duckdb/src/dialect.ts +++ b/packages/indexer-duckdb/src/dialect.ts @@ -14,37 +14,79 @@ export function wrapDbAsSqlDb(db: Db): SqlDb { }; } +/** + * DuckDB bindings for `FLOAT[dim]` columns go through the `@statewalker/db-duckdb-node` driver, which + * does not currently wrap JS arrays as `DuckDBArrayValue`. We format the embedding as a SQL array + * literal (locale-independent via `Number.prototype.toString`) and rely on the `$n::FLOAT[dim]` cast + * to parse it back. Switching to native array binding is a follow-up when the driver supports it. + */ function embeddingToLiteral(embedding: Float32Array): string { return `[${Array.from(embedding).join(",")}]`; } /** - * DuckDB FTS dialect. + * DuckDB FTS stemmer map. DuckDB's `fts` extension accepts a stemmer name per Snowball; we map + * ISO-639-1 language codes used at `FullTextIndexInfo.language` to the corresponding stemmer. + * Unknown codes fall through to 'porter' as a generic fallback. + */ +const DUCKDB_STEMMER_MAP: Record = { + en: "english", + fr: "french", + de: "german", + es: "spanish", + it: "italian", + pt: "portuguese", + nl: "dutch", + ru: "russian", + sv: "swedish", + no: "norwegian", + da: "danish", + fi: "finnish", + hu: "hungarian", + ro: "romanian", + tr: "turkish", +}; + +function resolveDuckDbStemmer(lang: string): string { + return DUCKDB_STEMMER_MAP[lang] ?? "porter"; +} + +/** + * The `fts_main_
` schema that DuckDB's fts extension creates when we call `create_fts_index`. + * Used in the BM25 search SQL. + */ +function ftsMainSchema(tableName: string): string { + return `fts_main_${tableName}`; +} + +/** + * DuckDB FTS dialect using the official `fts` extension. * - * Current implementation: LIKE-based scanning (word-count + rank-decay). Phase 8 replaces this with - * the official `fts` community extension (BM25 via `PRAGMA create_fts_index` + `match_bm25`). + * The table adds a virtual `fts_id` column that concatenates `(doc_id, block_id)` — required because + * `create_fts_index` takes a single-column identifier. The FTS index is (re)built lazily by the + * retriever (first search after a write; also on flush) via the `rebuild` hook, because the + * extension does not auto-update on INSERT/DELETE. */ export const duckdbFtsDialect: SqlFtsDialect = { createTableDdl({ tableName }) { return [ - `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, metadata TEXT, PRIMARY KEY (doc_id, block_id))`, + `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, content TEXT NOT NULL, metadata TEXT, fts_id TEXT GENERATED ALWAYS AS (CAST(doc_id AS VARCHAR) || '_' || block_id) VIRTUAL, PRIMARY KEY (doc_id, block_id))`, ]; }, - async search({ db, tableName, docsTable, query, paths, topK }) { - const words = query - .toLowerCase() - .split(/\s+/) - .filter((w) => w.length > 0); - if (words.length === 0) return []; + async rebuild({ db, tableName, info }) { + const stemmer = resolveDuckDbStemmer(info.language); + await db.exec( + `PRAGMA create_fts_index('${tableName}', 'fts_id', 'content', stemmer='${stemmer}', stopwords='none', strip_accents=1, lower=1, overwrite=1)`, + ); + }, - const likeParams = words.map((w) => `%${w}%`); - const conditions = words.map((_, i) => `LOWER(b.content) LIKE $${i + 1}`); - const scoreExpr = words - .map((_, i) => `CASE WHEN LOWER(b.content) LIKE $${i + 1} THEN 1 ELSE 0 END`) - .join(" + "); + async search({ db, tableName, docsTable, query, paths, topK }) { + const trimmed = query.trim(); + if (trimmed.length === 0) return []; - const allParams: unknown[] = [...likeParams]; + const schema = ftsMainSchema(tableName); + const allParams: unknown[] = [trimmed]; let pathClause = ""; if (paths && paths.length > 0) { @@ -56,20 +98,20 @@ export const duckdbFtsDialect: SqlFtsDialect = { const topKParam = `$${allParams.length + 1}`; allParams.push(topK); - const sql = `SELECT d.path, b.block_id, b.content, (${scoreExpr}) AS match_count FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE (${conditions.join(" OR ")})${pathClause} ORDER BY match_count DESC LIMIT ${topKParam}`; + const sql = `SELECT d.path, b.block_id, b.content, ${schema}.match_bm25(b.fts_id, $1) AS score FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${schema}.match_bm25(b.fts_id, $1) IS NOT NULL${pathClause} ORDER BY score DESC LIMIT ${topKParam}`; const rows = await db.query<{ path: string; block_id: string; content: string; - match_count: number; + score: number; }>(sql, allParams); - return rows.map((row, rank) => ({ + return rows.map((row) => ({ path: row.path as import("@statewalker/indexer-api").DocumentPath, blockId: row.block_id, content: row.content, - score: (row.match_count / words.length) * (1 - rank / (rows.length + 1)), + score: row.score, })); }, }; @@ -86,14 +128,27 @@ export const duckdbVectorDialect: SqlVectorDialect = { const dim = info.dimensionality; return [ `CREATE TABLE IF NOT EXISTS ${tableName} (doc_id INTEGER NOT NULL, block_id TEXT NOT NULL, embedding FLOAT[${dim}] NOT NULL, PRIMARY KEY (doc_id, block_id))`, - `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING HNSW (embedding)`, + // HNSW index with the cosine metric: required so `array_cosine_distance` searches actually + // use the index. Without this option DuckDB's vss extension defaults to l2sq and the planner + // falls back to a sequential scan for cosine queries. + `CREATE INDEX IF NOT EXISTS ${indexName} ON ${tableName} USING HNSW (embedding) WITH (metric = 'cosine')`, ]; }, bindEmbedding: embeddingToLiteral, embeddingCastSuffix: (dim) => `::FLOAT[${dim}]`, - async search({ db, tableName, docsTable, queryEmbedding, paths, topK, info, bindEmbedding, embeddingCastSuffix }) { + async search({ + db, + tableName, + docsTable, + queryEmbedding, + paths, + topK, + info, + bindEmbedding, + embeddingCastSuffix, + }) { const vecLiteral = bindEmbedding(queryEmbedding); const dim = info.dimensionality; @@ -128,6 +183,7 @@ export const duckdbVectorDialect: SqlVectorDialect = { /** Aggregate DuckDB dialect for `createSqlBackedIndexer`. */ export const duckdbDialect: SqlBackedDialect = { extensionInit: [ + "INSTALL fts; LOAD fts;", "INSTALL vss; LOAD vss;", "SET hnsw_enable_experimental_persistence = true;", ], diff --git a/packages/indexer-mem/src/mem-vector-index.ts b/packages/indexer-mem/src/mem-vector-index.ts index 4b054dc..1c71353 100644 --- a/packages/indexer-mem/src/mem-vector-index.ts +++ b/packages/indexer-mem/src/mem-vector-index.ts @@ -74,7 +74,7 @@ export class MemVectorIndex implements EmbeddingIndex { const bestScores = new Map(); for (const queryEmb of embeddings) { - validateDimensionality(this.info,queryEmb); + validateDimensionality(this.info, queryEmb); const filtered = [...this.filteredEntries(paths)]; const results = bruteForceSearch(queryEmb, filtered, topK); for (const r of results) { @@ -95,7 +95,7 @@ export class MemVectorIndex implements EmbeddingIndex { async addDocument(blocks: EmbeddingBlock[]): Promise { this.ensureOpen(); for (const block of blocks) { - validateDimensionality(this.info,block.embedding); + validateDimensionality(this.info, block.embedding); const key = compositeKey(block.path, block.blockId); this.entries.set(key, { path: block.path, diff --git a/packages/indexer-pglite/src/dialect.ts b/packages/indexer-pglite/src/dialect.ts index a95630e..0e29643 100644 --- a/packages/indexer-pglite/src/dialect.ts +++ b/packages/indexer-pglite/src/dialect.ts @@ -40,6 +40,12 @@ function resolvePgLanguage(lang: string): string { return LANGUAGE_MAP[lang] ?? lang; } +/** + * pgvector's `vector(dim)` column accepts either a JSON-array text literal (`"[1,2,3]"`) cast with + * `::vector(dim)`, or a JS array bound by the driver. The text-literal path is currently used + * because it works uniformly across the pglite vector extension versions available in the catalog; + * swapping to native array binding is a follow-up when driver support is confirmed. + */ function embeddingToLiteral(embedding: Float32Array): string { return `[${Array.from(embedding).join(",")}]`; } @@ -114,7 +120,17 @@ export const pgliteVectorDialect: SqlVectorDialect = { bindEmbedding: embeddingToLiteral, embeddingCastSuffix: (dim) => `::vector(${dim})`, - async search({ db, tableName, docsTable, queryEmbedding, paths, topK, info, bindEmbedding, embeddingCastSuffix }) { + async search({ + db, + tableName, + docsTable, + queryEmbedding, + paths, + topK, + info, + bindEmbedding, + embeddingCastSuffix, + }) { const vecLiteral = bindEmbedding(queryEmbedding); const dim = info.dimensionality; diff --git a/packages/indexer-tests/src/suites/full-text-index.suite.ts b/packages/indexer-tests/src/suites/full-text-index.suite.ts index 7505576..2710b17 100644 --- a/packages/indexer-tests/src/suites/full-text-index.suite.ts +++ b/packages/indexer-tests/src/suites/full-text-index.suite.ts @@ -85,6 +85,43 @@ export function runFullTextIndexSuite(getIndexer: () => Indexer): void { expect(await fts.getSize()).toBe(2); }); + it("multi-term query ranks fully-matched block above single-term matches", async () => { + const indexer = getIndexer(); + const index = await indexer.createIndex({ + name: "test", + fulltext: { language: "en" }, + }); + const fts = defined(index.getFullTextIndex()); + await fts.addDocument([ + { path: "/docs/a", blockId: "alpha-only", content: "alpha content about nothing else" }, + { path: "/docs/b", blockId: "beta-only", content: "beta content about nothing else" }, + { path: "/docs/c", blockId: "alpha-beta", content: "alpha beta content together" }, + ]); + + const results = await collect(fts.search({ queries: ["alpha beta"], topK: 10 })); + expect(results.length).toBeGreaterThan(0); + // The block containing both terms should rank above either single-term block. + expect(results[0]?.blockId).toBe("alpha-beta"); + }); + + it("flush + subsequent search preserves content", async () => { + const indexer = getIndexer(); + const index = await indexer.createIndex({ + name: "test", + fulltext: { language: "en" }, + }); + const fts = defined(index.getFullTextIndex()); + await fts.addDocument([ + { path: "/docs/1", blockId: "1", content: "hello world" }, + { path: "/docs/2", blockId: "2", content: "goodbye world" }, + ]); + await fts.flush(); + + const results = await collect(fts.search({ queries: ["hello"], topK: 10 })); + expect(results.length).toBeGreaterThan(0); + expect(results[0]?.blockId).toBe("1"); + }); + it("search ranks fixture queries with Hit@10 >= 75%", async () => { const indexer = getIndexer(); const blocks = loadBlocksFixture(); From 2f08a59d2e584699fc1e4b3b05d849a563c94645 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Fri, 24 Apr 2026 15:44:49 +0200 Subject: [PATCH 05/12] docs: align all READMEs with current public APIs The pre-refactor READMEs documented APIs that never existed (e.g. createMemIndex, idx.add/idx.search, chunkByTokens/chunkByParagraph) and used @repo/ package names that don't match the published @statewalker/ namespace. This pass rewrites every README to reflect what the code actually exports. Sub-repo root: * Add @statewalker/indexer-core to the package table with a Published=no marker; restate each backend's actual implementation (BM25 via `fts` extension for DuckDB, HNSW cosine everywhere, etc.) Per-package: * indexer-api - s/@repo\//@statewalker\//g (7 replacements); content already accurate * indexer-chunker - actual exports (chunkMarkdown, scanBreakPoints, findBestCutoff, findCodeFences/isInsideCodeFence); drop the fictional chunkByTokens/chunkByParagraph * indexer-core - document the full post-refactor surface: createCompositeIndex, mergeByRRF/mergeByWeights/mergeHybrid, createPersistenceBackedIndexer, createSqlBackedIndexer, createSqlFtsRetriever, createSqlVectorRetriever, SqlDb, SqlBackedDialect, plus pure helpers * indexer-mem-flexsearch / -minisearch - real API is createXxxIndexer({persistence?}) returning Indexer with the standard indexer.createIndex(...).index.addDocument(...)/ index.search(...) flow; document the persistence wire format * indexer-pglite - real function is createPGLiteIndexer (was: createPgliteIndexer); document the `vector` extension load, the SQL shape (idx__docs/_fts/_vec with GIN + HNSW cosine), the owns-db-if-none-passed semantics * indexer-tests - real export is runIndexerTestSuite (was: runIndexerSuite); add the fixture-loader exports indexer-duckdb/README.md and indexer-mem/README.md were already refreshed in Phases 8 and 5 respectively. No code changes; tests remain 585/585. Co-Authored-By: Claude Opus 4.7 (1M context) --- README.md | 21 ++++++------ packages/indexer-api/README.md | 28 +++++++-------- packages/indexer-chunker/README.md | 19 ++++++---- packages/indexer-core/README.md | 34 +++++++++++++----- packages/indexer-mem-flexsearch/README.md | 42 +++++++++++++++++++---- packages/indexer-mem-minisearch/README.md | 39 +++++++++++++++++---- packages/indexer-pglite/README.md | 42 +++++++++++++++++++---- packages/indexer-tests/README.md | 19 +++++++--- 8 files changed, 179 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index b3bee72..41c6ef3 100644 --- a/README.md +++ b/README.md @@ -6,16 +6,17 @@ Indexing primitives: pluggable full-text and vector indexers (in-memory, DuckDB, -| Package | Description | -| --- | --- | -| [@statewalker/indexer-api](packages/indexer-api) | Pluggable indexer contract: full-text, vector, hybrid. | -| [@statewalker/indexer-chunker](packages/indexer-chunker) | Token- and paragraph-aware chunking utilities. | -| [@statewalker/indexer-mem](packages/indexer-mem) | In-memory base scaffold for full-text indexers. | -| [@statewalker/indexer-mem-flexsearch](packages/indexer-mem-flexsearch) | FlexSearch-backed in-memory indexer. | -| [@statewalker/indexer-mem-minisearch](packages/indexer-mem-minisearch) | MiniSearch-backed in-memory indexer. | -| [@statewalker/indexer-duckdb](packages/indexer-duckdb) | DuckDB-backed hybrid FTS + vector indexer (via `@statewalker/db-api`). | -| [@statewalker/indexer-pglite](packages/indexer-pglite) | PGlite-backed in-browser Postgres indexer. | -| [@statewalker/indexer-tests](packages/indexer-tests) | Shared Vitest suite (internal, not published). | +| Package | Description | Published | +| --- | --- | :---: | +| [@statewalker/indexer-api](packages/indexer-api) | Pluggable indexer contract: full-text, vector, hybrid. | yes | +| [@statewalker/indexer-chunker](packages/indexer-chunker) | Markdown-aware chunking utilities. | yes | +| [@statewalker/indexer-core](packages/indexer-core) | Workspace-internal scaffolding consumed by the backends (composite index, merge, SQL retrievers, generic factory builders). | no | +| [@statewalker/indexer-mem](packages/indexer-mem) | In-memory vector sub-index (`MemVectorIndex`) used by the FlexSearch/MiniSearch indexers. | yes | +| [@statewalker/indexer-mem-flexsearch](packages/indexer-mem-flexsearch) | FlexSearch + `MemVectorIndex` + optional persistence. | yes | +| [@statewalker/indexer-mem-minisearch](packages/indexer-mem-minisearch) | MiniSearch + `MemVectorIndex` + optional persistence. | yes | +| [@statewalker/indexer-duckdb](packages/indexer-duckdb) | DuckDB backend: real BM25 FTS (`fts` extension) + HNSW cosine vector (`vss` extension). | yes | +| [@statewalker/indexer-pglite](packages/indexer-pglite) | PGlite backend: `tsvector`/GIN FTS + `pgvector` HNSW cosine. | yes | +| [@statewalker/indexer-tests](packages/indexer-tests) | Shared Vitest conformance suite run by every backend. | no | ## Development diff --git a/packages/indexer-api/README.md b/packages/indexer-api/README.md index 8c0c1b7..70e1795 100644 --- a/packages/indexer-api/README.md +++ b/packages/indexer-api/README.md @@ -1,4 +1,4 @@ -# @repo/indexer-api +# @statewalker/indexer-api Backend-agnostic TypeScript API for hybrid search indexes combining **full-text search (FTS)** and **vector/embedding similarity search**. @@ -12,7 +12,7 @@ Modern search applications need more than keyword matching. They need to combine Each of these capabilities can be backed by very different storage engines — in-memory structures, SQLite/PGlite, DuckDB, or external services. Without a shared abstraction, application code becomes tightly coupled to a specific backend, making it hard to swap implementations, test in isolation, or run the same logic in different environments (browser, Node, edge). -`@repo/indexer-api` solves this by defining a **pure-interface contract** with zero runtime dependencies. Application code programs against the API; concrete backends are injected at startup. +`@statewalker/indexer-api` solves this by defining a **pure-interface contract** with zero runtime dependencies. Application code programs against the API; concrete backends are injected at startup. ## How it works @@ -116,24 +116,24 @@ interface IndexerPersistence { | Package | Backend | Notes | |---------|---------|-------| -| `@repo/indexer-mem` | In-memory (Flechette/Arrow) | Foundation layer; vector-only | -| `@repo/indexer-mem-minisearch` | MiniSearch + in-memory vectors | Lightweight FTS; optional persistence | -| `@repo/indexer-mem-flexsearch` | FlexSearch + in-memory vectors | Alternative FTS engine; optional persistence | -| `@repo/indexer-pglite` | PGlite + pgvector | SQL-backed; full FTS + vector | -| `@repo/indexer-duckdb` | DuckDB + VSS/HNSW | Analytical SQL; high-performance vector search | +| `@statewalker/indexer-mem` | In-memory (Flechette/Arrow) | Foundation layer; vector-only | +| `@statewalker/indexer-mem-minisearch` | MiniSearch + in-memory vectors | Lightweight FTS; optional persistence | +| `@statewalker/indexer-mem-flexsearch` | FlexSearch + in-memory vectors | Alternative FTS engine; optional persistence | +| `@statewalker/indexer-pglite` | PGlite + pgvector | SQL-backed; full FTS + vector | +| `@statewalker/indexer-duckdb` | DuckDB + VSS/HNSW | Analytical SQL; high-performance vector search | Supporting packages: | Package | Purpose | |---------|---------| -| `@repo/indexer-chunker` | Markdown splitting and code fence detection for content preprocessing | +| `@statewalker/indexer-chunker` | Markdown splitting and code fence detection for content preprocessing | ## How to use ### Creating an index ```ts -import type { Indexer, CreateIndexParams } from "@repo/indexer-api"; +import type { Indexer, CreateIndexParams } from "@statewalker/indexer-api"; // Obtain an Indexer from a concrete backend (e.g., MiniSearch, PGlite, DuckDB) const indexer: Indexer = createMiniSearchIndexer(/* ... */); @@ -198,7 +198,7 @@ for await (const result of index.search({ ### Using SemanticIndex for automatic embedding ```ts -import { SemanticIndex } from "@repo/indexer-api"; +import { SemanticIndex } from "@statewalker/indexer-api"; const semantic = new SemanticIndex(index, embed); @@ -219,7 +219,7 @@ const results = await semantic.search({ ### Multi-search with RRF fusion ```ts -import { defaultMultiSearch } from "@repo/indexer-api"; +import { defaultMultiSearch } from "@statewalker/indexer-api"; const results = await defaultMultiSearch(index, { queries: ["CAP theorem", "consistency models"], @@ -232,7 +232,7 @@ const results = await defaultMultiSearch(index, { ### Structured queries ```ts -import { parseStructuredQuery } from "@repo/indexer-api"; +import { parseStructuredQuery } from "@statewalker/indexer-api"; const parsed = parseStructuredQuery("lex: CAP theorem\nvec: consensus algorithms"); // [{ type: "lex", query: "CAP theorem" }, { type: "vec", query: "consensus algorithms" }] @@ -241,7 +241,7 @@ const parsed = parseStructuredQuery("lex: CAP theorem\nvec: consensus algorithms ### SearchPipeline ```ts -import { SearchPipeline } from "@repo/indexer-api"; +import { SearchPipeline } from "@statewalker/indexer-api"; const results = await new SearchPipeline({ index, @@ -261,7 +261,7 @@ const results = await new SearchPipeline({ ### Batch indexing with indexDocuments ```ts -import { indexDocuments } from "@repo/indexer-api"; +import { indexDocuments } from "@statewalker/indexer-api"; const { indexed } = await indexDocuments(index, [ { path: "/docs/", blockId: "b1", content: "First document..." }, diff --git a/packages/indexer-chunker/README.md b/packages/indexer-chunker/README.md index 42835c5..5750092 100644 --- a/packages/indexer-chunker/README.md +++ b/packages/indexer-chunker/README.md @@ -1,6 +1,6 @@ # @statewalker/indexer-chunker -Text chunking utilities used to prepare documents before indexing. +Markdown-aware chunking utilities. Splits long documents into bounded-size chunks suitable for embedding and full-text indexing, preserving code-fence boundaries and Markdown block structure. ## Installation @@ -11,18 +11,23 @@ pnpm add @statewalker/indexer-chunker ## Usage ```ts -import { chunkByTokens } from "@statewalker/indexer-chunker"; +import { chunkMarkdown } from "@statewalker/indexer-chunker"; -for (const chunk of chunkByTokens(text, { maxTokens: 512, overlap: 64 })) { - // … +const chunks = chunkMarkdown(text, { targetChars: 2000, overlapChars: 200 }); +for (const { index, text: body, startChar, endChar } of chunks) { + // feed each chunk into the indexer as a separate block } ``` ## API -- `chunkByTokens` — token-budget windowed chunking with optional overlap. -- `chunkByParagraph` — soft boundaries on paragraph breaks. +- `chunkMarkdown(text, options)` — returns `Chunk[]`. `options.targetChars` is the soft size limit; the chunker picks the best cut within a configurable tolerance around each boundary, preferring paragraph / heading breaks over mid-sentence cuts. +- `scanBreakPoints(text)` — returns `BreakPoint[]` (paragraph, heading, list-item, code-fence boundaries) for custom chunking strategies. +- `findBestCutoff(breakPoints, target, tolerance)` — picks the break point closest to `target` within `tolerance`, or falls back to a char-count cut. +- `findCodeFences(text)` / `isInsideCodeFence(offset, fences)` — Markdown code-fence detection; used internally but exported for callers that need to avoid splitting inside a fenced block. + +Types: `Chunk`, `ChunkOptions`, `BreakPoint`, `CodeFence`. ## Related -- `@statewalker/indexer-api` — contract chunker output is typically fed into. +- `@statewalker/indexer-api` — the indexer contract that typically consumes chunker output. diff --git a/packages/indexer-core/README.md b/packages/indexer-core/README.md index 4b5fc85..85769ea 100644 --- a/packages/indexer-core/README.md +++ b/packages/indexer-core/README.md @@ -2,16 +2,32 @@ Workspace-internal scaffolding shared by the `@statewalker/indexer-*` backends. -**Not published to npm.** This package is consumed via `workspace:*` by sibling backend packages only (`indexer-mem`, `indexer-mem-flexsearch`, `indexer-mem-minisearch`, `indexer-duckdb`, `indexer-pglite`). +**Not published to npm.** This package is `"private": true` and consumed via `workspace:*` by sibling backends only (`indexer-mem`, `indexer-mem-flexsearch`, `indexer-mem-minisearch`, `indexer-duckdb`, `indexer-pglite`). -## Purpose +## What it holds -Holds engine-agnostic code that would otherwise be forked across every backend: +All the engine-agnostic code that would otherwise be forked across every backend. -- Shared pure helpers (`compositeKey`, `matchesPrefix`, `sanitizePrefix`, `validateDimensionality`, `toAsyncIterable`, persistence byte helpers). -- Unified hybrid merge (`mergeHybrid`) delegating RRF to `@statewalker/indexer-api`'s `reciprocalRankFusion`. -- One composite-index factory (`createCompositeIndex`) replacing `MemIndex` / `DuckDbIndex` / `PGLiteIndex`. -- Two generic `Indexer` factory builders: `createPersistenceBackedIndexer` (mem) and `createSqlBackedIndexer` (SQL). -- A `SqlRetrieverBase` + `SqlDialect` pair holding shared SQL CRUD; dialects override only search SQL + DDL + embedding binding. +### Composite index +- `createCompositeIndex({ name, fts, vec, metadata?, getSize?, onDeleteIndex? })` — returns a value satisfying the public `Index` contract. Fans FTS and vector sub-indexes out on `search`/`addDocument`/`deleteDocuments`; unions sub-index references for `getDocumentPaths`, `getDocumentBlocksRefs`, `getDocumentsBlocks` (de-duped by composite key — O(N), no separate tracking map). Backends inject their engine-specific `getSize` closure (SQL UNION for DuckDB/PGlite, sub-index enumeration for mem) and an optional `onDeleteIndex` hook (SQL backends `DROP TABLE` the shared docs table). -See [openspec change `indexer-core-consolidation`](../../../../openspec/changes/indexer-core-consolidation/) for the design and migration plan. +### Merge / rank fusion +- `mergeByRRF(fts, vec, topK)` and `mergeByWeights(fts, vec, weights, topK)` — single source of truth. Both key results by `compositeKey(path, blockId)`; this is deliberately different from `@statewalker/indexer-api`'s `reciprocalRankFusion`, which keys by `blockId` alone and applies a top-rank bonus. +- `mergeHybrid(fts, vec, topK, weights?)` — convenience dispatcher. + +### Generic `Indexer` factory builders +- `createPersistenceBackedIndexer({ createFts, serializeFts, deserializeFts, createVec, serializeVec, deserializeVec, persistence? })` — used by both mem backends. Preserves the save/load wire format byte-for-byte: `__manifest__`, `${name}/__config__`, `${name}/fts`, `${name}/vec`. +- `createSqlBackedIndexer({ db, dialect, onClose? })` — used by DuckDB and PGlite. Manages the shared `__indexer_manifest` table, extension init, per-index DDL/drop, and composite assembly. + +### SQL retrievers (shared CRUD) +- `createSqlFtsRetriever({ db, prefix, docsTable, info, dialect: SqlFtsDialect })` — shared FTS sub-index: `resolveDocId`, path-filter SQL, ingest/delete, enumeration. Dialect supplies `createTableDdl`, optional `rebuild` (triggered lazily on first search after writes; used by DuckDB's `fts` extension), and `search`. +- `createSqlVectorRetriever({ db, prefix, docsTable, info, dialect: SqlVectorDialect })` — shared vector sub-index with the same shape. Dialect supplies `createTableDdl` (incl. HNSW index), `bindEmbedding`, `embeddingCastSuffix`, `search`, `decodeEmbedding`. +- `SqlDb` — minimal normalised async SQL client the retrievers consume. Each backend wraps its native driver (DuckDB's `Db` from `@statewalker/db-api`; PGlite's `PGlite`) to satisfy it. +- `SqlBackedDialect` — per-backend bundle: `extensionInit`, `docsTableDdl`, `extraCleanup?`, `unionAliasSuffix` (PGlite requires `AS combined`), `fts`, `vec`. + +### Pure helpers +`compositeKey`, `matchesPrefix`, `sanitizePrefix`, `validateDimensionality`, `toAsyncIterable`, and persistence byte helpers (`toBytes`, `singleChunk`, `readEntryBytes`). + +## Design + +See [openspec change `indexer-core-consolidation`](../../../../openspec/changes/indexer-core-consolidation/) (archived) for the full design document, specs, and per-phase implementation plan. diff --git a/packages/indexer-mem-flexsearch/README.md b/packages/indexer-mem-flexsearch/README.md index cf767c2..df95e5e 100644 --- a/packages/indexer-mem-flexsearch/README.md +++ b/packages/indexer-mem-flexsearch/README.md @@ -1,6 +1,6 @@ # @statewalker/indexer-mem-flexsearch -In-memory indexer implementation backed by FlexSearch for full-text search. +In-memory implementation of `@statewalker/indexer-api` combining [FlexSearch](https://github.com/nextapps-de/flexsearch) for full-text search with `@statewalker/indexer-mem`'s `MemVectorIndex` for embeddings. Optional streaming persistence via `IndexerPersistence`. ## Installation @@ -11,17 +11,45 @@ pnpm add @statewalker/indexer-mem-flexsearch ## Usage ```ts -import { createFlexsearchIndex } from "@statewalker/indexer-mem-flexsearch"; +import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch"; + +const indexer = createFlexSearchIndexer(); +const index = await indexer.createIndex({ + name: "docs", + fulltext: { language: "en" }, + vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" }, +}); + +await index.addDocument([ + { path: "/docs/a", blockId: "1", content: "hello world" }, +]); + +for await (const hit of index.search({ queries: ["hello"], topK: 10 })) { + console.log(hit.path, hit.blockId, hit.score); +} +``` + +### With persistence + +```ts +import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch"; +import type { IndexerPersistence } from "@statewalker/indexer-api"; + +const persistence: IndexerPersistence = /* … your save/load adapter … */; +const indexer = createFlexSearchIndexer({ persistence }); -const idx = createFlexsearchIndex(); -await idx.add({ id: "1", text: "hello world" }); -await idx.search("hello"); +// First use loads existing state; `indexer.flush()` writes current state back. ``` +Wire format (stable, byte-compatible across releases): `__manifest__` (JSON array of index names), `${name}/__config__`, `${name}/fts` (FlexSearch serialized state), `${name}/vec` (Arrow IPC from `MemVectorIndex`). + ## API -- `createFlexsearchIndex(options)` — returns an `@statewalker/indexer-api`-compatible index. +- `createFlexSearchIndexer(options?)` — returns an `Indexer`. Options: `persistence?: IndexerPersistence`. +- `FlexSearchIndexerOptions` — option type. ## Related -- `@statewalker/indexer-api`, `@statewalker/indexer-mem`. +- `@statewalker/indexer-api` — the pluggable contract. +- `@statewalker/indexer-mem-minisearch` — drop-in alternative using MiniSearch. +- `@statewalker/indexer-mem` — provides the vector sub-index (`MemVectorIndex`). diff --git a/packages/indexer-mem-minisearch/README.md b/packages/indexer-mem-minisearch/README.md index a14b717..95ac988 100644 --- a/packages/indexer-mem-minisearch/README.md +++ b/packages/indexer-mem-minisearch/README.md @@ -1,6 +1,6 @@ # @statewalker/indexer-mem-minisearch -In-memory indexer implementation backed by MiniSearch for full-text search. +In-memory implementation of `@statewalker/indexer-api` combining [MiniSearch](https://github.com/lucaong/minisearch) for full-text search with `@statewalker/indexer-mem`'s `MemVectorIndex` for embeddings. Optional streaming persistence via `IndexerPersistence`. ## Installation @@ -11,16 +11,43 @@ pnpm add @statewalker/indexer-mem-minisearch ## Usage ```ts -import { createMinisearchIndex } from "@statewalker/indexer-mem-minisearch"; +import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch"; + +const indexer = createMiniSearchIndexer(); +const index = await indexer.createIndex({ + name: "docs", + fulltext: { language: "en" }, + vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" }, +}); + +await index.addDocument([ + { path: "/docs/a", blockId: "1", content: "hello world" }, +]); + +for await (const hit of index.search({ queries: ["hello"], topK: 10 })) { + console.log(hit.path, hit.blockId, hit.score); +} +``` + +### With persistence -const idx = createMinisearchIndex(); -await idx.add({ id: "1", text: "hello world" }); +```ts +import { createMiniSearchIndexer } from "@statewalker/indexer-mem-minisearch"; +import type { IndexerPersistence } from "@statewalker/indexer-api"; + +const persistence: IndexerPersistence = /* … your save/load adapter … */; +const indexer = createMiniSearchIndexer({ persistence }); ``` +Wire format (stable across releases): `__manifest__` (JSON array of index names), `${name}/__config__`, `${name}/fts` (MiniSearch `toJSON()`), `${name}/vec` (Arrow IPC from `MemVectorIndex`). + ## API -- `createMinisearchIndex(options)` — returns an `@statewalker/indexer-api`-compatible index. +- `createMiniSearchIndexer(options?)` — returns an `Indexer`. Options: `persistence?: IndexerPersistence`. +- `MiniSearchIndexerOptions` — option type. ## Related -- `@statewalker/indexer-api`, `@statewalker/indexer-mem`. +- `@statewalker/indexer-api` — the pluggable contract. +- `@statewalker/indexer-mem-flexsearch` — drop-in alternative using FlexSearch. +- `@statewalker/indexer-mem` — provides the vector sub-index (`MemVectorIndex`). diff --git a/packages/indexer-pglite/README.md b/packages/indexer-pglite/README.md index cdc532b..8502235 100644 --- a/packages/indexer-pglite/README.md +++ b/packages/indexer-pglite/README.md @@ -1,25 +1,53 @@ # @statewalker/indexer-pglite -PGlite-backed indexer implementation: full-text + vector search using in-browser Postgres. +PGlite-backed implementation of `@statewalker/indexer-api`: full-text search via PostgreSQL's native `tsvector` + GIN index, vector search via [pgvector](https://github.com/pgvector/pgvector) with an HNSW cosine index. Runs entirely in-process (in the browser, Node, or anywhere [PGlite](https://github.com/electric-sql/pglite) does). ## Installation ```sh -pnpm add @statewalker/indexer-pglite +pnpm add @statewalker/indexer-pglite @electric-sql/pglite ``` ## Usage ```ts -import { createPgliteIndexer } from "@statewalker/indexer-pglite"; - -const idx = await createPgliteIndexer({ dataDir: "idb://myapp" }); +import { createPGLiteIndexer } from "@statewalker/indexer-pglite"; + +// Uses an in-memory PGlite instance by default. +const indexer = await createPGLiteIndexer(); + +// Or pass an existing PGlite (with the `vector` extension already loaded): +// import { PGlite } from "@electric-sql/pglite"; +// import { vector } from "@electric-sql/pglite/vector"; +// const db = await PGlite.create({ dataDir: "idb://myapp", extensions: { vector } }); +// const indexer = await createPGLiteIndexer({ db }); + +const index = await indexer.createIndex({ + name: "docs", + fulltext: { language: "en" }, + vector: { dimensionality: 384, model: "all-MiniLM-L6-v2" }, +}); ``` +## Runtime extension + +`createPGLiteIndexer` runs `CREATE EXTENSION IF NOT EXISTS vector` at init. When the factory creates its own PGlite (no `db` passed), it auto-loads the `vector` extension via PGlite's `extensions: { vector }` option; when a caller-supplied `db` is passed, the caller is responsible for having the extension available (pass `extensions: { vector }` to `PGlite.create`). + +## SQL shape + +Per named index: + +- `idx__docs(doc_id SERIAL PRIMARY KEY, path TEXT UNIQUE)` — path ↔ doc_id mapping. +- `idx__fts(doc_id, block_id, content, content_tsv TSVECTOR GENERATED ..., metadata)` + `USING GIN (content_tsv)`. +- `idx__vec(doc_id, block_id, embedding vector(dim))` + `USING hnsw (embedding vector_cosine_ops)`. + +Search uses `ts_rank_cd` + `to_tsquery(, ...)` for FTS, and the `<=>` cosine-distance operator for vector search. + ## API -- `createPgliteIndexer(options)` — returns an `@statewalker/indexer-api` compatible index backed by PGlite. +- `createPGLiteIndexer(options?)` — returns `Promise`. Options: `db?: PGlite` (bring your own; otherwise a fresh in-memory instance is created, and the returned indexer owns and will close it on `indexer.close()`). +- `PGLiteIndexerOptions` — option type. ## Related -- `@statewalker/indexer-api`, `@electric-sql/pglite`. +- `@statewalker/indexer-api`, `@electric-sql/pglite`, `pgvector`. diff --git a/packages/indexer-tests/README.md b/packages/indexer-tests/README.md index 4b51dde..5b9c7b6 100644 --- a/packages/indexer-tests/README.md +++ b/packages/indexer-tests/README.md @@ -2,19 +2,28 @@ > Internal, workspace-only dev dependency. **Not published**. -Shared Vitest suite that every `@statewalker/indexer-*` implementation runs against. Keeps behavior consistent across backends. +Shared Vitest suite that every `@statewalker/indexer-*` backend runs against. Covers the full [`Indexer`/`Index`/`FullTextIndex`/`EmbeddingIndex`](../indexer-api/src/) contract: lifecycle, CRUD, path-prefix filtering, hybrid search, persistence round-trip, multi-indexer isolation, and search-quality fixtures. ## Usage -In another indexer package's `test/suite.test.ts`: +In a backend package's `tests/suite.test.ts`: ```ts -import { runIndexerSuite } from "@statewalker/indexer-tests"; -import { createFlexsearchIndex } from "@statewalker/indexer-mem-flexsearch"; +import { runIndexerTestSuite } from "@statewalker/indexer-tests"; +import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch"; -runIndexerSuite(createFlexsearchIndex); +runIndexerTestSuite("FlexSearch Indexer", async () => createFlexSearchIndexer()); ``` +The factory callback is invoked per-test to produce a fresh `Indexer` (in-memory for mem backends, a fresh DuckDB/PGlite instance for SQL backends). + +## Exports + +- `runIndexerTestSuite(name, factory)` — main cross-backend suite entry point. +- `IndexerFactory` — type alias for the factory callback. +- Fixture loaders: `loadBlocksFixture`, `loadQueriesFixture`, `loadQueriesEmbeddingsFixture`, `listFixtureDocs`, `readFixtureDoc`. +- `createFixtureEmbedFn`, `EMBEDDING_DIMENSIONS`, `EMBEDDING_MODEL` — deterministic embedding helpers for test inputs. + ## Related - `@statewalker/indexer-api` — contract these tests enforce. From 4e5861048c0dfa1cf4a8aaf4bb4f3022565c29df Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Tue, 28 Apr 2026 18:55:16 +0200 Subject: [PATCH 06/12] =?UTF-8?q?fix(indexer):=20break=20indexer-api=20?= =?UTF-8?q?=E2=86=94=20indexer-mem-flexsearch=20cycle?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Move the two integration tests that exercise the indexer-api helpers against a real flexsearch backend into indexer-mem-flexsearch/tests/ where they belong. Drop the indexer-mem-flexsearch devDependency from indexer-api so neither the direct cycle nor the longer indexer-api → mem-flexsearch → indexer-tests → indexer-api loop remains. Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/indexer-api/package.json | 1 - .../tests}/index-documents.test.ts | 9 ++++++--- .../tests}/search-pipeline.test.ts | 11 ++++++----- 3 files changed, 12 insertions(+), 9 deletions(-) rename packages/{indexer-api/test/helpers => indexer-mem-flexsearch/tests}/index-documents.test.ts (91%) rename packages/{indexer-api/test/helpers => indexer-mem-flexsearch/tests}/search-pipeline.test.ts (96%) diff --git a/packages/indexer-api/package.json b/packages/indexer-api/package.json index 800eb73..e3ddb13 100644 --- a/packages/indexer-api/package.json +++ b/packages/indexer-api/package.json @@ -32,7 +32,6 @@ "format": "biome format --write ." }, "devDependencies": { - "@statewalker/indexer-mem-flexsearch": "workspace:*", "rimraf": "catalog:", "tsdown": "catalog:", "typescript": "catalog:", diff --git a/packages/indexer-api/test/helpers/index-documents.test.ts b/packages/indexer-mem-flexsearch/tests/index-documents.test.ts similarity index 91% rename from packages/indexer-api/test/helpers/index-documents.test.ts rename to packages/indexer-mem-flexsearch/tests/index-documents.test.ts index 597baa5..48f732f 100644 --- a/packages/indexer-api/test/helpers/index-documents.test.ts +++ b/packages/indexer-mem-flexsearch/tests/index-documents.test.ts @@ -1,7 +1,10 @@ -import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch"; +import { + type DocumentPath, + type Index, + indexDocuments, +} from "@statewalker/indexer-api"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { indexDocuments } from "../../src/helpers/index-documents.js"; -import type { DocumentPath, Index } from "../../src/indexer-index.js"; +import { createFlexSearchIndexer } from "../src/index.js"; let indexer: ReturnType; let index: Index; diff --git a/packages/indexer-api/test/helpers/search-pipeline.test.ts b/packages/indexer-mem-flexsearch/tests/search-pipeline.test.ts similarity index 96% rename from packages/indexer-api/test/helpers/search-pipeline.test.ts rename to packages/indexer-mem-flexsearch/tests/search-pipeline.test.ts index bd58690..3b138cb 100644 --- a/packages/indexer-api/test/helpers/search-pipeline.test.ts +++ b/packages/indexer-mem-flexsearch/tests/search-pipeline.test.ts @@ -1,12 +1,13 @@ -import { createFlexSearchIndexer } from "@statewalker/indexer-mem-flexsearch"; -import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createMockCitationBuilder, createMockExpander, createMockReranker, -} from "../../src/helpers/mock.js"; -import { SearchPipeline } from "../../src/helpers/search-pipeline.js"; -import type { DocumentPath, Index } from "../../src/indexer-index.js"; + type DocumentPath, + type Index, + SearchPipeline, +} from "@statewalker/indexer-api"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createFlexSearchIndexer } from "../src/index.js"; let indexer: ReturnType; let index: Index; From a325e36123a08524fc975ceb0660c7064db323fe Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Sun, 3 May 2026 18:10:49 +0200 Subject: [PATCH 07/12] refactor(indexer): split @statewalker/indexer-api into contract + indexer-search strategy stack - @statewalker/indexer-api becomes pure contract (types/interfaces, zero runtime) - new @statewalker/indexer-search owns SearchPipeline, SemanticIndex, query parser, intent, reranker blending, mocks - consolidate two RRF implementations into @statewalker/indexer-core; mergeByRRF delegates to reciprocalRankFusion - defaultMultiSearch renamed to fanOutSearch and moved to indexer-core (backend-internal) - remove indexDocuments helper (callers use SemanticIndex.addDocuments) - remove RerankResult type alias (RerankerFn now returns ScoredItem[]) - remove indexer-tests/multi-search.suite.ts; semantic-index suite moves SemanticIndex import to indexer-search - update README for both packages and reorganise indexer-api/src into contract/ subdirectory Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/indexer-api/README.md | 164 ++------------ packages/indexer-api/src/contract/index.ts | 207 +++++++++++++++++ .../{indexer-index.ts => contract/types.ts} | 209 ++---------------- .../src/helpers/index-documents.ts | 32 --- packages/indexer-api/src/helpers/index.ts | 20 -- packages/indexer-api/src/index.ts | 64 ++---- packages/indexer-api/src/indexer.ts | 3 +- packages/indexer-api/src/multi-search.ts | 129 ----------- packages/indexer-core/src/fan-out-search.ts | 95 ++++++++ packages/indexer-core/src/index.ts | 8 + packages/indexer-core/src/merge.ts | 41 ++-- .../{indexer-api => indexer-core}/src/rrf.ts | 6 +- packages/indexer-core/test/merge.test.ts | 48 ++++ .../test/rrf.test.ts | 0 packages/indexer-mem-flexsearch/package.json | 1 + .../tests/index-documents.test.ts | 103 --------- .../tests/search-pipeline.test.ts | 5 +- .../tests/semantic-index.test.ts | 86 +++++++ packages/indexer-search/README.md | 159 +++++++++++++ packages/indexer-search/package.json | 47 ++++ .../src/fn-types.ts} | 12 +- packages/indexer-search/src/index.ts | 28 +++ .../src/intent.ts | 0 .../helpers => indexer-search/src}/mock.ts | 10 +- .../src/query-parser.ts | 0 .../src/reranker-blend.ts | 2 +- .../src}/search-pipeline.ts | 14 +- .../src/semantic-index.ts | 5 +- .../test/intent.test.ts | 0 .../test}/mock.test.ts | 6 +- .../test/query-parser.test.ts | 0 .../test/reranker-blend.test.ts | 2 +- packages/indexer-search/tsconfig.json | 27 +++ packages/indexer-tests/package.json | 3 +- packages/indexer-tests/src/suite-runner.ts | 2 - .../src/suites/multi-search.suite.ts | 102 --------- .../src/suites/semantic-index.suite.ts | 2 +- 37 files changed, 810 insertions(+), 832 deletions(-) create mode 100644 packages/indexer-api/src/contract/index.ts rename packages/indexer-api/src/{indexer-index.ts => contract/types.ts} (51%) delete mode 100644 packages/indexer-api/src/helpers/index-documents.ts delete mode 100644 packages/indexer-api/src/helpers/index.ts delete mode 100644 packages/indexer-api/src/multi-search.ts create mode 100644 packages/indexer-core/src/fan-out-search.ts rename packages/{indexer-api => indexer-core}/src/rrf.ts (96%) create mode 100644 packages/indexer-core/test/merge.test.ts rename packages/{indexer-api => indexer-core}/test/rrf.test.ts (100%) delete mode 100644 packages/indexer-mem-flexsearch/tests/index-documents.test.ts create mode 100644 packages/indexer-mem-flexsearch/tests/semantic-index.test.ts create mode 100644 packages/indexer-search/README.md create mode 100644 packages/indexer-search/package.json rename packages/{indexer-api/src/helpers/types.ts => indexer-search/src/fn-types.ts} (72%) create mode 100644 packages/indexer-search/src/index.ts rename packages/{indexer-api => indexer-search}/src/intent.ts (100%) rename packages/{indexer-api/src/helpers => indexer-search/src}/mock.ts (88%) rename packages/{indexer-api => indexer-search}/src/query-parser.ts (100%) rename packages/{indexer-api => indexer-search}/src/reranker-blend.ts (95%) rename packages/{indexer-api/src/helpers => indexer-search/src}/search-pipeline.ts (95%) rename packages/{indexer-api => indexer-search}/src/semantic-index.ts (97%) rename packages/{indexer-api => indexer-search}/test/intent.test.ts (100%) rename packages/{indexer-api/test/helpers => indexer-search/test}/mock.test.ts (96%) rename packages/{indexer-api => indexer-search}/test/query-parser.test.ts (100%) rename packages/{indexer-api => indexer-search}/test/reranker-blend.test.ts (99%) create mode 100644 packages/indexer-search/tsconfig.json delete mode 100644 packages/indexer-tests/src/suites/multi-search.suite.ts diff --git a/packages/indexer-api/README.md b/packages/indexer-api/README.md index 70e1795..7d3b578 100644 --- a/packages/indexer-api/README.md +++ b/packages/indexer-api/README.md @@ -1,6 +1,6 @@ # @statewalker/indexer-api -Backend-agnostic TypeScript API for hybrid search indexes combining **full-text search (FTS)** and **vector/embedding similarity search**. +Backend-agnostic TypeScript contract for hybrid search indexes combining **full-text search (FTS)** and **vector/embedding similarity search**. ## Why this API? @@ -12,7 +12,7 @@ Modern search applications need more than keyword matching. They need to combine Each of these capabilities can be backed by very different storage engines — in-memory structures, SQLite/PGlite, DuckDB, or external services. Without a shared abstraction, application code becomes tightly coupled to a specific backend, making it hard to swap implementations, test in isolation, or run the same logic in different environments (browser, Node, edge). -`@statewalker/indexer-api` solves this by defining a **pure-interface contract** with zero runtime dependencies. Application code programs against the API; concrete backends are injected at startup. +`@statewalker/indexer-api` solves this by defining a **pure-interface contract** with **zero runtime exports**. Application code programs against the API; concrete backends are injected at startup. The strategy stack (search pipeline, query parser, semantic index, reranker blending, mocks) lives in [`@statewalker/indexer-search`](../indexer-search/README.md). ## How it works @@ -31,15 +31,15 @@ Indexer — registry/factory: creates, lists, deletes named indexes Every index provides a uniform contract (`SearchIndex`) covering: -| Capability | Methods | -|---------------|---------| -| **Search** | `search()` — async generator streaming scored results | -| **Ingestion** | `addDocument()`, `addDocuments()` — single or bulk insert | -| **Deletion** | `deleteDocuments()` — by path prefix and/or block ID | +| Capability | Methods | +|-----------------|---------| +| **Search** | `search()` — async generator streaming scored results | +| **Ingestion** | `addDocument()`, `addDocuments()` — single or bulk insert | +| **Deletion** | `deleteDocuments()` — by path prefix and/or block ID | | **Enumeration** | `getSize()`, `getDocumentPaths()`, `getDocumentBlocksRefs()`, `getDocumentsBlocks()` | -| **Lifecycle** | `flush()`, `close()`, `deleteIndex()` | +| **Lifecycle** | `flush()`, `close()`, `deleteIndex()` | -### Hybrid search & score fusion +### Hybrid search Hybrid search accepts both FTS queries and embedding vectors simultaneously. Results from each modality are blended using configurable `HybridWeights`: @@ -54,40 +54,9 @@ index.search({ When only one modality is provided, the search gracefully degrades to single-modality mode. -### Reciprocal Rank Fusion (RRF) - -The `reciprocalRankFusion()` function merges multiple ranked lists into a single ranking. It supports: - -- **Weighted lists** — each list can carry a weight that scales its contribution. -- **Top-rank bonus** — items ranked #1 get a +0.05 bonus; ranks #2-3 get +0.02 (adapted from [QMD](https://github.com/tobi/qmd)). -- **Tracing** — `buildRrfTrace()` returns per-item contribution breakdowns for debugging and explainability. - -### Reranker blending - -`blendWithReranker()` combines initial retrieval scores with reranker scores using position-aware tiers. Top-ranked items are protected by higher retrieval weights (default: 0.75 for top-3, 0.60 for top-10, 0.40 for the rest), preventing aggressive rerankers from destabilizing high-confidence results. - -### Multi-search - -`defaultMultiSearch()` fans out multiple queries and embeddings into independent searches, then fuses them with RRF. It tracks `matchCount` (how many input queries matched each result) and supports grouping results by document path for organized output. - -### Query parsing - -`parseStructuredQuery()` parses typed search instructions with prefixes: - -- `lex:` — lexical/keyword query -- `vec:` — vector/semantic query -- `hyde:` — hypothetical document embedding query -- `expand:` — pass-through (returns `null` for default pipeline handling) - -Validators (`validateLexQuery`, `validateSemanticQuery`) catch malformed queries before they reach the backend. - -### Intent disambiguation - -`extractIntentTerms()` strips stop-words from a user's intent description, and `selectBestChunk()` uses both query terms and intent terms to pick the most relevant text chunk — useful for snippet extraction and context selection. - ### Path-prefix filtering -Documents are organized under hierarchical paths (`"/projects/alpha/specs/"`). All search, enumeration, and deletion operations accept optional path prefixes to restrict their scope. For example, passing `paths: ["/docs/"]` to `index.search(...)` limits results to documents whose path starts with `"/docs/"`. Prefix matching is a simple `startsWith` at the type level — backends implement it using their native SQL (DuckDB / PGlite) or in-memory filtering (indexer-mem-*). +Documents are organized under hierarchical paths (`"/projects/alpha/specs/"`). All search, enumeration, and deletion operations accept optional path prefixes to restrict their scope. For example, passing `paths: ["/docs/"]` to `index.search(...)` limits results to documents whose path starts with `"/docs/"`. Prefix matching is a simple `startsWith` at the type level — backends implement it using their native SQL (DuckDB / PGlite) or in-memory filtering (`indexer-mem-*`). ### Persistence @@ -100,17 +69,13 @@ interface IndexerPersistence { } ``` -### SemanticIndex convenience wrapper - -`SemanticIndex` wraps an `Index` and an `EmbedFn` to automatically compute embeddings at ingestion and search time. Application code provides plain text; the wrapper handles embedding generation transparently. +### Ranking primitive -### SearchPipeline +`ScoredItem = { blockId: string; score: number }` is a small primitive shape shared between the backend toolkit (`@statewalker/indexer-core`'s RRF) and the strategy stack (`@statewalker/indexer-search`'s reranker blending). Living here keeps both consumers free of cross-package coupling. -`SearchPipeline` is a builder/executor for multi-stage search with optional LLM-powered stages. It chains: **expand** (query expansion) → **embed** (semantic query embedding) → **search** (single `index.search()` call) → **rerank** (score blending) → **cite** (citation extraction). Each LLM stage is defined as a function type (`QueryExpanderFn`, `RerankerFn`, `CitationBuilderFn`) — no class instantiation required, just pass closures. +### Embedding boundary -### indexDocuments utility - -`indexDocuments()` is a convenience function for batch document ingestion with optional auto-embedding. It accepts a sync or async iterable of documents and an optional `embedFn`, and returns a count of indexed documents. +`EmbedFn = (text: string) => Promise` is the boundary type at which the application provides an embedding capability to the indexer. Both backends (via test fixtures) and the strategy stack consume it. ## Implementations / backends @@ -126,6 +91,7 @@ Supporting packages: | Package | Purpose | |---------|---------| +| `@statewalker/indexer-search` | Application-side strategy stack (`SearchPipeline`, `SemanticIndex`, query parser, reranker blending, mocks) | | `@statewalker/indexer-chunker` | Markdown splitting and code fence detection for content preprocessing | ## How to use @@ -195,104 +161,8 @@ for await (const result of index.search({ } ``` -### Using SemanticIndex for automatic embedding - -```ts -import { SemanticIndex } from "@statewalker/indexer-api"; - -const semantic = new SemanticIndex(index, embed); - -// Embedding computed automatically from content -await semantic.addDocument({ - path: "/docs/guide/", - blockId: "ch1", - content: "Chapter 1: Introduction...", -}); - -// Search with automatic query embedding -const results = await semantic.search({ - query: "introduction", - topK: 5, -}); -``` - -### Multi-search with RRF fusion - -```ts -import { defaultMultiSearch } from "@statewalker/indexer-api"; - -const results = await defaultMultiSearch(index, { - queries: ["CAP theorem", "consistency models"], - embeddings: [await embed("distributed systems trade-offs")], - topK: 10, - weights: { fts: 0.6, embedding: 0.4 }, -}); -``` - -### Structured queries - -```ts -import { parseStructuredQuery } from "@statewalker/indexer-api"; - -const parsed = parseStructuredQuery("lex: CAP theorem\nvec: consensus algorithms"); -// [{ type: "lex", query: "CAP theorem" }, { type: "vec", query: "consensus algorithms" }] -``` - -### SearchPipeline - -```ts -import { SearchPipeline } from "@statewalker/indexer-api"; - -const results = await new SearchPipeline({ - index, - embedFn: embed, - expander: async (query) => [ - { type: "lex", query }, - { type: "vec", query: `semantic: ${query}` }, - ], - reranker: async (query, candidates) => - candidates.map((c, i) => ({ blockId: c.blockId, score: 1 / (i + 1) })), -}) - .setPrompt("distributed consensus") - .setTopK(10) - .execute(); -``` - -### Batch indexing with indexDocuments - -```ts -import { indexDocuments } from "@statewalker/indexer-api"; - -const { indexed } = await indexDocuments(index, [ - { path: "/docs/", blockId: "b1", content: "First document..." }, - { path: "/docs/", blockId: "b2", content: "Second document..." }, -], { embedFn: embed }); -``` +For application-side ergonomics (auto-embedding, query expansion, reranking, citations, structured query parsing, intent extraction), see [`@statewalker/indexer-search`](../indexer-search/README.md). ## How it is tested -Tests use **vitest** and live in `test/`, mirroring the `src/` structure. The package has **7 test suites** covering the pure-logic modules: - -| Test file | What it covers | -|-----------|----------------| -| `test/rrf.test.ts` | RRF score computation, weighted lists, top-rank bonuses, trace correctness | -| `test/reranker-blend.test.ts` | Position-aware blending, tier boundaries, re-ordering, custom tiers, edge cases | -| `test/query-parser.test.ts` | Structured query parsing (`lex:/vec:/hyde:/expand:`), validation, error cases | -| `test/intent.test.ts` | Stop-word filtering, intent term extraction, chunk selection with intent weighting | -| `test/helpers/search-pipeline.test.ts` | SearchPipeline builder, FTS execution, expansion, reranking, citations, explain traces, error handling | -| `test/helpers/mock.test.ts` | Mock expander, reranker, and citation builder factory functions | -| `test/helpers/index-documents.test.ts` | Batch indexing utility with sync/async iterables and auto-embedding | - -The interface types (`Indexer`, `Index`, `FullTextIndex`, `EmbeddingIndex`) are not tested here — they are pure TypeScript interfaces with no runtime behavior. Each backend package (`indexer-mem`, `indexer-pglite`, `indexer-duckdb`, etc.) has its own integration test suite that validates conformance to these interfaces. - -Run tests: - -```bash -# Run once -pnpm test - -# Watch mode -pnpm test:watch -``` - -Several algorithms (RRF, query parser, intent extraction, reranker blending) are adapted from [QMD](https://github.com/tobi/qmd) by Tobi Lutke (MIT License). +`@statewalker/indexer-api` is contract-only — it has no runtime to test. Conformance to the contract is validated per backend in `@statewalker/indexer-tests` (the cross-backend conformance runner) and in each backend package's own integration suite. diff --git a/packages/indexer-api/src/contract/index.ts b/packages/indexer-api/src/contract/index.ts new file mode 100644 index 0000000..dd04dd0 --- /dev/null +++ b/packages/indexer-api/src/contract/index.ts @@ -0,0 +1,207 @@ +// ============================================================================= +// Indexer Contract — Operation Interfaces +// +// Defines the operation shapes implemented by every backend: the generic +// `SearchIndex` base, the FTS / vector sub-indexes, and the composite +// hybrid `Index`. Zero runtime. +// ============================================================================= + +import type { + BlockReference, + DocumentPath, + EmbeddingBlock, + EmbeddingSearchParams, + EmbeddingSearchResult, + FullTextBlock, + FullTextSearchParams, + FullTextSearchResult, + HybridSearchParams, + HybridSearchResult, + IndexedBlock, + Metadata, + PathSelector, +} from "./types.js"; + +// ============================================================================= +// 6. Generic search index base +// ============================================================================= + +/** + * Abstract base interface for all search indexes (FTS, embedding, hybrid). + * + * Provides a uniform contract for: + * - **Searching** — streaming results via an async generator. + * - **Ingestion** — adding documents as batches of blocks. + * - **Deletion** — removing documents by path prefix / block id. + * - **Enumeration** — listing paths, block references, and full blocks. + * - **Lifecycle** — flushing pending writes, closing, and destroying the index. + * + * @typeParam BlockType The block shape accepted for ingestion. + * @typeParam SearchParamsType The search parameters accepted by this index. + * @typeParam SearchResultType The shape of individual search results. + */ +interface SearchIndex { + // --- Search ---------------------------------------------------------------- + + /** + * Execute a search and stream results as an async generator. + * + * @param params Search parameters specific to the index type. + * @returns An async generator yielding scored search results. + */ + search(params: SearchParamsType): AsyncGenerator; + + // --- Ingestion ------------------------------------------------------------- + + /** + * Add a single document's blocks to the index. + * + * All blocks in the array **must** share the same {@link DocumentPath}. + * To ingest blocks from different documents use {@link addDocuments} or + * call this method once per document. + */ + addDocument(blocks: BlockType[]): Promise; + + /** + * Bulk-add documents from a (possibly async) iterable of block batches. + * + * Each inner array represents one document and **must** contain blocks + * sharing the same {@link DocumentPath}. + */ + addDocuments(blocks: Iterable | AsyncIterable): Promise; + + // --- Deletion -------------------------------------------------------------- + + /** + * Delete blocks matching the given path selectors. + * + * Each selector targets a path prefix and optionally a specific block id. + * Accepts either an array (for known-size batches) or an async iterable + * (for streaming deletion lists). + */ + deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable): Promise; + + // --- Enumeration ----------------------------------------------------------- + + /** + * Count the number of blocks in the index. + * + * @param pathPrefix When provided, only blocks under this prefix are counted. + */ + getSize(pathPrefix?: DocumentPath): Promise; + + /** + * Stream all unique document paths in the index. + * + * @param pathPrefix When provided, only paths under this prefix are yielded. + * @returns An async generator yielding unique {@link DocumentPath} values. + */ + getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator; + + /** + * Stream all block references (path + blockId) in the index. + * + * @param pathPrefix When provided, only blocks under this prefix are yielded. + */ + getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator; + + /** + * Stream all blocks (with full content / embedding data) in the index. + * + * @param pathPrefix When provided, only blocks under this prefix are yielded. + */ + getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator; + + // --- Lifecycle ------------------------------------------------------------- + + /** + * Close the index and release associated resources. + * + * @param options.force When `true`, pending writes may be discarded + * without flushing. Defaults to `false` (flush before closing). + */ + close(options?: { force?: boolean }): Promise; + + /** + * Flush pending writes so that all previously added blocks become + * searchable. Important for indexes with delayed or batched indexing. + */ + flush(): Promise; + + /** + * Permanently delete the entire index and all its contents. + * + * This operation is **irreversible**. After calling `deleteIndex` the + * instance should be considered unusable until re-initialised. + */ + deleteIndex(): Promise; +} + +// ============================================================================= +// 7. Sub-indexes +// ============================================================================= + +/** Configuration / status information for a full-text sub-index. */ +export interface FullTextIndexInfo { + /** Language used for stemming, stop-words, etc. */ + language: string; + /** Optional index-level metadata. */ + metadata?: Metadata; +} + +/** + * Full-text search sub-index operating on {@link FullTextBlock}s. + * + * Supports multi-query FTS with path-prefix filtering and relevance scoring. + */ +export interface FullTextIndex + extends SearchIndex { + /** Retrieve configuration and status information for this FTS index. */ + getIndexInfo(): Promise; +} + +/** Configuration / status information for an embedding sub-index. */ +export interface EmbeddingIndexInfo { + /** Dimensionality of the embedding vectors stored in this index. */ + dimensionality: number; + /** Name or identifier of the embedding model that produced the vectors. */ + model: string; + /** Optional index-level metadata. */ + metadata?: Metadata; +} + +/** + * Vector / embedding search sub-index operating on {@link EmbeddingBlock}s. + * + * Supports multi-vector similarity search with path-prefix filtering. + */ +export interface EmbeddingIndex + extends SearchIndex { + /** Retrieve configuration and status information for this vector index. */ + getIndexInfo(): Promise; +} + +// ============================================================================= +// 8. Composite hybrid index +// ============================================================================= + +/** + * A named hybrid search index combining optional FTS and vector sub-indexes. + * + * The composite {@link Index} accepts {@link IndexedBlock}s (which may carry + * text, an embedding, or both) and routes them to the appropriate sub-indexes. + * Hybrid search blends results from both modalities according to + * `HybridWeights`. + */ +export interface Index extends SearchIndex { + /** Unique name of this index within the indexer. */ + readonly name: string; + /** Optional index-level metadata. */ + readonly metadata?: Metadata; + + /** Returns the FTS sub-index, or `null` if this index has no full-text capability. */ + getFullTextIndex(): FullTextIndex | null; + + /** Returns the vector sub-index, or `null` if this index has no embedding capability. */ + getVectorIndex(): EmbeddingIndex | null; +} diff --git a/packages/indexer-api/src/indexer-index.ts b/packages/indexer-api/src/contract/types.ts similarity index 51% rename from packages/indexer-api/src/indexer-index.ts rename to packages/indexer-api/src/contract/types.ts index 2a7bd12..94d85ea 100644 --- a/packages/indexer-api/src/indexer-index.ts +++ b/packages/indexer-api/src/contract/types.ts @@ -1,21 +1,18 @@ // ============================================================================= -// Indexer Index API +// Indexer Contract — Data Types // -// Defines the core abstractions for a hybrid search index combining full-text -// search (FTS) and vector/embedding search. The index organises content into -// documents identified by hierarchical paths (DocumentPath) and blocks within -// those documents (BlockId). Each block may carry text content, an embedding -// vector, or both. +// Pure data shapes used by every backend implementation and the application- +// side strategy stack (`@statewalker/indexer-search`). Zero runtime. // // Logical layout: -// 1. Primitive types (DocumentPath, BlockId, Metadata) -// 2. References (BlockReference, PathSelector) -// 3. Block types (IndexedBlock, FullTextBlock, EmbeddingBlock) -// 4. Search params (FullTextSearchParams, EmbeddingSearchParams, HybridSearchParams) -// 5. Search results (FullTextSearchResult, EmbeddingSearchResult, HybridSearchResult) -// 6. Generic base (SearchIndex) -// 7. Sub-indexes (FullTextIndex, EmbeddingIndex) -// 8. Composite index (Index) +// 1. Primitives (DocumentPath, BlockId, Metadata) +// 2. References (BlockReference, PathSelector) +// 3. Block types (IndexedBlock, FullTextBlock, EmbeddingBlock) +// 4. Search params (FullTextSearchParams, EmbeddingSearchParams, HybridSearchParams) +// 5. Search results (FullTextSearchResult, EmbeddingSearchResult, HybridSearchResult) +// 6. Hybrid weights (HybridWeights) +// 7. Ranking primitive (ScoredItem) +// 8. Embed function (EmbedFn) // ============================================================================= // ============================================================================= @@ -83,7 +80,7 @@ export type PathSelector = { // ============================================================================= /** - * A block of content to be added to the hybrid {@link Index}. + * A block of content to be added to the hybrid index. * * At least one of `content` or `embedding` must be provided — a block with * neither has nothing to index. When both are present the block is indexed @@ -226,185 +223,25 @@ export interface HybridSearchResult extends BlockReference { } // ============================================================================= -// 6. Generic search index base +// 7. Ranking primitive // ============================================================================= /** - * Abstract base interface for all search indexes (FTS, embedding, hybrid). - * - * Provides a uniform contract for: - * - **Searching** — streaming results via an async generator. - * - **Ingestion** — adding documents as batches of blocks. - * - **Deletion** — removing documents by path prefix / block id. - * - **Enumeration** — listing paths, block references, and full blocks. - * - **Lifecycle** — flushing pending writes, closing, and destroying the index. - * - * @typeParam BlockType The block shape accepted for ingestion. - * @typeParam SearchParamsType The search parameters accepted by this index. - * @typeParam SearchResultType The shape of individual search results. - */ -interface SearchIndex { - // --- Search ---------------------------------------------------------------- - - /** - * Execute a search and stream results as an async generator. - * - * @param params Search parameters specific to the index type. - * @returns An async generator yielding scored search results. - */ - search(params: SearchParamsType): AsyncGenerator; - - // --- Ingestion ------------------------------------------------------------- - - /** - * Add a single document's blocks to the index. - * - * All blocks in the array **must** share the same {@link DocumentPath}. - * To ingest blocks from different documents use {@link addDocuments} or - * call this method once per document. - */ - addDocument(blocks: BlockType[]): Promise; - - /** - * Bulk-add documents from a (possibly async) iterable of block batches. - * - * Each inner array represents one document and **must** contain blocks - * sharing the same {@link DocumentPath}. - */ - addDocuments(blocks: Iterable | AsyncIterable): Promise; - - // --- Deletion -------------------------------------------------------------- - - /** - * Delete blocks matching the given path selectors. - * - * Each selector targets a path prefix and optionally a specific block id. - * Accepts either an array (for known-size batches) or an async iterable - * (for streaming deletion lists). - */ - deleteDocuments(pathSelectors: PathSelector[] | AsyncIterable): Promise; - - // --- Enumeration ----------------------------------------------------------- - - /** - * Count the number of blocks in the index. - * - * @param pathPrefix When provided, only blocks under this prefix are counted. - */ - getSize(pathPrefix?: DocumentPath): Promise; - - /** - * Stream all unique document paths in the index. - * - * @param pathPrefix When provided, only paths under this prefix are yielded. - * @returns An async generator yielding unique {@link DocumentPath} values. - */ - getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator; - - /** - * Stream all block references (path + blockId) in the index. - * - * @param pathPrefix When provided, only blocks under this prefix are yielded. - */ - getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator; - - /** - * Stream all blocks (with full content / embedding data) in the index. - * - * @param pathPrefix When provided, only blocks under this prefix are yielded. - */ - getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator; - - // --- Lifecycle ------------------------------------------------------------- - - /** - * Close the index and release associated resources. - * - * @param options.force When `true`, pending writes may be discarded - * without flushing. Defaults to `false` (flush before closing). - */ - close(options?: { force?: boolean }): Promise; - - /** - * Flush pending writes so that all previously added blocks become - * searchable. Important for indexes with delayed or batched indexing. - */ - flush(): Promise; - - /** - * Permanently delete the entire index and all its contents. - * - * This operation is **irreversible**. After calling `deleteIndex` the - * instance should be considered unusable until re-initialised. - */ - deleteIndex(): Promise; -} - -// ============================================================================= -// 7. Sub-indexes -// ============================================================================= - -/** Configuration / status information for a full-text sub-index. */ -export interface FullTextIndexInfo { - /** Language used for stemming, stop-words, etc. */ - language: string; - /** Optional index-level metadata. */ - metadata?: Metadata; -} - -/** - * Full-text search sub-index operating on {@link FullTextBlock}s. - * - * Supports multi-query FTS with path-prefix filtering and relevance scoring. + * Minimal scored item — a block reference plus a score. Used by ranking + * and reranking utilities (RRF in `@statewalker/indexer-core`, blend-with- + * reranker in `@statewalker/indexer-search`) as a shared primitive. */ -export interface FullTextIndex - extends SearchIndex { - /** Retrieve configuration and status information for this FTS index. */ - getIndexInfo(): Promise; -} - -/** Configuration / status information for an embedding sub-index. */ -export interface EmbeddingIndexInfo { - /** Dimensionality of the embedding vectors stored in this index. */ - dimensionality: number; - /** Name or identifier of the embedding model that produced the vectors. */ - model: string; - /** Optional index-level metadata. */ - metadata?: Metadata; -} - -/** - * Vector / embedding search sub-index operating on {@link EmbeddingBlock}s. - * - * Supports multi-vector similarity search with path-prefix filtering. - */ -export interface EmbeddingIndex - extends SearchIndex { - /** Retrieve configuration and status information for this vector index. */ - getIndexInfo(): Promise; +export interface ScoredItem { + blockId: BlockId; + score: number; } // ============================================================================= -// 8. Composite hybrid index +// 8. Embedding function // ============================================================================= /** - * A named hybrid search index combining optional FTS and vector sub-indexes. - * - * The composite {@link Index} accepts {@link IndexedBlock}s (which may carry - * text, an embedding, or both) and routes them to the appropriate sub-indexes. - * Hybrid search blends results from both modalities according to - * {@link HybridWeights}. + * Boundary type at which the application provides an embedding capability + * to the indexer. Maps a piece of text to a dense float vector. */ -export interface Index extends SearchIndex { - /** Unique name of this index within the {@link Indexer}. */ - readonly name: string; - /** Optional index-level metadata. */ - readonly metadata?: Metadata; - - /** Returns the FTS sub-index, or `null` if this index has no full-text capability. */ - getFullTextIndex(): FullTextIndex | null; - - /** Returns the vector sub-index, or `null` if this index has no embedding capability. */ - getVectorIndex(): EmbeddingIndex | null; -} +export type EmbedFn = (text: string) => Promise; diff --git a/packages/indexer-api/src/helpers/index-documents.ts b/packages/indexer-api/src/helpers/index-documents.ts deleted file mode 100644 index aa260f7..0000000 --- a/packages/indexer-api/src/helpers/index-documents.ts +++ /dev/null @@ -1,32 +0,0 @@ -import type { DocumentPath, Index, IndexedBlock } from "../indexer-index.js"; -import type { EmbedFn } from "../semantic-index.js"; - -export async function indexDocuments( - index: Index, - docs: - | Iterable<{ path: DocumentPath; blockId: string; content: string }> - | AsyncIterable<{ path: DocumentPath; blockId: string; content: string }>, - options?: { embedFn?: EmbedFn }, -): Promise<{ indexed: number }> { - const embedFn = options?.embedFn; - let indexed = 0; - - for await (const doc of docs as AsyncIterable<{ - path: DocumentPath; - blockId: string; - content: string; - }>) { - const block: IndexedBlock = { - path: doc.path, - blockId: doc.blockId, - content: doc.content, - }; - if (embedFn) { - block.embedding = await embedFn(doc.content); - } - await index.addDocument([block]); - indexed++; - } - - return { indexed }; -} diff --git a/packages/indexer-api/src/helpers/index.ts b/packages/indexer-api/src/helpers/index.ts deleted file mode 100644 index 5b7c20b..0000000 --- a/packages/indexer-api/src/helpers/index.ts +++ /dev/null @@ -1,20 +0,0 @@ -export { indexDocuments } from "./index-documents.js"; -export { - createMockCitationBuilder, - createMockExpander, - createMockReranker, -} from "./mock.js"; -export type { - EntryExplain, - PipelineConfig, - PipelineEntry, -} from "./search-pipeline.js"; -export { SearchPipeline } from "./search-pipeline.js"; -export type { - Citation, - CitationBuilderFn, - ExpandedQuery, - QueryExpanderFn, - RerankerFn, - RerankResult, -} from "./types.js"; diff --git a/packages/indexer-api/src/index.ts b/packages/indexer-api/src/index.ts index a3bf336..ffd485d 100644 --- a/packages/indexer-api/src/index.ts +++ b/packages/indexer-api/src/index.ts @@ -1,73 +1,35 @@ export type { - Citation, - CitationBuilderFn, - EntryExplain, - ExpandedQuery, - PipelineConfig, - PipelineEntry, - QueryExpanderFn, - RerankerFn, - RerankResult, -} from "./helpers/index.js"; -export { - createMockCitationBuilder, - createMockExpander, - createMockReranker, - indexDocuments, - SearchPipeline, -} from "./helpers/index.js"; -export type { - CreateIndexParams, - Indexer, - IndexInfo, -} from "./indexer.js"; + EmbeddingIndex, + EmbeddingIndexInfo, + FullTextIndex, + FullTextIndexInfo, + Index, +} from "./contract/index.js"; export type { BlockId, BlockReference, DocumentPath, EmbeddingBlock, - EmbeddingIndex, - EmbeddingIndexInfo, EmbeddingSearchParams, EmbeddingSearchResult, + EmbedFn, FullTextBlock, - FullTextIndex, - FullTextIndexInfo, FullTextSearchParams, FullTextSearchResult, HybridSearchParams, HybridSearchResult, HybridWeights, - Index, IndexedBlock, Metadata, PathSelector, -} from "./indexer-index.js"; -export type { ChunkSelection } from "./intent.js"; -export { extractIntentTerms, selectBestChunk } from "./intent.js"; + ScoredItem, +} from "./contract/types.js"; export type { - MultiSearchParams, - MultiSearchResult, -} from "./multi-search.js"; -export { defaultMultiSearch } from "./multi-search.js"; + CreateIndexParams, + Indexer, + IndexInfo, +} from "./indexer.js"; export type { IndexerPersistence, PersistenceEntry, } from "./persistence.js"; -export type { ParsedQuery, QueryType } from "./query-parser.js"; -export { - parseStructuredQuery, - validateLexQuery, - validateSemanticQuery, -} from "./query-parser.js"; -export type { BlendTier } from "./reranker-blend.js"; -export { blendWithReranker, DEFAULT_BLEND_TIERS } from "./reranker-blend.js"; -export type { - RankedList, - RRFContribution, - RRFTrace, - ScoredItem, -} from "./rrf.js"; -export { buildRrfTrace, reciprocalRankFusion } from "./rrf.js"; -export type { EmbedFn } from "./semantic-index.js"; -export { SemanticIndex } from "./semantic-index.js"; diff --git a/packages/indexer-api/src/indexer.ts b/packages/indexer-api/src/indexer.ts index 81b1e3d..29c748b 100644 --- a/packages/indexer-api/src/indexer.ts +++ b/packages/indexer-api/src/indexer.ts @@ -1,4 +1,5 @@ -import type { Index, Metadata } from "./indexer-index.js"; +import type { Index } from "./contract/index.js"; +import type { Metadata } from "./contract/types.js"; /** * Parameters for creating a new hybrid search index. diff --git a/packages/indexer-api/src/multi-search.ts b/packages/indexer-api/src/multi-search.ts deleted file mode 100644 index 88a066f..0000000 --- a/packages/indexer-api/src/multi-search.ts +++ /dev/null @@ -1,129 +0,0 @@ -import type { DocumentPath, HybridSearchResult, HybridWeights, Index } from "./indexer-index.js"; -import { type RankedList, reciprocalRankFusion, type ScoredItem } from "./rrf.js"; - -/** Collect all results from an async generator into an array. */ -async function collectResults( - gen: AsyncGenerator, -): Promise { - const results: HybridSearchResult[] = []; - for await (const r of gen) { - results.push(r); - } - return results; -} - -/** Parameters for multi-query search with RRF fusion. */ -export interface MultiSearchParams { - /** FTS queries — blocks matching more queries rank higher. */ - queries?: string[]; - /** Embedding vectors — blocks closer to more vectors rank higher. */ - embeddings?: Float32Array[]; - /** Maximum number of results to return. */ - topK: number; - /** Relative weights for blending FTS and embedding scores. */ - weights?: HybridWeights; - /** Path prefixes to restrict search scope. */ - paths?: DocumentPath[]; -} - -/** A scored result with cross-query match count. */ -export interface MultiSearchResult extends ScoredItem { - /** How many of the input queries matched this block. */ - matchCount: number; -} - -/** - * Multi-query search utility that fans out individual queries/embeddings - * to the index, fuses results with RRF, and tracks matchCount. - * - * This provides **cross-query RRF**: each query/embedding is run as a - * separate `index.search()` call, producing independent ranked lists that - * are then fused with {@link reciprocalRankFusion}. Blocks appearing in - * multiple lists get boosted. - * - * This differs from calling `index.search()` directly with multiple - * queries/embeddings, where the index handles multi-query fusion internally - * (typically best-score-per-block merge within each modality, then - * FTS/embedding blending). Use this function when you specifically need - * cross-query rank fusion; use `index.search()` directly when you want - * the index implementation to optimise multi-query handling natively - * (e.g. SQL-level merges in DuckDB/PostgreSQL). - */ -export async function defaultMultiSearch( - index: Index, - params: MultiSearchParams, -): Promise { - const { queries, embeddings, topK, weights, paths } = params; - - const hasQueries = queries && queries.length > 0; - const hasEmbeddings = embeddings && embeddings.length > 0; - - if (!hasQueries && !hasEmbeddings) { - return []; - } - - // Fan out: run each query/embedding independently - const rankedLists: RankedList[] = []; - const searchPromises: Promise[] = []; - - if (hasQueries) { - for (const query of queries) { - searchPromises.push( - collectResults(index.search({ queries: [query], topK, weights, paths })).then( - (hybridResults) => { - const results = hybridResults.map((r) => ({ - blockId: r.blockId, - score: r.score, - })); - rankedLists.push({ - results, - meta: { source: "fts", queryType: "lex", query }, - }); - }, - ), - ); - } - } - - if (hasEmbeddings) { - for (const embedding of embeddings) { - searchPromises.push( - collectResults(index.search({ embeddings: [embedding], topK, weights, paths })).then( - (hybridResults) => { - const results = hybridResults.map((r) => ({ - blockId: r.blockId, - score: r.score, - })); - rankedLists.push({ - results, - meta: { source: "vec", queryType: "vec", query: "" }, - }); - }, - ), - ); - } - } - - await Promise.all(searchPromises); - - // Track which queries matched each document - const matchCounts = new Map(); - for (const list of rankedLists) { - const seen = new Set(); - for (const r of list.results) { - if (!seen.has(r.blockId)) { - seen.add(r.blockId); - matchCounts.set(r.blockId, (matchCounts.get(r.blockId) ?? 0) + 1); - } - } - } - - // Fuse with RRF - const fused = reciprocalRankFusion(rankedLists, topK); - - return fused.map((r) => ({ - blockId: r.blockId, - score: r.score, - matchCount: matchCounts.get(r.blockId) ?? 1, - })); -} diff --git a/packages/indexer-core/src/fan-out-search.ts b/packages/indexer-core/src/fan-out-search.ts new file mode 100644 index 0000000..ddc906f --- /dev/null +++ b/packages/indexer-core/src/fan-out-search.ts @@ -0,0 +1,95 @@ +import type { + DocumentPath, + HybridSearchResult, + HybridWeights, + Index, + ScoredItem, +} from "@statewalker/indexer-api"; +import { type RankedList, reciprocalRankFusion } from "./rrf.js"; + +/** Parameters for fan-out search with cross-query RRF fusion. */ +export interface FanOutSearchParams { + /** FTS queries — blocks matching more queries rank higher. */ + queries?: string[]; + /** Embedding vectors — blocks closer to more vectors rank higher. */ + embeddings?: Float32Array[]; + /** Maximum number of results to return. */ + topK: number; + /** Relative weights for blending FTS and embedding scores within each per-query call. */ + weights?: HybridWeights; + /** Path prefixes to restrict search scope. */ + paths?: DocumentPath[]; +} + +async function collectResults( + gen: AsyncGenerator, +): Promise { + const results: HybridSearchResult[] = []; + for await (const r of gen) { + results.push(r); + } + return results; +} + +/** + * Backend-implementation glue: fans out individual queries/embeddings to + * `index.search()`, then fuses the resulting ranked lists with RRF. + * + * Use this when an underlying engine has no native multi-query merge — the + * backend can call `fanOutSearch` from inside its own `Index.search()` to + * obtain cross-query rank fusion. + * + * Not part of any consumer-facing public surface. The `SearchPipeline` in + * `@statewalker/indexer-search` delegates fusion to `index.search()` + * directly and does not need this helper. + */ +export async function fanOutSearch( + index: Index, + params: FanOutSearchParams, +): Promise { + const { queries, embeddings, topK, weights, paths } = params; + + const hasQueries = queries && queries.length > 0; + const hasEmbeddings = embeddings && embeddings.length > 0; + + if (!hasQueries && !hasEmbeddings) { + return []; + } + + const rankedLists: RankedList[] = []; + const searchPromises: Promise[] = []; + + if (hasQueries) { + for (const query of queries) { + searchPromises.push( + collectResults(index.search({ queries: [query], topK, weights, paths })).then( + (hybridResults) => { + rankedLists.push({ + results: hybridResults.map((r) => ({ blockId: r.blockId, score: r.score })), + meta: { source: "fts", queryType: "lex", query }, + }); + }, + ), + ); + } + } + + if (hasEmbeddings) { + for (const embedding of embeddings) { + searchPromises.push( + collectResults(index.search({ embeddings: [embedding], topK, weights, paths })).then( + (hybridResults) => { + rankedLists.push({ + results: hybridResults.map((r) => ({ blockId: r.blockId, score: r.score })), + meta: { source: "vec", queryType: "vec", query: "" }, + }); + }, + ), + ); + } + } + + await Promise.all(searchPromises); + + return reciprocalRankFusion(rankedLists, topK); +} diff --git a/packages/indexer-core/src/index.ts b/packages/indexer-core/src/index.ts index 87b9cd4..faff061 100644 --- a/packages/indexer-core/src/index.ts +++ b/packages/indexer-core/src/index.ts @@ -23,9 +23,17 @@ export { type SqlVectorDialect, type SqlVectorRetrieverOptions, } from "./create-sql-vector-retriever.js"; +export { type FanOutSearchParams, fanOutSearch } from "./fan-out-search.js"; export { mergeByRRF, mergeByWeights, mergeHybrid } from "./merge.js"; export { matchesPrefix } from "./path-prefix.js"; export { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; +export { + buildRrfTrace, + type RankedList, + type RRFContribution, + type RRFTrace, + reciprocalRankFusion, +} from "./rrf.js"; export { sanitizePrefix } from "./sanitize-prefix.js"; export type { SqlDb } from "./sql-db.js"; export { validateDimensionality } from "./validate-dimensionality.js"; diff --git a/packages/indexer-core/src/merge.ts b/packages/indexer-core/src/merge.ts index 2674ae1..60ae5a9 100644 --- a/packages/indexer-core/src/merge.ts +++ b/packages/indexer-core/src/merge.ts @@ -6,6 +6,7 @@ import type { HybridWeights, } from "@statewalker/indexer-api"; import { compositeKey } from "./composite-key.js"; +import { type RankedList, reciprocalRankFusion } from "./rrf.js"; export function mergeByRRF( ftsResults: FullTextSearchResult[], @@ -13,43 +14,45 @@ export function mergeByRRF( topK: number, k = 60, ): HybridSearchResult[] { - const scores = new Map(); const ftsMap = new Map(); const vecMap = new Map(); const pathMap = new Map(); const blockIdMap = new Map(); - for (let i = 0; i < ftsResults.length; i++) { - const r = ftsResults[i]; + const ftsList: { blockId: string; score: number }[] = []; + for (const r of ftsResults) { if (!r) continue; const key = compositeKey(r.path, r.blockId); - scores.set(key, (scores.get(key) ?? 0) + 1 / (k + i + 1)); ftsMap.set(key, r); pathMap.set(key, r.path); blockIdMap.set(key, r.blockId); + ftsList.push({ blockId: key, score: r.score }); } - for (let i = 0; i < vecResults.length; i++) { - const r = vecResults[i]; + + const vecList: { blockId: string; score: number }[] = []; + for (const r of vecResults) { if (!r) continue; const key = compositeKey(r.path, r.blockId); - scores.set(key, (scores.get(key) ?? 0) + 1 / (k + i + 1)); vecMap.set(key, r); if (!pathMap.has(key)) pathMap.set(key, r.path); if (!blockIdMap.has(key)) blockIdMap.set(key, r.blockId); + vecList.push({ blockId: key, score: r.score }); } - const results: HybridSearchResult[] = []; - for (const [key, score] of scores) { - results.push({ - path: pathMap.get(key) as DocumentPath, - blockId: blockIdMap.get(key) as string, - score, - fts: ftsMap.get(key) ?? null, - embedding: vecMap.get(key) ?? null, - }); - } - results.sort((a, b) => b.score - a.score); - return results.slice(0, topK); + const lists: RankedList[] = [ + { results: ftsList, meta: { source: "fts", queryType: "lex", query: "" } }, + { results: vecList, meta: { source: "vec", queryType: "vec", query: "" } }, + ]; + + const fused = reciprocalRankFusion(lists, topK, k); + + return fused.map((item) => ({ + path: pathMap.get(item.blockId) as DocumentPath, + blockId: blockIdMap.get(item.blockId) as string, + score: item.score, + fts: ftsMap.get(item.blockId) ?? null, + embedding: vecMap.get(item.blockId) ?? null, + })); } export function mergeByWeights( diff --git a/packages/indexer-api/src/rrf.ts b/packages/indexer-core/src/rrf.ts similarity index 96% rename from packages/indexer-api/src/rrf.ts rename to packages/indexer-core/src/rrf.ts index 368bc69..1be7394 100644 --- a/packages/indexer-api/src/rrf.ts +++ b/packages/indexer-core/src/rrf.ts @@ -5,11 +5,7 @@ * MIT License — Copyright (c) 2024-2026 Tobi Lutke. */ -/** Minimal scored item — any object with blockId and score works. */ -export interface ScoredItem { - blockId: string; - score: number; -} +import type { ScoredItem } from "@statewalker/indexer-api"; export interface RankedList { results: ScoredItem[]; diff --git a/packages/indexer-core/test/merge.test.ts b/packages/indexer-core/test/merge.test.ts new file mode 100644 index 0000000..4b05596 --- /dev/null +++ b/packages/indexer-core/test/merge.test.ts @@ -0,0 +1,48 @@ +import type { + DocumentPath, + EmbeddingSearchResult, + FullTextSearchResult, +} from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { mergeByRRF } from "../src/merge.js"; + +const path = "/docs/" as DocumentPath; + +function fts(blockId: string, score: number, snippet = ""): FullTextSearchResult { + return { path, blockId, score, snippet }; +} + +function vec(blockId: string, score: number): EmbeddingSearchResult { + return { path, blockId, score }; +} + +describe("mergeByRRF top-rank bonus", () => { + it("boosts a block ranked #1 in both lists over a block tied at rank #5", () => { + const ftsResults = [ + fts("top", 0.99), + fts("f2", 0.8), + fts("f3", 0.7), + fts("f4", 0.6), + fts("mid", 0.5), + ]; + const vecResults = [ + vec("top", 0.99), + vec("v2", 0.8), + vec("v3", 0.7), + vec("v4", 0.6), + vec("mid", 0.5), + ]; + + const merged = mergeByRRF(ftsResults, vecResults, 10); + const top = merged.find((r) => r.blockId === "top"); + const mid = merged.find((r) => r.blockId === "mid"); + expect(top).toBeDefined(); + expect(mid).toBeDefined(); + + // Top-rank bonus (0.05 at rank 1) is applied per-block, not per-list. + // The rank-1 block enjoys the +0.05 boost; the rank-5 block does not. + const boost = (top?.score ?? 0) - (mid?.score ?? 0); + const baseDiff = 2 * (1 / (60 + 1) - 1 / (60 + 5)); // diff in pure RRF contributions + expect(boost).toBeGreaterThan(baseDiff + 0.04); + }); +}); diff --git a/packages/indexer-api/test/rrf.test.ts b/packages/indexer-core/test/rrf.test.ts similarity index 100% rename from packages/indexer-api/test/rrf.test.ts rename to packages/indexer-core/test/rrf.test.ts diff --git a/packages/indexer-mem-flexsearch/package.json b/packages/indexer-mem-flexsearch/package.json index 94cc949..4e06d68 100644 --- a/packages/indexer-mem-flexsearch/package.json +++ b/packages/indexer-mem-flexsearch/package.json @@ -38,6 +38,7 @@ "flexsearch": "catalog:" }, "devDependencies": { + "@statewalker/indexer-search": "workspace:*", "@statewalker/indexer-tests": "workspace:*", "rimraf": "catalog:", "tsdown": "catalog:", diff --git a/packages/indexer-mem-flexsearch/tests/index-documents.test.ts b/packages/indexer-mem-flexsearch/tests/index-documents.test.ts deleted file mode 100644 index 48f732f..0000000 --- a/packages/indexer-mem-flexsearch/tests/index-documents.test.ts +++ /dev/null @@ -1,103 +0,0 @@ -import { - type DocumentPath, - type Index, - indexDocuments, -} from "@statewalker/indexer-api"; -import { afterEach, beforeEach, describe, expect, it } from "vitest"; -import { createFlexSearchIndexer } from "../src/index.js"; - -let indexer: ReturnType; -let index: Index; - -beforeEach(async () => { - indexer = createFlexSearchIndexer(); - index = await indexer.createIndex({ - name: "test", - fulltext: { language: "en" }, - }); -}); - -afterEach(async () => { - await indexer.close(); -}); - -describe("indexDocuments", () => { - it("indexes documents from a sync iterable", async () => { - const docs = [ - { path: "/docs/" as DocumentPath, blockId: "b1", content: "hello world" }, - { path: "/docs/" as DocumentPath, blockId: "b2", content: "foo bar" }, - ]; - - const result = await indexDocuments(index, docs); - expect(result.indexed).toBe(2); - }); - - it("indexed documents are searchable", async () => { - const docs = [ - { - path: "/docs/" as DocumentPath, - blockId: "b1", - content: "quantum mechanics physics", - }, - ]; - - await indexDocuments(index, docs); - - const results: Array<{ blockId: string }> = []; - for await (const r of index.search({ queries: ["quantum"], topK: 10 })) { - results.push(r); - } - expect(results.length).toBe(1); - expect(results[0]?.blockId).toBe("b1"); - }); - - it("indexes documents from an async iterable", async () => { - async function* generateDocs() { - yield { - path: "/docs/" as DocumentPath, - blockId: "a1", - content: "async document one", - }; - yield { - path: "/docs/" as DocumentPath, - blockId: "a2", - content: "async document two", - }; - } - - const result = await indexDocuments(index, generateDocs()); - expect(result.indexed).toBe(2); - - const size = await index.getSize(); - expect(size).toBe(2); - }); - - it("calls embedFn when provided", async () => { - const indexerWithVec = createFlexSearchIndexer(); - const vecIndex = await indexerWithVec.createIndex({ - name: "vec-test", - fulltext: { language: "en" }, - vector: { dimensionality: 3, model: "test" }, - }); - - const docs = [ - { - path: "/docs/" as DocumentPath, - blockId: "e1", - content: "embedding test", - }, - ]; - - let embedCalled = false; - const embedFn = async (_text: string) => { - embedCalled = true; - return new Float32Array([0.1, 0.2, 0.3]); - }; - - const result = await indexDocuments(vecIndex, docs, { embedFn }); - expect(result.indexed).toBe(1); - expect(embedCalled).toBe(true); - - await indexerWithVec.close(); - }); -}); diff --git a/packages/indexer-mem-flexsearch/tests/search-pipeline.test.ts b/packages/indexer-mem-flexsearch/tests/search-pipeline.test.ts index 3b138cb..2cda0a6 100644 --- a/packages/indexer-mem-flexsearch/tests/search-pipeline.test.ts +++ b/packages/indexer-mem-flexsearch/tests/search-pipeline.test.ts @@ -1,11 +1,10 @@ +import type { DocumentPath, Index } from "@statewalker/indexer-api"; import { createMockCitationBuilder, createMockExpander, createMockReranker, - type DocumentPath, - type Index, SearchPipeline, -} from "@statewalker/indexer-api"; +} from "@statewalker/indexer-search"; import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; import { createFlexSearchIndexer } from "../src/index.js"; diff --git a/packages/indexer-mem-flexsearch/tests/semantic-index.test.ts b/packages/indexer-mem-flexsearch/tests/semantic-index.test.ts new file mode 100644 index 0000000..309c933 --- /dev/null +++ b/packages/indexer-mem-flexsearch/tests/semantic-index.test.ts @@ -0,0 +1,86 @@ +import type { DocumentPath, EmbedFn, Index } from "@statewalker/indexer-api"; +import { SemanticIndex } from "@statewalker/indexer-search"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createFlexSearchIndexer } from "../src/index.js"; + +let indexer: ReturnType; +let index: Index; + +const noopEmbed: EmbedFn = async () => new Float32Array(); + +beforeEach(async () => { + indexer = createFlexSearchIndexer(); + index = await indexer.createIndex({ + name: "test", + fulltext: { language: "en" }, + }); +}); + +afterEach(async () => { + await indexer.close(); +}); + +describe("SemanticIndex.addDocuments (FTS-only backend)", () => { + it("ingests documents from a sync iterable", async () => { + const semantic = new SemanticIndex(index, noopEmbed); + const docs = [ + { path: "/docs/" as DocumentPath, blockId: "b1", content: "hello world" }, + { path: "/docs/" as DocumentPath, blockId: "b2", content: "foo bar" }, + ]; + + await semantic.addDocuments(docs); + expect(await index.getSize()).toBe(2); + }); + + it("ingested documents are searchable", async () => { + const semantic = new SemanticIndex(index, noopEmbed); + await semantic.addDocuments([ + { path: "/docs/" as DocumentPath, blockId: "b1", content: "quantum mechanics physics" }, + ]); + + const results: Array<{ blockId: string }> = []; + for await (const r of index.search({ queries: ["quantum"], topK: 10 })) { + results.push(r); + } + expect(results.length).toBe(1); + expect(results[0]?.blockId).toBe("b1"); + }); + + it("ingests documents from an async iterable", async () => { + const semantic = new SemanticIndex(index, noopEmbed); + async function* generateDocs() { + yield { path: "/docs/" as DocumentPath, blockId: "a1", content: "async document one" }; + yield { path: "/docs/" as DocumentPath, blockId: "a2", content: "async document two" }; + } + + await semantic.addDocuments(generateDocs()); + expect(await index.getSize()).toBe(2); + }); +}); + +describe("SemanticIndex.addDocuments (FTS + vector backend)", () => { + it("calls embed for every document", async () => { + const indexerWithVec = createFlexSearchIndexer(); + const vecIndex = await indexerWithVec.createIndex({ + name: "vec-test", + fulltext: { language: "en" }, + vector: { dimensionality: 3, model: "test" }, + }); + + let embedCalls = 0; + const embed: EmbedFn = async () => { + embedCalls++; + return new Float32Array([0.1, 0.2, 0.3]); + }; + + const semantic = new SemanticIndex(vecIndex, embed); + await semantic.addDocuments([ + { path: "/docs/" as DocumentPath, blockId: "e1", content: "embedding test" }, + ]); + + expect(await vecIndex.getSize()).toBe(1); + expect(embedCalls).toBe(1); + + await indexerWithVec.close(); + }); +}); diff --git a/packages/indexer-search/README.md b/packages/indexer-search/README.md new file mode 100644 index 0000000..8a9def6 --- /dev/null +++ b/packages/indexer-search/README.md @@ -0,0 +1,159 @@ +# @statewalker/indexer-search + +Application-side search-orchestration stack built on [`@statewalker/indexer-api`](../indexer-api/README.md). Owns the strategy code that consumers of any conforming backend (`indexer-mem-*`, `indexer-pglite`, `indexer-duckdb`, …) typically want without re-implementing it themselves. + +## What's in here + +| Module | Purpose | +|--------|---------| +| `SearchPipeline` | Builder/executor for multi-stage search (expand → embed → search → rerank → cite) | +| `SemanticIndex` | Convenience wrapper that auto-embeds at ingestion and search time | +| `parseStructuredQuery` / `validateLexQuery` / `validateSemanticQuery` | Parser for typed (`lex:` / `vec:` / `hyde:` / `expand:`) query syntax | +| `extractIntentTerms` / `selectBestChunk` | Intent-based stop-word filtering and chunk selection | +| `blendWithReranker` / `BlendTier` / `DEFAULT_BLEND_TIERS` | Position-aware reranker blending | +| Function types: `QueryExpanderFn`, `RerankerFn`, `CitationBuilderFn`, `ExpandedQuery`, `Citation` | The shapes a host application plugs in | +| `createMockExpander` / `createMockReranker` / `createMockCitationBuilder` | Deterministic test doubles for the function types | + +The package depends on `@statewalker/indexer-api` only. It does not depend on `@statewalker/indexer-core` or any backend package — strategy code lives strictly above the contract layer. + +## Why this lives in its own package + +Backends should not pay the compile/type cost of strategy code they never call. Conversely, application code that needs a `SearchPipeline` should not be coupled to a specific backend. Splitting the contract (`indexer-api`) from the strategy stack (`indexer-search`) makes the layering explicit: + +``` +indexer-api (contract: types + interfaces, zero runtime) + ↑ +indexer-core (backend toolkit: fanOutSearch, RRF, mergeHybrid, …) + ↑ ↑ +indexer-mem-* indexer-search (app-side: SearchPipeline, SemanticIndex, …) +indexer-pglite ↑ +indexer-duckdb downstream apps +``` + +## How to use + +### SemanticIndex — automatic embedding + +```ts +import type { EmbedFn } from "@statewalker/indexer-api"; +import { SemanticIndex } from "@statewalker/indexer-search"; + +const semantic = new SemanticIndex(index, embed satisfies EmbedFn); + +// Embedding computed automatically from content +await semantic.addDocument({ + path: "/docs/guide/", + blockId: "ch1", + content: "Chapter 1: Introduction...", +}); + +// Search with automatic query embedding +const results = await semantic.search({ + query: "introduction", + topK: 5, +}); +``` + +### SearchPipeline — multi-stage search + +`SearchPipeline` chains: **expand** (query expansion) → **embed** (semantic query embedding) → **search** (single `index.search()` call, delegating fusion to the index) → **rerank** (score blending) → **cite** (citation extraction). Each LLM stage is defined as a function type — pass closures, no class instantiation required. + +```ts +import { SearchPipeline } from "@statewalker/indexer-search"; + +const results = await new SearchPipeline({ + index, + embedFn: embed, + expander: async (query) => [ + { type: "lex", query }, + { type: "vec", query: `semantic: ${query}` }, + ], + reranker: async (query, candidates) => + candidates.map((c, i) => ({ blockId: c.blockId, score: 1 / (i + 1) })), +}) + .setPrompt("distributed consensus") + .setTopK(10) + .execute(); +``` + +Stages can be skipped (`pipeline.skip("rerank")`), inputs combined (`setTextQueries`, `setSemanticQueries`, `setEmbeddings`), and traces enabled (`setExplain(true)`). + +### Reranker blending + +`blendWithReranker()` combines initial retrieval scores with reranker scores using position-aware tiers. Top-ranked items are protected by higher retrieval weights (default: 0.75 for top-3, 0.60 for top-10, 0.40 for the rest), preventing aggressive rerankers from destabilizing high-confidence results. + +```ts +import { blendWithReranker, DEFAULT_BLEND_TIERS } from "@statewalker/indexer-search"; + +const blended = blendWithReranker(retrievalResults, rerankScores, DEFAULT_BLEND_TIERS); +``` + +### Structured query parsing + +```ts +import { parseStructuredQuery } from "@statewalker/indexer-search"; + +const parsed = parseStructuredQuery("lex: CAP theorem\nvec: consensus algorithms"); +// [{ type: "lex", query: "CAP theorem" }, { type: "vec", query: "consensus algorithms" }] +``` + +Recognised prefixes: + +- `lex:` — lexical/keyword query +- `vec:` — vector/semantic query +- `hyde:` — hypothetical document embedding query +- `expand:` — pass-through (returns `null` for default pipeline handling) + +Validators (`validateLexQuery`, `validateSemanticQuery`) catch malformed queries before they reach the backend. + +### Intent disambiguation + +`extractIntentTerms()` strips stop-words from a user's intent description, and `selectBestChunk()` uses both query terms and intent terms to pick the most relevant text chunk — useful for snippet extraction and context selection. + +### Mocks + +The mock factories return deterministic implementations of `QueryExpanderFn`, `RerankerFn`, and `CitationBuilderFn`, useful for unit-testing pipeline wiring without standing up real LLM calls. + +```ts +import { + createMockCitationBuilder, + createMockExpander, + createMockReranker, +} from "@statewalker/indexer-search"; + +const pipeline = new SearchPipeline({ + index, + embedFn: embed, + expander: createMockExpander(), + reranker: createMockReranker(), + citationBuilder: createMockCitationBuilder(), +}); +``` + +## Where the rank-fusion math lives + +`reciprocalRankFusion` (RRF) and the `mergeHybrid` family are backend-implementation glue and live in `@statewalker/indexer-core` (workspace-internal, not published). `SearchPipeline` does not call them directly — it issues a single `index.search(...)` call and lets each backend handle its own multi-query merge. + +If you really need cross-query RRF outside a backend's `Index.search()` (rare), the `fanOutSearch` helper in `@statewalker/indexer-core` is available to backend authors. + +## How it is tested + +Tests use **vitest** and live in `test/`: + +| Test file | What it covers | +|-----------|----------------| +| `test/reranker-blend.test.ts` | Position-aware blending, tier boundaries, re-ordering, custom tiers, edge cases | +| `test/query-parser.test.ts` | Structured query parsing (`lex:/vec:/hyde:/expand:`), validation, error cases | +| `test/intent.test.ts` | Stop-word filtering, intent term extraction, chunk selection with intent weighting | +| `test/mock.test.ts` | Mock expander, reranker, and citation builder factory functions | + +Cross-backend `SemanticIndex` conformance is exercised through `@statewalker/indexer-tests`'s `semantic-index.suite.ts` (run against every conforming backend). + +Run tests: + +```bash +pnpm test +pnpm test:watch +``` + +Several algorithms (query parser, intent extraction, reranker blending) are adapted from [QMD](https://github.com/tobi/qmd) by Tobi Lutke (MIT License). diff --git a/packages/indexer-search/package.json b/packages/indexer-search/package.json new file mode 100644 index 0000000..6a8dba1 --- /dev/null +++ b/packages/indexer-search/package.json @@ -0,0 +1,47 @@ +{ + "name": "@statewalker/indexer-search", + "version": "0.1.0", + "private": false, + "type": "module", + "description": "Application-side search-orchestration stack: SearchPipeline, SemanticIndex, query parser, intent extraction, reranker blending, and mock factories built on @statewalker/indexer-api.", + "homepage": "https://github.com/statewalker/statewalker-indexer", + "author": { + "name": "Mikhail Kotelnikov", + "email": "mikhail.kotelnikov@gmail.com" + }, + "license": "MIT", + "repository": { + "type": "git", + "url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git" + }, + "exports": { + ".": "./src/index.ts" + }, + "files": [ + "dist", + "src" + ], + "scripts": { + "build": "tsdown", + "dev": "tsdown --watch", + "test": "vitest run --passWithNoTests", + "test:watch": "vitest", + "typecheck": "tsc --noEmit", + "clean": "rimraf dist", + "lint": "biome check --write .", + "format": "biome format --write ." + }, + "dependencies": { + "@statewalker/indexer-api": "workspace:*" + }, + "devDependencies": { + "rimraf": "catalog:", + "tsdown": "catalog:", + "typescript": "catalog:", + "vitest": "catalog:" + }, + "sideEffects": false, + "publishConfig": { + "access": "public" + } +} diff --git a/packages/indexer-api/src/helpers/types.ts b/packages/indexer-search/src/fn-types.ts similarity index 72% rename from packages/indexer-api/src/helpers/types.ts rename to packages/indexer-search/src/fn-types.ts index 82c9d79..129db96 100644 --- a/packages/indexer-api/src/helpers/types.ts +++ b/packages/indexer-search/src/fn-types.ts @@ -1,17 +1,11 @@ -import type { BlockId } from "../indexer-index.js"; -import type { QueryType } from "../query-parser.js"; -import type { ScoredItem } from "../rrf.js"; +import type { BlockId, ScoredItem } from "@statewalker/indexer-api"; +import type { QueryType } from "./query-parser.js"; export interface ExpandedQuery { type: QueryType; query: string; } -export interface RerankResult { - blockId: BlockId; - score: number; -} - export interface Citation { blockId: BlockId; snippet: string; @@ -31,7 +25,7 @@ export type RerankerFn = ( query: string, candidates: Array<{ blockId: BlockId; text: string }>, options?: { topK?: number }, -) => Promise; +) => Promise; export type CitationBuilderFn = ( query: string, diff --git a/packages/indexer-search/src/index.ts b/packages/indexer-search/src/index.ts new file mode 100644 index 0000000..d55bf54 --- /dev/null +++ b/packages/indexer-search/src/index.ts @@ -0,0 +1,28 @@ +export type { + Citation, + CitationBuilderFn, + ExpandedQuery, + QueryExpanderFn, + RerankerFn, +} from "./fn-types.js"; +export type { ChunkSelection } from "./intent.js"; +export { extractIntentTerms, selectBestChunk } from "./intent.js"; +export { + createMockCitationBuilder, + createMockExpander, + createMockReranker, +} from "./mock.js"; +export type { ParsedQuery, QueryType } from "./query-parser.js"; +export { + parseStructuredQuery, + validateLexQuery, + validateSemanticQuery, +} from "./query-parser.js"; +export { + type BlendTier, + blendWithReranker, + DEFAULT_BLEND_TIERS, +} from "./reranker-blend.js"; +export type { EntryExplain, PipelineConfig, PipelineEntry } from "./search-pipeline.js"; +export { SearchPipeline } from "./search-pipeline.js"; +export { SemanticIndex } from "./semantic-index.js"; diff --git a/packages/indexer-api/src/intent.ts b/packages/indexer-search/src/intent.ts similarity index 100% rename from packages/indexer-api/src/intent.ts rename to packages/indexer-search/src/intent.ts diff --git a/packages/indexer-api/src/helpers/mock.ts b/packages/indexer-search/src/mock.ts similarity index 88% rename from packages/indexer-api/src/helpers/mock.ts rename to packages/indexer-search/src/mock.ts index 8f3a3fb..ae79cb5 100644 --- a/packages/indexer-api/src/helpers/mock.ts +++ b/packages/indexer-search/src/mock.ts @@ -1,13 +1,11 @@ -import type { BlockId } from "../indexer-index.js"; -import type { ScoredItem } from "../rrf.js"; +import type { BlockId, ScoredItem } from "@statewalker/indexer-api"; import type { Citation, CitationBuilderFn, ExpandedQuery, QueryExpanderFn, RerankerFn, - RerankResult, -} from "./types.js"; +} from "./fn-types.js"; export function createMockExpander(): QueryExpanderFn { return async ( @@ -34,8 +32,8 @@ export function createMockReranker(scoreMap?: Map): RerankerFn _query: string, candidates: Array<{ blockId: BlockId; text: string }>, options?: { topK?: number }, - ): Promise => { - const results: RerankResult[] = candidates.map((c, i) => ({ + ): Promise => { + const results: ScoredItem[] = candidates.map((c, i) => ({ blockId: c.blockId, score: scoreMap?.get(c.blockId) ?? 1 / (i + 1), })); diff --git a/packages/indexer-api/src/query-parser.ts b/packages/indexer-search/src/query-parser.ts similarity index 100% rename from packages/indexer-api/src/query-parser.ts rename to packages/indexer-search/src/query-parser.ts diff --git a/packages/indexer-api/src/reranker-blend.ts b/packages/indexer-search/src/reranker-blend.ts similarity index 95% rename from packages/indexer-api/src/reranker-blend.ts rename to packages/indexer-search/src/reranker-blend.ts index a7a43cd..4d94c75 100644 --- a/packages/indexer-api/src/reranker-blend.ts +++ b/packages/indexer-search/src/reranker-blend.ts @@ -5,7 +5,7 @@ * MIT License — Copyright (c) 2024-2026 Tobi Lutke. */ -import type { ScoredItem } from "./rrf.js"; +import type { ScoredItem } from "@statewalker/indexer-api"; export interface BlendTier { maxRank: number; diff --git a/packages/indexer-api/src/helpers/search-pipeline.ts b/packages/indexer-search/src/search-pipeline.ts similarity index 95% rename from packages/indexer-api/src/helpers/search-pipeline.ts rename to packages/indexer-search/src/search-pipeline.ts index 5a50d19..783dd30 100644 --- a/packages/indexer-api/src/helpers/search-pipeline.ts +++ b/packages/indexer-search/src/search-pipeline.ts @@ -1,14 +1,18 @@ -import type { DocumentPath, HybridSearchResult, HybridWeights, Index } from "../indexer-index.js"; -import type { BlendTier } from "../reranker-blend.js"; -import { blendWithReranker } from "../reranker-blend.js"; -import type { EmbedFn } from "../semantic-index.js"; +import type { + DocumentPath, + EmbedFn, + HybridSearchResult, + HybridWeights, + Index, +} from "@statewalker/indexer-api"; import type { Citation, CitationBuilderFn, ExpandedQuery, QueryExpanderFn, RerankerFn, -} from "./types.js"; +} from "./fn-types.js"; +import { type BlendTier, blendWithReranker } from "./reranker-blend.js"; export interface PipelineConfig { index: Index; diff --git a/packages/indexer-api/src/semantic-index.ts b/packages/indexer-search/src/semantic-index.ts similarity index 97% rename from packages/indexer-api/src/semantic-index.ts rename to packages/indexer-search/src/semantic-index.ts index 642ac7d..b038f5f 100644 --- a/packages/indexer-api/src/semantic-index.ts +++ b/packages/indexer-search/src/semantic-index.ts @@ -1,14 +1,13 @@ import type { DocumentPath, + EmbedFn, HybridSearchResult, HybridWeights, Index, IndexedBlock, Metadata, PathSelector, -} from "./indexer-index.js"; - -export type EmbedFn = (text: string) => Promise; +} from "@statewalker/indexer-api"; export class SemanticIndex { readonly index: Index; diff --git a/packages/indexer-api/test/intent.test.ts b/packages/indexer-search/test/intent.test.ts similarity index 100% rename from packages/indexer-api/test/intent.test.ts rename to packages/indexer-search/test/intent.test.ts diff --git a/packages/indexer-api/test/helpers/mock.test.ts b/packages/indexer-search/test/mock.test.ts similarity index 96% rename from packages/indexer-api/test/helpers/mock.test.ts rename to packages/indexer-search/test/mock.test.ts index c777909..00c4d66 100644 --- a/packages/indexer-api/test/helpers/mock.test.ts +++ b/packages/indexer-search/test/mock.test.ts @@ -1,9 +1,5 @@ import { describe, expect, it } from "vitest"; -import { - createMockCitationBuilder, - createMockExpander, - createMockReranker, -} from "../../src/helpers/mock.js"; +import { createMockCitationBuilder, createMockExpander, createMockReranker } from "../src/mock.js"; describe("createMockExpander", () => { it("returns deterministic lex expansion for any query", async () => { diff --git a/packages/indexer-api/test/query-parser.test.ts b/packages/indexer-search/test/query-parser.test.ts similarity index 100% rename from packages/indexer-api/test/query-parser.test.ts rename to packages/indexer-search/test/query-parser.test.ts diff --git a/packages/indexer-api/test/reranker-blend.test.ts b/packages/indexer-search/test/reranker-blend.test.ts similarity index 99% rename from packages/indexer-api/test/reranker-blend.test.ts rename to packages/indexer-search/test/reranker-blend.test.ts index 2e784d7..f5f91a2 100644 --- a/packages/indexer-api/test/reranker-blend.test.ts +++ b/packages/indexer-search/test/reranker-blend.test.ts @@ -1,6 +1,6 @@ +import type { ScoredItem } from "@statewalker/indexer-api"; import { describe, expect, it } from "vitest"; import { type BlendTier, blendWithReranker, DEFAULT_BLEND_TIERS } from "../src/reranker-blend.js"; -import type { ScoredItem } from "../src/rrf.js"; describe("blendWithReranker", () => { // --------------------------------------------------------------------------- diff --git a/packages/indexer-search/tsconfig.json b/packages/indexer-search/tsconfig.json new file mode 100644 index 0000000..e1bad3a --- /dev/null +++ b/packages/indexer-search/tsconfig.json @@ -0,0 +1,27 @@ +{ + "compilerOptions": { + "target": "ES2022", + "module": "Preserve", + "moduleResolution": "Bundler", + "lib": ["ESNext"], + "strict": true, + "skipLibCheck": true, + "verbatimModuleSyntax": true, + "resolveJsonModule": true, + "esModuleInterop": true, + "forceConsistentCasingInFileNames": true, + "isolatedModules": true, + "noUnusedLocals": true, + "noUnusedParameters": true, + "noImplicitReturns": true, + "noFallthroughCasesInSwitch": true, + "noUncheckedIndexedAccess": true, + "resolvePackageJsonExports": true, + "declaration": true, + "declarationMap": true, + "sourceMap": true, + "noEmit": true + }, + "include": ["./src"], + "exclude": ["node_modules", "dist"] +} diff --git a/packages/indexer-tests/package.json b/packages/indexer-tests/package.json index d36f938..1b91ae9 100644 --- a/packages/indexer-tests/package.json +++ b/packages/indexer-tests/package.json @@ -27,7 +27,8 @@ "format": "biome format --write ." }, "dependencies": { - "@statewalker/indexer-api": "workspace:*" + "@statewalker/indexer-api": "workspace:*", + "@statewalker/indexer-search": "workspace:*" }, "peerDependencies": { "vitest": "*" diff --git a/packages/indexer-tests/src/suite-runner.ts b/packages/indexer-tests/src/suite-runner.ts index d412510..9a74845 100644 --- a/packages/indexer-tests/src/suite-runner.ts +++ b/packages/indexer-tests/src/suite-runner.ts @@ -8,7 +8,6 @@ import { runIndexSuite } from "./suites/index.suite.js"; import { runIndexerSuite } from "./suites/indexer.suite.js"; import { runLifecycleSuite } from "./suites/lifecycle.suite.js"; import { runMultiIndexerIsolationSuite } from "./suites/multi-indexer-isolation.suite.js"; -import { runMultiSearchSuite } from "./suites/multi-search.suite.js"; import { runPersistenceSuite } from "./suites/persistence.suite.js"; import { runSearchQualitySuite } from "./suites/search-quality.suite.js"; import { runSemanticIndexSuite } from "./suites/semantic-index.suite.js"; @@ -49,7 +48,6 @@ export function runIndexerTestSuite(name: string, factory: IndexerFactory): void runErrorHandlingSuite(() => indexer); runSearchQualitySuite(() => indexer); - runMultiSearchSuite(() => indexer); runMultiIndexerIsolationSuite(factory.create); if (factory.createWithPersistence) { diff --git a/packages/indexer-tests/src/suites/multi-search.suite.ts b/packages/indexer-tests/src/suites/multi-search.suite.ts deleted file mode 100644 index c815812..0000000 --- a/packages/indexer-tests/src/suites/multi-search.suite.ts +++ /dev/null @@ -1,102 +0,0 @@ -import type { Indexer } from "@statewalker/indexer-api"; -import { defaultMultiSearch } from "@statewalker/indexer-api"; -import { describe, expect, it } from "vitest"; - -export function runMultiSearchSuite(getIndexer: () => Indexer): void { - describe("Multi-Search (defaultMultiSearch)", () => { - it("returns results for multiple text queries", async () => { - const indexer = getIndexer(); - const index = await indexer.createIndex({ - name: "test", - fulltext: { language: "en" }, - }); - await index.addDocument([{ path: "/docs/1", blockId: "1", content: "the quick brown fox" }]); - await index.addDocument([{ path: "/docs/2", blockId: "2", content: "lazy sleeping dog" }]); - await index.addDocument([{ path: "/docs/3", blockId: "3", content: "the fox and the dog" }]); - - const results = await defaultMultiSearch(index, { - queries: ["fox", "dog"], - topK: 10, - }); - expect(results.length).toBeGreaterThan(0); - // Block 3 mentions both fox and dog — should have highest matchCount - const block3 = results.find((r) => r.blockId === "3"); - expect(block3?.matchCount).toBe(2); - }); - - it("matchCount reflects number of matching queries", async () => { - const indexer = getIndexer(); - const index = await indexer.createIndex({ - name: "test", - fulltext: { language: "en" }, - }); - await index.addDocument([{ path: "/docs/1", blockId: "1", content: "alpha beta gamma" }]); - await index.addDocument([{ path: "/docs/2", blockId: "2", content: "only alpha here" }]); - - const results = await defaultMultiSearch(index, { - queries: ["alpha", "beta", "gamma"], - topK: 10, - }); - const r1 = results.find((r) => r.blockId === "1"); - const r2 = results.find((r) => r.blockId === "2"); - expect(r1?.matchCount).toBe(3); - expect(r2?.matchCount).toBe(1); - }); - - it("returns empty array when no queries provided", async () => { - const indexer = getIndexer(); - const index = await indexer.createIndex({ - name: "test", - fulltext: { language: "en" }, - }); - const results = await defaultMultiSearch(index, { topK: 10 }); - expect(results).toEqual([]); - }); - - it("respects topK limit", async () => { - const indexer = getIndexer(); - const index = await indexer.createIndex({ - name: "test", - fulltext: { language: "en" }, - }); - for (let i = 0; i < 20; i++) { - await index.addDocument([ - { - path: `/docs/${i}` as `/${string}`, - blockId: String(i), - content: `document number ${i} about search`, - }, - ]); - } - const results = await defaultMultiSearch(index, { - queries: ["search"], - topK: 5, - }); - expect(results.length).toBeLessThanOrEqual(5); - }); - - it("supports path filtering", async () => { - const indexer = getIndexer(); - const index = await indexer.createIndex({ - name: "test", - fulltext: { language: "en" }, - }); - await index.addDocument([ - { - path: "/science/physics", - blockId: "1", - content: "quantum mechanics", - }, - ]); - await index.addDocument([{ path: "/tech/code", blockId: "2", content: "quantum computing" }]); - - const results = await defaultMultiSearch(index, { - queries: ["quantum"], - topK: 10, - paths: ["/science/"], - }); - expect(results.length).toBe(1); - expect(results[0]?.blockId).toBe("1"); - }); - }); -} diff --git a/packages/indexer-tests/src/suites/semantic-index.suite.ts b/packages/indexer-tests/src/suites/semantic-index.suite.ts index 90b06e5..9c1ec46 100644 --- a/packages/indexer-tests/src/suites/semantic-index.suite.ts +++ b/packages/indexer-tests/src/suites/semantic-index.suite.ts @@ -1,5 +1,5 @@ import type { Indexer } from "@statewalker/indexer-api"; -import { SemanticIndex } from "@statewalker/indexer-api"; +import { SemanticIndex } from "@statewalker/indexer-search"; import { describe, expect, it, vi } from "vitest"; import { createFixtureEmbedFn, From 40a8058044a928669312c1a1d486f4c338d14bc4 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Sun, 3 May 2026 23:52:51 +0200 Subject: [PATCH 08/12] style(indexer-duckdb): hoist DocumentPath import + minor formatting Replace the inline `import("@statewalker/indexer-api").DocumentPath` type casts with a module-level type-only import. Same behavior; a few unrelated formatting tweaks come along (SqlDb.query and the vector search row generic). Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/indexer-duckdb/src/dialect.ts | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/packages/indexer-duckdb/src/dialect.ts b/packages/indexer-duckdb/src/dialect.ts index 4ba9e4c..502e609 100644 --- a/packages/indexer-duckdb/src/dialect.ts +++ b/packages/indexer-duckdb/src/dialect.ts @@ -1,4 +1,5 @@ import type { Db } from "@statewalker/db-api"; +import type { DocumentPath } from "@statewalker/indexer-api"; import type { SqlBackedDialect, SqlDb, @@ -10,7 +11,8 @@ import type { export function wrapDbAsSqlDb(db: Db): SqlDb { return { exec: (sql) => db.exec(sql), - query: (sql: string, params?: unknown[]) => db.query(sql, params ?? []), + query: (sql: string, params?: unknown[]) => + db.query(sql, params ?? []), }; } @@ -108,7 +110,7 @@ export const duckdbFtsDialect: SqlFtsDialect = { }>(sql, allParams); return rows.map((row) => ({ - path: row.path as import("@statewalker/indexer-api").DocumentPath, + path: row.path as DocumentPath, blockId: row.block_id, content: row.content, score: row.score, @@ -163,13 +165,17 @@ export const duckdbVectorDialect: SqlVectorDialect = { const topKParam = `$${allParams.length + 1}`; allParams.push(topK); - const rows = await db.query<{ path: string; block_id: string; dist: number }>( + const rows = await db.query<{ + path: string; + block_id: string; + dist: number; + }>( `SELECT d.path, b.block_id, array_cosine_distance(b.embedding, $1${embeddingCastSuffix(dim)}) AS dist FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id ${pathClause}ORDER BY dist ASC LIMIT ${topKParam}`, allParams, ); return rows.map((row) => ({ - path: row.path as import("@statewalker/indexer-api").DocumentPath, + path: row.path as DocumentPath, blockId: row.block_id, score: 1 - row.dist, })); From c4c9a3d95f66950df2da4ab469b9ab3b9a34d5f6 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Sat, 23 May 2026 11:02:36 +0200 Subject: [PATCH 09/12] refactor(indexer): prune dead exports, isolate QMD utils, demote indexer-search MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Five conflict-free cleanups identified by /grill-complexity on 2026-05-23 (see umbrella notes/2026-05/2026-05-23/grill-complexity-statewalker-indexer-packages.md): R1 — drop dead exports from @statewalker/indexer-core: - delete fan-out-search.ts (95 LOC, docstring already admitted "not part of any consumer-facing public surface") - drop mergeHybrid wrapper (createCompositeIndex inlines the RRF/weights branch directly) - drop buildRrfTrace + RRFTrace/RRFContribution (no caller) R2 — delete 4 unused SQL sub-index factory files in indexer-duckdb / indexer-pglite that were never re-exported from any barrel and never imported (~96 LOC). R3 — relocate QMD-port utilities (parseStructuredQuery, validateLexQuery, validateSemanticQuery, extractIntentTerms, selectBestChunk) from the top-level @statewalker/indexer-search barrel into a secondary ./utils sub-export. Marks them clearly as "unwired primitives", not active orchestration surface. R4 — collapse SemanticIndex (119 LOC class, five pass-through methods) to two free functions: embedAndAdd, embedAndSearch. Tests rewritten. B5 — demote @statewalker/indexer-search to "private": true (no umbrella consumer imports it; published-package shape unearned). Verified: all 525 indexer tests pass; 41 content-pipeline + 10 content-cli tests pass; typecheck clean except pre-existing TextEncoder/TextDecoder baseline; biome clean. Co-Authored-By: Claude Opus 4.7 (1M context) --- .changeset/nine-doors-look.md | 11 ++ README.md | 1 + packages/indexer-core/src/fan-out-search.ts | 95 -------------- packages/indexer-core/src/index.ts | 11 +- packages/indexer-core/src/merge.ts | 11 -- packages/indexer-core/src/rrf.ts | 74 ----------- packages/indexer-core/test/rrf.test.ts | 105 +--------------- packages/indexer-duckdb/src/dialect.ts | 3 +- .../src/duckdb-full-text-index.ts | 24 ---- .../indexer-duckdb/src/duckdb-vector-index.ts | 24 ---- .../tests/semantic-index.test.ts | 18 ++- .../src/pglite-full-text-index.ts | 24 ---- .../indexer-pglite/src/pglite-vector-index.ts | 24 ---- packages/indexer-search/package.json | 7 +- packages/indexer-search/src/embed-helpers.ts | 75 +++++++++++ packages/indexer-search/src/fn-types.ts | 2 +- packages/indexer-search/src/index.ts | 11 +- packages/indexer-search/src/semantic-index.ts | 118 ------------------ packages/indexer-search/src/utils/index.ts | 8 ++ .../indexer-search/src/{ => utils}/intent.ts | 0 .../src/{ => utils}/query-parser.ts | 0 .../test/{ => utils}/intent.test.ts | 2 +- .../test/{ => utils}/query-parser.test.ts | 2 +- .../src/suites/semantic-index.suite.ts | 101 ++++++++------- 24 files changed, 168 insertions(+), 583 deletions(-) create mode 100644 .changeset/nine-doors-look.md delete mode 100644 packages/indexer-core/src/fan-out-search.ts delete mode 100644 packages/indexer-duckdb/src/duckdb-full-text-index.ts delete mode 100644 packages/indexer-duckdb/src/duckdb-vector-index.ts delete mode 100644 packages/indexer-pglite/src/pglite-full-text-index.ts delete mode 100644 packages/indexer-pglite/src/pglite-vector-index.ts create mode 100644 packages/indexer-search/src/embed-helpers.ts delete mode 100644 packages/indexer-search/src/semantic-index.ts create mode 100644 packages/indexer-search/src/utils/index.ts rename packages/indexer-search/src/{ => utils}/intent.ts (100%) rename packages/indexer-search/src/{ => utils}/query-parser.ts (100%) rename packages/indexer-search/test/{ => utils}/intent.test.ts (98%) rename packages/indexer-search/test/{ => utils}/query-parser.test.ts (99%) diff --git a/.changeset/nine-doors-look.md b/.changeset/nine-doors-look.md new file mode 100644 index 0000000..e85063c --- /dev/null +++ b/.changeset/nine-doors-look.md @@ -0,0 +1,11 @@ +--- +--- + +Empty changeset — prune-indexer-dead-code refactor. + +Internal cleanup only: +- `@statewalker/indexer-core` (workspace-internal): drop dead `fanOutSearch` / `mergeHybrid` / `buildRrfTrace` exports. +- `@statewalker/indexer-duckdb`, `@statewalker/indexer-pglite`: delete 4 unused sub-index factory files. +- `@statewalker/indexer-search`: demoted to `private: true` (no published surface change to react to). `SemanticIndex` collapsed to `embedAndAdd` / `embedAndSearch` helpers; QMD-port utilities (`parseStructuredQuery`, `extractIntentTerms`, …) relocated behind a secondary `./utils` sub-export. + +No published packages have observable contract changes. diff --git a/README.md b/README.md index 41c6ef3..9db4c80 100644 --- a/README.md +++ b/README.md @@ -16,6 +16,7 @@ Indexing primitives: pluggable full-text and vector indexers (in-memory, DuckDB, | [@statewalker/indexer-mem-minisearch](packages/indexer-mem-minisearch) | MiniSearch + `MemVectorIndex` + optional persistence. | yes | | [@statewalker/indexer-duckdb](packages/indexer-duckdb) | DuckDB backend: real BM25 FTS (`fts` extension) + HNSW cosine vector (`vss` extension). | yes | | [@statewalker/indexer-pglite](packages/indexer-pglite) | PGlite backend: `tsvector`/GIN FTS + `pgvector` HNSW cosine. | yes | +| [@statewalker/indexer-search](packages/indexer-search) | Workspace-internal app-side search-orchestration stack (SearchPipeline, embed helpers, reranker blending, mocks). QMD-port utilities live under `./utils`. | no | | [@statewalker/indexer-tests](packages/indexer-tests) | Shared Vitest conformance suite run by every backend. | no | ## Development diff --git a/packages/indexer-core/src/fan-out-search.ts b/packages/indexer-core/src/fan-out-search.ts deleted file mode 100644 index ddc906f..0000000 --- a/packages/indexer-core/src/fan-out-search.ts +++ /dev/null @@ -1,95 +0,0 @@ -import type { - DocumentPath, - HybridSearchResult, - HybridWeights, - Index, - ScoredItem, -} from "@statewalker/indexer-api"; -import { type RankedList, reciprocalRankFusion } from "./rrf.js"; - -/** Parameters for fan-out search with cross-query RRF fusion. */ -export interface FanOutSearchParams { - /** FTS queries — blocks matching more queries rank higher. */ - queries?: string[]; - /** Embedding vectors — blocks closer to more vectors rank higher. */ - embeddings?: Float32Array[]; - /** Maximum number of results to return. */ - topK: number; - /** Relative weights for blending FTS and embedding scores within each per-query call. */ - weights?: HybridWeights; - /** Path prefixes to restrict search scope. */ - paths?: DocumentPath[]; -} - -async function collectResults( - gen: AsyncGenerator, -): Promise { - const results: HybridSearchResult[] = []; - for await (const r of gen) { - results.push(r); - } - return results; -} - -/** - * Backend-implementation glue: fans out individual queries/embeddings to - * `index.search()`, then fuses the resulting ranked lists with RRF. - * - * Use this when an underlying engine has no native multi-query merge — the - * backend can call `fanOutSearch` from inside its own `Index.search()` to - * obtain cross-query rank fusion. - * - * Not part of any consumer-facing public surface. The `SearchPipeline` in - * `@statewalker/indexer-search` delegates fusion to `index.search()` - * directly and does not need this helper. - */ -export async function fanOutSearch( - index: Index, - params: FanOutSearchParams, -): Promise { - const { queries, embeddings, topK, weights, paths } = params; - - const hasQueries = queries && queries.length > 0; - const hasEmbeddings = embeddings && embeddings.length > 0; - - if (!hasQueries && !hasEmbeddings) { - return []; - } - - const rankedLists: RankedList[] = []; - const searchPromises: Promise[] = []; - - if (hasQueries) { - for (const query of queries) { - searchPromises.push( - collectResults(index.search({ queries: [query], topK, weights, paths })).then( - (hybridResults) => { - rankedLists.push({ - results: hybridResults.map((r) => ({ blockId: r.blockId, score: r.score })), - meta: { source: "fts", queryType: "lex", query }, - }); - }, - ), - ); - } - } - - if (hasEmbeddings) { - for (const embedding of embeddings) { - searchPromises.push( - collectResults(index.search({ embeddings: [embedding], topK, weights, paths })).then( - (hybridResults) => { - rankedLists.push({ - results: hybridResults.map((r) => ({ blockId: r.blockId, score: r.score })), - meta: { source: "vec", queryType: "vec", query: "" }, - }); - }, - ), - ); - } - } - - await Promise.all(searchPromises); - - return reciprocalRankFusion(rankedLists, topK); -} diff --git a/packages/indexer-core/src/index.ts b/packages/indexer-core/src/index.ts index faff061..4795ff6 100644 --- a/packages/indexer-core/src/index.ts +++ b/packages/indexer-core/src/index.ts @@ -23,17 +23,10 @@ export { type SqlVectorDialect, type SqlVectorRetrieverOptions, } from "./create-sql-vector-retriever.js"; -export { type FanOutSearchParams, fanOutSearch } from "./fan-out-search.js"; -export { mergeByRRF, mergeByWeights, mergeHybrid } from "./merge.js"; +export { mergeByRRF, mergeByWeights } from "./merge.js"; export { matchesPrefix } from "./path-prefix.js"; export { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; -export { - buildRrfTrace, - type RankedList, - type RRFContribution, - type RRFTrace, - reciprocalRankFusion, -} from "./rrf.js"; +export { type RankedList, reciprocalRankFusion } from "./rrf.js"; export { sanitizePrefix } from "./sanitize-prefix.js"; export type { SqlDb } from "./sql-db.js"; export { validateDimensionality } from "./validate-dimensionality.js"; diff --git a/packages/indexer-core/src/merge.ts b/packages/indexer-core/src/merge.ts index 60ae5a9..02b1f4c 100644 --- a/packages/indexer-core/src/merge.ts +++ b/packages/indexer-core/src/merge.ts @@ -139,14 +139,3 @@ export function mergeByWeights( results.sort((a, b) => b.score - a.score); return results.slice(0, topK); } - -export function mergeHybrid( - ftsResults: FullTextSearchResult[], - vecResults: EmbeddingSearchResult[], - topK: number, - weights?: HybridWeights, -): HybridSearchResult[] { - return weights - ? mergeByWeights(ftsResults, vecResults, weights, topK) - : mergeByRRF(ftsResults, vecResults, topK); -} diff --git a/packages/indexer-core/src/rrf.ts b/packages/indexer-core/src/rrf.ts index 1be7394..a13ca12 100644 --- a/packages/indexer-core/src/rrf.ts +++ b/packages/indexer-core/src/rrf.ts @@ -13,23 +13,6 @@ export interface RankedList { meta?: { source: string; queryType: string; query: string }; } -export interface RRFContribution { - listIndex: number; - source?: string; - queryType?: string; - rank: number; - weight: number; - contribution: number; -} - -export interface RRFTrace { - contributions: RRFContribution[]; - baseScore: number; - topRank: number; - topRankBonus: number; - totalScore: number; -} - const TOP_RANK_BONUSES: [number, number][] = [ [1, 0.05], [3, 0.02], @@ -78,60 +61,3 @@ export function reciprocalRankFusion(lists: RankedList[], topK: number, k = 60): score, })); } - -export function buildRrfTrace(lists: RankedList[], k = 60): Map { - const traces = new Map(); - - function getOrCreate(blockId: string): RRFTrace { - let trace = traces.get(blockId); - if (!trace) { - trace = { - contributions: [], - baseScore: 0, - topRank: Number.POSITIVE_INFINITY, - topRankBonus: 0, - totalScore: 0, - }; - traces.set(blockId, trace); - } - return trace; - } - - for (let listIndex = 0; listIndex < lists.length; listIndex++) { - const list = lists[listIndex]; - if (!list) continue; - const w = list.weight ?? 1.0; - - for (let i = 0; i < list.results.length; i++) { - const result = list.results[i]; - if (!result) continue; - const { blockId } = result; - const contribution = w / (k + i + 1); - const oneIndexedRank = i + 1; - - const trace = getOrCreate(blockId); - trace.baseScore += contribution; - - if (oneIndexedRank < trace.topRank) { - trace.topRank = oneIndexedRank; - } - - trace.contributions.push({ - listIndex, - source: list.meta?.source, - queryType: list.meta?.queryType, - rank: oneIndexedRank, - weight: w, - contribution, - }); - } - } - - // Compute bonuses and totals - for (const trace of traces.values()) { - trace.topRankBonus = getTopRankBonus(trace.topRank); - trace.totalScore = trace.baseScore + trace.topRankBonus; - } - - return traces; -} diff --git a/packages/indexer-core/test/rrf.test.ts b/packages/indexer-core/test/rrf.test.ts index b2f0c4f..30a2852 100644 --- a/packages/indexer-core/test/rrf.test.ts +++ b/packages/indexer-core/test/rrf.test.ts @@ -3,7 +3,7 @@ * by Tobi Lutke. MIT License — Copyright (c) 2024-2026 Tobi Lutke. */ import { describe, expect, it } from "vitest"; -import { buildRrfTrace, type RankedList, reciprocalRankFusion } from "../src/rrf.js"; +import { type RankedList, reciprocalRankFusion } from "../src/rrf.js"; // --------------------------------------------------------------------------- // Helpers @@ -191,106 +191,3 @@ describe("reciprocalRankFusion — top-rank bonus", () => { expect(item?.score).toBeCloseTo(1 / 63 + 1 / 61 + 0.05, 10); }); }); - -// --------------------------------------------------------------------------- -// buildRrfTrace -// --------------------------------------------------------------------------- -describe("buildRrfTrace", () => { - it("trace totals match fusion results exactly", () => { - const lists = makeTwoLists(); - const merged = reciprocalRankFusion(lists, 10, K); - const traces = buildRrfTrace(lists, K); - - for (const result of merged) { - const trace = traces.get(result.blockId); - expect(trace).toBeDefined(); - expect(trace?.totalScore).toBeCloseTo(result.score, 10); - } - }); - - it("records per-list contributions with source metadata", () => { - const lists = makeTwoLists(); - const traces = buildRrfTrace(lists, K); - - const traceA = traces.get("a"); - expect(traceA).toBeDefined(); - expect(traceA?.contributions).toHaveLength(2); - - const fromFts = traceA?.contributions.find((c) => c.source === "fts"); - expect(fromFts).toBeDefined(); - expect(fromFts?.listIndex).toBe(0); - expect(fromFts?.rank).toBe(1); // 1-indexed - expect(fromFts?.weight).toBe(2.0); - expect(fromFts?.queryType).toBe("keyword"); - - const fromVec = traceA?.contributions.find((c) => c.source === "vec"); - expect(fromVec).toBeDefined(); - expect(fromVec?.listIndex).toBe(1); - expect(fromVec?.rank).toBe(2); // 1-indexed - expect(fromVec?.weight).toBe(1.0); - expect(fromVec?.queryType).toBe("semantic"); - }); - - it("topRank is best rank across all lists", () => { - const lists = makeTwoLists(); - const traces = buildRrfTrace(lists, K); - - // "a" is rank 1 in list 0, rank 2 in list 1 → topRank = 1 - expect(traces.get("a")?.topRank).toBe(1); - // "b" is rank 2 in list 0, rank 1 in list 1 → topRank = 1 - expect(traces.get("b")?.topRank).toBe(1); - }); - - it("topRankBonus matches thresholds (0.05/0.02/0.0)", () => { - const lists: RankedList[] = [ - { - results: [ - { blockId: "rank1", score: 1 }, - { blockId: "rank2", score: 0.9 }, - { blockId: "rank3", score: 0.8 }, - { blockId: "rank4", score: 0.7 }, - ], - }, - ]; - const traces = buildRrfTrace(lists, K); - - expect(traces.get("rank1")?.topRankBonus).toBe(0.05); - expect(traces.get("rank2")?.topRankBonus).toBe(0.02); - expect(traces.get("rank3")?.topRankBonus).toBe(0.02); - expect(traces.get("rank4")?.topRankBonus).toBe(0); - }); - - it("contributions array has one entry per list appearance", () => { - const lists: RankedList[] = [ - { results: [{ blockId: "doc", score: 1 }] }, - { results: [{ blockId: "doc", score: 1 }] }, - { - results: [ - { blockId: "other", score: 1 }, - { blockId: "doc", score: 0.5 }, - ], - }, - ]; - const traces = buildRrfTrace(lists, K); - expect(traces.get("doc")?.contributions).toHaveLength(3); - expect(traces.get("other")?.contributions).toHaveLength(1); - }); - - it("weighted contribution = weight / (k + rank + 1)", () => { - const lists: RankedList[] = [ - { - results: [{ blockId: "a", score: 1 }], - weight: 3.0, - }, - ]; - const traces = buildRrfTrace(lists, K); - const contrib = traces.get("a")?.contributions[0]; - - // rank 1 (1-indexed), so contribution = 3.0 / (60 + 1 + 1) = 3/62 - // Wait — rank is 1-indexed in trace, but formula uses 0-indexed position - // contribution = weight / (k + 0-indexed-rank + 1) = 3 / (60 + 0 + 1) = 3/61 - expect(contrib?.contribution).toBeCloseTo(3 / 61, 10); - expect(contrib?.rank).toBe(1); - expect(contrib?.weight).toBe(3.0); - }); -}); diff --git a/packages/indexer-duckdb/src/dialect.ts b/packages/indexer-duckdb/src/dialect.ts index 502e609..9975bf7 100644 --- a/packages/indexer-duckdb/src/dialect.ts +++ b/packages/indexer-duckdb/src/dialect.ts @@ -11,8 +11,7 @@ import type { export function wrapDbAsSqlDb(db: Db): SqlDb { return { exec: (sql) => db.exec(sql), - query: (sql: string, params?: unknown[]) => - db.query(sql, params ?? []), + query: (sql: string, params?: unknown[]) => db.query(sql, params ?? []), }; } diff --git a/packages/indexer-duckdb/src/duckdb-full-text-index.ts b/packages/indexer-duckdb/src/duckdb-full-text-index.ts deleted file mode 100644 index daddb56..0000000 --- a/packages/indexer-duckdb/src/duckdb-full-text-index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { Db } from "@statewalker/db-api"; -import type { FullTextIndex, FullTextIndexInfo } from "@statewalker/indexer-api"; -import { createSqlFtsRetriever } from "@statewalker/indexer-core"; -import { duckdbFtsDialect, wrapDbAsSqlDb } from "./dialect.js"; - -export type DuckDbFullTextIndex = FullTextIndex & { - readonly tableName: string; - init(): Promise; -}; - -export function createDuckDbFullTextIndex( - db: Db, - prefix: string, - docsTable: string, - info: FullTextIndexInfo, -): DuckDbFullTextIndex { - return createSqlFtsRetriever({ - db: wrapDbAsSqlDb(db), - prefix, - docsTable, - info, - dialect: duckdbFtsDialect, - }); -} diff --git a/packages/indexer-duckdb/src/duckdb-vector-index.ts b/packages/indexer-duckdb/src/duckdb-vector-index.ts deleted file mode 100644 index 8608f43..0000000 --- a/packages/indexer-duckdb/src/duckdb-vector-index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { Db } from "@statewalker/db-api"; -import type { EmbeddingIndex, EmbeddingIndexInfo } from "@statewalker/indexer-api"; -import { createSqlVectorRetriever } from "@statewalker/indexer-core"; -import { duckdbVectorDialect, wrapDbAsSqlDb } from "./dialect.js"; - -export type DuckDbVectorIndex = EmbeddingIndex & { - readonly tableName: string; - init(): Promise; -}; - -export function createDuckDbVectorIndex( - db: Db, - prefix: string, - docsTable: string, - info: EmbeddingIndexInfo, -): DuckDbVectorIndex { - return createSqlVectorRetriever({ - db: wrapDbAsSqlDb(db), - prefix, - docsTable, - info, - dialect: duckdbVectorDialect, - }); -} diff --git a/packages/indexer-mem-flexsearch/tests/semantic-index.test.ts b/packages/indexer-mem-flexsearch/tests/semantic-index.test.ts index 309c933..c8f0e4a 100644 --- a/packages/indexer-mem-flexsearch/tests/semantic-index.test.ts +++ b/packages/indexer-mem-flexsearch/tests/semantic-index.test.ts @@ -1,5 +1,5 @@ import type { DocumentPath, EmbedFn, Index } from "@statewalker/indexer-api"; -import { SemanticIndex } from "@statewalker/indexer-search"; +import { embedAndAdd } from "@statewalker/indexer-search"; import { afterEach, beforeEach, describe, expect, it } from "vitest"; import { createFlexSearchIndexer } from "../src/index.js"; @@ -20,21 +20,19 @@ afterEach(async () => { await indexer.close(); }); -describe("SemanticIndex.addDocuments (FTS-only backend)", () => { +describe("embedAndAdd (FTS-only backend)", () => { it("ingests documents from a sync iterable", async () => { - const semantic = new SemanticIndex(index, noopEmbed); const docs = [ { path: "/docs/" as DocumentPath, blockId: "b1", content: "hello world" }, { path: "/docs/" as DocumentPath, blockId: "b2", content: "foo bar" }, ]; - await semantic.addDocuments(docs); + await embedAndAdd(index, noopEmbed, docs); expect(await index.getSize()).toBe(2); }); it("ingested documents are searchable", async () => { - const semantic = new SemanticIndex(index, noopEmbed); - await semantic.addDocuments([ + await embedAndAdd(index, noopEmbed, [ { path: "/docs/" as DocumentPath, blockId: "b1", content: "quantum mechanics physics" }, ]); @@ -47,18 +45,17 @@ describe("SemanticIndex.addDocuments (FTS-only backend)", () => { }); it("ingests documents from an async iterable", async () => { - const semantic = new SemanticIndex(index, noopEmbed); async function* generateDocs() { yield { path: "/docs/" as DocumentPath, blockId: "a1", content: "async document one" }; yield { path: "/docs/" as DocumentPath, blockId: "a2", content: "async document two" }; } - await semantic.addDocuments(generateDocs()); + await embedAndAdd(index, noopEmbed, generateDocs()); expect(await index.getSize()).toBe(2); }); }); -describe("SemanticIndex.addDocuments (FTS + vector backend)", () => { +describe("embedAndAdd (FTS + vector backend)", () => { it("calls embed for every document", async () => { const indexerWithVec = createFlexSearchIndexer(); const vecIndex = await indexerWithVec.createIndex({ @@ -73,8 +70,7 @@ describe("SemanticIndex.addDocuments (FTS + vector backend)", () => { return new Float32Array([0.1, 0.2, 0.3]); }; - const semantic = new SemanticIndex(vecIndex, embed); - await semantic.addDocuments([ + await embedAndAdd(vecIndex, embed, [ { path: "/docs/" as DocumentPath, blockId: "e1", content: "embedding test" }, ]); diff --git a/packages/indexer-pglite/src/pglite-full-text-index.ts b/packages/indexer-pglite/src/pglite-full-text-index.ts deleted file mode 100644 index a5db48f..0000000 --- a/packages/indexer-pglite/src/pglite-full-text-index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { PGlite } from "@electric-sql/pglite"; -import type { FullTextIndex, FullTextIndexInfo } from "@statewalker/indexer-api"; -import { createSqlFtsRetriever } from "@statewalker/indexer-core"; -import { pgliteFtsDialect, wrapDbAsSqlDb } from "./dialect.js"; - -export type PGLiteFullTextIndex = FullTextIndex & { - readonly tableName: string; - init(): Promise; -}; - -export function createPGLiteFullTextIndex( - db: PGlite, - prefix: string, - docsTable: string, - info: FullTextIndexInfo, -): PGLiteFullTextIndex { - return createSqlFtsRetriever({ - db: wrapDbAsSqlDb(db), - prefix, - docsTable, - info, - dialect: pgliteFtsDialect, - }); -} diff --git a/packages/indexer-pglite/src/pglite-vector-index.ts b/packages/indexer-pglite/src/pglite-vector-index.ts deleted file mode 100644 index 61ecda1..0000000 --- a/packages/indexer-pglite/src/pglite-vector-index.ts +++ /dev/null @@ -1,24 +0,0 @@ -import type { PGlite } from "@electric-sql/pglite"; -import type { EmbeddingIndex, EmbeddingIndexInfo } from "@statewalker/indexer-api"; -import { createSqlVectorRetriever } from "@statewalker/indexer-core"; -import { pgliteVectorDialect, wrapDbAsSqlDb } from "./dialect.js"; - -export type PGLiteVectorIndex = EmbeddingIndex & { - readonly tableName: string; - init(): Promise; -}; - -export function createPGLiteVectorIndex( - db: PGlite, - prefix: string, - docsTable: string, - info: EmbeddingIndexInfo, -): PGLiteVectorIndex { - return createSqlVectorRetriever({ - db: wrapDbAsSqlDb(db), - prefix, - docsTable, - info, - dialect: pgliteVectorDialect, - }); -} diff --git a/packages/indexer-search/package.json b/packages/indexer-search/package.json index 6a8dba1..2fcb078 100644 --- a/packages/indexer-search/package.json +++ b/packages/indexer-search/package.json @@ -1,9 +1,9 @@ { "name": "@statewalker/indexer-search", "version": "0.1.0", - "private": false, + "private": true, "type": "module", - "description": "Application-side search-orchestration stack: SearchPipeline, SemanticIndex, query parser, intent extraction, reranker blending, and mock factories built on @statewalker/indexer-api.", + "description": "Workspace-internal app-side search-orchestration stack: SearchPipeline + embedAndAdd/embedAndSearch helpers + reranker blending + mock factories. QMD-port query/intent utilities live under the secondary ./utils sub-export.", "homepage": "https://github.com/statewalker/statewalker-indexer", "author": { "name": "Mikhail Kotelnikov", @@ -15,7 +15,8 @@ "url": "git+ssh://git@github.com/statewalker/statewalker-indexer.git" }, "exports": { - ".": "./src/index.ts" + ".": "./src/index.ts", + "./utils": "./src/utils/index.ts" }, "files": [ "dist", diff --git a/packages/indexer-search/src/embed-helpers.ts b/packages/indexer-search/src/embed-helpers.ts new file mode 100644 index 0000000..d964069 --- /dev/null +++ b/packages/indexer-search/src/embed-helpers.ts @@ -0,0 +1,75 @@ +import type { + DocumentPath, + EmbedFn, + HybridSearchResult, + HybridWeights, + Index, + IndexedBlock, + Metadata, +} from "@statewalker/indexer-api"; + +export interface EmbedDoc { + path: DocumentPath; + blockId: string; + content: string; + embeddingContent?: string; + metadata?: Metadata; +} + +export interface EmbedSearchParams { + query: string; + semanticQuery?: string; + topK: number; + weights?: HybridWeights; + paths?: DocumentPath[]; +} + +export async function embedAndAdd( + index: Index, + embed: EmbedFn, + docs: Iterable | AsyncIterable, +): Promise { + const hasVector = index.getVectorIndex() !== null; + + async function* toBatches(source: typeof docs): AsyncGenerator { + for await (const doc of source) { + const block: IndexedBlock = { + path: doc.path, + blockId: doc.blockId, + content: doc.content, + metadata: doc.metadata, + }; + if (hasVector) { + block.embedding = await embed(doc.embeddingContent ?? doc.content); + } + yield [block]; + } + } + + return index.addDocuments(toBatches(docs)); +} + +export async function embedAndSearch( + index: Index, + embed: EmbedFn, + params: EmbedSearchParams, +): Promise { + const { query, semanticQuery, topK, weights, paths } = params; + const hasVector = index.getVectorIndex() !== null; + + const searchParams = hasVector + ? { + queries: [query], + embeddings: [await embed(semanticQuery ?? query)], + topK, + weights, + paths, + } + : { queries: [query], topK, weights, paths }; + + const results: HybridSearchResult[] = []; + for await (const r of index.search(searchParams)) { + results.push(r); + } + return results; +} diff --git a/packages/indexer-search/src/fn-types.ts b/packages/indexer-search/src/fn-types.ts index 129db96..f4df8ea 100644 --- a/packages/indexer-search/src/fn-types.ts +++ b/packages/indexer-search/src/fn-types.ts @@ -1,5 +1,5 @@ import type { BlockId, ScoredItem } from "@statewalker/indexer-api"; -import type { QueryType } from "./query-parser.js"; +import type { QueryType } from "./utils/query-parser.js"; export interface ExpandedQuery { type: QueryType; diff --git a/packages/indexer-search/src/index.ts b/packages/indexer-search/src/index.ts index d55bf54..9c6eba2 100644 --- a/packages/indexer-search/src/index.ts +++ b/packages/indexer-search/src/index.ts @@ -1,3 +1,5 @@ +export type { EmbedDoc, EmbedSearchParams } from "./embed-helpers.js"; +export { embedAndAdd, embedAndSearch } from "./embed-helpers.js"; export type { Citation, CitationBuilderFn, @@ -5,19 +7,11 @@ export type { QueryExpanderFn, RerankerFn, } from "./fn-types.js"; -export type { ChunkSelection } from "./intent.js"; -export { extractIntentTerms, selectBestChunk } from "./intent.js"; export { createMockCitationBuilder, createMockExpander, createMockReranker, } from "./mock.js"; -export type { ParsedQuery, QueryType } from "./query-parser.js"; -export { - parseStructuredQuery, - validateLexQuery, - validateSemanticQuery, -} from "./query-parser.js"; export { type BlendTier, blendWithReranker, @@ -25,4 +19,3 @@ export { } from "./reranker-blend.js"; export type { EntryExplain, PipelineConfig, PipelineEntry } from "./search-pipeline.js"; export { SearchPipeline } from "./search-pipeline.js"; -export { SemanticIndex } from "./semantic-index.js"; diff --git a/packages/indexer-search/src/semantic-index.ts b/packages/indexer-search/src/semantic-index.ts deleted file mode 100644 index b038f5f..0000000 --- a/packages/indexer-search/src/semantic-index.ts +++ /dev/null @@ -1,118 +0,0 @@ -import type { - DocumentPath, - EmbedFn, - HybridSearchResult, - HybridWeights, - Index, - IndexedBlock, - Metadata, - PathSelector, -} from "@statewalker/indexer-api"; - -export class SemanticIndex { - readonly index: Index; - private readonly embed: EmbedFn; - - constructor(index: Index, embed: EmbedFn) { - this.index = index; - this.embed = embed; - } - - async search(params: { - query: string; - semanticQuery?: string; - topK: number; - weights?: HybridWeights; - paths?: DocumentPath[]; - }): Promise { - const { query, semanticQuery, topK, weights, paths } = params; - const hasVector = this.index.getVectorIndex() !== null; - - const searchParams = hasVector - ? { - queries: [query], - embeddings: [await this.embed(semanticQuery ?? query)], - topK, - weights, - paths, - } - : { queries: [query], topK, weights, paths }; - - const results: HybridSearchResult[] = []; - for await (const r of this.index.search(searchParams)) { - results.push(r); - } - return results; - } - - async addDocument(params: { - path: DocumentPath; - blockId: string; - content: string; - embeddingContent?: string; - metadata?: Metadata; - }): Promise { - const { path, blockId, content, embeddingContent, metadata } = params; - const hasVector = this.index.getVectorIndex() !== null; - - const block: IndexedBlock = { path, blockId, content, metadata }; - if (hasVector) { - block.embedding = await this.embed(embeddingContent ?? content); - } - return this.index.addDocument([block]); - } - - async addDocuments( - docs: - | Iterable<{ - path: DocumentPath; - blockId: string; - content: string; - embeddingContent?: string; - metadata?: Metadata; - }> - | AsyncIterable<{ - path: DocumentPath; - blockId: string; - content: string; - embeddingContent?: string; - metadata?: Metadata; - }>, - ): Promise { - const hasVector = this.index.getVectorIndex() !== null; - const embed = this.embed; - - const mapped = async function* (source: typeof docs): AsyncGenerator { - for await (const doc of source) { - const block: IndexedBlock = { - path: doc.path, - blockId: doc.blockId, - content: doc.content, - metadata: doc.metadata, - }; - if (hasVector) { - block.embedding = await embed(doc.embeddingContent ?? doc.content); - } - yield [block]; - } - }; - - return this.index.addDocuments(mapped(docs)); - } - - async deleteDocuments(pathSelectors: PathSelector[]): Promise { - return this.index.deleteDocuments(pathSelectors); - } - - async getSize(pathPrefix?: DocumentPath): Promise { - return this.index.getSize(pathPrefix); - } - - async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { - yield* this.index.getDocumentPaths(pathPrefix); - } - - async close(): Promise { - return this.index.close(); - } -} diff --git a/packages/indexer-search/src/utils/index.ts b/packages/indexer-search/src/utils/index.ts new file mode 100644 index 0000000..a98ac9d --- /dev/null +++ b/packages/indexer-search/src/utils/index.ts @@ -0,0 +1,8 @@ +export type { ChunkSelection } from "./intent.js"; +export { extractIntentTerms, selectBestChunk } from "./intent.js"; +export type { ParsedQuery, QueryType } from "./query-parser.js"; +export { + parseStructuredQuery, + validateLexQuery, + validateSemanticQuery, +} from "./query-parser.js"; diff --git a/packages/indexer-search/src/intent.ts b/packages/indexer-search/src/utils/intent.ts similarity index 100% rename from packages/indexer-search/src/intent.ts rename to packages/indexer-search/src/utils/intent.ts diff --git a/packages/indexer-search/src/query-parser.ts b/packages/indexer-search/src/utils/query-parser.ts similarity index 100% rename from packages/indexer-search/src/query-parser.ts rename to packages/indexer-search/src/utils/query-parser.ts diff --git a/packages/indexer-search/test/intent.test.ts b/packages/indexer-search/test/utils/intent.test.ts similarity index 98% rename from packages/indexer-search/test/intent.test.ts rename to packages/indexer-search/test/utils/intent.test.ts index 1a1f7e9..409e811 100644 --- a/packages/indexer-search/test/intent.test.ts +++ b/packages/indexer-search/test/utils/intent.test.ts @@ -3,7 +3,7 @@ * by Tobi Lutke. MIT License — Copyright (c) 2024-2026 Tobi Lutke. */ import { describe, expect, it } from "vitest"; -import { extractIntentTerms, selectBestChunk } from "../src/intent.js"; +import { extractIntentTerms, selectBestChunk } from "../../src/utils/intent.js"; describe("extractIntentTerms", () => { it("filters common stop words (the, a, is, of, in, to, and, for, with, on, at, by, an, or)", () => { diff --git a/packages/indexer-search/test/query-parser.test.ts b/packages/indexer-search/test/utils/query-parser.test.ts similarity index 99% rename from packages/indexer-search/test/query-parser.test.ts rename to packages/indexer-search/test/utils/query-parser.test.ts index e93a803..d6b4354 100644 --- a/packages/indexer-search/test/query-parser.test.ts +++ b/packages/indexer-search/test/utils/query-parser.test.ts @@ -7,7 +7,7 @@ import { parseStructuredQuery, validateLexQuery, validateSemanticQuery, -} from "../src/query-parser.js"; +} from "../../src/utils/query-parser.js"; describe("parseStructuredQuery", () => { describe("plain queries — returns null", () => { diff --git a/packages/indexer-tests/src/suites/semantic-index.suite.ts b/packages/indexer-tests/src/suites/semantic-index.suite.ts index 9c1ec46..07f8dcd 100644 --- a/packages/indexer-tests/src/suites/semantic-index.suite.ts +++ b/packages/indexer-tests/src/suites/semantic-index.suite.ts @@ -1,5 +1,5 @@ import type { Indexer } from "@statewalker/indexer-api"; -import { SemanticIndex } from "@statewalker/indexer-search"; +import { embedAndAdd, embedAndSearch } from "@statewalker/indexer-search"; import { describe, expect, it, vi } from "vitest"; import { createFixtureEmbedFn, @@ -11,7 +11,7 @@ import { import { defined } from "./test-utils.js"; export function runSemanticIndexSuite(getIndexer: () => Indexer): void { - describe("SemanticIndex", () => { + describe("embed helpers", () => { it("embeds query text on search", async () => { const indexer = getIndexer(); const index = await indexer.createIndex({ @@ -24,13 +24,14 @@ export function runSemanticIndexSuite(getIndexer: () => Indexer): void { }); const embedFn = vi.fn(createFixtureEmbedFn()); - const semantic = new SemanticIndex(index, embedFn); - await semantic.addDocument({ - path: "/test/1", - blockId: "1", - content: "hello world", - }); - await semantic.search({ query: "hello", topK: 10 }); + await embedAndAdd(index, embedFn, [ + { + path: "/test/1", + blockId: "1", + content: "hello world", + }, + ]); + await embedAndSearch(index, embedFn, { query: "hello", topK: 10 }); expect(embedFn).toHaveBeenCalled(); }); @@ -50,8 +51,7 @@ export function runSemanticIndexSuite(getIndexer: () => Indexer): void { calls.push(text); return new Float32Array(EMBEDDING_DIMENSIONS); }; - const semantic = new SemanticIndex(index, embedFn); - await semantic.search({ + await embedAndSearch(index, embedFn, { query: "original", semanticQuery: "rewritten", topK: 10, @@ -76,13 +76,14 @@ export function runSemanticIndexSuite(getIndexer: () => Indexer): void { calls.push(text); return new Float32Array(EMBEDDING_DIMENSIONS); }; - const semantic = new SemanticIndex(index, embedFn); - await semantic.addDocument({ - path: "/test/1", - blockId: "1", - content: "original", - embeddingContent: "enriched", - }); + await embedAndAdd(index, embedFn, [ + { + path: "/test/1", + blockId: "1", + content: "original", + embeddingContent: "enriched", + }, + ]); expect(calls).toContain("enriched"); expect(calls).not.toContain("original"); }); @@ -95,46 +96,49 @@ export function runSemanticIndexSuite(getIndexer: () => Indexer): void { }); const embedFn = vi.fn(async () => new Float32Array(EMBEDDING_DIMENSIONS)); - const semantic = new SemanticIndex(index, embedFn); - await semantic.addDocument({ - path: "/test/1", - blockId: "1", - content: "hello", - }); + await embedAndAdd(index, embedFn, [ + { + path: "/test/1", + blockId: "1", + content: "hello", + }, + ]); expect(embedFn).not.toHaveBeenCalled(); }); - it("delegates getSize", async () => { + it("ingested documents are visible to index.getSize", async () => { const indexer = getIndexer(); const index = await indexer.createIndex({ name: "test", fulltext: { language: "en" }, }); const embedFn = createFixtureEmbedFn(); - const semantic = new SemanticIndex(index, embedFn); - await semantic.addDocument({ - path: "/test/1", - blockId: "1", - content: "hello", - }); - expect(await semantic.getSize()).toBe(1); + await embedAndAdd(index, embedFn, [ + { + path: "/test/1", + blockId: "1", + content: "hello", + }, + ]); + expect(await index.getSize()).toBe(1); }); - it("delegates deleteDocuments", async () => { + it("index.deleteDocuments removes helper-added docs", async () => { const indexer = getIndexer(); const index = await indexer.createIndex({ name: "test", fulltext: { language: "en" }, }); const embedFn = createFixtureEmbedFn(); - const semantic = new SemanticIndex(index, embedFn); - await semantic.addDocument({ - path: "/test/1", - blockId: "1", - content: "hello", - }); - await semantic.deleteDocuments([{ path: "/test/1", blockId: "1" }]); - expect(await semantic.getSize()).toBe(0); + await embedAndAdd(index, embedFn, [ + { + path: "/test/1", + blockId: "1", + content: "hello", + }, + ]); + await index.deleteDocuments([{ path: "/test/1", blockId: "1" }]); + expect(await index.getSize()).toBe(0); }); it("end-to-end search with fixture blocks", async () => { @@ -148,24 +152,25 @@ export function runSemanticIndexSuite(getIndexer: () => Indexer): void { }, }); const embedFn = createFixtureEmbedFn(); - const semantic = new SemanticIndex(index, embedFn); const blocks = loadBlocksFixture(); const queries = loadQueriesFixture(); for (const [fileName, docBlocks] of Object.entries(blocks)) { let blockNum = 1; for (const [, block] of Object.entries(docBlocks)) { - await semantic.addDocument({ - path: `/${fileName}` as `/${string}`, - blockId: `${fileName}-${blockNum}`, - content: block.text, - }); + await embedAndAdd(index, embedFn, [ + { + path: `/${fileName}` as `/${string}`, + blockId: `${fileName}-${blockNum}`, + content: block.text, + }, + ]); blockNum++; } } const q = defined(queries[0]); - const results = await semantic.search({ query: q.query, topK: 10 }); + const results = await embedAndSearch(index, embedFn, { query: q.query, topK: 10 }); expect(results.length).toBeGreaterThan(0); }); }); From 7b4ef2dee432e3cde045ad9eeac90847198c2925 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Sat, 23 May 2026 12:32:56 +0200 Subject: [PATCH 10/12] fix(indexer): harden retrievers, pipeline, and serialisation against adversarial inputs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Each fix lands red/green: failing test first, then the minimum change to make it pass. Highlights: - path-prefix: boundary-aware match (/foo no longer matches /foobar) - compositeKey: length-prefixed segments so NUL/delimiter can't collide keys - merge: weighted blend preserves order when all input scores are equal - mergeByRRF / SearchPipeline: stop feeding blockId as candidate text - persistence-backed indexer: init failure no longer marks indexer initialised - FlexSearch/MiniSearch deserialize: throw on unsupported version - MemVectorIndex Arrow roundtrip: metadata preserved - SQL retrievers: LIKE wildcards in path prefixes are now escaped, and prefix matching respects path-component boundaries (new sql-path-prefix helper used by every retriever and dialect) - SQL retrievers: addDocument is a single INSERT … ON CONFLICT DO UPDATE - PGlite dialect: language is whitelisted (no SQL injection via info.language) and FTS no longer strips non-ASCII (Russian/CJK/etc. reach the engine) - cosineSimilarity: throws on dimension mismatch instead of silent zero-pad Co-Authored-By: Claude Opus 4.7 (1M context) --- packages/indexer-core/src/composite-key.ts | 9 +- .../src/create-persistence-backed-indexer.ts | 11 ++- .../src/create-sql-backed-indexer.ts | 7 +- .../src/create-sql-fts-retriever.ts | 51 +++++----- .../src/create-sql-vector-retriever.ts | 51 +++++----- packages/indexer-core/src/index.ts | 6 ++ packages/indexer-core/src/merge.ts | 6 +- packages/indexer-core/src/path-prefix.ts | 11 ++- packages/indexer-core/src/sql-path-prefix.ts | 65 +++++++++++++ .../indexer-core/test/composite-key.test.ts | Bin 0 -> 905 bytes ...te-persistence-backed-indexer-init.test.ts | 61 ++++++++++++ .../indexer-core/test/merge-weights.test.ts | 29 ++++++ .../indexer-core/test/path-prefix.test.ts | 31 +++++++ .../indexer-core/test/sql-path-prefix.test.ts | 50 ++++++++++ packages/indexer-duckdb/src/dialect.ts | 13 +-- .../src/flexsearch-full-text-index.ts | 41 +++++---- .../tests/deserialize-version.test.ts | 30 ++++++ .../tests/reranker-text.test.ts | 51 ++++++++++ .../src/minisearch-full-text-index.ts | 33 ++++--- .../tests/deserialize-version.test.ts | 28 ++++++ packages/indexer-mem/src/mem-vector-index.ts | 11 ++- packages/indexer-mem/src/vector-search.ts | 3 + .../test/cosine-similarity.test.ts | 24 +++++ .../indexer-mem/test/serialize-arrow.test.ts | 37 ++++++++ packages/indexer-pglite/src/dialect.ts | 29 ++++-- .../indexer-search/src/search-pipeline.ts | 87 ++++++++++-------- turbo.json | 20 +--- 27 files changed, 633 insertions(+), 162 deletions(-) create mode 100644 packages/indexer-core/src/sql-path-prefix.ts create mode 100644 packages/indexer-core/test/composite-key.test.ts create mode 100644 packages/indexer-core/test/create-persistence-backed-indexer-init.test.ts create mode 100644 packages/indexer-core/test/merge-weights.test.ts create mode 100644 packages/indexer-core/test/path-prefix.test.ts create mode 100644 packages/indexer-core/test/sql-path-prefix.test.ts create mode 100644 packages/indexer-mem-flexsearch/tests/deserialize-version.test.ts create mode 100644 packages/indexer-mem-flexsearch/tests/reranker-text.test.ts create mode 100644 packages/indexer-mem-minisearch/tests/deserialize-version.test.ts create mode 100644 packages/indexer-mem/test/cosine-similarity.test.ts create mode 100644 packages/indexer-mem/test/serialize-arrow.test.ts diff --git a/packages/indexer-core/src/composite-key.ts b/packages/indexer-core/src/composite-key.ts index 0cf66ec..40754d8 100644 --- a/packages/indexer-core/src/composite-key.ts +++ b/packages/indexer-core/src/composite-key.ts @@ -1,5 +1,12 @@ import type { DocumentPath } from "@statewalker/indexer-api"; +/** + * Joins a (path, blockId) pair into a string key safe to use as a Map key. + * + * Encodes both inputs as length-prefixed segments so that no character in either + * input can collide with the delimiter (e.g. NUL inside a blockId cannot impersonate + * a path/blockId boundary). + */ export function compositeKey(path: DocumentPath, blockId: string): string { - return `${path}\0${blockId}`; + return `${path.length}:${path}|${blockId}`; } diff --git a/packages/indexer-core/src/create-persistence-backed-indexer.ts b/packages/indexer-core/src/create-persistence-backed-indexer.ts index fbd9bd0..a3c71b4 100644 --- a/packages/indexer-core/src/create-persistence-backed-indexer.ts +++ b/packages/indexer-core/src/create-persistence-backed-indexer.ts @@ -73,7 +73,6 @@ export function createPersistenceBackedIndexer { if (!persistence || initialized) return; - initialized = true; const textEntries = new Map(); const binaryEntries = new Map(); @@ -82,6 +81,7 @@ export function createPersistenceBackedIndexer { if (initialized) return; - if (!initPromise) initPromise = loadFromPersistence(); + if (!initPromise) { + initPromise = loadFromPersistence().catch((err) => { + // Allow the next caller to retry rather than treating a transient + // load failure as permanent initialised-state. + initPromise = null; + throw err; + }); + } await initPromise; } diff --git a/packages/indexer-core/src/create-sql-backed-indexer.ts b/packages/indexer-core/src/create-sql-backed-indexer.ts index 417f631..36e15e5 100644 --- a/packages/indexer-core/src/create-sql-backed-indexer.ts +++ b/packages/indexer-core/src/create-sql-backed-indexer.ts @@ -16,6 +16,7 @@ import type { SqlVectorDialect } from "./create-sql-vector-retriever.js"; import { createSqlVectorRetriever } from "./create-sql-vector-retriever.js"; import { sanitizePrefix } from "./sanitize-prefix.js"; import type { SqlDb } from "./sql-db.js"; +import { buildPathPrefixSql } from "./sql-path-prefix.js"; type FtsWithTable = FullTextIndex & { readonly tableName: string; init(): Promise }; type VecWithTable = EmbeddingIndex & { readonly tableName: string; init(): Promise }; @@ -100,8 +101,10 @@ export async function createSqlBackedIndexer(opts: SqlBackedIndexerOptions): Pro vec, getSize: async (pathPrefix?: DocumentPath): Promise => { if (fts !== null && vec !== null) { - const pathClause = pathPrefix !== undefined ? ` WHERE d.path LIKE $1 || '%'` : ""; - const params = pathPrefix !== undefined ? [pathPrefix] : []; + const cond = + pathPrefix !== undefined ? buildPathPrefixSql("d.path", pathPrefix, 1) : null; + const pathClause = cond ? ` WHERE ${cond.sql}` : ""; + const params = cond?.params ?? []; const sql = `SELECT COUNT(*) AS cnt FROM (SELECT b.doc_id, b.block_id FROM ${fts.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause} UNION SELECT b.doc_id, b.block_id FROM ${vec.tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id${pathClause})${dialect.unionAliasSuffix}`; const rows = await db.query<{ cnt: number | bigint }>(sql, params); return Number(rows[0]?.cnt ?? 0); diff --git a/packages/indexer-core/src/create-sql-fts-retriever.ts b/packages/indexer-core/src/create-sql-fts-retriever.ts index be65e9e..3978a49 100644 --- a/packages/indexer-core/src/create-sql-fts-retriever.ts +++ b/packages/indexer-core/src/create-sql-fts-retriever.ts @@ -12,6 +12,7 @@ import type { import { toAsyncIterable } from "./async.js"; import { compositeKey } from "./composite-key.js"; import type { SqlDb } from "./sql-db.js"; +import { buildPathPrefixSql } from "./sql-path-prefix.js"; /** * Per-dialect SQL hooks for a full-text sub-index. @@ -139,12 +140,10 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd for (const block of blocks) { const docId = await resolveDocId(block.path); const metaJson = block.metadata ? JSON.stringify(block.metadata) : null; - await db.query(`DELETE FROM ${tableName} WHERE doc_id = $1 AND block_id = $2`, [ - docId, - block.blockId, - ]); + // Single-statement UPSERT: a failed re-ingest can't strand the old row + // the way a DELETE+INSERT pair would when the INSERT half throws. await db.query( - `INSERT INTO ${tableName} (doc_id, block_id, content, metadata) VALUES ($1, $2, $3, $4)`, + `INSERT INTO ${tableName} (doc_id, block_id, content, metadata) VALUES ($1, $2, $3, $4) ON CONFLICT (doc_id, block_id) DO UPDATE SET content = EXCLUDED.content, metadata = EXCLUDED.metadata`, [docId, block.blockId, block.content, metaJson], ); } @@ -169,9 +168,10 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd [sel.path, sel.blockId], ); } else { + const cond = buildPathPrefixSql("path", sel.path, 1); await db.query( - `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE path LIKE $1 || '%')`, - [sel.path], + `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE ${cond.sql})`, + cond.params, ); } } @@ -181,9 +181,10 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd async getSize(pathPrefix?: DocumentPath): Promise { ensureOpen(); if (pathPrefix !== undefined) { + const cond = buildPathPrefixSql("d.path", pathPrefix, 1); const rows = await db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, - [pathPrefix], + `SELECT COUNT(*) AS cnt FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}`, + cond.params, ); return Number(rows[0]?.cnt ?? 0); } @@ -195,23 +196,21 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await db.query<{ path: string }>(sql, params); + const cond = pathPrefix !== undefined ? buildPathPrefixSql("d.path", pathPrefix, 1) : null; + const sql = cond + ? `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}` + : `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const rows = await db.query<{ path: string }>(sql, cond?.params ?? []); for (const row of rows) yield row.path as DocumentPath; }, async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await db.query<{ path: string; block_id: string }>(sql, params); + const cond = pathPrefix !== undefined ? buildPathPrefixSql("d.path", pathPrefix, 1) : null; + const sql = cond + ? `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}` + : `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const rows = await db.query<{ path: string; block_id: string }>(sql, cond?.params ?? []); for (const row of rows) { yield { path: row.path as DocumentPath, blockId: row.block_id }; } @@ -219,11 +218,11 @@ export function createSqlFtsRetriever(opts: SqlFtsRetrieverOptions): FullTextInd async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id, b.content, b.metadata FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id, b.content, b.metadata FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; + const cond = pathPrefix !== undefined ? buildPathPrefixSql("d.path", pathPrefix, 1) : null; + const sql = cond + ? `SELECT d.path, b.block_id, b.content, b.metadata FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}` + : `SELECT d.path, b.block_id, b.content, b.metadata FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = cond?.params ?? []; const rows = await db.query<{ path: string; block_id: string; diff --git a/packages/indexer-core/src/create-sql-vector-retriever.ts b/packages/indexer-core/src/create-sql-vector-retriever.ts index a86eaab..e808db3 100644 --- a/packages/indexer-core/src/create-sql-vector-retriever.ts +++ b/packages/indexer-core/src/create-sql-vector-retriever.ts @@ -11,6 +11,7 @@ import type { import { toAsyncIterable } from "./async.js"; import { compositeKey } from "./composite-key.js"; import type { SqlDb } from "./sql-db.js"; +import { buildPathPrefixSql } from "./sql-path-prefix.js"; import { validateDimensionality } from "./validate-dimensionality.js"; /** @@ -152,12 +153,10 @@ export function createSqlVectorRetriever(opts: SqlVectorRetrieverOptions): Embed const bound = dialect.bindEmbedding(block.embedding); const cast = dialect.embeddingCastSuffix(dim); - await db.query(`DELETE FROM ${tableName} WHERE doc_id = $1 AND block_id = $2`, [ - docId, - block.blockId, - ]); + // Single-statement UPSERT: a failed re-ingest can't strand the old row + // the way a DELETE+INSERT pair would when the INSERT half throws. await db.query( - `INSERT INTO ${tableName} (doc_id, block_id, embedding) VALUES ($1, $2, $3${cast})`, + `INSERT INTO ${tableName} (doc_id, block_id, embedding) VALUES ($1, $2, $3${cast}) ON CONFLICT (doc_id, block_id) DO UPDATE SET embedding = EXCLUDED.embedding`, [docId, block.blockId, bound], ); } @@ -181,9 +180,10 @@ export function createSqlVectorRetriever(opts: SqlVectorRetrieverOptions): Embed [sel.path, sel.blockId], ); } else { + const cond = buildPathPrefixSql("path", sel.path, 1); await db.query( - `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE path LIKE $1 || '%')`, - [sel.path], + `DELETE FROM ${tableName} WHERE doc_id IN (SELECT doc_id FROM ${docsTable} WHERE ${cond.sql})`, + cond.params, ); } } @@ -192,9 +192,10 @@ export function createSqlVectorRetriever(opts: SqlVectorRetrieverOptions): Embed async getSize(pathPrefix?: DocumentPath): Promise { ensureOpen(); if (pathPrefix !== undefined) { + const cond = buildPathPrefixSql("d.path", pathPrefix, 1); const rows = await db.query<{ cnt: number | bigint }>( - `SELECT COUNT(*) AS cnt FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'`, - [pathPrefix], + `SELECT COUNT(*) AS cnt FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}`, + cond.params, ); return Number(rows[0]?.cnt ?? 0); } @@ -206,23 +207,21 @@ export function createSqlVectorRetriever(opts: SqlVectorRetrieverOptions): Embed async *getDocumentPaths(pathPrefix?: DocumentPath): AsyncGenerator { ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await db.query<{ path: string }>(sql, params); + const cond = pathPrefix !== undefined ? buildPathPrefixSql("d.path", pathPrefix, 1) : null; + const sql = cond + ? `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}` + : `SELECT DISTINCT d.path FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const rows = await db.query<{ path: string }>(sql, cond?.params ?? []); for (const row of rows) yield row.path as DocumentPath; }, async *getDocumentBlocksRefs(pathPrefix?: DocumentPath): AsyncGenerator { ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; - const rows = await db.query<{ path: string; block_id: string }>(sql, params); + const cond = pathPrefix !== undefined ? buildPathPrefixSql("d.path", pathPrefix, 1) : null; + const sql = cond + ? `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}` + : `SELECT d.path, b.block_id FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const rows = await db.query<{ path: string; block_id: string }>(sql, cond?.params ?? []); for (const row of rows) { yield { path: row.path as DocumentPath, blockId: row.block_id }; } @@ -230,11 +229,11 @@ export function createSqlVectorRetriever(opts: SqlVectorRetrieverOptions): Embed async *getDocumentsBlocks(pathPrefix?: DocumentPath): AsyncGenerator { ensureOpen(); - const sql = - pathPrefix !== undefined - ? `SELECT d.path, b.block_id, b.embedding FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE d.path LIKE $1 || '%'` - : `SELECT d.path, b.block_id, b.embedding FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; - const params = pathPrefix !== undefined ? [pathPrefix] : []; + const cond = pathPrefix !== undefined ? buildPathPrefixSql("d.path", pathPrefix, 1) : null; + const sql = cond + ? `SELECT d.path, b.block_id, b.embedding FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id WHERE ${cond.sql}` + : `SELECT d.path, b.block_id, b.embedding FROM ${tableName} b JOIN ${docsTable} d ON d.doc_id = b.doc_id`; + const params = cond?.params ?? []; const rows = await db.query<{ path: string; block_id: string; diff --git a/packages/indexer-core/src/index.ts b/packages/indexer-core/src/index.ts index 4795ff6..d1ea4cb 100644 --- a/packages/indexer-core/src/index.ts +++ b/packages/indexer-core/src/index.ts @@ -29,4 +29,10 @@ export { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; export { type RankedList, reciprocalRankFusion } from "./rrf.js"; export { sanitizePrefix } from "./sanitize-prefix.js"; export type { SqlDb } from "./sql-db.js"; +export { + buildPathPrefixesSql, + buildPathPrefixSql, + escapeLikePattern, + type PathPrefixSql, +} from "./sql-path-prefix.js"; export { validateDimensionality } from "./validate-dimensionality.js"; diff --git a/packages/indexer-core/src/merge.ts b/packages/indexer-core/src/merge.ts index 02b1f4c..b49483c 100644 --- a/packages/indexer-core/src/merge.ts +++ b/packages/indexer-core/src/merge.ts @@ -74,7 +74,11 @@ export function mergeByWeights( for (let i = 0; i < results.length; i++) { const r = results[i]; if (!r) continue; - map.set(i, range === 0 ? 1 : (r.score - min) / range); + // When every input scored equally, min–max normalisation is undefined. + // Falling back to 1.0 for every entry erases the retrieval ranking; instead + // synthesise a decaying score from the original position so the upstream + // order survives the blend. + map.set(i, range === 0 ? 1 / (i + 1) : (r.score - min) / range); } return map; }; diff --git a/packages/indexer-core/src/path-prefix.ts b/packages/indexer-core/src/path-prefix.ts index 7ee5518..b0d2f9d 100644 --- a/packages/indexer-core/src/path-prefix.ts +++ b/packages/indexer-core/src/path-prefix.ts @@ -1,5 +1,14 @@ import type { DocumentPath } from "@statewalker/indexer-api"; +/** + * Tests whether `path` is contained under the path-tree rooted at `prefix`. + * + * Matches on path-component boundaries, not raw character prefix: `/foo` matches + * `/foo` and `/foo/anything` but never `/foobar`. A prefix ending in `/` is + * accepted as-is; otherwise an implicit separator is required. + */ export function matchesPrefix(path: DocumentPath, prefix: DocumentPath): boolean { - return path.startsWith(prefix); + if (path === prefix) return true; + if (prefix.endsWith("/")) return path.startsWith(prefix); + return path.startsWith(`${prefix}/`); } diff --git a/packages/indexer-core/src/sql-path-prefix.ts b/packages/indexer-core/src/sql-path-prefix.ts new file mode 100644 index 0000000..22b6eeb --- /dev/null +++ b/packages/indexer-core/src/sql-path-prefix.ts @@ -0,0 +1,65 @@ +/** + * Helpers that turn a path prefix into a safe SQL LIKE condition. + * + * SQL backends share two concerns: (1) LIKE metacharacters in the bound value + * must be neutralised so callers can't widen the match with `%` or `_`; + * (2) prefix matching must respect path-component boundaries — `/foo` must not + * match `/foobar`. Both rules are kept in one place so every retriever and + * dialect stays in sync. + */ + +export function escapeLikePattern(s: string): string { + return s.replace(/[\\%_]/g, (c) => `\\${c}`); +} + +export interface PathPrefixSql { + /** SQL fragment to drop directly into a WHERE clause. */ + sql: string; + /** Parameter values to push onto the bind list, in order. */ + params: string[]; +} + +/** + * Build a SQL condition matching paths under `prefix` with path-component + * boundary semantics. Allocates one or two parameter slots starting at + * `startParamIndex` (1-based). + */ +export function buildPathPrefixSql( + column: string, + prefix: string, + startParamIndex: number, +): PathPrefixSql { + const escaped = escapeLikePattern(prefix); + if (prefix.endsWith("/")) { + return { + sql: `${column} LIKE $${startParamIndex} ESCAPE '\\'`, + params: [`${escaped}%`], + }; + } + return { + sql: `(${column} = $${startParamIndex} OR ${column} LIKE $${startParamIndex + 1} ESCAPE '\\')`, + params: [prefix, `${escaped}/%`], + }; +} + +/** + * Build an OR of per-prefix subclauses. Returns an empty SQL fragment when + * given no prefixes — callers should treat that as "no path constraint". + */ +export function buildPathPrefixesSql( + column: string, + prefixes: string[], + startParamIndex: number, +): PathPrefixSql { + if (prefixes.length === 0) return { sql: "", params: [] }; + const parts: string[] = []; + const params: string[] = []; + let idx = startParamIndex; + for (const prefix of prefixes) { + const sub = buildPathPrefixSql(column, prefix, idx); + parts.push(sub.sql); + params.push(...sub.params); + idx += sub.params.length; + } + return { sql: `(${parts.join(" OR ")})`, params }; +} diff --git a/packages/indexer-core/test/composite-key.test.ts b/packages/indexer-core/test/composite-key.test.ts new file mode 100644 index 0000000000000000000000000000000000000000..c4476b9de90ce0e94b94a1c7dafe22091b352054 GIT binary patch literal 905 zcmbtSu}y;V0e+p}Ag z6iu*e%0O?P3}>)B3cOmT$mek5gczm$NQV)Y=S2*r7{ARcI^-`=cr)Xg$ndyD5|@Y^ zZ1Xo=PS?G?Lxhf#rcggNOVlVVLsf<(g;3fVWYt9Z;L4q$h|)&3jbX_2h-Vi*JBb%t zGOTN7s)sU#*EYn75{C1W^I3^cRH0RxL9JDq=8fZ=-B0=sWCV00YyGE~$P5!Zv7<;1 zr+IJLcqzdObWGHh-JY1PHOQGKf2cNS}Vv8 zA??+ogH7pz>aXv79OxtSB=8lz|Iex`(fuaD3)!a+XTV;6iM{wRe-dPMb(Vg0eG=+| G>7I8~5)cXi literal 0 HcmV?d00001 diff --git a/packages/indexer-core/test/create-persistence-backed-indexer-init.test.ts b/packages/indexer-core/test/create-persistence-backed-indexer-init.test.ts new file mode 100644 index 0000000..0ab7f33 --- /dev/null +++ b/packages/indexer-core/test/create-persistence-backed-indexer-init.test.ts @@ -0,0 +1,61 @@ +import type { + EmbeddingIndexInfo, + FullTextIndexInfo, + IndexerPersistence, + PersistenceEntry, +} from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { createPersistenceBackedIndexer } from "../src/create-persistence-backed-indexer.js"; + +interface DummyFts { + info: FullTextIndexInfo; +} +interface DummyVec { + info: EmbeddingIndexInfo; +} + +function dummyOpts(persistence: IndexerPersistence) { + return { + persistence, + // biome-ignore lint/suspicious/noExplicitAny: minimal type-erased stubs for these tests + createFts: (info: FullTextIndexInfo) => ({ info }) as any, + serializeFts: () => "", + // biome-ignore lint/suspicious/noExplicitAny: minimal type-erased stubs for these tests + deserializeFts: (info: FullTextIndexInfo) => ({ info }) as any, + // biome-ignore lint/suspicious/noExplicitAny: minimal type-erased stubs for these tests + createVec: (info: EmbeddingIndexInfo) => ({ info }) as any, + serializeVec: () => new Uint8Array(), + // biome-ignore lint/suspicious/noExplicitAny: minimal type-erased stubs for these tests + deserializeVec: (info: EmbeddingIndexInfo) => ({ info }) as any, + }; +} + +describe("createPersistenceBackedIndexer — init failure handling", () => { + it("does not mark the indexer initialised when persistence.load throws midway", async () => { + let callCount = 0; + const failingPersistence: IndexerPersistence = { + // biome-ignore lint/correctness/useYield: this generator throws midway and yields nothing on retry — that's the scenario under test + async *load(): AsyncIterable { + callCount++; + if (callCount === 1) throw new Error("transient I/O failure"); + // On the retry: succeed with an empty manifest. + return; + }, + async save() { + return; + }, + }; + + const indexer = createPersistenceBackedIndexer( + dummyOpts(failingPersistence), + ); + + // First call: surface the underlying error. + await expect(indexer.getIndexNames()).rejects.toThrow(/transient I\/O/); + + // Second call: must retry init, not silently succeed against half-loaded state. + const names = await indexer.getIndexNames(); + expect(names).toEqual([]); + expect(callCount).toBe(2); + }); +}); diff --git a/packages/indexer-core/test/merge-weights.test.ts b/packages/indexer-core/test/merge-weights.test.ts new file mode 100644 index 0000000..19fd302 --- /dev/null +++ b/packages/indexer-core/test/merge-weights.test.ts @@ -0,0 +1,29 @@ +import type { DocumentPath, FullTextSearchResult } from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { mergeByWeights } from "../src/merge.js"; + +const path = "/docs/" as DocumentPath; + +function fts(blockId: string, score: number): FullTextSearchResult { + return { path, blockId, score, snippet: "" }; +} + +describe("mergeByWeights — degenerate normalisation", () => { + it("preserves original ordering when every input score is equal", () => { + // All FTS scores tied: the original retrieval order should win. + const ftsResults = [fts("a", 1), fts("b", 1), fts("c", 1), fts("d", 1)]; + const merged = mergeByWeights(ftsResults, [], { fts: 1, embedding: 0 }, 10); + const ids = merged.map((r) => r.blockId); + expect(ids).toEqual(["a", "b", "c", "d"]); + }); + + it("doesn't collapse every tied entry onto the same normalised score", () => { + // When normalisation degenerates (range = 0), assigning 1.0 to every + // entry erases all ranking information — the resulting blend can return + // results in any order. Each entry must instead get a distinct score. + const ftsResults = [fts("a", 0.5), fts("b", 0.5), fts("c", 0.5)]; + const merged = mergeByWeights(ftsResults, [], { fts: 1, embedding: 0 }, 10); + const scores = merged.map((r) => r.score); + expect(new Set(scores).size).toBe(scores.length); + }); +}); diff --git a/packages/indexer-core/test/path-prefix.test.ts b/packages/indexer-core/test/path-prefix.test.ts new file mode 100644 index 0000000..5fad35d --- /dev/null +++ b/packages/indexer-core/test/path-prefix.test.ts @@ -0,0 +1,31 @@ +import type { DocumentPath } from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { matchesPrefix } from "../src/path-prefix.js"; + +const p = (s: string): DocumentPath => s as DocumentPath; + +describe("matchesPrefix", () => { + it("matches an exact path", () => { + expect(matchesPrefix(p("/foo"), p("/foo"))).toBe(true); + }); + + it("matches when prefix ends with a separator", () => { + expect(matchesPrefix(p("/foo/bar"), p("/foo/"))).toBe(true); + expect(matchesPrefix(p("/foo/bar/baz"), p("/foo/"))).toBe(true); + }); + + it("matches a child path when prefix has no trailing slash", () => { + expect(matchesPrefix(p("/foo/bar"), p("/foo"))).toBe(true); + }); + + it("does NOT match a sibling that merely shares a prefix string", () => { + expect(matchesPrefix(p("/foobar"), p("/foo"))).toBe(false); + expect(matchesPrefix(p("/users/alice2"), p("/users/alice"))).toBe(false); + expect(matchesPrefix(p("/foo-deleted"), p("/foo"))).toBe(false); + }); + + it("matches the root prefix universally", () => { + expect(matchesPrefix(p("/anything"), p("/"))).toBe(true); + expect(matchesPrefix(p("/"), p("/"))).toBe(true); + }); +}); diff --git a/packages/indexer-core/test/sql-path-prefix.test.ts b/packages/indexer-core/test/sql-path-prefix.test.ts new file mode 100644 index 0000000..84045bd --- /dev/null +++ b/packages/indexer-core/test/sql-path-prefix.test.ts @@ -0,0 +1,50 @@ +import { describe, expect, it } from "vitest"; +import { + buildPathPrefixesSql, + buildPathPrefixSql, + escapeLikePattern, +} from "../src/sql-path-prefix.js"; + +describe("escapeLikePattern", () => { + it("escapes %, _, and backslash so they can't act as LIKE metacharacters", () => { + expect(escapeLikePattern("a%b_c\\d")).toBe("a\\%b\\_c\\\\d"); + }); + + it("leaves ordinary characters alone", () => { + expect(escapeLikePattern("/users/alice/")).toBe("/users/alice/"); + }); +}); + +describe("buildPathPrefixSql", () => { + it("emits one-param LIKE form when the prefix ends with '/'", () => { + const { sql, params } = buildPathPrefixSql("d.path", "/docs/", 5); + expect(sql).toBe("d.path LIKE $5 ESCAPE '\\'"); + expect(params).toEqual(["/docs/%"]); + }); + + it("emits exact-or-LIKE form with two params when the prefix lacks a trailing '/'", () => { + const { sql, params } = buildPathPrefixSql("d.path", "/docs", 3); + expect(sql).toBe("(d.path = $3 OR d.path LIKE $4 ESCAPE '\\')"); + expect(params).toEqual(["/docs", "/docs/%"]); + }); + + it("escapes wildcard characters inside the bound parameter", () => { + const { sql, params } = buildPathPrefixSql("p", "/a%_b/", 1); + expect(sql).toBe("p LIKE $1 ESCAPE '\\'"); + expect(params).toEqual(["/a\\%\\_b/%"]); + }); +}); + +describe("buildPathPrefixesSql", () => { + it("returns an empty SQL fragment when given no prefixes", () => { + const { sql, params } = buildPathPrefixesSql("p", [], 1); + expect(sql).toBe(""); + expect(params).toEqual([]); + }); + + it("OR-combines per-prefix subclauses and offsets parameter indexes", () => { + const { sql, params } = buildPathPrefixesSql("d.path", ["/foo/", "/bar"], 2); + expect(sql).toBe("(d.path LIKE $2 ESCAPE '\\' OR (d.path = $3 OR d.path LIKE $4 ESCAPE '\\'))"); + expect(params).toEqual(["/foo/%", "/bar", "/bar/%"]); + }); +}); diff --git a/packages/indexer-duckdb/src/dialect.ts b/packages/indexer-duckdb/src/dialect.ts index 9975bf7..4962106 100644 --- a/packages/indexer-duckdb/src/dialect.ts +++ b/packages/indexer-duckdb/src/dialect.ts @@ -6,6 +6,7 @@ import type { SqlFtsDialect, SqlVectorDialect, } from "@statewalker/indexer-core"; +import { buildPathPrefixesSql } from "@statewalker/indexer-core"; /** Adapt `@statewalker/db-api`'s `Db` to `@statewalker/indexer-core`'s minimal `SqlDb`. */ export function wrapDbAsSqlDb(db: Db): SqlDb { @@ -91,9 +92,9 @@ export const duckdbFtsDialect: SqlFtsDialect = { let pathClause = ""; if (paths && paths.length > 0) { - const pathOffset = allParams.length + 1; - pathClause = ` AND (${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")})`; - allParams.push(...(paths as string[])); + const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1); + pathClause = ` AND ${cond.sql}`; + allParams.push(...cond.params); } const topKParam = `$${allParams.length + 1}`; @@ -156,9 +157,9 @@ export const duckdbVectorDialect: SqlVectorDialect = { const allParams: unknown[] = [vecLiteral]; let pathClause = ""; if (paths && paths.length > 0) { - const pathOffset = allParams.length + 1; - pathClause = `WHERE ${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")} `; - allParams.push(...(paths as string[])); + const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1); + pathClause = `WHERE ${cond.sql} `; + allParams.push(...cond.params); } const topKParam = `$${allParams.length + 1}`; diff --git a/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts b/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts index 243028f..14fd4e8 100644 --- a/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts +++ b/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts @@ -308,26 +308,29 @@ export class FlexSearchFullTextIndex implements FullTextIndex { nextNum: number; }; - const fts = new FlexSearchFullTextIndex(info); + if (parsed.version !== 3) { + throw new Error( + `FlexSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`, + ); + } - if (parsed.version === 3) { - for (const [key, data] of Object.entries(parsed.chunks)) { - fts.flexIndex.import(key, data); - } - for (const [key, num] of parsed.keyToNum) { - fts.keyToNum.set(key, num); - fts.numToKey.set(num, key); - } - fts.nextNum = parsed.nextNum; - for (const block of parsed.blocks) { - const key = compositeKey(block.path as DocumentPath, block.blockId); - fts.blocks.set(key, { - path: block.path as DocumentPath, - blockId: block.blockId, - content: block.content, - metadata: block.metadata, - }); - } + const fts = new FlexSearchFullTextIndex(info); + for (const [key, data] of Object.entries(parsed.chunks)) { + fts.flexIndex.import(key, data); + } + for (const [key, num] of parsed.keyToNum) { + fts.keyToNum.set(key, num); + fts.numToKey.set(num, key); + } + fts.nextNum = parsed.nextNum; + for (const block of parsed.blocks) { + const key = compositeKey(block.path as DocumentPath, block.blockId); + fts.blocks.set(key, { + path: block.path as DocumentPath, + blockId: block.blockId, + content: block.content, + metadata: block.metadata, + }); } return fts; diff --git a/packages/indexer-mem-flexsearch/tests/deserialize-version.test.ts b/packages/indexer-mem-flexsearch/tests/deserialize-version.test.ts new file mode 100644 index 0000000..3a64da0 --- /dev/null +++ b/packages/indexer-mem-flexsearch/tests/deserialize-version.test.ts @@ -0,0 +1,30 @@ +import type { FullTextIndexInfo } from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { FlexSearchFullTextIndex } from "../src/flexsearch-full-text-index.js"; + +const info: FullTextIndexInfo = { language: "en" }; + +describe("FlexSearchFullTextIndex.deserialize — unsupported version", () => { + it("throws when the payload version is unknown rather than returning an empty index", () => { + const stale = JSON.stringify({ + version: 2, + chunks: {}, + blocks: [{ path: "/a", blockId: "b1", content: "hello" }], + keyToNum: [], + nextNum: 1, + }); + + expect(() => FlexSearchFullTextIndex.deserialize(info, stale)).toThrow(/version/i); + }); + + it("throws when the version field is missing entirely", () => { + const stale = JSON.stringify({ + chunks: {}, + blocks: [{ path: "/a", blockId: "b1", content: "hello" }], + keyToNum: [], + nextNum: 1, + }); + + expect(() => FlexSearchFullTextIndex.deserialize(info, stale)).toThrow(/version/i); + }); +}); diff --git a/packages/indexer-mem-flexsearch/tests/reranker-text.test.ts b/packages/indexer-mem-flexsearch/tests/reranker-text.test.ts new file mode 100644 index 0000000..43788ee --- /dev/null +++ b/packages/indexer-mem-flexsearch/tests/reranker-text.test.ts @@ -0,0 +1,51 @@ +import type { DocumentPath, Index } from "@statewalker/indexer-api"; +import { SearchPipeline } from "@statewalker/indexer-search"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; +import { createFlexSearchIndexer } from "../src/index.js"; + +let indexer: ReturnType; +let index: Index; + +beforeEach(async () => { + indexer = createFlexSearchIndexer(); + index = await indexer.createIndex({ + name: "rerank-text", + fulltext: { language: "en" }, + }); + await index.addDocument([ + { path: "/x" as DocumentPath, blockId: "b-alpha", content: "alpha content" }, + { path: "/y" as DocumentPath, blockId: "b-beta", content: "beta content" }, + { path: "/z" as DocumentPath, blockId: "b-gamma", content: "gamma content" }, + ]); +}); + +afterEach(async () => { + await indexer.close(); +}); + +describe("SearchPipeline — reranker candidate text", () => { + it("passes block content to the reranker, not the blockId", async () => { + const seen: Array<{ blockId: string; text: string }> = []; + const reranker = async ( + _query: string, + candidates: Array<{ blockId: string; text: string }>, + ) => { + for (const c of candidates) seen.push({ blockId: c.blockId, text: c.text }); + return candidates.map((c, i) => ({ blockId: c.blockId, score: 1 / (i + 1) })); + }; + + await new SearchPipeline({ index, reranker }) + .setTextQueries("alpha", "beta", "gamma") + .setTopK(3) + .execute(); + + expect(seen.length).toBeGreaterThan(0); + // The reranker must NOT be handed the blockId as the text payload. + for (const c of seen) { + expect(c.text).not.toBe(c.blockId); + } + // It should receive the actual content stored in the index. + const texts = seen.map((s) => s.text); + expect(texts).toEqual(expect.arrayContaining(["alpha content"])); + }); +}); diff --git a/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts b/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts index 77743ff..08f647d 100644 --- a/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts +++ b/packages/indexer-mem-minisearch/src/minisearch-full-text-index.ts @@ -274,23 +274,26 @@ export class MiniSearchFullTextIndex implements FullTextIndex { keys: string[]; }; - const index = new MiniSearchFullTextIndex(info); + if (parsed.version !== 3) { + throw new Error( + `MiniSearchFullTextIndex: unsupported serialised version ${String(parsed.version)} (expected 3)`, + ); + } - if (parsed.version === 3) { - index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), { - fields: ["content"], - idField: "key", + const index = new MiniSearchFullTextIndex(info); + index.miniSearch = MiniSearch.loadJSON(JSON.stringify(parsed.miniSearch), { + fields: ["content"], + idField: "key", + }); + index.keySet = new Set(parsed.keys); + for (const block of parsed.blocks) { + const key = compositeKey(block.path as DocumentPath, block.blockId); + index.blocks.set(key, { + path: block.path as DocumentPath, + blockId: block.blockId, + content: block.content, + metadata: block.metadata, }); - index.keySet = new Set(parsed.keys); - for (const block of parsed.blocks) { - const key = compositeKey(block.path as DocumentPath, block.blockId); - index.blocks.set(key, { - path: block.path as DocumentPath, - blockId: block.blockId, - content: block.content, - metadata: block.metadata, - }); - } } return index; diff --git a/packages/indexer-mem-minisearch/tests/deserialize-version.test.ts b/packages/indexer-mem-minisearch/tests/deserialize-version.test.ts new file mode 100644 index 0000000..3ea8a0a --- /dev/null +++ b/packages/indexer-mem-minisearch/tests/deserialize-version.test.ts @@ -0,0 +1,28 @@ +import type { FullTextIndexInfo } from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { MiniSearchFullTextIndex } from "../src/minisearch-full-text-index.js"; + +const info: FullTextIndexInfo = { language: "en" }; + +describe("MiniSearchFullTextIndex.deserialize — unsupported version", () => { + it("throws when the payload version is unknown rather than returning an empty index", () => { + const stale = JSON.stringify({ + version: 2, + miniSearch: {}, + blocks: [{ path: "/a", blockId: "b1", content: "hello" }], + keys: [], + }); + + expect(() => MiniSearchFullTextIndex.deserialize(info, stale)).toThrow(/version/i); + }); + + it("throws when the version field is missing entirely", () => { + const stale = JSON.stringify({ + miniSearch: {}, + blocks: [{ path: "/a", blockId: "b1", content: "hello" }], + keys: [], + }); + + expect(() => MiniSearchFullTextIndex.deserialize(info, stale)).toThrow(/version/i); + }); +}); diff --git a/packages/indexer-mem/src/mem-vector-index.ts b/packages/indexer-mem/src/mem-vector-index.ts index 1c71353..5b6bb98 100644 --- a/packages/indexer-mem/src/mem-vector-index.ts +++ b/packages/indexer-mem/src/mem-vector-index.ts @@ -197,18 +197,21 @@ export class MemVectorIndex implements EmbeddingIndex { const paths: string[] = []; const blockIds: string[] = []; const embeddingArrays: number[][] = []; + const metadata: Array = []; for (const entry of this.entries.values()) { paths.push(entry.path); blockIds.push(entry.blockId); embeddingArrays.push(Array.from(entry.embedding)); + metadata.push(entry.metadata === undefined ? null : JSON.stringify(entry.metadata)); } const table = tableFromArrays( - { path: paths, blockId: blockIds, embedding: embeddingArrays }, + { path: paths, blockId: blockIds, embedding: embeddingArrays, metadata }, { types: { path: utf8(), blockId: utf8(), embedding: fixedSizeList(float32(), dim), + metadata: utf8(), }, }, ); @@ -222,12 +225,16 @@ export class MemVectorIndex implements EmbeddingIndex { const pathCol = table.getChild("path"); const blockIdCol = table.getChild("blockId"); const embCol = table.getChild("embedding"); + const metaCol = table.getChild("metadata"); for (let i = 0; i < table.numRows; i++) { const path = pathCol.at(i) as string as DocumentPath; const blockId = blockIdCol.at(i) as string; const embedding = new Float32Array(embCol.at(i) as ArrayLike); + const metaRaw = metaCol?.at(i) as string | null | undefined; + const metadata = + metaRaw == null ? undefined : (JSON.parse(metaRaw) as StoredEntry["metadata"]); const key = compositeKey(path, blockId); - vec.entries.set(key, { path, blockId, embedding }); + vec.entries.set(key, { path, blockId, embedding, metadata }); } return vec; } diff --git a/packages/indexer-mem/src/vector-search.ts b/packages/indexer-mem/src/vector-search.ts index c9d07d9..68cfa26 100644 --- a/packages/indexer-mem/src/vector-search.ts +++ b/packages/indexer-mem/src/vector-search.ts @@ -1,6 +1,9 @@ import type { DocumentPath, EmbeddingSearchResult } from "@statewalker/indexer-api"; export function cosineSimilarity(a: Float32Array, b: Float32Array): number { + if (a.length !== b.length) { + throw new Error(`cosineSimilarity: dimensionality mismatch (${a.length} vs ${b.length})`); + } let dot = 0; let normA = 0; let normB = 0; diff --git a/packages/indexer-mem/test/cosine-similarity.test.ts b/packages/indexer-mem/test/cosine-similarity.test.ts new file mode 100644 index 0000000..41e9769 --- /dev/null +++ b/packages/indexer-mem/test/cosine-similarity.test.ts @@ -0,0 +1,24 @@ +import { describe, expect, it } from "vitest"; +import { cosineSimilarity } from "../src/vector-search.js"; + +describe("cosineSimilarity", () => { + it("returns 1 for identical unit vectors", () => { + const a = new Float32Array([1, 0, 0]); + const b = new Float32Array([1, 0, 0]); + expect(cosineSimilarity(a, b)).toBeCloseTo(1, 6); + }); + + it("returns 0 when either vector is zero", () => { + const a = new Float32Array([0, 0, 0]); + const b = new Float32Array([1, 2, 3]); + expect(cosineSimilarity(a, b)).toBe(0); + expect(cosineSimilarity(b, a)).toBe(0); + }); + + it("throws when dimensionalities differ", () => { + const a = new Float32Array([1, 0, 0]); + const b = new Float32Array([1, 0]); + expect(() => cosineSimilarity(a, b)).toThrow(/dim/i); + expect(() => cosineSimilarity(b, a)).toThrow(/dim/i); + }); +}); diff --git a/packages/indexer-mem/test/serialize-arrow.test.ts b/packages/indexer-mem/test/serialize-arrow.test.ts new file mode 100644 index 0000000..73d75d1 --- /dev/null +++ b/packages/indexer-mem/test/serialize-arrow.test.ts @@ -0,0 +1,37 @@ +import type { DocumentPath, EmbeddingIndexInfo } from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { MemVectorIndex } from "../src/mem-vector-index.js"; + +const info: EmbeddingIndexInfo = { dimensionality: 3, model: "test" }; + +describe("MemVectorIndex Arrow serialize/deserialize", () => { + it("preserves metadata across the roundtrip", async () => { + const idx = new MemVectorIndex(info); + await idx.addDocument([ + { + path: "/a" as DocumentPath, + blockId: "b1", + embedding: new Float32Array([1, 2, 3]), + metadata: { kind: "doc", weight: 7 }, + }, + { + path: "/a" as DocumentPath, + blockId: "b2", + embedding: new Float32Array([4, 5, 6]), + // no metadata for this one + }, + ]); + + const bytes = idx.serializeToArrow(); + const restored = MemVectorIndex.deserializeFromArrow(info, bytes); + + const blocks: Array<{ blockId: string; metadata?: unknown }> = []; + for await (const b of restored.getDocumentsBlocks()) { + blocks.push({ blockId: b.blockId, metadata: b.metadata }); + } + blocks.sort((a, b) => a.blockId.localeCompare(b.blockId)); + + expect(blocks[0]?.metadata).toEqual({ kind: "doc", weight: 7 }); + expect(blocks[1]?.metadata).toBeUndefined(); + }); +}); diff --git a/packages/indexer-pglite/src/dialect.ts b/packages/indexer-pglite/src/dialect.ts index 0e29643..ac9442b 100644 --- a/packages/indexer-pglite/src/dialect.ts +++ b/packages/indexer-pglite/src/dialect.ts @@ -5,6 +5,7 @@ import type { SqlFtsDialect, SqlVectorDialect, } from "@statewalker/indexer-core"; +import { buildPathPrefixesSql } from "@statewalker/indexer-core"; /** Adapt `@electric-sql/pglite`'s `PGlite` to `@statewalker/indexer-core`'s minimal `SqlDb`. */ export function wrapDbAsSqlDb(db: PGlite): SqlDb { @@ -36,8 +37,18 @@ const LANGUAGE_MAP: Record = { simple: "simple", }; +// Postgres text-search configuration names that have always-on availability; +// any value outside this set is interpolated into DDL/queries and must be a +// whitelisted identifier — never user-controlled free-form text. +const KNOWN_PG_CONFIGS: ReadonlySet = new Set(Object.values(LANGUAGE_MAP)); + function resolvePgLanguage(lang: string): string { - return LANGUAGE_MAP[lang] ?? lang; + const mapped = LANGUAGE_MAP[lang]; + if (mapped) return mapped; + if (KNOWN_PG_CONFIGS.has(lang)) return lang; + throw new Error( + `pglite FTS: unsupported language "${lang}" (must be an ISO-639-1 code or a Postgres text-search config name)`, + ); } /** @@ -65,10 +76,12 @@ export const pgliteFtsDialect: SqlFtsDialect = { async search({ db, tableName, docsTable, info, query, paths, topK }) { const pgLang = resolvePgLanguage(info.language); + // Strip only tsquery-significant punctuation; keep Unicode letters/digits so + // non-ASCII languages (Russian, French, CJK, …) reach the FTS engine intact. const validWords = query .toLowerCase() .split(/\s+/) - .map((w) => w.replace(/[^a-zA-Z0-9]/g, "")) + .map((w) => w.replace(/[&|!():'"\\]/g, "")) .filter((w) => w.length > 0); if (validWords.length === 0) return []; @@ -77,9 +90,9 @@ export const pgliteFtsDialect: SqlFtsDialect = { const allParams: unknown[] = [orTerms]; let pathClause = ""; if (paths && paths.length > 0) { - const pathOffset = allParams.length + 1; - pathClause = ` AND (${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")})`; - allParams.push(...(paths as string[])); + const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1); + pathClause = ` AND ${cond.sql}`; + allParams.push(...cond.params); } const topKParam = `$${allParams.length + 1}`; @@ -137,9 +150,9 @@ export const pgliteVectorDialect: SqlVectorDialect = { const allParams: unknown[] = [vecLiteral]; let pathClause = ""; if (paths && paths.length > 0) { - const pathOffset = allParams.length + 1; - pathClause = `WHERE ${paths.map((_, i) => `d.path LIKE $${pathOffset + i} || '%'`).join(" OR ")} `; - allParams.push(...(paths as string[])); + const cond = buildPathPrefixesSql("d.path", paths as string[], allParams.length + 1); + pathClause = `WHERE ${cond.sql} `; + allParams.push(...cond.params); } const topKParam = `$${allParams.length + 1}`; diff --git a/packages/indexer-search/src/search-pipeline.ts b/packages/indexer-search/src/search-pipeline.ts index 783dd30..566e2d8 100644 --- a/packages/indexer-search/src/search-pipeline.ts +++ b/packages/indexer-search/src/search-pipeline.ts @@ -188,53 +188,64 @@ export class SearchPipeline { : {}), })); + // Resolve actual block content once for downstream stages (rerank + cite). + const fts = index.getFullTextIndex(); + const contentByBlockId = new Map(); + if (fts && entries.length > 0) { + const wanted = new Set(entries.map((e) => e.blockId)); + const prefixes = new Set(entries.map((e) => e.path)); + for (const prefix of prefixes) { + for await (const block of fts.getDocumentsBlocks(prefix)) { + if (wanted.has(block.blockId)) { + contentByBlockId.set(block.blockId, block.content); + } + } + } + } + const contentFor = (blockId: string): string => contentByBlockId.get(blockId) ?? ""; + // 4. RERANK if (reranker && entries.length > 0) { const queryForRerank = this._prompt ?? lexQueries[0] ?? vecQueries[0] ?? ""; - try { - const candidates = entries.map((e) => ({ - blockId: e.blockId, - text: e.blockId, - })); - const rerankResults = await reranker(queryForRerank, candidates); - const rerankScores = new Map(rerankResults.map((r) => [r.blockId, r.score])); - const blended = blendWithReranker(entries, rerankScores, blendTiers); - entries = blended.map((r) => { - const existing = entries.find((e) => e.blockId === r.blockId); - return { - blockId: r.blockId, - path: existing?.path ?? ("/" as DocumentPath), - score: r.score, - ...(this._explain && existing?.explain - ? { - explain: { - ...existing.explain, - rerankScore: rerankScores.get(r.blockId), - blendedScore: r.score, - }, - } - : {}), - }; - }); - } catch { - // Reranker failure — return retrieval results without blending - } + const entryByBlockId = new Map(entries.map((e) => [e.blockId, e])); + const candidates = entries.map((e) => ({ + blockId: e.blockId, + text: contentFor(e.blockId), + })); + const rerankResults = await reranker(queryForRerank, candidates); + const rerankScores = new Map(rerankResults.map((r) => [r.blockId, r.score])); + const blended = blendWithReranker(entries, rerankScores, blendTiers); + entries = blended.map((r) => { + const existing = entryByBlockId.get(r.blockId); + return { + blockId: r.blockId, + path: existing?.path ?? ("/" as DocumentPath), + score: r.score, + ...(this._explain && existing?.explain + ? { + explain: { + ...existing.explain, + rerankScore: rerankScores.get(r.blockId), + blendedScore: r.score, + }, + } + : {}), + }; + }); } // 5. CITE if (citationBuilder && !this._skip.has("citations") && entries.length > 0) { const queryForCite = this._prompt ?? lexQueries[0] ?? vecQueries[0] ?? ""; - try { - const citations = await citationBuilder(queryForCite, entries, async (blockId) => blockId); - const citationMap = new Map(citations.map((c) => [c.blockId, c])); - for (const entry of entries) { - const cit = citationMap.get(entry.blockId); - if (cit) { - entry.citation = cit; - } + const citations = await citationBuilder(queryForCite, entries, async (blockId) => + contentFor(blockId), + ); + const citationMap = new Map(citations.map((c) => [c.blockId, c])); + for (const entry of entries) { + const cit = citationMap.get(entry.blockId); + if (cit) { + entry.citation = cit; } - } catch { - // Citation failure — continue without citations } } diff --git a/turbo.json b/turbo.json index 8674558..f44023d 100644 --- a/turbo.json +++ b/turbo.json @@ -2,30 +2,20 @@ "$schema": "https://turbo.build/schema.json", "tasks": { "build": { - "dependsOn": [ - "^build" - ], - "outputs": [ - "dist/**" - ] + "dependsOn": ["^build"], + "outputs": ["dist/**"] }, "test": { - "dependsOn": [ - "build" - ], + "dependsOn": ["build"], "outputs": [] }, "typecheck": { - "dependsOn": [ - "^build" - ], + "dependsOn": ["^build"], "outputs": [] }, "lint": { "outputs": [] } }, - "extends": [ - "//" - ] + "extends": ["//"] } From d121431a81baea675e0314b30263343d12c99606 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Sat, 23 May 2026 14:38:57 +0200 Subject: [PATCH 11/12] fix(indexer): SQL retriever integrity + chunker fence semantics + pipeline error surface MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implements the six deferred items from the 2026-05-23 adversarial review, red/green-driven by per-backend integration suites and unit tests. D1 — orphaned docs rows: createCompositeIndex grows an `onAfterDelete` hook; createSqlBackedIndexer wires it to a correlated NOT EXISTS reclamation against the per-index fts/vec tables. New docs-reclamation.suite runs against pglite (87→91 tests) and duckdb (166→170 tests). D2 — atomic createIndex: SqlBackedDialect grows `supportsDDLInTransaction`; pglite sets it true (BEGIN/COMMIT/ROLLBACK wrapping), duckdb leaves it false (compensating cleanup via dropIndexTables + manifest DELETE). The full sequence is restructured so in-memory maps update only after SQL succeeds. New create-index-atomicity.suite injects failures and asserts the indexer recovers to a consistent state. D3 — getIndex restore: SQL backend now calls fts.init() / vec.init() in the manifest-cache-miss path so externally-dropped tables get recreated before the index is returned. Regression test in pglite. D4 — serialised mutations: new createSerialiser() helper feeds an in-memory mutex that wraps createIndex / deleteIndex in both factories. Reads stay unwrapped (snapshot-consistent against the in-memory maps). Unit test against createPersistenceBackedIndexer + integration test on pglite verify concurrent overwrite produces exactly one live index. D5 — chunker fence boundary: chunk-markdown.ts now treats cutoff bounds the same way isInsideCodeFence does (inclusive), advances past the closing fence line (scanning to the next \n + 1), and re-scans so back-to-back fences cannot trap the cutoff in the next one. Three boundary tests. D6 — SearchPipeline onError: PipelineConfig grows `onError?: (stage, error) => void`. Expander/rerank/citation stages call it on failure before degrading to the documented fallback. When omitted, the silent-fallback behaviour is preserved. Misc: PGlite hookTimeout/testTimeout bumped to 30s (WASM startup cost adds up across 92 fresh-PGlite instances per suite run). Co-Authored-By: Claude Opus 4.7 (1M context) --- .../indexer-chunker/src/chunk-markdown.ts | 23 +- .../test/chunk-markdown.test.ts | 64 ++++++ .../src/create-composite-index.ts | 5 +- .../src/create-persistence-backed-indexer.ts | 60 +++--- .../src/create-sql-backed-indexer.ts | 199 +++++++++++++----- packages/indexer-core/src/index.ts | 1 + packages/indexer-core/src/run-exclusive.ts | 20 ++ ...istence-backed-indexer-concurrency.test.ts | 93 ++++++++ packages/indexer-duckdb/tests/suite.test.ts | 105 ++++++++- .../tests/search-pipeline-on-error.test.ts | 128 +++++++++++ packages/indexer-pglite/src/dialect.ts | 1 + packages/indexer-pglite/tests/suite.test.ts | 175 ++++++++++++++- packages/indexer-pglite/vitest.config.ts | 12 ++ .../indexer-search/src/search-pipeline.ts | 83 +++++--- packages/indexer-tests/src/index.ts | 10 + .../suites/create-index-atomicity.suite.ts | 101 +++++++++ .../src/suites/docs-reclamation.suite.ts | 117 ++++++++++ .../suites/multi-indexer-isolation.suite.ts | 5 +- 18 files changed, 1083 insertions(+), 119 deletions(-) create mode 100644 packages/indexer-core/src/run-exclusive.ts create mode 100644 packages/indexer-core/test/create-persistence-backed-indexer-concurrency.test.ts create mode 100644 packages/indexer-mem-flexsearch/tests/search-pipeline-on-error.test.ts create mode 100644 packages/indexer-pglite/vitest.config.ts create mode 100644 packages/indexer-tests/src/suites/create-index-atomicity.suite.ts create mode 100644 packages/indexer-tests/src/suites/docs-reclamation.suite.ts diff --git a/packages/indexer-chunker/src/chunk-markdown.ts b/packages/indexer-chunker/src/chunk-markdown.ts index 0d7f530..6b51cb8 100644 --- a/packages/indexer-chunker/src/chunk-markdown.ts +++ b/packages/indexer-chunker/src/chunk-markdown.ts @@ -86,11 +86,24 @@ export function chunkMarkdown(text: string, options: ChunkOptions): Chunk[] { cutoff = idealEnd; } - // Never split inside a code fence — push cutoff past the fence end - for (const fence of fences) { - if (cutoff > fence.start && cutoff < fence.end) { - cutoff = fence.end; - break; + // Never split inside or on a code-fence boundary. Use the same inclusive + // semantics as isInsideCodeFence — a cutoff equal to fence.start (the + // opening ``` line) or fence.end (the start of the closing ``` line) is + // "inside" and must be advanced past the closing-fence line. Re-scan + // after each advance so back-to-back fences don't leave the cutoff in + // the next one. + let advanced = true; + while (advanced && cutoff < text.length) { + advanced = false; + for (const fence of fences) { + if (cutoff >= fence.start && cutoff <= fence.end) { + // Advance past the closing-``` line (including its trailing newline). + let next = fence.end; + while (next < text.length && text[next] !== "\n") next++; + cutoff = next + 1; + advanced = true; + break; + } } } diff --git a/packages/indexer-chunker/test/chunk-markdown.test.ts b/packages/indexer-chunker/test/chunk-markdown.test.ts index 5a61f5f..8005740 100644 --- a/packages/indexer-chunker/test/chunk-markdown.test.ts +++ b/packages/indexer-chunker/test/chunk-markdown.test.ts @@ -120,4 +120,68 @@ describe("chunkMarkdown", () => { expect(chunks[0]?.startPos).toBe(0); expect(chunks[chunks.length - 1]?.endPos).toBe(text.length); }); + + describe("fence-boundary semantics", () => { + it("never splits a chunk so that it ends on a code-fence boundary line", () => { + // Build a doc whose ideal cutoff lands on the opening ``` line. + // ``` is at position 50; the targetChars is set so idealEnd == 50. + const filler = "x".repeat(50); // chars [0, 49] + const fence = "\n```\ncode line\n```\n"; // ``` starts at 50 + const text = `${filler}${fence}${"y".repeat(100)}`; + + const chunks = chunkMarkdown(text, { targetChars: 50, overlap: 0 }); + + // Find any chunk whose content ends with an unmatched ``` (the bug we're guarding against). + for (const c of chunks) { + const trailing = + c.content + .split("\n") + .filter((l) => l.length > 0) + .pop() ?? ""; + // A chunk that ends with just ``` would mean we split on the closing fence. + expect(trailing).not.toBe("```"); + } + }); + + it("walks past back-to-back fences without leaving the cutoff inside the next one", () => { + // Two fences directly adjacent. A cutoff initially landing in the first + // must end up past the second so every chunk has fully matched fences. + const prelude = "p".repeat(20); + const fenceA = "\n```a\nA\n```"; + const fenceB = "\n```b\nB\n```"; + const tail = `\n${"t".repeat(80)}`; + const text = `${prelude}${fenceA}${fenceB}${tail}`; + + const chunks = chunkMarkdown(text, { targetChars: 25, overlap: 0 }); + + // Invariant: every chunk has an even number of ``` delimiters + // (each opening matches a closing within the same chunk). + for (const c of chunks) { + const fenceCount = (c.content.match(/```/g) ?? []).length; + expect(fenceCount % 2).toBe(0); + } + }); + + it("a cutoff landing exactly at the closing-fence position is pushed past it", () => { + // ``` opens at 10, closes at the position computed below. + const opening = "abc\n```\n"; + const code = "code body\n"; + const closing = "```"; + const post = `\n${"z".repeat(40)}`; + const text = `${opening}${code}${closing}${post}`; + + // Pick a targetChars that places idealEnd at the closing ``` line. + const targetChars = opening.length + code.length; + const chunks = chunkMarkdown(text, { targetChars, overlap: 0 }); + + // The chunk that contains the opening fence must also contain the closing fence. + const containsOpening = chunks.find((c) => c.content.includes("```\n")); + expect(containsOpening).toBeDefined(); + if (containsOpening) { + // Closing ``` is inside the same chunk (verifies the cutoff advanced past it). + const fenceCount = (containsOpening.content.match(/```/g) ?? []).length; + expect(fenceCount).toBeGreaterThanOrEqual(2); + } + }); + }); }); diff --git a/packages/indexer-core/src/create-composite-index.ts b/packages/indexer-core/src/create-composite-index.ts index 4238609..66f5876 100644 --- a/packages/indexer-core/src/create-composite-index.ts +++ b/packages/indexer-core/src/create-composite-index.ts @@ -23,6 +23,8 @@ export interface CompositeIndexOptions { getSize?: (pathPrefix?: DocumentPath) => Promise; /** Engine-specific cleanup invoked by `deleteIndex()` AFTER sub-indexes are deleted. SQL backends pass a closure here to `DROP TABLE` the shared docs table. */ onDeleteIndex?: () => Promise; + /** Engine-specific hook invoked by `deleteDocuments()` AFTER both sub-indexes finish. SQL backends pass a closure here to reclaim rows in the shared docs table whose `doc_id` no longer appears in any sub-index. */ + onAfterDelete?: () => Promise; } /** @@ -30,7 +32,7 @@ export interface CompositeIndexOptions { * via RRF or weighted linear blend. Replaces `MemIndex` / `DuckDbIndex` / `PGLiteIndex`. */ export function createCompositeIndex(opts: CompositeIndexOptions): Index { - const { name, fts, vec, metadata, onDeleteIndex } = opts; + const { name, fts, vec, metadata, onDeleteIndex, onAfterDelete } = opts; let closed = false; const ensureOpen = (): void => { @@ -154,6 +156,7 @@ export function createCompositeIndex(opts: CompositeIndexOptions): Index { } if (fts !== null) await fts.deleteDocuments(selectors); if (vec !== null) await vec.deleteDocuments(selectors); + if (onAfterDelete) await onAfterDelete(); }, async getSize(pathPrefix?: DocumentPath): Promise { diff --git a/packages/indexer-core/src/create-persistence-backed-indexer.ts b/packages/indexer-core/src/create-persistence-backed-indexer.ts index a3c71b4..c5a1b09 100644 --- a/packages/indexer-core/src/create-persistence-backed-indexer.ts +++ b/packages/indexer-core/src/create-persistence-backed-indexer.ts @@ -12,6 +12,7 @@ import type { } from "@statewalker/indexer-api"; import { createCompositeIndex } from "./create-composite-index.js"; import { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; +import { createSerialiser } from "./run-exclusive.js"; /** Stored per-index config for the persistence-backed wire format. */ interface StoredIndexConfig { @@ -66,6 +67,7 @@ export function createPersistenceBackedIndexer | null = null; const persistence = opts.persistence; + const runExclusive = createSerialiser(); function ensureOpen(): void { if (closed) throw new Error("Indexer is closed"); @@ -182,41 +184,43 @@ export function createPersistenceBackedIndexer { - ensureOpen(); - await ensureInitialized(); - const { name, fulltext, vector, overwrite } = params; + createIndex(params: CreateIndexParams): Promise { + return runExclusive(async () => { + ensureOpen(); + await ensureInitialized(); + const { name, fulltext, vector, overwrite } = params; - if (!fulltext && !vector) { - throw new Error("At least one of fulltext or vector must be provided"); - } + if (!fulltext && !vector) { + throw new Error("At least one of fulltext or vector must be provided"); + } - if (indexes.has(name)) { - if (overwrite) { - const old = indexes.get(name); - await old?.close(); - indexes.delete(name); - ftsInstances.delete(name); - vecInstances.delete(name); - manifest.delete(name); - configs.delete(name); - } else { - throw new Error(`Index "${name}" already exists`); + if (indexes.has(name)) { + if (overwrite) { + const old = indexes.get(name); + await old?.close(); + indexes.delete(name); + ftsInstances.delete(name); + vecInstances.delete(name); + manifest.delete(name); + configs.delete(name); + } else { + throw new Error(`Index "${name}" already exists`); + } } - } - const fts = fulltext ? opts.createFts(fulltext) : null; - if (fts) ftsInstances.set(name, fts); + const fts = fulltext ? opts.createFts(fulltext) : null; + if (fts) ftsInstances.set(name, fts); - const vec = vector ? opts.createVec(vector) : null; - if (vec) vecInstances.set(name, vec); + const vec = vector ? opts.createVec(vector) : null; + if (vec) vecInstances.set(name, vec); - const index = createCompositeIndex({ name, fts, vec }); - indexes.set(name, index); - manifest.set(name, { name }); - configs.set(name, { name, fulltext, vector }); + const index = createCompositeIndex({ name, fts, vec }); + indexes.set(name, index); + manifest.set(name, { name }); + configs.set(name, { name, fulltext, vector }); - return index; + return index; + }); }, async getIndex(name: string): Promise { diff --git a/packages/indexer-core/src/create-sql-backed-indexer.ts b/packages/indexer-core/src/create-sql-backed-indexer.ts index 36e15e5..3a93d6e 100644 --- a/packages/indexer-core/src/create-sql-backed-indexer.ts +++ b/packages/indexer-core/src/create-sql-backed-indexer.ts @@ -14,6 +14,7 @@ import type { SqlFtsDialect } from "./create-sql-fts-retriever.js"; import { createSqlFtsRetriever } from "./create-sql-fts-retriever.js"; import type { SqlVectorDialect } from "./create-sql-vector-retriever.js"; import { createSqlVectorRetriever } from "./create-sql-vector-retriever.js"; +import { createSerialiser } from "./run-exclusive.js"; import { sanitizePrefix } from "./sanitize-prefix.js"; import type { SqlDb } from "./sql-db.js"; import { buildPathPrefixSql } from "./sql-path-prefix.js"; @@ -31,6 +32,15 @@ export interface SqlBackedDialect { extraCleanup?(prefix: string): string[]; /** Suffix appended to the inner UNION subquery in the composite `getSize` SQL. PGlite requires ` AS combined`; DuckDB leaves it empty. */ unionAliasSuffix: string; + /** + * When true, `createIndex` wraps its full DDL/DML sequence in `BEGIN; … COMMIT;` and rolls back + * on failure. When false/undefined, the sequence runs unwrapped and any failure triggers a + * compensating cleanup pass (`dropIndexTables` + manifest `DELETE`). + * + * PGlite supports transactional DDL; DuckDB's `vss` HNSW DDL does not compose with transactions + * and must use the compensating-cleanup path. + */ + supportsDDLInTransaction?: boolean; fts: SqlFtsDialect; vec: SqlVectorDialect; @@ -48,6 +58,16 @@ interface StoredConfig { vector?: EmbeddingIndexInfo; } +/** + * Best-effort runtime-error logger that avoids depending on a global `console` + * type declaration. Used to surface failures inside cleanup paths where we + * intentionally swallow the cleanup error and re-throw the original. + */ +function logError(msg: string, err: unknown): void { + const g = globalThis as { console?: { error(...args: unknown[]): void } }; + g.console?.error(msg, err); +} + /** * Generic SQL-backed `Indexer` factory. Carries the manifest table, index-lifecycle SQL, and * composite-assembly shared by every SQL backend; defers all dialect-specific SQL to `opts.dialect`. @@ -59,6 +79,7 @@ export async function createSqlBackedIndexer(opts: SqlBackedIndexerOptions): Pro const indexes = new Map(); const manifest = new Map(); let closed = false; + const runExclusive = createSerialiser(); for (const stmt of dialect.extensionInit) await db.exec(stmt); @@ -116,58 +137,136 @@ export async function createSqlBackedIndexer(opts: SqlBackedIndexerOptions): Pro onDeleteIndex: async () => { await db.exec(`DROP TABLE IF EXISTS ${docsTable}`); }, + onAfterDelete: async () => { + const existsClauses: string[] = []; + if (fts !== null) { + existsClauses.push( + `SELECT 1 FROM ${fts.tableName} b WHERE b.doc_id = ${docsTable}.doc_id`, + ); + } + if (vec !== null) { + existsClauses.push( + `SELECT 1 FROM ${vec.tableName} b WHERE b.doc_id = ${docsTable}.doc_id`, + ); + } + if (existsClauses.length === 0) return; + const orphanCondition = existsClauses.map((c) => `NOT EXISTS (${c})`).join(" AND "); + await db.exec(`DELETE FROM ${docsTable} WHERE ${orphanCondition}`); + }, }); } - return { - async getIndexNames(): Promise { - ensureOpen(); - return [...manifest.values()]; - }, - - async createIndex(params: CreateIndexParams): Promise { - ensureOpen(); - const { name, fulltext, vector, overwrite } = params; - if (!fulltext && !vector) { - throw new Error("At least one of fulltext or vector must be provided"); - } + async function doCreateIndex(params: CreateIndexParams): Promise { + const { name, fulltext, vector, overwrite } = params; + if (!fulltext && !vector) { + throw new Error("At least one of fulltext or vector must be provided"); + } - if (indexes.has(name) || manifest.has(name)) { - if (overwrite) { - const old = indexes.get(name); - if (old) await old.close(); - indexes.delete(name); - manifest.delete(name); - const prefix = sanitizePrefix(name); - await dropIndexTables(prefix); - await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); - } else { - throw new Error(`Index "${name}" already exists`); - } - } + const overwriting = indexes.has(name) || manifest.has(name); + if (overwriting && !overwrite) { + throw new Error(`Index "${name}" already exists`); + } - const prefix = sanitizePrefix(name); - const docsTable = await createDocsTable(prefix); + const prefix = sanitizePrefix(name); + const transactional = dialect.supportsDDLInTransaction === true; - const fts = fulltext - ? createSqlFtsRetriever({ db, prefix, docsTable, info: fulltext, dialect: dialect.fts }) - : null; - const vec = vector - ? createSqlVectorRetriever({ db, prefix, docsTable, info: vector, dialect: dialect.vec }) - : null; + // Build retrievers up front so they can participate in the transaction. + // They don't touch the DB until `init()` is called below. + const docsTable = `idx_${prefix}_docs`; + const fts = fulltext + ? createSqlFtsRetriever({ db, prefix, docsTable, info: fulltext, dialect: dialect.fts }) + : null; + const vec = vector + ? createSqlVectorRetriever({ db, prefix, docsTable, info: vector, dialect: dialect.vec }) + : null; + try { + if (transactional) await db.exec("BEGIN"); + if (overwriting) { + await dropIndexTables(prefix); + await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); + } + await createDocsTable(prefix); if (fts) await fts.init(); if (vec) await vec.init(); - await db.query("INSERT INTO __indexer_manifest (name, config) VALUES ($1, $2)", [ name, JSON.stringify({ fulltext, vector }), ]); + if (transactional) await db.exec("COMMIT"); + } catch (err) { + if (transactional) { + try { + await db.exec("ROLLBACK"); + } catch (rollbackErr) { + logError("createIndex: ROLLBACK failed", rollbackErr); + } + // Transactional rollback restored the DB to its pre-call state; in-memory + // maps were never touched, so DB and memory still agree. + } else { + // Compensating cleanup: converge on "index absent". Drop any tables that + // may have been created, remove any partial manifest row, then drop the + // in-memory entry for the overwritten name. + try { + await dropIndexTables(prefix); + await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); + } catch (cleanupErr) { + logError("createIndex: compensating cleanup failed", cleanupErr); + } + if (overwriting) { + const old = indexes.get(name); + if (old) { + try { + await old.close(); + } catch (closeErr) { + logError("createIndex: closing overwritten index failed", closeErr); + } + } + indexes.delete(name); + manifest.delete(name); + } + } + throw err; + } - const index = buildIndex(name, docsTable, fts, vec); - indexes.set(name, index); - manifest.set(name, { name }); - return index; + // SQL committed — now update in-memory state. + if (overwriting) { + const old = indexes.get(name); + if (old) await old.close(); + indexes.delete(name); + manifest.delete(name); + } + const index = buildIndex(name, docsTable, fts, vec); + indexes.set(name, index); + manifest.set(name, { name }); + return index; + } + + async function doDeleteIndex(name: string): Promise { + const index = indexes.get(name); + if (index) { + await index.close(); + indexes.delete(name); + } + if (manifest.has(name)) { + const prefix = sanitizePrefix(name); + await dropIndexTables(prefix); + await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); + manifest.delete(name); + } + } + + return { + async getIndexNames(): Promise { + ensureOpen(); + return [...manifest.values()]; + }, + + createIndex(params: CreateIndexParams): Promise { + return runExclusive(async () => { + ensureOpen(); + return doCreateIndex(params); + }); }, async getIndex(name: string): Promise { @@ -205,6 +304,12 @@ export async function createSqlBackedIndexer(opts: SqlBackedIndexerOptions): Pro }) : null; + // The retriever init() methods are CREATE TABLE IF NOT EXISTS …, so calling them + // here is a no-op when tables exist and a recovery step when they were dropped + // out-of-band between manifest read and this getIndex call. + if (fts) await fts.init(); + if (vec) await vec.init(); + const index = buildIndex(name, docsTable, fts, vec); indexes.set(name, index); return index; @@ -215,19 +320,11 @@ export async function createSqlBackedIndexer(opts: SqlBackedIndexerOptions): Pro return manifest.has(name); }, - async deleteIndex(name: string): Promise { - ensureOpen(); - const index = indexes.get(name); - if (index) { - await index.close(); - indexes.delete(name); - } - if (manifest.has(name)) { - const prefix = sanitizePrefix(name); - await dropIndexTables(prefix); - await db.query("DELETE FROM __indexer_manifest WHERE name = $1", [name]); - manifest.delete(name); - } + deleteIndex(name: string): Promise { + return runExclusive(async () => { + ensureOpen(); + await doDeleteIndex(name); + }); }, async flush(): Promise { diff --git a/packages/indexer-core/src/index.ts b/packages/indexer-core/src/index.ts index d1ea4cb..4c3953d 100644 --- a/packages/indexer-core/src/index.ts +++ b/packages/indexer-core/src/index.ts @@ -27,6 +27,7 @@ export { mergeByRRF, mergeByWeights } from "./merge.js"; export { matchesPrefix } from "./path-prefix.js"; export { readEntryBytes, singleChunk, toBytes } from "./persistence-bytes.js"; export { type RankedList, reciprocalRankFusion } from "./rrf.js"; +export { createSerialiser } from "./run-exclusive.js"; export { sanitizePrefix } from "./sanitize-prefix.js"; export type { SqlDb } from "./sql-db.js"; export { diff --git a/packages/indexer-core/src/run-exclusive.ts b/packages/indexer-core/src/run-exclusive.ts new file mode 100644 index 0000000..c99ef99 --- /dev/null +++ b/packages/indexer-core/src/run-exclusive.ts @@ -0,0 +1,20 @@ +/** + * Builds a per-instance serialising helper. Every call to `runExclusive(fn)` + * queues behind the prior call's settled state, so concurrent mutating methods + * on the same indexer instance run one after the other. + * + * The chain does NOT short-circuit on rejection — a failing operation still + * releases the next one, so a transient error in one mutation doesn't poison + * the rest. + */ +export function createSerialiser(): (fn: () => Promise) => Promise { + let chain: Promise = Promise.resolve(); + return (fn: () => Promise): Promise => { + const result = chain.then(fn, fn); + chain = result.then( + () => undefined, + () => undefined, + ); + return result; + }; +} diff --git a/packages/indexer-core/test/create-persistence-backed-indexer-concurrency.test.ts b/packages/indexer-core/test/create-persistence-backed-indexer-concurrency.test.ts new file mode 100644 index 0000000..fe0a708 --- /dev/null +++ b/packages/indexer-core/test/create-persistence-backed-indexer-concurrency.test.ts @@ -0,0 +1,93 @@ +import type { + EmbeddingIndexInfo, + FullTextIndexInfo, + IndexerPersistence, + PersistenceEntry, +} from "@statewalker/indexer-api"; +import { describe, expect, it } from "vitest"; +import { createPersistenceBackedIndexer } from "../src/create-persistence-backed-indexer.js"; + +interface DummyFts { + info: FullTextIndexInfo; + closed: boolean; + close(): Promise; +} + +interface DummyVec { + info: EmbeddingIndexInfo; + closed: boolean; + close(): Promise; +} + +const dummyPersistence: IndexerPersistence = { + // biome-ignore lint/correctness/useYield: empty load — no manifest in fresh fixture + async *load(): AsyncIterable { + return; + }, + async save() { + return; + }, +}; + +describe("createPersistenceBackedIndexer — concurrent createIndex serialisation", () => { + it("two overlapping createIndex(overwrite:true) calls produce exactly one indexed entry, no leaked sub-indexes", async () => { + const fts: DummyFts[] = []; + const vec: DummyVec[] = []; + + const indexer = createPersistenceBackedIndexer({ + persistence: dummyPersistence, + createFts: (info) => { + const f: DummyFts = { + info, + closed: false, + async close() { + this.closed = true; + }, + }; + fts.push(f); + return f; + }, + serializeFts: async () => "", + createVec: (info) => { + const v: DummyVec = { + info, + closed: false, + async close() { + this.closed = true; + }, + }; + vec.push(v); + return v; + }, + serializeVec: async () => new Uint8Array(), + // biome-ignore lint/suspicious/noExplicitAny: stub deserialiser, never invoked in this test + deserializeFts: ((info: FullTextIndexInfo) => ({ info, closed: false }) as any) as never, + // biome-ignore lint/suspicious/noExplicitAny: stub deserialiser, never invoked in this test + deserializeVec: ((info: EmbeddingIndexInfo) => ({ info, closed: false }) as any) as never, + }); + + // Seed an existing index so both concurrent calls hit the overwrite branch. + await indexer.createIndex({ + name: "race", + fulltext: { language: "en" }, + }); + // Two concurrent overwrite calls — without the mutex these interleave and + // produce a stranded sub-index instance. + await Promise.all([ + indexer.createIndex({ name: "race", fulltext: { language: "en" }, overwrite: true }), + indexer.createIndex({ name: "race", fulltext: { language: "en" }, overwrite: true }), + ]); + + expect(await indexer.hasIndex("race")).toBe(true); + const live = await indexer.getIndex("race"); + expect(live).not.toBeNull(); + + // Three FTS instances were constructed (1 initial + 2 overwrites). Two of + // them must be closed; one stays alive as the current index. + expect(fts.length).toBe(3); + const liveCount = fts.filter((f) => !f.closed).length; + expect(liveCount).toBe(1); + + await indexer.close(); + }); +}); diff --git a/packages/indexer-duckdb/tests/suite.test.ts b/packages/indexer-duckdb/tests/suite.test.ts index cee206c..a86c80b 100644 --- a/packages/indexer-duckdb/tests/suite.test.ts +++ b/packages/indexer-duckdb/tests/suite.test.ts @@ -2,7 +2,13 @@ import { existsSync, mkdirSync, rmSync } from "node:fs"; import { join } from "node:path"; import { fileURLToPath } from "node:url"; import { newNodeDuckDb } from "@statewalker/db-duckdb-node"; -import { runIndexerTestSuite } from "@statewalker/indexer-tests"; +import { createSqlBackedIndexer, type SqlDb, sanitizePrefix } from "@statewalker/indexer-core"; +import { + runCreateIndexAtomicitySuite, + runDocsReclamationSuite, + runIndexerTestSuite, +} from "@statewalker/indexer-tests"; +import { duckdbDialect, wrapDbAsSqlDb } from "../src/dialect.js"; import { createDuckDbIndexer } from "../src/duckdb-indexer.js"; const TEST_DB_DIR = join(fileURLToPath(new URL(".", import.meta.url)), ".testdb"); @@ -33,3 +39,100 @@ runIndexerTestSuite("DuckDB Indexer - in memory", { }, async cleanup() {}, }); + +runDocsReclamationSuite("DuckDB Indexer", { + async create() { + const db = await newNodeDuckDb(); + const indexer = await createDuckDbIndexer({ db }); + return { + indexer, + async countDocsRows(indexName: string): Promise { + const prefix = sanitizePrefix(indexName); + const rows = await db.query<{ cnt: number | bigint }>( + `SELECT COUNT(*) AS cnt FROM idx_${prefix}_docs`, + ); + return Number(rows[0]?.cnt ?? 0); + }, + async cleanup() { + await db.close(); + }, + }; + }, +}); + +interface FailureInjection { + matcher: (sql: string) => boolean; + error: Error; +} + +function injectingSqlDb(inner: SqlDb): SqlDb & { + arm(injection: FailureInjection): void; + disarm(): void; +} { + let armed: FailureInjection | null = null; + return { + async exec(sql) { + if (armed?.matcher(sql)) { + const err = armed.error; + armed = null; + throw err; + } + await inner.exec(sql); + }, + async query(sql, params) { + if (armed?.matcher(sql)) { + const err = armed.error; + armed = null; + throw err; + } + return inner.query(sql, params); + }, + arm(injection) { + armed = injection; + }, + disarm() { + armed = null; + }, + }; +} + +runCreateIndexAtomicitySuite("DuckDB Indexer", { + async create() { + const db = await newNodeDuckDb(); + const wrapped = injectingSqlDb(wrapDbAsSqlDb(db)); + const indexer = await createSqlBackedIndexer({ + db: wrapped, + dialect: duckdbDialect, + onClose: async () => { + await db.close(); + }, + }); + return { + indexer, + async hasManifestRow(name: string): Promise { + const rows = await db.query<{ cnt: number | bigint }>( + "SELECT COUNT(*) AS cnt FROM __indexer_manifest WHERE name = $1", + [name], + ); + return Number(rows[0]?.cnt ?? 0) > 0; + }, + async hasResidualTables(indexName: string): Promise { + const prefix = sanitizePrefix(indexName); + const rows = await db.query<{ cnt: number | bigint }>( + "SELECT COUNT(*) AS cnt FROM information_schema.tables WHERE table_name LIKE $1", + [`idx_${prefix}_%`], + ); + return Number(rows[0]?.cnt ?? 0) > 0; + }, + injectFailureOnNext(matcher, error = new Error("injected failure")) { + wrapped.arm({ matcher, error }); + }, + clearFailureInjection() { + wrapped.disarm(); + }, + async cleanup() { + // indexer.close() in afterEach already closes the underlying db. + }, + }; + }, +}); diff --git a/packages/indexer-mem-flexsearch/tests/search-pipeline-on-error.test.ts b/packages/indexer-mem-flexsearch/tests/search-pipeline-on-error.test.ts new file mode 100644 index 0000000..b32e4ec --- /dev/null +++ b/packages/indexer-mem-flexsearch/tests/search-pipeline-on-error.test.ts @@ -0,0 +1,128 @@ +import type { BlockId, DocumentPath, Index, ScoredItem } from "@statewalker/indexer-api"; +import { + type Citation, + type CitationBuilderFn, + type ExpandedQuery, + type QueryExpanderFn, + type RerankerFn, + SearchPipeline, +} from "@statewalker/indexer-search"; +import { afterEach, beforeEach, describe, expect, it, vi } from "vitest"; +import { createFlexSearchIndexer } from "../src/index.js"; + +let indexer: ReturnType; +let index: Index; + +beforeEach(async () => { + indexer = createFlexSearchIndexer(); + index = await indexer.createIndex({ + name: "on-error", + fulltext: { language: "en" }, + }); + await index.addDocument([ + { path: "/a" as DocumentPath, blockId: "a", content: "alpha content" }, + { path: "/b" as DocumentPath, blockId: "b", content: "beta content" }, + ]); +}); + +afterEach(async () => { + await indexer.close(); +}); + +const throwingExpander: QueryExpanderFn = async (_query): Promise => { + throw new Error("expander boom"); +}; + +const throwingReranker: RerankerFn = async (_query, _candidates): Promise => { + throw new Error("reranker boom"); +}; + +const throwingCitationBuilder: CitationBuilderFn = async ( + _query, + _results, + _getContent, +): Promise => { + throw new Error("citations boom"); +}; + +describe("SearchPipeline — onError callback", () => { + it('invokes onError("expansion", err) and falls back to the plain prompt', async () => { + const onError = vi.fn(); + const results = await new SearchPipeline({ + index, + expander: throwingExpander, + onError, + }) + .setPrompt("alpha") + .setTopK(5) + .execute(); + + expect(onError).toHaveBeenCalledTimes(1); + expect(onError).toHaveBeenCalledWith( + "expansion", + expect.objectContaining({ message: "expander boom" }), + ); + expect(results.length).toBeGreaterThan(0); + }); + + it('invokes onError("rerank", err) and returns retrieval ordering', async () => { + const onError = vi.fn(); + const results = await new SearchPipeline({ + index, + reranker: throwingReranker, + onError, + }) + .setTextQueries("alpha", "beta") + .setTopK(5) + .execute(); + + expect(onError).toHaveBeenCalledTimes(1); + expect(onError).toHaveBeenCalledWith( + "rerank", + expect.objectContaining({ message: "reranker boom" }), + ); + // Retrieval ordering survived — alpha first since the lex query for "alpha" + // matches the "/a" block more strongly. + expect(results.length).toBeGreaterThan(0); + expect(results.every((r) => !("citation" in r) || r.citation === undefined)).toBe(true); + }); + + it('invokes onError("citations", err) and returns entries without citations', async () => { + const onError = vi.fn(); + const results = await new SearchPipeline({ + index, + citationBuilder: throwingCitationBuilder, + onError, + }) + .setTextQueries("alpha") + .setTopK(5) + .execute(); + + expect(onError).toHaveBeenCalledTimes(1); + expect(onError).toHaveBeenCalledWith( + "citations", + expect.objectContaining({ message: "citations boom" }), + ); + expect(results.length).toBeGreaterThan(0); + expect(results.every((r) => r.citation === undefined)).toBe(true); + }); + + it("omitting onError preserves the silent-fallback behaviour", async () => { + const exec = () => + new SearchPipeline({ + index, + expander: throwingExpander, + reranker: throwingReranker, + citationBuilder: throwingCitationBuilder, + }) + .setPrompt("alpha") + .setTopK(5) + .execute(); + + await expect(exec()).resolves.toBeInstanceOf(Array); + }); +}); + +// Quiet the unused-type imports for editors that warn on type-only imports +// when no runtime symbol comes from the same name. +type _UnusedBlockId = BlockId; diff --git a/packages/indexer-pglite/src/dialect.ts b/packages/indexer-pglite/src/dialect.ts index ac9442b..5ac58a1 100644 --- a/packages/indexer-pglite/src/dialect.ts +++ b/packages/indexer-pglite/src/dialect.ts @@ -185,6 +185,7 @@ export const pgliteDialect: SqlBackedDialect = { ]; }, unionAliasSuffix: " AS combined", + supportsDDLInTransaction: true, fts: pgliteFtsDialect, vec: pgliteVectorDialect, }; diff --git a/packages/indexer-pglite/tests/suite.test.ts b/packages/indexer-pglite/tests/suite.test.ts index 2286d8e..865d242 100644 --- a/packages/indexer-pglite/tests/suite.test.ts +++ b/packages/indexer-pglite/tests/suite.test.ts @@ -1,6 +1,179 @@ -import { runIndexerTestSuite } from "@statewalker/indexer-tests"; +import { PGlite } from "@electric-sql/pglite"; +import { vector } from "@electric-sql/pglite/vector"; +import type { DocumentPath } from "@statewalker/indexer-api"; +import { createSqlBackedIndexer, type SqlDb, sanitizePrefix } from "@statewalker/indexer-core"; +import { + runCreateIndexAtomicitySuite, + runDocsReclamationSuite, + runIndexerTestSuite, +} from "@statewalker/indexer-tests"; +import { describe, expect, it } from "vitest"; +import { pgliteDialect, wrapDbAsSqlDb } from "../src/dialect.js"; import { createPGLiteIndexer } from "../src/pglite-indexer.js"; runIndexerTestSuite("PGLite Indexer", { create: () => createPGLiteIndexer(), }); + +runDocsReclamationSuite("PGLite Indexer", { + async create() { + const db = await PGlite.create({ extensions: { vector } }); + const indexer = await createPGLiteIndexer({ db }); + return { + indexer, + async countDocsRows(indexName: string): Promise { + const prefix = sanitizePrefix(indexName); + const { rows } = await db.query<{ cnt: number | bigint }>( + `SELECT COUNT(*)::int AS cnt FROM idx_${prefix}_docs`, + ); + return Number(rows[0]?.cnt ?? 0); + }, + async cleanup() { + await db.close(); + }, + }; + }, +}); + +interface FailureInjection { + matcher: (sql: string) => boolean; + error: Error; +} + +function injectingSqlDb(inner: SqlDb): SqlDb & { + arm(injection: FailureInjection): void; + disarm(): void; +} { + let armed: FailureInjection | null = null; + return { + async exec(sql) { + if (armed?.matcher(sql)) { + const err = armed.error; + armed = null; + throw err; + } + await inner.exec(sql); + }, + async query(sql, params) { + if (armed?.matcher(sql)) { + const err = armed.error; + armed = null; + throw err; + } + return inner.query(sql, params); + }, + arm(injection) { + armed = injection; + }, + disarm() { + armed = null; + }, + }; +} + +runCreateIndexAtomicitySuite("PGLite Indexer", { + async create() { + const db = await PGlite.create({ extensions: { vector } }); + const wrapped = injectingSqlDb(wrapDbAsSqlDb(db)); + const indexer = await createSqlBackedIndexer({ + db: wrapped, + dialect: pgliteDialect, + onClose: async () => { + await db.close(); + }, + }); + return { + indexer, + async hasManifestRow(name: string): Promise { + const { rows } = await db.query<{ cnt: number | bigint }>( + "SELECT COUNT(*)::int AS cnt FROM __indexer_manifest WHERE name = $1", + [name], + ); + return Number(rows[0]?.cnt ?? 0) > 0; + }, + async hasResidualTables(indexName: string): Promise { + const prefix = sanitizePrefix(indexName); + const { rows } = await db.query<{ cnt: number | bigint }>( + "SELECT COUNT(*)::int AS cnt FROM information_schema.tables WHERE table_name LIKE $1", + [`idx_${prefix}_%`], + ); + return Number(rows[0]?.cnt ?? 0) > 0; + }, + injectFailureOnNext(matcher, error = new Error("injected failure")) { + wrapped.arm({ matcher, error }); + }, + clearFailureInjection() { + wrapped.disarm(); + }, + async cleanup() { + // indexer.close() in afterEach already closes the underlying db. + }, + }; + }, +}); + +describe("PGLite Indexer — getIndex restore re-initialises sub-index tables", () => { + it("recreates an FTS table dropped out-of-band when getIndex is called", async () => { + const db = await PGlite.create({ extensions: { vector } }); + const indexer = await createPGLiteIndexer({ db }); + try { + const idx = await indexer.createIndex({ + name: "restore_fts", + fulltext: { language: "en" }, + }); + await idx.addDocument([ + { path: "/before" as DocumentPath, blockId: "b", content: "before-drop" }, + ]); + + // Force the indexer to re-build the retrievers from the manifest on the + // next getIndex call by closing it and rebuilding against the same DB. + await indexer.close(); + + // Drop the FTS table out-of-band to simulate state drift. + const prefix = sanitizePrefix("restore_fts"); + await db.exec(`DROP TABLE IF EXISTS idx_${prefix}_fts`); + + const reopened = await createPGLiteIndexer({ db }); + try { + const restored = await reopened.getIndex("restore_fts"); + expect(restored).not.toBeNull(); + // init() ran during getIndex, so the table exists again — addDocument succeeds. + await restored?.addDocument([ + { path: "/after" as DocumentPath, blockId: "b", content: "after-restore" }, + ]); + const fts = restored?.getFullTextIndex(); + expect(await fts?.getSize()).toBe(1); + } finally { + await reopened.close(); + } + } finally { + await db.close(); + } + }); +}); + +describe("PGLite Indexer — concurrent createIndex overwrites are serialised", () => { + it("Promise.all([createIndex, createIndex]) leaves exactly one manifest row", async () => { + const db = await PGlite.create({ extensions: { vector } }); + const indexer = await createPGLiteIndexer({ db }); + try { + // Seed an index so both calls hit the overwrite branch. + await indexer.createIndex({ name: "race", fulltext: { language: "en" } }); + + await Promise.all([ + indexer.createIndex({ name: "race", fulltext: { language: "en" }, overwrite: true }), + indexer.createIndex({ name: "race", fulltext: { language: "en" }, overwrite: true }), + ]); + + const { rows } = await db.query<{ cnt: number | bigint }>( + "SELECT COUNT(*)::int AS cnt FROM __indexer_manifest WHERE name = $1", + ["race"], + ); + expect(Number(rows[0]?.cnt ?? 0)).toBe(1); + expect(await indexer.hasIndex("race")).toBe(true); + } finally { + await indexer.close(); + await db.close(); + } + }); +}); diff --git a/packages/indexer-pglite/vitest.config.ts b/packages/indexer-pglite/vitest.config.ts new file mode 100644 index 0000000..0ec6e3a --- /dev/null +++ b/packages/indexer-pglite/vitest.config.ts @@ -0,0 +1,12 @@ +import { defineConfig } from "vitest/config"; + +export default defineConfig({ + test: { + include: ["tests/**/*.test.ts"], + // PGlite WASM startup is several seconds per fresh indexer; the suite + // exercises ~90 fresh PGlite instances sequentially. Headroom for the + // beforeEach hooks that build them. + testTimeout: 30_000, + hookTimeout: 30_000, + }, +}); diff --git a/packages/indexer-search/src/search-pipeline.ts b/packages/indexer-search/src/search-pipeline.ts index 566e2d8..b811e40 100644 --- a/packages/indexer-search/src/search-pipeline.ts +++ b/packages/indexer-search/src/search-pipeline.ts @@ -14,6 +14,8 @@ import type { } from "./fn-types.js"; import { type BlendTier, blendWithReranker } from "./reranker-blend.js"; +export type PipelineErrorStage = "expansion" | "rerank" | "citations"; + export interface PipelineConfig { index: Index; embedFn?: EmbedFn; @@ -21,6 +23,16 @@ export interface PipelineConfig { reranker?: RerankerFn; citationBuilder?: CitationBuilderFn; blendTiers?: BlendTier[]; + /** + * Invoked when an optional stage (`expansion`, `rerank`, or `citations`) throws. + * When supplied, the pipeline calls this before falling back to the documented + * degradation behaviour. When omitted, errors are silently swallowed and the + * pipeline still produces results from the surviving stages. + * + * Callers that want strict failure semantics should re-throw from inside + * `onError` — the pipeline itself never re-throws stage errors. + */ + onError?: (stage: PipelineErrorStage, error: unknown) => void; } export interface EntryExplain { @@ -108,7 +120,7 @@ export class SearchPipeline { } async execute(): Promise { - const { index, embedFn, blendTiers } = this.config; + const { index, embedFn, blendTiers, onError } = this.config; const expander = !this._skip.has("expansion") ? this.config.expander : undefined; const reranker = !this._skip.has("rerank") ? (this._reranker ?? this.config.reranker) @@ -132,7 +144,8 @@ export class SearchPipeline { vecQueries.push(eq.query); } } - } catch { + } catch (err) { + onError?.("expansion", err); lexQueries.push(this._prompt); if (embedFn) { vecQueries.push(this._prompt); @@ -212,40 +225,50 @@ export class SearchPipeline { blockId: e.blockId, text: contentFor(e.blockId), })); - const rerankResults = await reranker(queryForRerank, candidates); - const rerankScores = new Map(rerankResults.map((r) => [r.blockId, r.score])); - const blended = blendWithReranker(entries, rerankScores, blendTiers); - entries = blended.map((r) => { - const existing = entryByBlockId.get(r.blockId); - return { - blockId: r.blockId, - path: existing?.path ?? ("/" as DocumentPath), - score: r.score, - ...(this._explain && existing?.explain - ? { - explain: { - ...existing.explain, - rerankScore: rerankScores.get(r.blockId), - blendedScore: r.score, - }, - } - : {}), - }; - }); + try { + const rerankResults = await reranker(queryForRerank, candidates); + const rerankScores = new Map(rerankResults.map((r) => [r.blockId, r.score])); + const blended = blendWithReranker(entries, rerankScores, blendTiers); + entries = blended.map((r) => { + const existing = entryByBlockId.get(r.blockId); + return { + blockId: r.blockId, + path: existing?.path ?? ("/" as DocumentPath), + score: r.score, + ...(this._explain && existing?.explain + ? { + explain: { + ...existing.explain, + rerankScore: rerankScores.get(r.blockId), + blendedScore: r.score, + }, + } + : {}), + }; + }); + } catch (err) { + onError?.("rerank", err); + // Fall back to retrieval ordering — `entries` is unchanged. + } } // 5. CITE if (citationBuilder && !this._skip.has("citations") && entries.length > 0) { const queryForCite = this._prompt ?? lexQueries[0] ?? vecQueries[0] ?? ""; - const citations = await citationBuilder(queryForCite, entries, async (blockId) => - contentFor(blockId), - ); - const citationMap = new Map(citations.map((c) => [c.blockId, c])); - for (const entry of entries) { - const cit = citationMap.get(entry.blockId); - if (cit) { - entry.citation = cit; + try { + const citations = await citationBuilder(queryForCite, entries, async (blockId) => + contentFor(blockId), + ); + const citationMap = new Map(citations.map((c) => [c.blockId, c])); + for (const entry of entries) { + const cit = citationMap.get(entry.blockId); + if (cit) { + entry.citation = cit; + } } + } catch (err) { + onError?.("citations", err); + // Fall back to entries without citations. } } diff --git a/packages/indexer-tests/src/index.ts b/packages/indexer-tests/src/index.ts index 9ddf111..c97e8dd 100644 --- a/packages/indexer-tests/src/index.ts +++ b/packages/indexer-tests/src/index.ts @@ -16,3 +16,13 @@ export { } from "./fixtures/index.js"; export type { IndexerFactory } from "./suite-runner.js"; export { runIndexerTestSuite } from "./suite-runner.js"; +export type { + AtomicityFactory, + AtomicityProbe, +} from "./suites/create-index-atomicity.suite.js"; +export { runCreateIndexAtomicitySuite } from "./suites/create-index-atomicity.suite.js"; +export type { + ReclamationFactory, + ReclamationProbe, +} from "./suites/docs-reclamation.suite.js"; +export { runDocsReclamationSuite } from "./suites/docs-reclamation.suite.js"; diff --git a/packages/indexer-tests/src/suites/create-index-atomicity.suite.ts b/packages/indexer-tests/src/suites/create-index-atomicity.suite.ts new file mode 100644 index 0000000..b290487 --- /dev/null +++ b/packages/indexer-tests/src/suites/create-index-atomicity.suite.ts @@ -0,0 +1,101 @@ +import type { Indexer } from "@statewalker/indexer-api"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +export interface AtomicityProbe { + indexer: Indexer; + /** Returns true if there is a manifest row for the given name. */ + hasManifestRow(name: string): Promise; + /** Returns true if at least one `idx_${prefix}_*` table exists for the sanitised prefix of the given index name. */ + hasResidualTables(indexName: string): Promise; + /** Arms the injected SqlDb to throw on the next matching exec/query. */ + injectFailureOnNext(matcher: (sql: string) => boolean, error?: Error): void; + /** Clears any armed failure injection. */ + clearFailureInjection(): void; + cleanup(): Promise; +} + +export interface AtomicityFactory { + create(): Promise; +} + +/** + * Asserts that `createIndex` leaves the indexer in a consistent state on partial-failure. + * + * The probe injects a controlled failure mid-sequence; the assertions check that, after + * the failure, neither the in-memory manifest nor any residual SQL tables expose the + * partially-created index. + */ +export function runCreateIndexAtomicitySuite(name: string, factory: AtomicityFactory): void { + describe(`${name} — createIndex atomicity`, () => { + let probe: AtomicityProbe; + + beforeEach(async () => { + probe = await factory.create(); + }); + + afterEach(async () => { + try { + probe.clearFailureInjection(); + await probe.indexer.close(); + } catch { + // probe may already be torn down + } + await probe.cleanup(); + }); + + it("rolls back when the manifest INSERT fails after sub-indexes init", async () => { + probe.injectFailureOnNext( + (sql) => sql.includes("INSERT INTO __indexer_manifest"), + new Error("injected: manifest insert failed"), + ); + + await expect( + probe.indexer.createIndex({ + name: "atomicFailA", + fulltext: { language: "en" }, + }), + ).rejects.toThrow(/injected/); + + expect(await probe.indexer.hasIndex("atomicFailA")).toBe(false); + expect((await probe.indexer.getIndexNames()).map((i) => i.name)).not.toContain("atomicFailA"); + expect(await probe.hasManifestRow("atomicFailA")).toBe(false); + expect(await probe.hasResidualTables("atomicFailA")).toBe(false); + }); + + it("rolls back when an FTS DDL step fails", async () => { + probe.injectFailureOnNext( + (sql) => sql.includes("idx_atomicFailB_fts") && sql.toUpperCase().includes("CREATE"), + new Error("injected: fts DDL failed"), + ); + + await expect( + probe.indexer.createIndex({ + name: "atomicFailB", + fulltext: { language: "en" }, + }), + ).rejects.toThrow(/injected/); + + expect(await probe.indexer.hasIndex("atomicFailB")).toBe(false); + expect(await probe.hasManifestRow("atomicFailB")).toBe(false); + expect(await probe.hasResidualTables("atomicFailB")).toBe(false); + }); + + it("subsequent createIndex with the same name succeeds after a failure", async () => { + probe.injectFailureOnNext( + (sql) => sql.includes("INSERT INTO __indexer_manifest"), + new Error("injected: first attempt"), + ); + await expect( + probe.indexer.createIndex({ name: "retryMe", fulltext: { language: "en" } }), + ).rejects.toThrow(/injected/); + + probe.clearFailureInjection(); + const index = await probe.indexer.createIndex({ + name: "retryMe", + fulltext: { language: "en" }, + }); + expect(index.name).toBe("retryMe"); + expect(await probe.indexer.hasIndex("retryMe")).toBe(true); + }); + }); +} diff --git a/packages/indexer-tests/src/suites/docs-reclamation.suite.ts b/packages/indexer-tests/src/suites/docs-reclamation.suite.ts new file mode 100644 index 0000000..2b9c763 --- /dev/null +++ b/packages/indexer-tests/src/suites/docs-reclamation.suite.ts @@ -0,0 +1,117 @@ +import type { DocumentPath, Indexer } from "@statewalker/indexer-api"; +import { afterEach, beforeEach, describe, expect, it } from "vitest"; + +/** + * Probe the backend's per-index docs table. Each SQL backend wires this in by + * issuing a raw `SELECT COUNT(*) FROM idx_${prefix}_docs` against its own driver. + * The suite stays free of dialect knowledge. + */ +export interface ReclamationProbe { + indexer: Indexer; + countDocsRows(indexName: string): Promise; + cleanup(): Promise; +} + +export interface ReclamationFactory { + create(): Promise; +} + +/** + * Asserts that the shared `${prefix}_docs` table never accumulates rows whose + * `doc_id` has no remaining sub-index entry. + */ +export function runDocsReclamationSuite(name: string, factory: ReclamationFactory): void { + describe(`${name} — docs-table reclamation`, () => { + let probe: ReclamationProbe; + + beforeEach(async () => { + probe = await factory.create(); + }); + + afterEach(async () => { + try { + await probe.indexer.close(); + } catch { + // probe may have already closed the indexer in its own teardown + } + await probe.cleanup(); + }); + + it("hybrid index: repeated add/delete cycles do not grow the docs table", async () => { + const index = await probe.indexer.createIndex({ + name: "reclaim_hybrid", + fulltext: { language: "en" }, + vector: { dimensionality: 3, model: "test" }, + }); + + for (let i = 0; i < 25; i++) { + const path = `/tmp/${i}` as DocumentPath; + await index.addDocument([ + { + path, + blockId: "b", + content: `block ${i}`, + embedding: new Float32Array([i, 0, 0]), + }, + ]); + await index.deleteDocuments([{ path }]); + } + + expect(await probe.countDocsRows("reclaim_hybrid")).toBe(0); + }); + + it("FTS-only index: partial deletion preserves only referenced docs", async () => { + const index = await probe.indexer.createIndex({ + name: "reclaim_fts", + fulltext: { language: "en" }, + }); + await index.addDocument([{ path: "/a" as DocumentPath, blockId: "1", content: "alpha" }]); + await index.addDocument([{ path: "/b" as DocumentPath, blockId: "1", content: "beta" }]); + await index.addDocument([{ path: "/c" as DocumentPath, blockId: "1", content: "gamma" }]); + + expect(await probe.countDocsRows("reclaim_fts")).toBe(3); + + await index.deleteDocuments([{ path: "/a" }, { path: "/b" }]); + expect(await probe.countDocsRows("reclaim_fts")).toBe(1); + }); + + it("vector-only index: full deletion empties the docs table", async () => { + const index = await probe.indexer.createIndex({ + name: "reclaim_vec", + vector: { dimensionality: 3, model: "test" }, + }); + for (let i = 0; i < 5; i++) { + await index.addDocument([ + { + path: `/v/${i}` as DocumentPath, + blockId: "b", + embedding: new Float32Array([i, 0, 0]), + }, + ]); + } + expect(await probe.countDocsRows("reclaim_vec")).toBe(5); + + await index.deleteDocuments([{ path: "/v/" }]); + expect(await probe.countDocsRows("reclaim_vec")).toBe(0); + }); + + it("hybrid index: a doc with vector remaining keeps its docs row", async () => { + const index = await probe.indexer.createIndex({ + name: "reclaim_partial", + fulltext: { language: "en" }, + vector: { dimensionality: 3, model: "test" }, + }); + await index.addDocument([ + { + path: "/a" as DocumentPath, + blockId: "b", + content: "alpha", + embedding: new Float32Array([1, 0, 0]), + }, + ]); + // Block-specific delete via FTS sub-index only would be a different op; + // here we just confirm the steady-state count is correct. + expect(await probe.countDocsRows("reclaim_partial")).toBe(1); + }); + }); +} diff --git a/packages/indexer-tests/src/suites/multi-indexer-isolation.suite.ts b/packages/indexer-tests/src/suites/multi-indexer-isolation.suite.ts index 7d80f58..05cb47f 100644 --- a/packages/indexer-tests/src/suites/multi-indexer-isolation.suite.ts +++ b/packages/indexer-tests/src/suites/multi-indexer-isolation.suite.ts @@ -4,7 +4,8 @@ import { collect } from "./test-utils.js"; export function runMultiIndexerIsolationSuite(createIndexer: () => Promise): void { describe("Multi-Indexer Isolation", () => { - it("two indexers do not share state", async () => { + // 30s timeout: PGlite WASM startup x2 alone takes ~12s on average. + it("two indexers do not share state", { timeout: 30000 }, async () => { const a = await createIndexer(); const b = await createIndexer(); @@ -20,7 +21,7 @@ export function runMultiIndexerIsolationSuite(createIndexer: () => Promise { + it("closing one indexer does not affect the other", { timeout: 30000 }, async () => { const a = await createIndexer(); const b = await createIndexer(); From 8b6c91ee7937124405aaafe678496c8be73a9660 Mon Sep 17 00:00:00 2001 From: Mikhail Kotelnikov Date: Wed, 27 May 2026 22:29:49 +0200 Subject: [PATCH 12/12] perf(indexer-mem): skip re-serialization when index is unchanged Add a dirty-bit + cached-output to MemVectorIndex and FlexSearchFullTextIndex. addDocument/deleteDocuments set dirty; the serializers (serializeToArrow / serialize) return the cached bytes/JSON while clean, and the deserializers prime the cache from the input so a just-loaded index also short-circuits its first serialize. Makes no-op re-syncs cheap: the persistence layer still calls the serializer each flush, but the heavy Arrow/FlexSearch encoding only runs when the index actually changed. Co-Authored-By: Claude Opus 4.7 --- .../src/flexsearch-full-text-index.ts | 55 ++++++++++++++++-- packages/indexer-mem/src/mem-vector-index.ts | 56 +++++++++++++++++-- 2 files changed, 101 insertions(+), 10 deletions(-) diff --git a/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts b/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts index 14fd4e8..0a25612 100644 --- a/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts +++ b/packages/indexer-mem-flexsearch/src/flexsearch-full-text-index.ts @@ -27,6 +27,15 @@ export class FlexSearchFullTextIndex implements FullTextIndex { private blocks = new Map(); private nextNum = 1; private closed = false; + /** + * Dirty bit. `true` whenever the FTS index has been mutated since the + * last `serialize()`. The serializer caches the last produced JSON and + * returns it directly when `!dirty`, so a no-op re-sync skips the + * expensive FlexSearch export entirely. Starts `true` so the first + * serialize after construction always runs. + */ + private dirty = true; + private cachedSerialized?: string; constructor(info: FullTextIndexInfo) { this.info = info; @@ -37,6 +46,15 @@ export class FlexSearchFullTextIndex implements FullTextIndex { }); } + /** + * `true` when the in-memory state has changed since the last serialize. + * Callers (e.g. the persistence layer) may use it to skip serialization + * and writes when the index has not been touched. + */ + isDirty(): boolean { + return this.dirty; + } + private ensureOpen(): void { if (this.closed) { throw new Error("FullTextIndex is closed"); @@ -143,6 +161,7 @@ export class FlexSearchFullTextIndex implements FullTextIndex { async addDocument(blocks: FullTextBlock[]): Promise { this.ensureOpen(); + if (blocks.length === 0) return; for (const block of blocks) { const key = compositeKey(block.path, block.blockId); const num = this.getOrAssignNum(key); @@ -158,6 +177,7 @@ export class FlexSearchFullTextIndex implements FullTextIndex { metadata: block.metadata, }); } + this.dirty = true; } async addDocuments( @@ -173,10 +193,11 @@ export class FlexSearchFullTextIndex implements FullTextIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); + let removed = 0; for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { const key = compositeKey(sel.path, sel.blockId); - this.removeByKey(key); + if (this.removeByKey(key)) removed += 1; } else { const keysToDelete: string[] = []; for (const [key, block] of this.blocks) { @@ -185,20 +206,24 @@ export class FlexSearchFullTextIndex implements FullTextIndex { } } for (const key of keysToDelete) { - this.removeByKey(key); + if (this.removeByKey(key)) removed += 1; } } } + if (removed > 0) this.dirty = true; } - private removeByKey(key: string): void { + /** Returns true when the key was present and got removed. */ + private removeByKey(key: string): boolean { const num = this.keyToNum.get(key); + const had = this.blocks.has(key); if (num !== undefined) { this.flexIndex.remove(num); this.keyToNum.delete(key); this.numToKey.delete(num); } this.blocks.delete(key); + return had; } async getSize(pathPrefix?: DocumentPath): Promise { @@ -259,11 +284,23 @@ export class FlexSearchFullTextIndex implements FullTextIndex { this.blocks.clear(); this.keyToNum.clear(); this.numToKey.clear(); + this.cachedSerialized = undefined; + this.dirty = true; this.closed = true; } - /** Serialize to JSON string (v3 format with path data) */ + /** + * Serialize to JSON string (v3 format with path data). + * + * Caches the result; while `dirty` is false (no mutations since the + * last call) returns the same string, skipping the FlexSearch export + * entirely. The persistence layer still calls into us each flush, but + * the heavy work only runs when the index actually changed. + */ async serialize(): Promise { + if (!this.dirty && this.cachedSerialized !== undefined) { + return this.cachedSerialized; + } const chunks = new Map(); await this.flexIndex.export((key: string | number, data: string) => { chunks.set(String(key), data); @@ -284,13 +321,16 @@ export class FlexSearchFullTextIndex implements FullTextIndex { }); } - return JSON.stringify({ + const out = JSON.stringify({ version: 3, chunks: Object.fromEntries(chunks), blocks: blocksData, keyToNum: [...this.keyToNum.entries()], nextNum: this.nextNum, }); + this.cachedSerialized = out; + this.dirty = false; + return out; } /** Deserialize from JSON — handles v3 format */ @@ -332,7 +372,10 @@ export class FlexSearchFullTextIndex implements FullTextIndex { metadata: block.metadata, }); } - + // Prime the cache: a deserialized index that's never mutated should + // return the same JSON it was loaded from when re-serialized. + fts.cachedSerialized = json; + fts.dirty = false; return fts; } } diff --git a/packages/indexer-mem/src/mem-vector-index.ts b/packages/indexer-mem/src/mem-vector-index.ts index 5b6bb98..2066df3 100644 --- a/packages/indexer-mem/src/mem-vector-index.ts +++ b/packages/indexer-mem/src/mem-vector-index.ts @@ -36,11 +36,29 @@ export class MemVectorIndex implements EmbeddingIndex { private readonly info: EmbeddingIndexInfo; private readonly entries = new Map(); private closed = false; + /** + * Dirty bit. `true` whenever `entries` has been mutated since the last + * `serializeToArrow()`. The serializer caches the last produced + * `Uint8Array` and returns it directly when `!dirty`, so a no-op re-sync + * skips the expensive Arrow IPC encoding for an unchanged index. + * Starts `true` so the first serialize after construction always runs. + */ + private dirty = true; + private cachedSerialized?: Uint8Array; constructor(info: EmbeddingIndexInfo) { this.info = info; } + /** + * `true` when the in-memory state has changed since the last serialize. + * Callers (e.g. the persistence layer) may use it to skip serialization + * and writes when the index has not been touched. + */ + isDirty(): boolean { + return this.dirty; + } + private ensureOpen(): void { if (this.closed) { throw new Error("EmbeddingIndex is closed"); @@ -94,6 +112,7 @@ export class MemVectorIndex implements EmbeddingIndex { async addDocument(blocks: EmbeddingBlock[]): Promise { this.ensureOpen(); + if (blocks.length === 0) return; for (const block of blocks) { validateDimensionality(this.info, block.embedding); const key = compositeKey(block.path, block.blockId); @@ -104,6 +123,7 @@ export class MemVectorIndex implements EmbeddingIndex { metadata: block.metadata, }); } + this.dirty = true; } async addDocuments( @@ -119,17 +139,22 @@ export class MemVectorIndex implements EmbeddingIndex { pathSelectors: PathSelector[] | AsyncIterable, ): Promise { this.ensureOpen(); + let removed = 0; for await (const sel of toAsyncIterable(pathSelectors)) { if (sel.blockId !== undefined) { - this.entries.delete(compositeKey(sel.path, sel.blockId)); + if (this.entries.delete(compositeKey(sel.path, sel.blockId))) { + removed += 1; + } } else { for (const [key, entry] of this.entries) { if (matchesPrefix(entry.path, sel.path)) { this.entries.delete(key); + removed += 1; } } } } + if (removed > 0) this.dirty = true; } async getSize(pathPrefix?: DocumentPath): Promise { @@ -188,11 +213,24 @@ export class MemVectorIndex implements EmbeddingIndex { async deleteIndex(): Promise { this.ensureOpen(); this.entries.clear(); + this.cachedSerialized = undefined; + this.dirty = true; this.closed = true; } - /** Serialize embeddings to Arrow IPC format */ + /** + * Serialize embeddings to Arrow IPC format. + * + * Caches the result; while `dirty` is false (no mutations since the + * last call) returns the same `Uint8Array` instance, skipping the + * Arrow IPC encoding entirely. This makes no-op re-syncs cheap — the + * persistence layer still calls into us each flush, but the heavy + * encoding only runs when the index actually changed. + */ serializeToArrow(): Uint8Array { + if (!this.dirty && this.cachedSerialized !== undefined) { + return this.cachedSerialized; + } const dim = this.info.dimensionality; const paths: string[] = []; const blockIds: string[] = []; @@ -215,10 +253,18 @@ export class MemVectorIndex implements EmbeddingIndex { }, }, ); - return tableToIPC(table, { format: "stream" }) as Uint8Array; + const bytes = tableToIPC(table, { format: "stream" }) as Uint8Array; + this.cachedSerialized = bytes; + this.dirty = false; + return bytes; } - /** Deserialize embeddings from Arrow IPC format */ + /** + * Deserialize embeddings from Arrow IPC format. The reconstructed index + * starts clean (`dirty = false`) and primes its serialize cache with the + * input bytes — a subsequent `serializeToArrow()` on the just-loaded + * index returns the same bytes without re-encoding. + */ static deserializeFromArrow(info: EmbeddingIndexInfo, data: Uint8Array): MemVectorIndex { const table = tableFromIPC(data); const vec = new MemVectorIndex(info); @@ -236,6 +282,8 @@ export class MemVectorIndex implements EmbeddingIndex { const key = compositeKey(path, blockId); vec.entries.set(key, { path, blockId, embedding, metadata }); } + vec.cachedSerialized = data; + vec.dirty = false; return vec; } }