Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 5 additions & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
- Writing back a file read with `repo_read_file` dropped its final newline. The read returns the lines without a newline after the last one, so the written content had none, and the dry run's diff showed it only as a removed line that is easy to miss. A file that ends with a newline now keeps it (CRLF for a CRLF file) unless the new content is empty.
- A file that is not UTF-8 was corrupted by a repo write. It was read as UTF-8 with its other bytes replaced, then written back that way. Found 2026-11-09: in an EUC-KR file, editing only its last line destroyed the Korean text of the first, and the dry run's diff showed that line as unchanged. A UTF-16 file was rewritten as UTF-8 without its BOM. The dry run now refuses a file that is not UTF-8 text and says why.
- `repo_write_file_dry_run` accepted content that `repo_write_file_apply` then refused as too large, blaming the target file. The size limit (1,000,000 bytes) was checked before the CRLF endings and BOM the file keeps were restored, which add bytes. It is now checked on the content that will be written.
- MCP `repo_search` never searched hidden files: `.github/` workflows, `.eslintrc.cjs`, `.husky/` and the like came back with no matches, even with a glob such as `.github/**`, although `repo_read_file` reads them. Found 2026-11-09. They are now searched; `.env` files, `.git`, `.bridge` and key material stay out of the results as before.
- `repo_search` dropped a match on a line that is not UTF-8 (a Latin-1 `café`, say), and every match in a file whose name is not UTF-8. Such a line now comes back with the unreadable bytes shown as replacement characters.
- One directory `repo_search` could not read (a database volume owned by another user, for one) failed the whole search with the raw ripgrep command line, and the matches found everywhere else were thrown away. Those matches are now returned.
- A broad `repo_search` (a few thousand matching lines) failed as "returned too many matches" instead of returning its first 100 marked `truncated: true`. A query containing a line break failed with the raw ripgrep command line; it now says that a search matches one line at a time.
- `repo_read_file` and the repo write tools refused a top-level file whose name starts with two dots, such as `..notes.md`, as escaping the repository.

## 0.40.50

Expand Down
98 changes: 67 additions & 31 deletions src/repo.ts
Original file line number Diff line number Diff line change
Expand Up @@ -67,12 +67,16 @@ export interface SearchRepoResult {
limit: number;
}

// ripgrep names a path or line that is not UTF-8 as base64 "bytes" instead
// of "text".
type RipgrepJsonText = { text: string } | { bytes: string };

interface RipgrepJsonMatch {
type: "match";
data: {
path: { text: string };
path: RipgrepJsonText;
line_number: number;
lines: { text: string };
lines: RipgrepJsonText;
};
}

Expand All @@ -93,11 +97,17 @@ export function assertRepoRelativePath(repoPath: string): void {
assertNotSensitiveRepoPath(normalized);
}

// "..notes.md" is a file in the root; only ".." itself or a path through it
// leaves it.
function escapesRoot(relative: string): boolean {
return relative === ".." || relative.startsWith(`..${path.sep}`) || path.isAbsolute(relative);
}

export function resolveRepoPath(root: string, repoPath: string): string {
assertRepoRelativePath(repoPath);
const resolved = path.resolve(root, repoPath);
const relative = path.relative(root, resolved);
if (relative.startsWith("..") || path.isAbsolute(relative)) {
if (escapesRoot(relative)) {
throw new Error(`Path ${repoPath} escapes the repository root`);
}
return resolved;
Expand Down Expand Up @@ -141,19 +151,30 @@ export async function searchRepoWithMetadata(root: string, query: string, glob?:
if (!query.trim()) {
throw new Error("Search query must not be empty");
}
if (/[\r\n]/.test(query)) {
throw new Error("repo_search matches one line at a time, so a query cannot contain a line break; search for one of its lines.");
}
if (glob) {
assertRepoRelativeGlob(glob);
}
const args = [
"--no-config",
"--no-follow",
"--json",
// repo_read_file reads .github/, .eslintrc and the like, and without this
// ripgrep skipped every hidden file. Results still pass the same path
// policy as reads, and the globs below keep the sensitive ones unopened.
"--hidden",
...(glob ? ["--glob", glob] : []),
"--glob",
"!.bridge/**",
"--glob",
"!**/.bridge/**",
"--glob",
"!.git",
"--glob",
"!**/.git",
"--glob",
"!.git/**",
"--glob",
"!**/.git/**",
Expand Down Expand Up @@ -192,32 +213,27 @@ export async function searchRepoWithMetadata(root: string, query: string, glob?:
timeout: 10_000,
maxBuffer: 1024 * 1024
});
const parsedMatches = stdout
.split("\n")
.filter(Boolean)
.map(parseRipgrepJsonMatch)
.filter((match): match is SearchResult => match !== undefined);
const allowedMatches: SearchResult[] = [];
let truncated = false;
for (const match of parsedMatches) {
if (await isAllowedRepoSearchResult(root, match.path)) {
if (allowedMatches.length >= MAX_REPO_SEARCH_MATCHES) {
truncated = true;
break;
}
allowedMatches.push(match);
}
}
return { matches: allowedMatches, truncated, limit: MAX_REPO_SEARCH_MATCHES };
return await allowedSearchResult(root, stdout, false);
} catch (error) {
const maybe = error as { code?: number | string; stdout?: string };
if (maybe.code === "ENOENT") {
throw new Error("ripgrep (rg) is required on PATH for repo_search");
}
if (isSearchOutputTooLarge(error)) {
// A broad query outgrew the buffer: return the matches read so far,
// marked truncated, rather than none.
const partial = await allowedSearchResult(root, maybe.stdout ?? "", true);
if (partial.matches.length > 0) return partial;
throw new Error("repo_search returned too many matches; narrow the query or glob and try again");
}
if (maybe.code === 1) return { matches: [], truncated: false, limit: MAX_REPO_SEARCH_MATCHES };
// ripgrep exits 2 when it could not read some path (a directory it may
// not enter, say) but still prints what it found elsewhere; that used to
// fail the whole search with the raw command line.
if (maybe.code === 2 && !ripgrepRegexError((error as { stderr?: string }).stderr)) {
const partial = await allowedSearchResult(root, maybe.stdout ?? "", false);
if (partial.matches.length > 0) return partial;
}
// An invalid pattern used to surface as "Command failed: <rg path> ..."
// with rg's parse error buried in it (measured 2026-10-21, query "(").
const reason = ripgrepRegexError((error as { stderr?: string }).stderr);
Expand All @@ -230,6 +246,25 @@ export async function searchRepoWithMetadata(root: string, query: string, glob?:
}
}

async function allowedSearchResult(root: string, stdout: string, truncated: boolean): Promise<SearchRepoResult> {
const parsedMatches = stdout
.split("\n")
.filter(Boolean)
.map(parseRipgrepJsonMatch)
.filter((match): match is SearchResult => match !== undefined);
const allowedMatches: SearchResult[] = [];
for (const match of parsedMatches) {
if (await isAllowedRepoSearchResult(root, match.path)) {
if (allowedMatches.length >= MAX_REPO_SEARCH_MATCHES) {
truncated = true;
break;
}
allowedMatches.push(match);
}
}
return { matches: allowedMatches, truncated, limit: MAX_REPO_SEARCH_MATCHES };
}

function ripgrepRegexError(stderr: string | undefined): string | undefined {
if (!stderr || !/regex parse error/i.test(stderr)) return undefined;
const detail = stderr.split("\n").map((line) => line.trim()).find((line) => /^error:/i.test(line));
Expand All @@ -245,27 +280,28 @@ function parseRipgrepJsonMatch(line: string): SearchResult | undefined {
}
if (!isRipgrepJsonMatch(event)) return undefined;
return {
path: normalizeRipgrepMatchPath(event.data.path.text),
path: normalizeRipgrepMatchPath(ripgrepText(event.data.path)),

Copy link
Copy Markdown

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

P1 Badge Validate the original path bytes before lossy decoding

On Unix, lossily decoding a non-UTF-8 filename before passing it to isAllowedRepoSearchResult makes the security checks inspect a different file. I reproduced this with an invalid-byte filename bad-\xff.txt hard-linked to an outside secret and an ordinary, single-link file named bad-�.txt: searching for TOKEN returns the outside secret under the ordinary file's name because the realpath and hard-link checks validate that ordinary file instead. Previously these byte-path events were discarded. Preserve the original path bytes for filesystem validation, or reject paths that cannot round-trip through UTF-8, rather than using the replacement-character path for authorization.

Useful? React with 👍 / 👎.

line: event.data.line_number,
text: stripOneTrailingLineEnding(event.data.lines.text)
text: stripOneTrailingLineEnding(ripgrepText(event.data.lines))
};
}

function ripgrepText(value: RipgrepJsonText): string {
return "text" in value ? value.text : Buffer.from(value.bytes, "base64").toString("utf8");
}

function isRipgrepJsonText(value: unknown): value is RipgrepJsonText {
return isRecord(value) && (typeof value.text === "string" || typeof value.bytes === "string");
}

function normalizeRipgrepMatchPath(matchPath: string): string {
return matchPath.split(path.sep).join("/").replace(/^\.\//, "");
}

function isRipgrepJsonMatch(value: unknown): value is RipgrepJsonMatch {
if (!isRecord(value) || value.type !== "match" || !isRecord(value.data)) return false;
const { path: matchPath, line_number: lineNumber, lines } = value.data;
return (
isRecord(matchPath) &&
typeof matchPath.text === "string" &&
typeof lineNumber === "number" &&
Number.isFinite(lineNumber) &&
isRecord(lines) &&
typeof lines.text === "string"
);
return isRipgrepJsonText(matchPath) && typeof lineNumber === "number" && Number.isFinite(lineNumber) && isRipgrepJsonText(lines);
}

function isRecord(value: unknown): value is Record<string, unknown> {
Expand Down Expand Up @@ -542,7 +578,7 @@ export async function assertResolvedRepoPathAllowed(root: string, resolved: stri
throw error;
}
const relative = path.relative(realRoot, realTarget);
if (relative.startsWith("..") || path.isAbsolute(relative)) {
if (escapesRoot(relative)) {
throw new Error(`Path ${repoPath} escapes the repository root after resolving symlinks`);
}
if (relative) {
Expand Down
74 changes: 74 additions & 0 deletions tests/repo-search-gaps.test.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,74 @@
import { chmod, mkdir, mkdtemp, writeFile } from "node:fs/promises";
import { tmpdir } from "node:os";
import path from "node:path";
import { describe, expect, it } from "vitest";

import { readRepoFile, searchRepo, searchRepoWithMetadata } from "../src/repo.js";

// Found 2026-11-09 driving the MCP repo tools against throwaway repos.
async function repo(files: Record<string, string | Buffer>): Promise<string> {
const root = await mkdtemp(path.join(tmpdir(), "prodex-repo-search-"));
for (const [file, content] of Object.entries(files)) {
await mkdir(path.dirname(path.join(root, file)), { recursive: true });
await writeFile(path.join(root, file), content);
}
return root;
}

describe("repo search", () => {
// repo_read_file reads .github/workflows/ci.yml, but ripgrep skips hidden
// files unless asked, so a search never found anything in it.
it("searches dot-files and dot-directories that repo_read_file can read", async () => {
const root = await repo({ ".github/workflows/ci.yml": "run: echo needle\n", ".eslintrc.cjs": "// needle\n", "src/a.ts": "needle\n" });
const paths = (await searchRepo(root, "needle")).map((match) => match.path).sort();
expect(paths).toEqual([".eslintrc.cjs", ".github/workflows/ci.yml", "src/a.ts"]);
expect((await searchRepo(root, "needle", ".github/**")).map((match) => match.path)).toEqual([".github/workflows/ci.yml"]);
});

it("still keeps sensitive dot-files out of the results", async () => {
const root = await repo({ ".env": "needle\n", ".git/config": "needle\n", ".bridge/x.json": "needle\n", ".ssh/id": "needle\n", "ok.md": "needle\n" });
expect((await searchRepo(root, "needle")).map((match) => match.path)).toEqual(["ok.md"]);
});

// ripgrep reports a line that is not UTF-8 as base64 "bytes" instead of
// "text", and such matches were dropped.
it("returns a match on a line that is not UTF-8", async () => {
const root = await repo({ "legacy.properties": Buffer.concat([Buffer.from("label=caf"), Buffer.from([0xe9]), Buffer.from(" needle\nother needle\n")]) });
const matches = await searchRepo(root, "needle");
expect(matches.map((match) => match.line)).toEqual([1, 2]);
expect(matches[0].text).toBe("label=caf� needle");
});

// One unreadable directory (a docker volume, say) made ripgrep exit 2, and
// every match it had found was thrown away with the raw command line.
it.skipIf(process.platform === "win32" || process.getuid?.() === 0)("returns the matches it found when a directory cannot be read", async () => {
const root = await repo({ "pgdata/base": "needle\n", "src/a.ts": "needle\n" });
await chmod(path.join(root, "pgdata"), 0o000);
try {
expect((await searchRepo(root, "needle")).map((match) => match.path)).toEqual(["src/a.ts"]);
} finally {
await chmod(path.join(root, "pgdata"), 0o755);
}
});

it("explains that a query cannot span lines", async () => {
const root = await repo({ "a.ts": "const\nneedle\n" });
await expect(searchRepo(root, "const\nneedle")).rejects.toThrow(/one line at a time/);
});

// ripgrep's output for a broad query outgrew the 1 MB buffer, and the search
// failed instead of returning its first matches marked truncated.
it("returns the first matches of a broad search, marked truncated", async () => {
const root = await repo({ "big.txt": `${Array.from({ length: 6000 }, (_, index) => `needle line ${index + 1}`).join("\n")}\n` });
const result = await searchRepoWithMetadata(root, "needle");
expect(result.matches).toHaveLength(100);
expect(result.truncated).toBe(true);
});
});

describe("repo paths", () => {
it("reads a top-level file whose name starts with two dots", async () => {
const root = await repo({ "..notes.md": "hello\n" });
await expect(readRepoFile(root, "..notes.md")).resolves.toMatchObject({ content: "hello" });
});
});
Loading