feat(index): bound indexing resources

Makefile.cbm

@@ -191,6 +191,7 @@ FOUNDATION_SRCS = \
     src/foundation/profile.c \
     src/foundation/dump_verify.c \
     src/foundation/limits.c \
+    src/foundation/index_limits.c \
     src/foundation/subprocess.c \
     src/foundation/sha256.c \
     src/foundation/macos_acl.c \
@@ -430,6 +431,7 @@ TEST_FOUNDATION_SRCS = \
     tests/test_platform.c \
     tests/test_diagnostics.c \
     tests/test_dump_verify.c \
+    tests/test_index_limits.c \
     tests/test_subprocess.c \
     tests/test_private_file_lock.c \
     tests/test_lock_registry.c

README.md

@@ -661,6 +661,12 @@ codebase-memory-mcp config set auto_watch false          # don't register backgr
 codebase-memory-mcp config reset auto_index              # reset to default
 ```
 
+Repository indexing also has bounded defaults for root breadth, discovery,
+worker count, memory, duration, temporary files, database size, cache admission,
+and reserved free disk. See
+[docs/INDEX_RESOURCE_LIMITS.md](docs/INDEX_RESOURCE_LIMITS.md) for all keys and
+failure semantics.
+
 ### Environment Variables
 
 | Variable | Default | Description |

docs/INDEX_RESOURCE_LIMITS.md

@@ -0,0 +1,81 @@
+# Index Resource Limits
+
+Indexing is intentionally bounded. Before parsing begins, the server validates
+the canonical repository root and performs a bounded discovery walk. The worker
+and persistence stages continue enforcing the same resolved policy.
+
+If a hard boundary is reached, `index_repository` returns
+`resource_limit_exceeded` with the boundary name, observed value, configured
+limit, and a remediation hint. A partial index is never published as a complete
+one.
+
+## Defaults
+
+| Configuration key | Default | Protects |
+|---|---:|---|
+| `index_max_files` | `100000` | Accepted source-file count |
+| `index_max_directories` | `20000` | Traversed directory count |
+| `index_max_entries` | `500000` | Examined filesystem entries |
+| `index_max_depth` | `64` | Directory nesting depth |
+| `index_max_source_mb` | `4096` | Aggregate accepted source size |
+| `index_max_file_mb` | `64` | One source file |
+| `index_scan_timeout_seconds` | `30` | Discovery time |
+| `index_cpu_cores` | `4` | Indexing worker threads |
+| `index_concurrent_jobs` | `1` | Simultaneous physical index workers |
+| `index_memory_limit_mb` | `8192` | Worker resident memory, capped by detected budget |
+| `index_max_db_mb` | `16384` | Published SQLite database |
+| `index_max_staging_mb` | `20480` | Staging database and sidecars |
+| `index_max_task_temp_mb` | `24576` | Staging, worker log, and response temporary files |
+| `index_cache_max_mb` | `32768` | Total index cache admission |
+| `index_min_free_disk_mb` | `4096` | Free disk kept in reserve |
+| `index_max_duration_seconds` | `3600` | Worker wall-clock time |
+| `index_low_priority` | `true` | Best-effort reduced scheduling priority |
+| `index_denied_roots` | empty | Additional exact canonical roots |
+
+Automatic indexing keeps its separate, stricter `auto_index_limit` default of
+`50000`.
+
+Set a value with:
+
+```sh
+codebase-memory-mcp config set index_max_files 75000
+codebase-memory-mcp config set index_memory_limit_mb 4096
+```
+
+Separate additional denied roots with semicolons on every platform:
+
+```sh
+codebase-memory-mcp config set index_denied_roots '/path/to/aggregate;/path/to/archive'
+```
+
+Numeric values must be positive and within the ranges shown by
+`codebase-memory-mcp config --help`. `index_denied_roots` may be empty. Invalid
+or unknown keys are rejected.
+
+## Root safety
+
+The filesystem root, current user's home directory, and the Codebase Memory
+cache directory are always refused as repository roots. This built-in check
+cannot be disabled. `index_denied_roots` adds exact canonical roots; it does not
+deny all descendants, so individual repositories below an aggregation
+directory can still be indexed.
+
+`CBM_ALLOWED_ROOT` remains available for deployments that also need an outer
+containment boundary.
+
+`cross-repo-intelligence` is not a source-indexing mode: it reads and updates
+already published project databases and does not walk `repo_path`. It therefore
+keeps the session containment and cross-repository target-count checks, but does
+not apply source-root or discovery limits.
+
+## Failure and recovery
+
+Discovery violations stop before parsing. Worker memory, time, and temporary
+output violations terminate the contained process tree. Persistence violations
+remove the current operation's staging files and preserve the previously
+published database. Reduced scheduling priority is applied where the operating
+system permits it; a refusal is logged and does not silently change the other
+limits.
+
+The cache limit is an admission check, not an eviction policy. The indexer never
+deletes unrelated project indexes automatically.

docs/changes/index-resource-guards.md

@@ -0,0 +1,171 @@
+# Index Resource Guard Design
+
+## Problem
+
+An explicit `index_repository` request currently trusts the requested directory
+and lets discovery, parsing, and persistence grow until the operation finishes
+or the host runs out of resources. This is unsafe for aggregation directories,
+accidental filesystem roots, and repositories with unexpectedly large generated
+trees.
+
+## Behavior contract
+
+The indexer resolves one immutable resource policy at admission time. The same
+policy is passed to discovery, the worker process, and persistence. A limit
+failure is reported as `resource_limit_exceeded`; it is never reported as a
+successful or merely empty index.
+
+### BDD scenarios
+
+1. **Dangerous root**
+   - Given an explicit request whose canonical path is the filesystem root,
+     current user's home directory, or the cache directory
+   - When `index_repository` validates the request
+   - Then it refuses the request before discovery or a worker is started
+   - And the response identifies the `repository_root` policy
+
+2. **Bounded discovery**
+   - Given a tree that exceeds the configured file, directory, entry, depth, or
+     aggregate indexable-source-byte limit
+   - When discovery reaches that boundary
+   - Then discovery stops immediately and identifies the exact boundary
+   - And no partial file list is passed to parsing
+
+3. **Bounded compute**
+   - Given default configuration on a large host
+   - When an index job starts
+   - Then at most four indexing workers are used
+   - And only one physical indexing job runs at a time
+   - And the worker runs at reduced scheduling priority where supported
+
+4. **Bounded memory and time**
+   - Given a supervised worker that stays above its hard memory limit or runs
+     longer than its duration limit
+   - When the supervisor polls it
+   - Then the process tree is terminated
+   - And the operation returns `resource_limit_exceeded`, including the limit
+     name and configured value
+
+5. **Bounded persistence**
+   - Given an index whose staging database, final database, temporary task
+     footprint, or cache footprint would exceed policy
+   - When persistence checks the next publish step
+   - Then the new index is not published
+   - And the previously published database remains usable
+   - And abandoned staging and sidecar files for this operation are removed
+
+6. **Configuration validation**
+   - Given an unknown resource key, a non-integer value, zero, or a value outside
+     the supported range
+   - When `config set` is invoked
+   - Then configuration is rejected without changing the previous value
+
+7. **Public-repository hygiene**
+   - Given the final change
+   - When tracked content is scanned
+   - Then it contains only generic examples such as `/path/to/repo`
+   - And it contains no developer usernames, workstation paths, private
+     repository names, tokens, or index contents
+
+8. **Cross-repository database linking**
+   - Given `mode="cross-repo-intelligence"` and existing source and target
+     databases
+   - When `index_repository` handles the request
+   - Then it preserves session containment and target validation
+   - And it does not apply source-root or discovery limits because it does not
+     walk the source directory
+
+## Resource policy
+
+All byte-size settings use MiB at the user-facing configuration boundary and
+are converted with overflow checks.
+
+| Key | Default | Decision |
+|---|---:|---|
+| `index_max_files` | 100,000 | Accepted source files |
+| `auto_index_limit` | 50,000 | Stricter automatic-admission file limit |
+| `index_max_directories` | 20,000 | Traversed directories, including root |
+| `index_max_entries` | 500,000 | Filesystem entries examined |
+| `index_max_depth` | 64 | Root is depth 0 |
+| `index_max_source_mb` | 4,096 | Aggregate accepted source bytes |
+| `index_max_file_mb` | 64 | Oversized files are skipped and reported |
+| `index_scan_timeout_seconds` | 30 | Discovery deadline |
+| `index_cpu_cores` | 4 | Maximum indexing worker threads |
+| `index_concurrent_jobs` | 1 | Maximum physical index workers |
+| `index_memory_limit_mb` | 8,192 | Also capped by detected available memory |
+| `index_max_db_mb` | 16,384 | Published database ceiling |
+| `index_max_staging_mb` | 20,480 | Staging DB plus SQLite sidecars |
+| `index_max_task_temp_mb` | 24,576 | Staging plus worker log/response footprint |
+| `index_cache_max_mb` | 32,768 | Admission ceiling; no automatic eviction |
+| `index_min_free_disk_mb` | 4,096 | Reserved free space |
+| `index_max_duration_seconds` | 3,600 | Wall-clock worker deadline |
+| `index_low_priority` | `true` | Best-effort lower scheduling priority |
+| `index_denied_roots` | empty | Additional canonical roots, semicolon-separated |
+
+Built-in dangerous roots cannot be disabled. Operators may lower or raise
+numeric values within documented validation ranges. `0` never means unlimited;
+it is rejected.
+
+## Data model and state transitions
+
+`cbm_index_limits_t` is the resolved, byte-based policy. Discovery additionally
+returns counters and a stable violation enum. The index request progresses:
+
+`admitted -> preflight discovery -> worker -> staging persistence -> publish`
+
+Any resource violation transitions to `resource_limit_exceeded`. Before publish,
+failure removes only files owned by the current staging operation. After a
+successful atomic rename, the old database is already replaced and later
+best-effort artifact export cannot claim rollback.
+
+## Shared entry paths
+
+- MCP and one-shot CLI both enter through `handle_index_repository`.
+- Daemon admission resolves concurrency and memory caps once.
+- Supervised workers receive only resolved internal numeric arguments, not
+  caller-controlled overrides.
+- Automatic indexing uses the same policy with the stricter
+  `auto_index_limit`.
+- `cross-repo-intelligence` remains a database-linking path with its existing
+  target-count and mutation guards; it does not enter source discovery.
+
+## Tests and fixtures
+
+- Discovery fixtures use temporary generic directory trees.
+- MCP integration tests exercise dangerous-root rejection and structured error
+  output without spawning a worker.
+- Pipeline tests create a valid old database, force a small staging/DB limit,
+  and verify the old database remains.
+- Supervisor tests use deterministic process/resource probes rather than large
+  real allocations, including worker-owned temporary output.
+- A real supervised-worker integration test proves the parent's resolved policy
+  reaches the child and replaces caller-supplied internal policy data.
+- Configuration tests verify valid overrides and fail-closed parsing.
+- Existing cross-repository integration tests verify target validation,
+  cancellation, deduplication, missing databases, and source-name overrides.
+
+## Installation and migration
+
+No database migration is required. Existing indexes remain readable. The new
+limits affect the next indexing operation. Existing environment variables remain
+compatible, but the resolved policy always chooses the safer lower effective
+limit.
+
+## Non-goals
+
+- Exact percentage CPU throttling. Worker-count tokens, single-job admission,
+  and lower process priority provide portable bounded CPU use.
+- Automatic deletion of unrelated cache entries. Cache excess blocks new work
+  and reports remediation; it does not delete another project's data.
+- Following symlinks, junctions, or paths outside the canonical requested root.
+- Per-request public overrides that let an untrusted MCP caller weaken operator
+  policy.
+
+## Directly applicable repository rules
+
+- "All code must be properly formatted before committing."
+- "Maintain appropriate tests for your changes following the test hierarchy."
+- "Prefer explicit error handling."
+- "Do NOT create directories or files directly in the repository root."
+- Behavior changes follow red, green, refactor: tests precede production code.
+- Tracked content must be suitable for a public open-source repository.

src/cli/cli.c

@@ -6407,10 +6407,42 @@ int cbm_config_get_int(cbm_config_t *cfg, const char *key, int default_val) {
     return (int)v;
 }
 
+bool cbm_config_load_index_limits(cbm_config_t *cfg, cbm_index_limits_t *limits, char *error,
+                                  size_t error_size) {
+    if (!limits) {
+        if (error && error_size > 0) {
+            (void)snprintf(error, error_size, "missing index resource policy");
+        }
+        return false;
+    }
+    cbm_index_limits_defaults(limits);
+    if (!cfg) {
+        return true;
+    }
+    for (size_t i = 0; i < cbm_index_limits_config_key_count(); i++) {
+        const char *key = cbm_index_limits_config_key_at(i);
+        const char *value = cbm_config_get(cfg, key, NULL);
+        if (value && !cbm_index_limits_set(limits, key, value, error, error_size)) {
+            return false;
+        }
+    }
+    return true;
+}
+
 int cbm_config_set(cbm_config_t *cfg, const char *key, const char *value) {
     if (!cfg || !key || !value) {
         return CLI_ERR;
     }
+    if (cbm_index_limits_is_config_key(key)) {
+        cbm_index_limits_t validated;
+        char error[CLI_BUF_256];
+        cbm_index_limits_defaults(&validated);
+        if (!cbm_index_limits_set(&validated, key, value, error, sizeof(error))) {
+            return CLI_ERR;
+        }
+    } else if (strncmp(key, "index_", sizeof("index_") - 1U) == 0) {
+        return CLI_ERR;
+    }
 
     sqlite3_stmt *stmt = NULL;
     if (sqlite3_prepare_v2(cfg->db, "INSERT OR REPLACE INTO config (key, value) VALUES (?, ?)",
@@ -6461,6 +6493,15 @@ int cbm_cmd_config(int argc, char **argv) {
                "Register background git watcher on session connect");
         printf("  %-25s  default=%-10s  %s\n", CBM_CONFIG_UI_LANG, "auto",
                "Pin graph UI language: en, zh, or auto");
+        printf("\nIndex resource keys (sizes are MiB, durations are seconds):\n");
+        for (size_t i = 0; i < cbm_index_limits_config_key_count(); i++) {
+            const char *key = cbm_index_limits_config_key_at(i);
+            char default_value[CLI_BUF_256];
+            if (cbm_index_limits_default_value(key, default_value, sizeof(default_value))) {
+                printf("  %-32s  default=%s\n", key, default_value[0] ? default_value : "(empty)");
+            }
+        }
+        printf("See docs/INDEX_RESOURCE_LIMITS.md for boundary semantics and ranges.\n");
         return 0;
     }
 
@@ -6490,6 +6531,13 @@ int cbm_cmd_config(int argc, char **argv) {
                cbm_config_get(cfg, CBM_CONFIG_AUTO_WATCH, "true"));
         printf("  %-25s = %-10s\n", CBM_CONFIG_UI_LANG,
                cbm_config_get(cfg, CBM_CONFIG_UI_LANG, "auto"));
+        for (size_t i = 0; i < cbm_index_limits_config_key_count(); i++) {
+            const char *key = cbm_index_limits_config_key_at(i);
+            char default_value[CLI_BUF_256];
+            if (cbm_index_limits_default_value(key, default_value, sizeof(default_value))) {
+                printf("  %-32s = %s\n", key, cbm_config_get(cfg, key, default_value));
+            }
+        }
     } else if (strcmp(argv[0], "get") == 0) {
         if (argc < MIN_ARGC_GET) {
             (void)fprintf(stderr, "Usage: config get <key>\n");
@@ -6502,7 +6550,24 @@ int cbm_cmd_config(int argc, char **argv) {
             (void)fprintf(stderr, "Usage: config set <key> <value>\n");
             rc = CLI_TRUE;
         } else {
-            if (cbm_config_set(cfg, argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]) == 0) {
+            const char *key = argv[CLI_SKIP_ONE];
+            const char *value = argv[CLI_PAIR_LEN];
+            char validation_error[CLI_BUF_256] = {0};
+            bool resource_value_valid = true;
+            if (cbm_index_limits_is_config_key(key)) {
+                cbm_index_limits_t validated;
+                cbm_index_limits_defaults(&validated);
+                resource_value_valid = cbm_index_limits_set(
+                    &validated, key, value, validation_error, sizeof(validation_error));
+            } else if (strncmp(key, "index_", sizeof("index_") - 1U) == 0) {
+                resource_value_valid = false;
+                (void)snprintf(validation_error, sizeof(validation_error),
+                               "unknown index resource key: %s", key);
+            }
+            if (!resource_value_valid) {
+                (void)fprintf(stderr, "error: %s\n", validation_error);
+                rc = CLI_TRUE;
+            } else if (cbm_config_set(cfg, key, value) == 0) {
                 printf("%s = %s\n", argv[CLI_SKIP_ONE], argv[CLI_PAIR_LEN]);
             } else {
                 (void)fprintf(stderr, "error: failed to set %s\n", argv[CLI_SKIP_ONE]);