diff --git a/Cargo.lock b/Cargo.lock index 4e717161..f3dde8f8 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -482,6 +482,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foldhash" version = "0.2.0" @@ -527,6 +533,41 @@ dependencies = [ "winapi", ] +[[package]] +name = "futures-core" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7e3450815272ef58cec6d564423f6e755e25379b217b0bc688e295ba24df6b1d" + +[[package]] +name = "futures-executor" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "baf29c38818342a3b26b5b923639e7b1f4a61fc5e76102d4b1981c6dc7a7579d" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-task" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "037711b3d59c33004d3856fbdc83b99d4ff37a24768fa1be9ce3538a1cde4393" + +[[package]] +name = "futures-util" +version = "0.3.32" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "389ca41296e6190b48053de0321d02a77f32f8a5d2461dd38762c0593805c6d6" +dependencies = [ + "futures-core", + "futures-task", + "pin-project-lite", + "slab", +] + [[package]] name = "fxhash" version = "0.2.1" @@ -565,10 +606,23 @@ checksum = "899def5c37c4fd7b2664648c28120ecec138e4d395b459e5ca34f9cce2dd77fd" dependencies = [ "cfg-if", "libc", - "r-efi", + "r-efi 5.3.0", "wasip2", ] +[[package]] +name = "getrandom" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0de51e6874e94e7bf76d726fc5d13ba782deca734ff60d5bb2fb2607c7406555" +dependencies = [ + "cfg-if", + "libc", + "r-efi 6.0.0", + "wasip2", + "wasip3", +] + [[package]] name = "git2" version = "0.20.4" @@ -601,10 +655,19 @@ version = "0.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0c7ed2f2edad8a14c8186b847909a41fbb9c3eafa44f88bd891114ed5019da09" dependencies = [ - "hashbrown", + "hashbrown 0.16.1", "serde", ] +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" +dependencies = [ + "foldhash 0.1.5", +] + [[package]] name = "hashbrown" version = "0.16.1" @@ -613,7 +676,7 @@ checksum = "841d1cc9bed7f9236f321df977030373f4a4163ae1a7dbfe1a51a2c1a51d9100" dependencies = [ "allocator-api2", "equivalent", - "foldhash", + "foldhash 0.2.0", ] [[package]] @@ -622,7 +685,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ea0b22561a9c04a7cb1a302c013e0259cd3b4bb619f145b32f72b8b4bcbed230" dependencies = [ - "hashbrown", + "hashbrown 0.16.1", ] [[package]] @@ -643,15 +706,18 @@ dependencies = [ "prunist", "rand", "rand_chacha", + "rayon", "regex", "serde", "serde_json", "serde_yaml", + "serial_test", "simd-json", "syntect", "tempfile", "test_each_file", "unicode-segmentation", + "uuid", "yaml-rust2", ] @@ -782,6 +848,12 @@ dependencies = [ "zerovec", ] +[[package]] +name = "id-arena" +version = "2.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "3d3067d79b975e8844ca9eb072e16b31c3c1c36928edf9c6789548c524d0d954" + [[package]] name = "idna" version = "1.1.0" @@ -826,16 +898,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "7714e70437a7dc3ac8eb7e6f8df75fd8eb422675fc7678aff7364301092b1017" dependencies = [ "equivalent", - "hashbrown", -] - -[[package]] -name = "indoc" -version = "2.0.7" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "79cf5c93f93228cf8efb3ba362535fb11199ac548a09ce117c9b1adc3030d706" -dependencies = [ - "rustversion", + "hashbrown 0.16.1", + "serde", + "serde_core", ] [[package]] @@ -891,6 +956,12 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "leb128fmt" +version = "0.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "09edd9e8b54e49e587e4f6295a7d29c3ea94d469cb40ab8ca70b288248a81db2" + [[package]] name = "libc" version = "0.2.180" @@ -971,15 +1042,6 @@ version = "2.8.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f8ca58f447f06ed17d5fc4043ce1b10dd205e060fb3ce5b979b8ed8e59ff3f79" -[[package]] -name = "memoffset" -version = "0.9.1" -source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "488016bfae457b036d996092f6cb448677611ce4449e970ceaf42695203f218a" -dependencies = [ - "autocfg", -] - [[package]] name = "miniz_oxide" version = "0.8.9" @@ -1053,7 +1115,17 @@ checksum = "7d17b78036a60663b797adeaee46f5c9dfebb86948d1255007a1d6be0271ff99" dependencies = [ "instant", "lock_api", - "parking_lot_core", + "parking_lot_core 0.8.6", +] + +[[package]] +name = "parking_lot" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "93857453250e3077bd71ff98b6a65ea6621a19bb0f559a85248955ac12c45a1a" +dependencies = [ + "lock_api", + "parking_lot_core 0.9.12", ] [[package]] @@ -1070,12 +1142,31 @@ dependencies = [ "winapi", ] +[[package]] +name = "parking_lot_core" +version = "0.9.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2621685985a2ebf1c516881c026032ac7deafcda1a2c9b7850dc81e3dfcb64c1" +dependencies = [ + "cfg-if", + "libc", + "redox_syscall 0.5.18", + "smallvec", + "windows-link", +] + [[package]] name = "percent-encoding" version = "2.3.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9b4f627cb1b25917193a259e49bdad08f671f8d9708acfd5fe0a8c1455d87220" +[[package]] +name = "pin-project-lite" +version = "0.2.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a89322df9ebe1c1578d689c92318e070967d1042b512afbe49518723f4e6d5cd" + [[package]] name = "pkg-config" version = "0.3.32" @@ -1152,6 +1243,16 @@ dependencies = [ "termtree", ] +[[package]] +name = "prettyplease" +version = "0.2.37" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "479ca8adacdd7ce8f1fb39ce9ecccbfe93a3f1344b3d0d97f20bc0196208f62b" +dependencies = [ + "proc-macro2", + "syn", +] + [[package]] name = "proc-macro2" version = "1.0.106" @@ -1167,35 +1268,32 @@ version = "0.17.0" [[package]] name = "pyo3" -version = "0.27.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ab53c047fcd1a1d2a8820fe84f05d6be69e9526be40cb03b73f86b6b03e6d87d" +checksum = "cd274650b21d4bfc26a0a47587962c1edb425f69287324355cd040c3ea66071c" dependencies = [ - "indoc", "libc", - "memoffset", "once_cell", "portable-atomic", "pyo3-build-config", "pyo3-ffi", "pyo3-macros", - "unindent", ] [[package]] name = "pyo3-build-config" -version = "0.27.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b455933107de8642b4487ed26d912c2d899dec6114884214a0b3bb3be9261ea6" +checksum = "c5e2a7d2f0d013342f295c048ad19237add5154a55b1c5a254c0ec93d4109078" dependencies = [ "target-lexicon", ] [[package]] name = "pyo3-ffi" -version = "0.27.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "1c85c9cbfaddf651b1221594209aed57e9e5cff63c4d11d1feead529b872a089" +checksum = "ca85c467da1bbc8d866eea5deff9cf29ea5f7785054a17da36e65bda9c05845b" dependencies = [ "libc", "pyo3-build-config", @@ -1203,9 +1301,9 @@ dependencies = [ [[package]] name = "pyo3-macros" -version = "0.27.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0a5b10c9bf9888125d917fb4d2ca2d25c8df94c7ab5a52e13313a07e050a3b02" +checksum = "9ac53762fd065daa3194dd09337a38bd793a188100fd1a9304c4ab312d901771" dependencies = [ "proc-macro2", "pyo3-macros-backend", @@ -1215,13 +1313,12 @@ dependencies = [ [[package]] name = "pyo3-macros-backend" -version = "0.27.2" +version = "0.29.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "03b51720d314836e53327f5871d4c0cfb4fb37cc2c4a11cc71907a86342c40f9" +checksum = "4ca3a1557399783172dc5bf39cfca835157732532cba56b71d2292161e53b362" dependencies = [ "heck", "proc-macro2", - "pyo3-build-config", "quote", "syn", ] @@ -1250,6 +1347,12 @@ version = "5.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "69cdb34c158ceb288df11e18b4bd39de994f6657d83847bdffdbd7f346754b0f" +[[package]] +name = "r-efi" +version = "6.0.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8dcc9c7d52a811697d2151c701e0d08956f92b0e24136cf4cf27b57a6a0d9bf" + [[package]] name = "rand" version = "0.9.2" @@ -1308,6 +1411,15 @@ dependencies = [ "bitflags 1.3.2", ] +[[package]] +name = "redox_syscall" +version = "0.5.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ed2bf2547551a7053d6fdfafda3f938979645c44812fbfcda098faae3f1a362d" +dependencies = [ + "bitflags 2.10.0", +] + [[package]] name = "redox_syscall" version = "0.7.0" @@ -1417,12 +1529,33 @@ dependencies = [ "winapi-util", ] +[[package]] +name = "scc" +version = "2.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "46e6f046b7fef48e2660c57ed794263155d713de679057f2d0c169bfc6e756cc" +dependencies = [ + "sdd", +] + [[package]] name = "scopeguard" version = "1.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "94143f37725109f92c262ed2cf5e59bce7498c01bcc1502d7b9afe439a4e9f49" +[[package]] +name = "sdd" +version = "3.0.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "490dcfcbfef26be6800d11870ff2df8774fa6e86d047e3e8c8a76b25655e41ca" + +[[package]] +name = "semver" +version = "1.0.28" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8a7852d02fc848982e0c167ef163aaff9cd91dc640ba85e263cb1ce46fae51cd" + [[package]] name = "serde" version = "1.0.228" @@ -1479,6 +1612,32 @@ dependencies = [ "unsafe-libyaml", ] +[[package]] +name = "serial_test" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "911bd979bf1070a3f3aa7b691a3b3e9968f339ceeec89e08c280a8a22207a32f" +dependencies = [ + "futures-executor", + "futures-util", + "log", + "once_cell", + "parking_lot 0.12.5", + "scc", + "serial_test_derive", +] + +[[package]] +name = "serial_test_derive" +version = "3.4.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0a7d91949b85b0d2fb687445e448b40d322b6b3e4af6b44a29b21d9a5f33e6d9" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "sha2" version = "0.10.9" @@ -1528,6 +1687,12 @@ version = "2.7.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bbbb5d9659141646ae647b42fe094daf6c6192d1620870b449d9557f748b2daa" +[[package]] +name = "slab" +version = "0.4.12" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c790de23124f9ab44544d7ac05d60440adc586479ce501c1d6d7da3cd8c9cf5" + [[package]] name = "sled" version = "0.34.7" @@ -1541,7 +1706,7 @@ dependencies = [ "fxhash", "libc", "log", - "parking_lot", + "parking_lot 0.11.2", ] [[package]] @@ -1722,10 +1887,10 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "f6ccf251212114b54433ec949fd6a7841275f9ada20dddd2f29e9ceea4501493" [[package]] -name = "unindent" -version = "0.2.4" +name = "unicode-xid" +version = "0.2.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "7264e107f553ccae879d21fbea1d6724ac785e8c3bfc762137959b5802826ef3" +checksum = "ebc1c04c71510c7f702b52b7c350734c9ff1295c464a03335b00bb84fc54f853" [[package]] name = "unsafe-libyaml" @@ -1757,6 +1922,17 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.23.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ddd74a9687298c6858e9b88ec8935ec45d22e8fd5e6394fa1bd4e99a87789c76" +dependencies = [ + "getrandom 0.4.2", + "js-sys", + "wasm-bindgen", +] + [[package]] name = "value-trait" version = "0.12.1" @@ -1815,6 +1991,15 @@ dependencies = [ "wit-bindgen", ] +[[package]] +name = "wasip3" +version = "0.4.0+wasi-0.3.0-rc-2026-01-06" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5428f8bf88ea5ddc08faddef2ac4a67e390b88186c703ce6dbd955e1c145aca5" +dependencies = [ + "wit-bindgen", +] + [[package]] name = "wasm-bindgen" version = "0.2.108" @@ -1860,6 +2045,40 @@ dependencies = [ "unicode-ident", ] +[[package]] +name = "wasm-encoder" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "990065f2fe63003fe337b932cfb5e3b80e0b4d0f5ff650e6985b1048f62c8319" +dependencies = [ + "leb128fmt", + "wasmparser", +] + +[[package]] +name = "wasm-metadata" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "bb0e353e6a2fbdc176932bbaab493762eb1255a7900fe0fea1a2f96c296cc909" +dependencies = [ + "anyhow", + "indexmap", + "wasm-encoder", + "wasmparser", +] + +[[package]] +name = "wasmparser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "47b807c72e1bac69382b3a6fb3dbe8ea4c0ed87ff5629b8685ae6b9a611028fe" +dependencies = [ + "bitflags 2.10.0", + "hashbrown 0.15.5", + "indexmap", + "semver", +] + [[package]] name = "winapi" version = "0.3.9" @@ -2037,6 +2256,88 @@ name = "wit-bindgen" version = "0.51.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "d7249219f66ced02969388cf2bb044a09756a083d0fab1e566056b04d9fbcaa5" +dependencies = [ + "wit-bindgen-rust-macro", +] + +[[package]] +name = "wit-bindgen-core" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ea61de684c3ea68cb082b7a88508a8b27fcc8b797d738bfc99a82facf1d752dc" +dependencies = [ + "anyhow", + "heck", + "wit-parser", +] + +[[package]] +name = "wit-bindgen-rust" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b7c566e0f4b284dd6561c786d9cb0142da491f46a9fbed79ea69cdad5db17f21" +dependencies = [ + "anyhow", + "heck", + "indexmap", + "prettyplease", + "syn", + "wasm-metadata", + "wit-bindgen-core", + "wit-component", +] + +[[package]] +name = "wit-bindgen-rust-macro" +version = "0.51.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c0f9bfd77e6a48eccf51359e3ae77140a7f50b1e2ebfe62422d8afdaffab17a" +dependencies = [ + "anyhow", + "prettyplease", + "proc-macro2", + "quote", + "syn", + "wit-bindgen-core", + "wit-bindgen-rust", +] + +[[package]] +name = "wit-component" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d66ea20e9553b30172b5e831994e35fbde2d165325bec84fc43dbf6f4eb9cb2" +dependencies = [ + "anyhow", + "bitflags 2.10.0", + "indexmap", + "log", + "serde", + "serde_derive", + "serde_json", + "wasm-encoder", + "wasm-metadata", + "wasmparser", + "wit-parser", +] + +[[package]] +name = "wit-parser" +version = "0.244.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ecc8ac4bc1dc3381b7f59c34f00b67e18f910c2c0f50015669dde7def656a736" +dependencies = [ + "anyhow", + "id-arena", + "indexmap", + "log", + "semver", + "serde", + "serde_derive", + "serde_json", + "unicode-xid", + "wasmparser", +] [[package]] name = "writeable" diff --git a/Cargo.toml b/Cargo.toml index 0d4a2afd..156e3302 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -32,7 +32,7 @@ include = [ [dependencies] anyhow = "1.0.100" -clap = { version = "4.5.49", features = ["derive"] } +clap = { version = "4.5.49", features = ["derive", "env"] } serde = { version = "1.0.228", features = ["derive"] } serde_json = "1.0.145" unicode-segmentation = "1.12.0" @@ -45,11 +45,11 @@ ignore = "0.4" yaml-rust2 = "0.11" once_cell = "1.19" syntect = "5" +rayon = "1" regex = "1.11" clap_complete = "4.5.62" prunist = { path = "crates/prunist", version = "0.17.0" } - - +uuid = { version = "1", features = ["v4"] } [features] default = [] @@ -70,6 +70,7 @@ rand = "0.9" rand_chacha = "0.9" tempfile = "3" serde_yaml = "0.9" +serial_test = "3" [profile.release] # Prioritize runtime speed for large inputs diff --git a/README.md b/README.md index 60859657..84a441f2 100644 --- a/README.md +++ b/README.md @@ -32,6 +32,7 @@ Available as: - Multi-file mode: preview many files at once (paths, `--glob ...`, or `--recursive` on directories) with shared or per-file budgets - Repo-aware ordering: in git repos, frequent+recent files show up first (rarely touched files drift to the end; mtime fallback) - `grep`-like search and `tree`-like view: `--grep ` and `--tree` emulate the workflows while still summarizing file contents inline +- Explore sessions: with an active session (`hson explore start`), repeated runs under tight budgets surface fresh content instead of repeating the same preview - Fast: processes gigabyte‑scale files in seconds (mostly disk‑bound) - Available as a CLI app and as a Python library @@ -143,6 +144,7 @@ hson -n 20 src/main.py - [Budget modes](#budget-modes) - [Text mode](#text-mode) - [Source code support](#source-code-support) +- [Explore sessions](#explore-sessions) #### Common flags @@ -273,6 +275,43 @@ For source code files, headson uses an indentation-aware heuristic to build an o - Under tight budgets, it tends to keep block-introducing lines (like function/class headers) and omit less relevant blocks from the middle. - With colors enabled, you also get syntax highlighting and line numbers. +#### Explore sessions + +With an active explore session, `hson` remembers which leaves it already showed (its “breadcrumbs”) and penalizes them in priority scoring. Repeated runs under a tight budget surface fresh content instead of the same preview every time; under a loose budget previously-seen items still appear — the penalty is soft, never an exclusion. + +```bash +export HSON_SESSION=$(hson explore start) # create a session; prints a bare UUID +hson src/ -C 8000 --tree # first look +hson src/ -C 8000 --tree # same command, fresher content +hson explore status # id, label, step count, breadcrumbs, last activity +hson explore list # chronological query log +hson explore clear # forget breadcrumbs; keep the query log +``` + +Subcommands: + +- `hson explore start [label]`: create a new session and print its UUID to stdout (bare, so it can be assigned directly to `HSON_SESSION`). The label defaults to `Explore session started originally in `. +- `hson explore status`: show the active session: id, label, step count, breadcrumb count, and last activity. +- `hson explore clear`: clear the breadcrumbs and reset the step count; the query log and label are preserved. +- `hson explore list`: print the session’s query log in chronological order. + +Flags (on normal `hson` invocations): + +- `--session `: activate a session for this invocation only, overriding `HSON_SESSION`. +- `--no-record`: apply the session’s penalties but write nothing back (no breadcrumbs, no query log entry, no step increment). +- `--explore-decay `: per-step penalty decay factor (default: 0.5). With the default, content you saw a few steps ago is already mostly “forgiven”; use `1.0` for no decay (seen content keeps its full penalty for the rest of the session). +- `--explore-memory `: maximum breadcrumbs retained per session (default: 10000). Oldest entries are evicted first. + +Notes: + +- Sessions are only created by `hson explore start`. An unknown session ID — via `--session` or `HSON_SESSION` — is an error, so a typo can’t silently start a fresh session and lose the bias context of the one you meant. An empty `HSON_SESSION` is treated as unset, and an invalid one never blocks `hson explore start` (you only get a warning). +- The `explore` subcommand shadows an input literally named `explore`: `hson explore` in a directory containing `explore/` shows the subcommand help. Use `hson ./explore` to preview such a file or directory. +- Stdin input is never tracked: no breadcrumbs are recorded and no penalties apply. +- Session state lives at `$XDG_STATE_HOME/headson/sessions/.json` (falling back to `~/.local/state` when `XDG_STATE_HOME` is unset). +- Breadcrumbs are content-addressed: when a value changes on disk, its old breadcrumb stops matching and the penalty disappears automatically. +- Breadcrumbs are also keyed by the file’s resolved absolute path, so identical values in two different files never share a penalty, and the same file stays recognized across working directories and invocation styles. +- Concurrent invocations on the same session are safe (a session lock plus atomic writes). + Show help: ```sh diff --git a/python/Cargo.toml b/python/Cargo.toml index 2bed38b3..c275b645 100644 --- a/python/Cargo.toml +++ b/python/Cargo.toml @@ -10,6 +10,6 @@ crate-type = ["cdylib"] [dependencies] anyhow = "1" -pyo3 = { version = "0.27", features = ["extension-module", "abi3-py310"] } +pyo3 = { version = "0.29", features = ["extension-module", "abi3-py310"] } headson_core = { package = "headson", path = ".." } regex = "1" diff --git a/src/cli/args.rs b/src/cli/args.rs index bedb31f1..d818004b 100644 --- a/src/cli/args.rs +++ b/src/cli/args.rs @@ -1,8 +1,38 @@ use std::path::PathBuf; -use clap::{ArgAction, Parser, ValueEnum}; +use clap::{ArgAction, Parser, Subcommand, ValueEnum}; use clap_complete::Shell; +fn parse_session_id(s: &str) -> Result { + uuid::Uuid::parse_str(s) + .map(|u| u.to_string()) + .map_err(|e| format!("invalid session ID (must be a UUID): {e}")) +} + +fn parse_explore_decay(s: &str) -> Result { + let alpha: f64 = s + .parse() + .map_err(|e| format!("invalid decay factor: {e}"))?; + if alpha > 0.0 && alpha <= 1.0 { + Ok(alpha) + } else { + Err(format!( + "decay factor must satisfy 0.0 < ALPHA <= 1.0 (got {alpha})" + )) + } +} + +fn parse_explore_memory(s: &str) -> Result { + let n: usize = s + .parse() + .map_err(|e| format!("invalid breadcrumb capacity: {e}"))?; + if n >= 1 { + Ok(n) + } else { + Err("breadcrumb capacity must be at least 1".to_string()) + } +} + /// Top-level CLI flags and enums. #[derive(Parser, Debug)] #[command( @@ -268,6 +298,44 @@ pub struct Cli { help_heading = "Filtering" )] pub grep_show: GrepShowArg, + // HSON_SESSION is intentionally NOT wired through clap's `env` attribute: + // env resolution happens in `session_middleware::resolve_session_id` so an + // empty value acts as unset and an invalid value cannot fail parsing for + // every invocation (including `hson explore start`, the escape hatch). + #[arg( + long = "session", + value_name = "SESSION_ID", + global = true, + value_parser = parse_session_id, + help = "Activate an explore session by ID (UUID). Falls back to a non-empty HSON_SESSION environment variable.", + help_heading = "Explore" + )] + pub session: Option, + #[arg( + long = "no-record", + action = ArgAction::SetTrue, + help = "Apply session penalty without recording breadcrumbs or incrementing step count.", + help_heading = "Explore" + )] + pub no_record: bool, + #[arg( + long = "explore-decay", + value_name = "ALPHA", + default_value_t = crate::cli::session_middleware::DEFAULT_ALPHA, + value_parser = parse_explore_decay, + help = "Decay factor per step for session novelty penalties (0 < ALPHA <= 1; 1.0 = no decay). Only takes effect with an active session.", + help_heading = "Explore" + )] + pub explore_decay: f64, + #[arg( + long = "explore-memory", + value_name = "N", + default_value_t = crate::cli::session_middleware::BREADCRUMB_CAP, + value_parser = parse_explore_memory, + help = "Maximum breadcrumbs retained per session; older entries are evicted. Only takes effect with an active session.", + help_heading = "Explore" + )] + pub explore_memory: usize, #[arg( long = "completions", value_name = "SHELL", @@ -275,6 +343,35 @@ pub struct Cli { help = "Print shell completions for the given shell" )] pub completions: Option, + #[command(subcommand)] + pub subcommand: Option, +} + +#[derive(Subcommand, Debug)] +pub enum TopSubcommand { + /// Manage explore sessions (novelty-bias mode) + Explore(ExploreArgs), +} + +#[derive(clap::Args, Debug)] +pub struct ExploreArgs { + #[command(subcommand)] + pub command: ExploreSubcommand, +} + +#[derive(Subcommand, Debug)] +pub enum ExploreSubcommand { + /// Start a new explore session and print the session ID to stdout + Start { + /// Optional human-readable label for this session + label: Option, + }, + /// Show the current session status + Status, + /// Clear breadcrumb memory (query log and label are preserved) + Clear, + /// Print the query log for the current session in chronological order + List, } #[derive(Copy, Clone, Debug, ValueEnum)] @@ -403,3 +500,136 @@ pub fn print_completions( &mut std::io::stdout(), ); } + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn session_flag_can_appear_after_explore_subcommand() { + use clap::Parser; + let result = Cli::try_parse_from([ + "hson", + "explore", + "status", + "--session", + "00000000-0000-0000-0000-000000000000", + ]); + assert!( + result.is_ok(), + "--session must be accepted after `explore status`; got: {:?}", + result.err() + ); + let cli = result.unwrap(); + assert_eq!( + cli.session.as_deref(), + Some("00000000-0000-0000-0000-000000000000") + ); + } + + #[test] + fn non_uuid_session_value_fails_to_parse() { + use clap::Parser; + let result = + Cli::try_parse_from(["hson", "--session", "not-a-uuid", "file"]); + assert!( + result.is_err(), + "non-UUID session value must fail at parse time; got Ok with cli.session={:?}", + result.ok().and_then(|c| c.session) + ); + } + + #[test] + fn empty_session_value_fails_to_parse() { + use clap::Parser; + let result = Cli::try_parse_from(["hson", "--session", "", "file"]); + assert!( + result.is_err(), + "empty session value must fail at parse time" + ); + } + + #[test] + fn path_traversal_session_value_fails_to_parse() { + use clap::Parser; + let result = Cli::try_parse_from([ + "hson", + "--session", + "../../etc/passwd", + "file", + ]); + assert!( + result.is_err(), + "session value containing path separators must fail at parse time" + ); + } + + #[test] + fn explore_decay_and_memory_use_documented_defaults() { + use clap::Parser; + let cli = Cli::try_parse_from(["hson", "file"]) + .expect("plain invocation must parse"); + assert!( + (cli.explore_decay - 0.5).abs() < f64::EPSILON, + "default --explore-decay must be 0.5; got {}", + cli.explore_decay + ); + assert_eq!( + cli.explore_memory, 10_000, + "default --explore-memory must be 10000" + ); + } + + #[test] + fn explore_decay_rejects_out_of_range_values() { + use clap::Parser; + for bad in ["0", "0.0", "1.5", "nan"] { + let result = + Cli::try_parse_from(["hson", "--explore-decay", bad, "file"]); + assert!( + result.is_err(), + "--explore-decay {bad} must fail at parse time" + ); + } + } + + #[test] + fn explore_decay_accepts_boundary_and_interior_values() { + use clap::Parser; + for good in ["1.0", "0.5", "0.001"] { + let result = + Cli::try_parse_from(["hson", "--explore-decay", good, "file"]); + assert!( + result.is_ok(), + "--explore-decay {good} must parse; got: {:?}", + result.err() + ); + } + } + + #[test] + fn explore_memory_rejects_zero_and_accepts_one() { + use clap::Parser; + assert!( + Cli::try_parse_from(["hson", "--explore-memory", "0", "file"]) + .is_err(), + "--explore-memory 0 must fail at parse time" + ); + let cli = + Cli::try_parse_from(["hson", "--explore-memory", "1", "file"]) + .expect("--explore-memory 1 must parse"); + assert_eq!(cli.explore_memory, 1); + } + + #[test] + fn valid_uuid_session_value_parses_successfully() { + use clap::Parser; + let result = Cli::try_parse_from([ + "hson", + "--session", + "00000000-0000-0000-0000-000000000000", + "file", + ]); + assert!(result.is_ok(), "valid UUID must parse: {:?}", result.err()); + } +} diff --git a/src/cli/explore.rs b/src/cli/explore.rs new file mode 100644 index 00000000..f0ca9087 --- /dev/null +++ b/src/cli/explore.rs @@ -0,0 +1,752 @@ +use anyhow::Result; +use uuid::Uuid; + +use crate::cli::args::{Cli, ExploreSubcommand}; +use crate::cli::session_middleware::{ + require_session_exists, resolve_session_id, session_file_path, +}; + +const NO_SESSION_MSG: &str = + "No active session. Run `hson explore start` to begin."; + +fn load_active_session( + session_id: Option<&str>, +) -> Result> { + let Some(id) = session_id else { + return Ok(None); + }; + let path = session_file_path(id)?; + // `require_session_exists` runs before this on every path; a missing file + // here is a race (deleted in between) and keeps the friendly "no session" + // behavior. A file that exists but fails to parse is corruption — surface + // it loudly instead of masking it as "no active session". + if !path.exists() { + return Ok(None); + } + let session = crate::session::io::load_from_path(&path).map_err(|e| { + anyhow::anyhow!( + "session file {} is corrupt or unreadable: {e}", + path.display() + ) + })?; + Ok(Some((id.to_string(), session))) +} + +/// Format argv for display: relativize paths that are under `cwd`, and +/// for argv[0] (the binary) use just the filename when not under `cwd`. +fn display_argv_relative(argv: &[String], cwd: &str) -> String { + let cwd_path = std::path::Path::new(cwd); + argv.iter() + .enumerate() + .map(|(i, arg)| { + let p = std::path::Path::new(arg); + if let Ok(rel) = p.strip_prefix(cwd_path) { + rel.to_string_lossy().into_owned() + } else if i == 0 { + p.file_name() + .map(|n| n.to_string_lossy().into_owned()) + .unwrap_or_else(|| arg.clone()) + } else { + arg.clone() + } + }) + .collect::>() + .join(" ") +} + +pub(crate) fn run_subcommand( + cmd: &ExploreSubcommand, + cli: &Cli, +) -> Result { + match cmd { + ExploreSubcommand::Start { label } => { + // `explore start` is the escape hatch out of a broken + // HSON_SESSION: an invalid env value must never block creating a + // fresh session, so surface it as a warning only. + if let Err(e) = resolve_session_id(cli) { + eprintln!("warning: {e}"); + } + let id = Uuid::new_v4().to_string(); + let path = session_file_path(&id)?; + if let Some(parent) = path.parent() { + let _ = std::fs::create_dir_all(parent); + } + let cwd = std::env::current_dir() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + let resolved_label = match label { + Some(l) => l.clone(), + None => format!("Explore session started originally in {cwd}"), + }; + let session = + crate::session::Session::new(id.clone(), resolved_label); + crate::session::io::save_to_path(&session, &path).map_err( + |e| anyhow::anyhow!("failed to create session file: {e}"), + )?; + Ok(id) + } + ExploreSubcommand::Status => { + let active = resolve_session_id(cli)?; + require_session_exists(active.as_deref())?; + let Some((_, session)) = load_active_session(active.as_deref())? + else { + return Ok(NO_SESSION_MSG.to_string()); + }; + let last_active = session + .queries + .last() + .map_or("never", |q| q.timestamp.as_str()); + Ok(format!( + "Session: {}\n\ + Label: {}\n\ + Steps: {}\n\ + Breadcrumbs: {}\n\ + Last active: {}", + session.id, + session.label, + session.step_count, + session.breadcrumbs.len(), + last_active, + )) + } + ExploreSubcommand::Clear => { + let active = resolve_session_id(cli)?; + require_session_exists(active.as_deref())?; + let Some(id) = active else { + return Ok(NO_SESSION_MSG.to_string()); + }; + let path = session_file_path(&id)?; + // Clearing is a read-modify-write; hold the session lock so a + // concurrent `record_step_atomic` writer can't be lost between + // our load and save. + let _lock = crate::session::io::acquire_session_lock(&path) + .map_err(|e| anyhow::anyhow!("failed to lock session: {e}"))?; + let Some((_, mut session)) = load_active_session(Some(&id))? + else { + return Ok(NO_SESSION_MSG.to_string()); + }; + session.clear(); + crate::session::io::save_to_path(&session, &path) + .map_err(|e| anyhow::anyhow!("failed to save session: {e}"))?; + Ok(String::new()) + } + ExploreSubcommand::List => { + let active = resolve_session_id(cli)?; + require_session_exists(active.as_deref())?; + let Some((_, session)) = load_active_session(active.as_deref())? + else { + return Ok(NO_SESSION_MSG.to_string()); + }; + let lines: Vec = session + .queries + .iter() + .map(|q| { + let display_argv = display_argv_relative(&q.argv, &q.cwd); + format!("[{}] {} {}", q.timestamp, q.cwd, display_argv) + }) + .collect(); + Ok(lines.join("\n")) + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::cli::args::ExploreSubcommand; + use crate::session::io::{load_from_path, save_to_path}; + use crate::session::{QueryEntry, Session}; + use clap::Parser; + use serial_test::serial; + use tempfile::tempdir; + + use crate::cli::test_helpers::IsolatedEnv; + + /// Build a minimal Cli with an optional --session value and no inputs. + /// XDG_STATE_HOME must already be set before this is called. + fn make_cli(session_id: Option<&str>) -> Cli { + let mut args = vec!["hson"]; + if let Some(id) = session_id { + args.push("--session"); + args.push(id); + } + Cli::parse_from(args) + } + + /// Step 36: `explore start` with no label returns a bare UUID string — no + /// trailing whitespace or newline, matches the canonical UUID pattern. + #[test] + #[serial] + fn explore_start_returns_uuid_string() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(None); + let result = + run_subcommand(&ExploreSubcommand::Start { label: None }, &cli); + + let output = result.expect("run_subcommand(Start) must succeed"); + let uuid_re = regex::Regex::new( + r"^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$", + ) + .unwrap(); + assert!( + uuid_re.is_match(output.trim()), + "expected UUID string (no trailing whitespace/newline), got: {output:?}" + ); + assert_eq!( + output, + output.trim(), + "output must have no leading/trailing whitespace; got: {output:?}" + ); + } + + /// Step 37: `explore start --label "my label"` stores `label = Some("my label")` + /// in the session file on disk. + #[test] + #[serial] + fn explore_start_with_label_stores_label_in_session_file() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(None); + let result = run_subcommand( + &ExploreSubcommand::Start { + label: Some("my label".to_string()), + }, + &cli, + ); + + let output = result.expect("run_subcommand(Start) must succeed"); + + // Load the session file that was just created + let session_path = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{output}.json")); + + assert!( + session_path.exists(), + "session file must exist at {session_path:?}" + ); + let session = load_from_path(&session_path) + .expect("session file must be valid JSON"); + assert_eq!( + session.label, "my label", + "session.label must be 'my label'; got: {:?}", + session.label + ); + } + + /// Step 38: `explore start` with no label stores a label that contains the + /// last path component of the current working directory. + #[test] + #[serial] + fn explore_start_no_label_stores_cwd_derived_label() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cwd = std::env::current_dir() + .expect("must be able to get cwd") + .to_string_lossy() + .into_owned(); + let last_component = std::path::Path::new(&cwd) + .file_name() + .expect("cwd must have a last component") + .to_string_lossy() + .into_owned(); + + let cli = make_cli(None); + let result = + run_subcommand(&ExploreSubcommand::Start { label: None }, &cli); + + let output = result.expect("run_subcommand(Start) must succeed"); + + let session_path = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{output}.json")); + + assert!( + session_path.exists(), + "session file must exist at {session_path:?}" + ); + let session = load_from_path(&session_path) + .expect("session file must be valid JSON"); + assert!( + session.label.contains(&last_component), + "session.label must contain last cwd component '{last_component}'; got: {:?}", + session.label + ); + } + + /// Step 39: `explore status` with a session containing step_count=3 returns + /// a string that mentions the step count ("3") and the session ID. + #[test] + #[serial] + fn explore_status_shows_step_count_and_uuid() { + let state_dir = tempdir().unwrap(); + let session_id = "39000000-0000-0000-0000-000000000000"; + + // Write a session file with step_count=3 + let sessions_dir = state_dir.path().join("headson").join("sessions"); + std::fs::create_dir_all(&sessions_dir).unwrap(); + let session_path = sessions_dir.join(format!("{session_id}.json")); + let mut session = + Session::new(session_id.to_string(), "test label".to_string()); + session.step_count = 3; + save_to_path(&session, &session_path).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(Some(session_id)); + let result = run_subcommand(&ExploreSubcommand::Status, &cli); + + let output = result.expect("run_subcommand(Status) must succeed"); + // The fixture session ID contains '3' too, so assert the exact + // labeled line rather than a bare contains('3'). + assert!( + output.lines().any(|l| l == "Steps: 3"), + "status output must contain the labeled step count line \ + 'Steps: 3'; got: {output:?}" + ); + assert!( + output.contains(session_id), + "status output must contain the session ID '{session_id}'; got: {output:?}" + ); + } + + /// Step 40: `explore status` with no active session returns Ok and a helpful + /// message that includes "start" (suggesting `hson explore start`). + #[test] + #[serial] + fn explore_status_no_session_prints_helpful_message() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(None); + let result = run_subcommand(&ExploreSubcommand::Status, &cli); + + let output = result.expect( + "run_subcommand(Status) with no session must return Ok, not Err", + ); + assert!( + output.contains("start"), + "no-session status output must contain 'start' (suggesting hson explore start); got: {output:?}" + ); + } + + /// Step 41: `explore clear` zeroes breadcrumbs and step_count but + /// PRESERVES the query log, label, and id — matching the `clear` help + /// text ("query log and label are preserved"). + #[test] + #[serial] + fn explore_clear_zeroes_breadcrumbs_and_step_count_preserves_queries() { + let state_dir = tempdir().unwrap(); + let session_id = "41000000-0000-0000-0000-000000000000"; + + // Create a session with breadcrumbs and a query + let sessions_dir = state_dir.path().join("headson").join("sessions"); + std::fs::create_dir_all(&sessions_dir).unwrap(); + let session_path = sessions_dir.join(format!("{session_id}.json")); + let mut session = + Session::new(session_id.to_string(), "clear test".to_string()); + session.record_breadcrumb("a.json", "users.0", 1); + session.record_breadcrumb("b.json", "items.1", 2); + session.queries.push(QueryEntry { + step: 1, + timestamp: "2026-01-01T00:00:00Z".to_string(), + cwd: "/home/user/project".to_string(), + argv: vec!["src/".to_string()], + }); + save_to_path(&session, &session_path).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(Some(session_id)); + let result = run_subcommand(&ExploreSubcommand::Clear, &cli); + + result.expect("run_subcommand(Clear) must succeed"); + + // Reload and verify + let updated = load_from_path(&session_path) + .expect("session file must still be valid after clear"); + assert!( + updated.breadcrumbs.is_empty(), + "breadcrumbs must be empty after clear; got: {:?}", + updated.breadcrumbs + ); + assert_eq!( + updated.step_count, 0, + "step_count must be 0 after clear; got: {}", + updated.step_count + ); + assert_eq!( + updated.queries.len(), + 1, + "query log must be preserved by clear; got: {:?}", + updated.queries + ); + assert_eq!( + updated.id, session_id, + "session id must be preserved by clear" + ); + assert_eq!( + updated.label, "clear test", + "session label must be preserved by clear" + ); + } + + /// Issue #513: a session file that EXISTS but fails to parse must produce + /// a clear error naming the file path — not the misleading "No active + /// session" success that masks corruption. + #[test] + #[serial] + fn corrupt_session_file_errors_and_mentions_path() { + let state_dir = tempdir().unwrap(); + let session_id = "55555555-5555-5555-5555-555555555555"; + + let sessions_dir = state_dir.path().join("headson").join("sessions"); + std::fs::create_dir_all(&sessions_dir).unwrap(); + let session_path = sessions_dir.join(format!("{session_id}.json")); + std::fs::write(&session_path, "{not valid json").unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(Some(session_id)); + let result = run_subcommand(&ExploreSubcommand::Status, &cli); + + let err = result.expect_err( + "explore status with a corrupt session file must return Err, \ + not the friendly no-session message", + ); + let msg = format!("{err:#}"); + assert!( + msg.contains(session_path.to_str().unwrap()), + "error must mention the corrupt session file path \ + {session_path:?}; got: {msg}" + ); + } + + /// Regression (issue #513): `explore status` with an unknown session ID + /// must error (e.g. "Session '' not found. Run `hson explore start` + /// to create one.") instead of silently treating the unknown ID as a + /// brand-new empty session. + #[test] + #[serial] + fn unknown_session_id_in_explore_status_errors() { + let state_dir = tempdir().unwrap(); + + let unknown_id = "22222222-3333-4444-5555-666666666666"; + let _env = IsolatedEnv::new(state_dir.path(), Some(unknown_id)); + + let cli = make_cli(None); + let result = run_subcommand(&ExploreSubcommand::Status, &cli); + + assert!( + result.is_err(), + "explore status with an unknown session ID must return Err; \ + today it silently fabricates an empty session and returns Ok. \ + got: {result:?}" + ); + } + + /// Regression (issue #513): `explore list` with an unknown session ID + /// must error instead of silently treating the unknown ID as a brand-new + /// empty session and returning an empty query list. + #[test] + #[serial] + fn unknown_session_id_in_explore_list_errors() { + let state_dir = tempdir().unwrap(); + + let unknown_id = "33333333-4444-5555-6666-777777777777"; + let _env = IsolatedEnv::new(state_dir.path(), Some(unknown_id)); + + let cli = make_cli(None); + let result = run_subcommand(&ExploreSubcommand::List, &cli); + + assert!( + result.is_err(), + "explore list with an unknown session ID must return Err; \ + today it silently fabricates an empty session and returns Ok. \ + got: {result:?}" + ); + } + + /// Regression (issue #513): `explore clear` with an unknown session ID + /// must error instead of silently materializing a session file at that + /// path. The user likely typo'd; auto-creating overwrites their intent. + #[test] + #[serial] + fn unknown_session_id_in_explore_clear_errors() { + let state_dir = tempdir().unwrap(); + // Pre-create the sessions directory so save_to_path won't fail for an + // unrelated reason (missing parent dir). We want the test to exercise + // the unknown-ID semantic, not a filesystem-error red herring. + let sessions_dir = state_dir.path().join("headson").join("sessions"); + std::fs::create_dir_all(&sessions_dir).unwrap(); + + let unknown_id = "44444444-5555-6666-7777-888888888888"; + let _env = IsolatedEnv::new(state_dir.path(), Some(unknown_id)); + + let cli = make_cli(None); + let result = run_subcommand(&ExploreSubcommand::Clear, &cli); + + let expected = sessions_dir.join(format!("{unknown_id}.json")); + assert!( + result.is_err(), + "explore clear with an unknown session ID must error; got: {result:?}" + ); + assert!( + !expected.exists(), + "explore clear with an unknown session ID must not create the \ + session file at {expected:?}" + ); + } + + /// Step 42: `explore list` returns all recorded queries in chronological + /// order (ascending timestamp / step order). + #[test] + #[serial] + fn explore_list_prints_queries_chronologically() { + let state_dir = tempdir().unwrap(); + let session_id = "42000000-0000-0000-0000-000000000000"; + + // Create a session with 3 queries at distinct timestamps + let sessions_dir = state_dir.path().join("headson").join("sessions"); + std::fs::create_dir_all(&sessions_dir).unwrap(); + let session_path = sessions_dir.join(format!("{session_id}.json")); + let mut session = + Session::new(session_id.to_string(), "list test".to_string()); + session.queries.push(QueryEntry { + step: 1, + timestamp: "2026-01-01T10:00:00Z".to_string(), + cwd: "/home/user/alpha".to_string(), + argv: vec![], + }); + session.queries.push(QueryEntry { + step: 2, + timestamp: "2026-01-01T11:00:00Z".to_string(), + cwd: "/home/user/beta".to_string(), + argv: vec![], + }); + session.queries.push(QueryEntry { + step: 3, + timestamp: "2026-01-01T12:00:00Z".to_string(), + cwd: "/home/user/gamma".to_string(), + argv: vec![], + }); + save_to_path(&session, &session_path).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(Some(session_id)); + let result = run_subcommand(&ExploreSubcommand::List, &cli); + + let output = result.expect("run_subcommand(List) must succeed"); + + // All 3 cwd values must appear + assert!( + output.contains("/home/user/alpha"), + "output must contain '/home/user/alpha'; got: {output:?}" + ); + assert!( + output.contains("/home/user/beta"), + "output must contain '/home/user/beta'; got: {output:?}" + ); + assert!( + output.contains("/home/user/gamma"), + "output must contain '/home/user/gamma'; got: {output:?}" + ); + + // Chronological order: alpha before beta before gamma + let pos_alpha = output + .find("/home/user/alpha") + .expect("alpha must be in output"); + let pos_beta = output + .find("/home/user/beta") + .expect("beta must be in output"); + let pos_gamma = output + .find("/home/user/gamma") + .expect("gamma must be in output"); + assert!( + pos_alpha < pos_beta, + "alpha (step 1) must appear before beta (step 2) in output; alpha@{pos_alpha}, beta@{pos_beta}" + ); + assert!( + pos_beta < pos_gamma, + "beta (step 2) must appear before gamma (step 3) in output; beta@{pos_beta}, gamma@{pos_gamma}" + ); + } + + /// Issue #513: `explore status` must surface the number of breadcrumbs in + /// the active session so users can gauge how much novelty bias has built + /// up. The existing output only shows Session/Label/Steps and hides this. + #[test] + #[serial] + fn explore_status_shows_breadcrumb_count() { + let state_dir = tempdir().unwrap(); + let session_id = "33333333-3333-3333-3333-333333333333"; + + let path = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{session_id}.json")); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut session = + Session::new(session_id.to_string(), "lbl".to_string()); + session.record_breadcrumb("a.json", "x", 1); + session.record_breadcrumb("b.json", "y", 2); + session.record_breadcrumb("c.json", "z", 3); + session.step_count = 3; + save_to_path(&session, &path).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), Some(session_id)); + + let cli = make_cli(Some(session_id)); + let out = run_subcommand(&ExploreSubcommand::Status, &cli) + .expect("run_subcommand(Status) must succeed"); + + // The fixture session ID contains '3' too, so assert the exact + // labeled line rather than a bare contains('3'). + assert!( + out.lines().any(|l| l == "Breadcrumbs: 3"), + "status output must include the labeled breadcrumb count line \ + 'Breadcrumbs: 3'; got:\n{out}" + ); + } + + /// Issue #513: `explore status` must surface the timestamp of the most + /// recent recorded query so users can tell when a session was last active. + #[test] + #[serial] + fn explore_status_shows_last_active_timestamp() { + let state_dir = tempdir().unwrap(); + let session_id = "44444444-4444-4444-4444-444444444444"; + + let path = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{session_id}.json")); + std::fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut session = + Session::new(session_id.to_string(), "lbl".to_string()); + session.record_query("2026-05-08T12:34:56Z", "/cwd", &[]); + save_to_path(&session, &path).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), Some(session_id)); + + let cli = make_cli(Some(session_id)); + let out = run_subcommand(&ExploreSubcommand::Status, &cli) + .expect("run_subcommand(Status) must succeed"); + + assert!( + out.contains("Last active") || out.contains("last active"), + "status output must include a Last active line; got:\n{out}" + ); + assert!( + out.contains("2026-05-08T12:34:56Z"), + "status output must include the most recent query timestamp; got:\n{out}" + ); + } + + /// Escape hatch: an invalid (non-UUID) HSON_SESSION must NOT block + /// `hson explore start` — it's the very command users need to fix their + /// environment. At most a warning goes to stderr. + #[test] + #[serial] + fn invalid_hson_session_env_does_not_break_explore_start() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), Some("not-a-uuid")); + + let cli = make_cli(None); + let result = + run_subcommand(&ExploreSubcommand::Start { label: None }, &cli); + + let output = result.expect( + "explore start must succeed despite an invalid HSON_SESSION", + ); + let session_path = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{output}.json")); + assert!( + session_path.exists(), + "explore start must create the new session file at {session_path:?}" + ); + } + + /// An invalid HSON_SESSION must produce a clear error naming HSON_SESSION + /// for `explore status` (and the other read subcommands). + #[test] + #[serial] + fn invalid_hson_session_env_errors_in_explore_status() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), Some("not-a-uuid")); + + let cli = make_cli(None); + let result = run_subcommand(&ExploreSubcommand::Status, &cli); + + let err = result.expect_err( + "explore status with an invalid HSON_SESSION must error", + ); + let msg = format!("{err:#}"); + assert!( + msg.contains("HSON_SESSION"), + "error must name HSON_SESSION; got: {msg}" + ); + } + + /// An empty HSON_SESSION is treated as unset: `explore status` reports + /// "no active session" instead of failing. + #[test] + #[serial] + fn empty_hson_session_env_is_unset_for_explore_status() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), Some("")); + + let cli = make_cli(None); + let result = run_subcommand(&ExploreSubcommand::Status, &cli); + + let output = result + .expect("explore status with empty HSON_SESSION must succeed"); + assert_eq!( + output, NO_SESSION_MSG, + "empty HSON_SESSION must behave exactly as unset" + ); + } + + /// `explore clear` must release its session lock: a leftover `.lock` + /// sibling would block all subsequent recording on the session. + #[test] + #[serial] + fn explore_clear_releases_session_lock() { + let state_dir = tempdir().unwrap(); + let session_id = "43000000-0000-0000-0000-000000000000"; + + let sessions_dir = state_dir.path().join("headson").join("sessions"); + std::fs::create_dir_all(&sessions_dir).unwrap(); + let session_path = sessions_dir.join(format!("{session_id}.json")); + let session = + Session::new(session_id.to_string(), "lock test".to_string()); + save_to_path(&session, &session_path).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = make_cli(Some(session_id)); + run_subcommand(&ExploreSubcommand::Clear, &cli) + .expect("run_subcommand(Clear) must succeed"); + + let lock_path = session_path.with_extension("lock"); + assert!( + !lock_path.exists(), + "session lock must be released after clear; found {lock_path:?}" + ); + } +} diff --git a/src/cli/mod.rs b/src/cli/mod.rs index ad4073ca..5e3e9d0e 100644 --- a/src/cli/mod.rs +++ b/src/cli/mod.rs @@ -1,3 +1,8 @@ pub mod args; pub mod budget; +pub(crate) mod explore; pub mod run; +pub(crate) mod session_middleware; +#[cfg(test)] +pub(crate) mod test_helpers; +pub(crate) mod timestamp; diff --git a/src/cli/run.rs b/src/cli/run.rs index 8cb07b5b..d01ccb58 100644 --- a/src/cli/run.rs +++ b/src/cli/run.rs @@ -4,6 +4,10 @@ use std::fs::File; use std::io::{self, Read}; use std::path::{Path, PathBuf}; +use crate::cli::session_middleware::{ + maybe_record_session, require_session_exists, resolve_session_id, +}; + use anyhow::{Context, Result, bail}; use content_inspector::{ContentType, inspect}; use ignore::WalkBuilder; @@ -18,6 +22,12 @@ use crate::sorting::sort_paths_for_fileset; type InputEntry = (String, Vec); type InputEntries = Vec; pub(crate) type CliWarnings = Vec; +type RenderResult = ( + String, + CliWarnings, + Option, + Vec, +); fn build_grep_config_from_cli( cli: &Cli, @@ -41,13 +51,15 @@ fn build_effective_configs( cli: &Cli, mut render_cfg: headson::RenderConfig, input_count: usize, + explore_ctx: Option<&headson::ExploreContext>, ) -> ( headson::RenderConfig, headson::PriorityConfig, headson::Budgets, ) { let effective = budget::compute_effective(cli, input_count); - let prio = budget::build_priority_config(cli, &effective); + let mut prio = budget::build_priority_config(cli, &effective); + prio.explore = explore_ctx.cloned(); render_cfg = budget::render_config_for_budgets(render_cfg, &effective); (render_cfg, prio, effective.budgets) } @@ -56,12 +68,53 @@ fn needs_fileset(cli: &Cli, inputs_len: usize) -> bool { inputs_len > 1 || cli.tree } +fn load_explore_context( + session_id: Option<&str>, + alpha: f64, +) -> Option { + let id = session_id?; + let path = match crate::cli::session_middleware::session_file_path(id) { + Ok(p) => p, + Err(e) => { + eprintln!( + "warning: cannot resolve session path, ignoring session: {e}" + ); + return None; + } + }; + let session = if path.exists() { + match crate::session::io::load_from_path(&path) { + Ok(s) => s, + Err(e) => { + eprintln!("warning: session file unreadable, ignoring: {e}"); + return None; + } + } + } else { + return None; + }; + let breadcrumbs = session.breadcrumbs; + Some(headson::ExploreContext { + breadcrumbs, + current_step: session.step_count + 1, + alpha, + // Filled in per input: single-file renders resolve their path into + // it; fileset slots resolve their own paths from input names. + file: None, + }) +} + pub(crate) fn run(cli: &Cli) -> Result<(String, CliWarnings)> { budget::validate(cli)?; + let session_id = resolve_session_id(cli)?; + require_session_exists(session_id.as_deref())?; let render_cfg = get_render_config_from(cli); let grep_cfg = build_grep_config_from_cli(cli)?; let resolved_inputs = resolve_inputs(cli)?; - let (out, mut warnings, match_summary) = if resolved_inputs.is_empty() { + let explore_ctx = + load_explore_context(session_id.as_deref(), cli.explore_decay); + let from_stdin = resolved_inputs.is_empty(); + let (out, mut warnings, match_summary, shown_leaves) = if from_stdin { if !cli.globs.is_empty() || cli.recursive { return Ok(( String::new(), @@ -71,9 +124,16 @@ pub(crate) fn run(cli: &Cli) -> Result<(String, CliWarnings)> { if cli.tree { bail!("--tree requires file inputs; stdin mode is not supported"); } - run_from_stdin(cli, &render_cfg, &grep_cfg)? + let (text, w, ms) = run_from_stdin(cli, &render_cfg, &grep_cfg)?; + (text, w, ms, vec![]) } else { - run_from_paths(cli, &render_cfg, &grep_cfg, &resolved_inputs)? + run_from_paths( + cli, + &render_cfg, + &grep_cfg, + &resolved_inputs, + explore_ctx.as_ref(), + )? }; if cli.count_matches { if let Some(summary) = match_summary { @@ -83,6 +143,12 @@ pub(crate) fn run(cli: &Cli) -> Result<(String, CliWarnings)> { )); } } + maybe_record_session( + cli, + session_id.as_deref(), + from_stdin, + &shown_leaves, + ); Ok((out, warnings)) } @@ -113,9 +179,10 @@ fn run_from_stdin( let input_count = 1usize; let mut cfg = render_cfg.clone(); cfg.template = resolve_effective_template_for_stdin(cli.format, cfg.style); - let (cfg, prio, budgets) = build_effective_configs(cli, cfg, input_count); + let (cfg, prio, budgets) = + build_effective_configs(cli, cfg, input_count, None); let chosen_input = cli.input_format.unwrap_or(InputFormat::Json); - let (out, warnings, match_summary) = render_single_input( + let (out, warnings, match_summary, _) = render_single_input( chosen_input, input_bytes, &cfg, @@ -126,34 +193,57 @@ fn run_from_stdin( Ok((out, warnings, match_summary)) } -fn run_from_paths( +fn prepare_file_entries( cli: &Cli, - render_cfg: &headson::RenderConfig, - grep_cfg: &headson::GrepConfig, inputs: &[PathBuf], -) -> Result<(String, CliWarnings, Option)> { +) -> Result<(InputEntries, CliWarnings)> { let sorted_inputs = if needs_fileset(cli, inputs.len()) && !cli.no_sort { sort_paths_for_fileset(inputs) } else { inputs.to_vec() }; if std::env::var_os("HEADSON_FRECEN_TRACE").is_some() { - eprintln!("run_from_paths sorted_inputs={sorted_inputs:?}"); + eprintln!("prepare_file_entries sorted_inputs={sorted_inputs:?}"); } let (entries, warnings) = ingest_paths(&sorted_inputs)?; if std::env::var_os("HEADSON_FRECEN_TRACE").is_some() { eprintln!( - "run_from_paths ingested={:?}", + "prepare_file_entries ingested={:?}", entries.iter().map(|(n, _)| n).collect::>() ); } + Ok((entries, warnings)) +} + +fn run_from_paths( + cli: &Cli, + render_cfg: &headson::RenderConfig, + grep_cfg: &headson::GrepConfig, + inputs: &[PathBuf], + explore_ctx: Option<&headson::ExploreContext>, +) -> Result { + let (entries, warnings) = prepare_file_entries(cli, inputs)?; if needs_fileset(cli, inputs.len()) { - return render_fileset(entries, warnings, cli, render_cfg, grep_cfg); + return render_fileset( + entries, + warnings, + cli, + render_cfg, + grep_cfg, + explore_ctx, + ); } if entries.is_empty() { - return Ok((String::new(), warnings, None)); + return Ok((String::new(), warnings, None, vec![])); } - render_single_entry(entries, warnings, cli, render_cfg, grep_cfg) + render_single_entry( + entries, + warnings, + cli, + render_cfg, + grep_cfg, + explore_ctx, + ) } fn read_stdin() -> Result> { @@ -206,7 +296,7 @@ fn ingest_paths(paths: &[PathBuf]) -> Result<(InputEntries, CliWarnings)> { let display = path.display().to_string(); if let Ok(meta) = std::fs::metadata(path) { if meta.is_dir() { - warnings.push(format!("Ignored directory: {display}")); + warnings.push(format!("Ignored directory: {display} (use -r/--recursive to include directory contents)")); continue; } } @@ -281,7 +371,7 @@ impl InputCollector { no_sort: bool, ) -> Result<()> { let dir = ensure_recursive_dir(&self.display_root, path)?; - let dir_norm = normalize_path(&dir); + let dir_norm = headson::node_path::lexically_normalized(&dir); self.expand_globs_in_root(&dir_norm, &["**/*".to_string()], no_sort) } @@ -447,30 +537,6 @@ where builder.build().context("failed to compile glob overrides") } -fn normalize_path(path: &Path) -> PathBuf { - let mut out = PathBuf::new(); - let mut has_root = false; - for comp in path.components() { - match comp { - std::path::Component::Prefix(prefix) => { - out.push(prefix.as_os_str()); - } - std::path::Component::RootDir => { - out.push(comp.as_os_str()); - has_root = true; - } - std::path::Component::CurDir => {} - std::path::Component::ParentDir => { - if !out.pop() && !has_root { - out.push(comp.as_os_str()); - } - } - std::path::Component::Normal(part) => out.push(part), - } - } - out -} - fn render_single_input( input_format: InputFormat, bytes: Vec, @@ -478,7 +544,7 @@ fn render_single_input( prio: &headson::PriorityConfig, grep_cfg: &headson::GrepConfig, budgets: headson::Budgets, -) -> Result<(String, CliWarnings, Option)> { +) -> Result { let text_mode = if matches!(cfg.template, headson::OutputTemplate::Code) { headson::TextMode::CodeLike } else { @@ -517,7 +583,7 @@ fn render_single_input( budgets, ), } - .map(|out| (out.text, out.warnings, out.match_summary)) + .map(|out| (out.text, out.warnings, out.match_summary, out.shown_leaves)) } fn resolve_effective_template_for_stdin( @@ -564,7 +630,8 @@ fn render_fileset( cli: &Cli, render_cfg: &headson::RenderConfig, grep_cfg: &headson::GrepConfig, -) -> Result<(String, CliWarnings, Option)> { + explore_ctx: Option<&headson::ExploreContext>, +) -> Result { if !matches!(cli.format, OutputFormat::Auto) { bail!( "--format cannot be customized for filesets; remove it or set to auto" @@ -573,7 +640,8 @@ fn render_fileset( let mut cfg = render_cfg.clone(); cfg.template = headson::OutputTemplate::Auto; let input_count = entries.len().max(1); - let (cfg, prio, budgets) = build_effective_configs(cli, cfg, input_count); + let (cfg, prio, budgets) = + build_effective_configs(cli, cfg, input_count, explore_ctx); let files: Vec = entries .into_iter() .map(|(name, bytes)| { @@ -585,6 +653,7 @@ fn render_fileset( text: out, warnings: fallback_warnings, match_summary, + shown_leaves, } = headson::headson( headson::InputKind::Fileset(files), &cfg, @@ -599,7 +668,7 @@ fn render_fileset( { warnings.push("No grep matches found".to_string()); } - Ok((out, warnings, match_summary)) + Ok((out, warnings, match_summary, shown_leaves)) } fn render_single_entry( @@ -608,7 +677,8 @@ fn render_single_entry( cli: &Cli, render_cfg: &headson::RenderConfig, grep_cfg: &headson::GrepConfig, -) -> Result<(String, CliWarnings, Option)> { + explore_ctx: Option<&headson::ExploreContext>, +) -> Result { let (name, bytes) = entries .pop() .expect("single-entry render expects one ingested input"); @@ -621,21 +691,32 @@ fn render_single_entry( &name, chosen_input, ); - let (cfg_for_render, prio, budgets) = - build_effective_configs(cli, cfg_for_render, 1usize); - let (out, mut fallback_warnings, match_summary) = render_single_input( - chosen_input, - bytes, - &cfg_for_render, - &prio, - grep_cfg, - budgets, - ) - .with_context(|| format!("failed to parse input file: {name}"))?; + // Single inputs lose their filename before reaching the library, so the + // breadcrumb file identity is resolved here and threaded via the context. + let explore_ctx = explore_ctx.cloned().map(|mut ctx| { + ctx.file = Some(headson::resolve_breadcrumb_file(&name)); + ctx + }); + let (cfg_for_render, prio, budgets) = build_effective_configs( + cli, + cfg_for_render, + 1usize, + explore_ctx.as_ref(), + ); + let (out, mut fallback_warnings, match_summary, shown_leaves) = + render_single_input( + chosen_input, + bytes, + &cfg_for_render, + &prio, + grep_cfg, + budgets, + ) + .with_context(|| format!("failed to parse input file: {name}"))?; if !fallback_warnings.is_empty() { warnings.append(&mut fallback_warnings); } - Ok((out, warnings, match_summary)) + Ok((out, warnings, match_summary, shown_leaves)) } fn build_single_render_config( @@ -686,6 +767,7 @@ mod tests { use super::*; use crate::cli::args::Cli; use clap::Parser; + use serial_test::serial; use std::fs; use tempfile::tempdir; @@ -790,8 +872,13 @@ mod tests { /// With --count-matches and --grep, loose budget: all matches shown. /// The warnings vec must contain exactly one line "N matches shown, 0 hidden". #[test] + #[serial] fn count_matches_summary_in_warnings_when_all_shown() { let dir = tempdir().unwrap(); + // Isolate HSON_SESSION so explore tests running in parallel don't leak + // a non-existent session ID into this test's env. + let _env = + crate::cli::test_helpers::IsolatedEnv::new(dir.path(), None); let path = dir.path().join("data.json"); // Three keys that each contain "match" somewhere in their string value; // 10 000-byte budget is ample so nothing is hidden. @@ -837,8 +924,11 @@ mod tests { /// With --count-matches and --weak-grep under a very tight line budget, /// some matches must be hidden (N hidden > 0). #[test] + #[serial] fn count_matches_summary_in_warnings_when_some_hidden() { let dir = tempdir().unwrap(); + let _env = + crate::cli::test_helpers::IsolatedEnv::new(dir.path(), None); let path = dir.path().join("data.json"); // Six matches spread across many lines; --lines 1 forces tight truncation. fs::write( @@ -922,4 +1012,6 @@ mod tests { "zero-match summary must say '0 hidden'; got: {summary:?}" ); } + + // Session middleware tests (steps 31–35) live in session_middleware.rs. } diff --git a/src/cli/session_middleware.rs b/src/cli/session_middleware.rs new file mode 100644 index 00000000..68c69411 --- /dev/null +++ b/src/cli/session_middleware.rs @@ -0,0 +1,799 @@ +use std::env; +use std::path::PathBuf; + +use crate::cli::args::Cli; + +/// Default maximum breadcrumbs kept per session (`--explore-memory` default); +/// older entries are evicted. +pub(crate) const BREADCRUMB_CAP: usize = 10_000; +/// Maximum number of query log entries kept; oldest are dropped when exceeded. +pub(crate) const QUERY_LOG_CAP: usize = 1000; +/// Default decay factor per step (`--explore-decay` default) — matches the +/// alpha used by the penalty engine. +pub(crate) const DEFAULT_ALPHA: f64 = 0.5; + +/// Session-control flags that take a value; stripped together with their +/// value from recorded argv so the query log reflects only the preview +/// request itself. +const STRIPPED_VALUE_FLAGS: [&str; 3] = + ["--session", "--explore-decay", "--explore-memory"]; +/// Session-control boolean flags stripped from recorded argv. +const STRIPPED_BOOL_FLAGS: [&str; 1] = ["--no-record"]; + +/// Resolve the active session ID with precedence: an explicit `--session` +/// flag (already UUID-validated by clap) wins over the `HSON_SESSION` +/// environment variable. +pub(crate) fn resolve_session_id(cli: &Cli) -> anyhow::Result> { + if let Some(id) = &cli.session { + return Ok(Some(id.clone())); + } + session_id_from_env() +} + +/// Read `HSON_SESSION`. Empty or whitespace-only values act as unset (so +/// `export HSON_SESSION=""` doesn't break every invocation); any other +/// non-UUID value errors, naming the env var rather than the --session flag. +fn session_id_from_env() -> anyhow::Result> { + let Some(raw) = env::var_os("HSON_SESSION") else { + return Ok(None); + }; + let raw = raw.to_string_lossy(); + let value = raw.trim(); + if value.is_empty() { + return Ok(None); + } + match uuid::Uuid::parse_str(value) { + Ok(uuid) => Ok(Some(uuid.to_string())), + Err(e) => anyhow::bail!( + "invalid HSON_SESSION environment variable {value:?} \ + (must be a UUID): {e}. Unset HSON_SESSION or run \ + `hson explore start` to create a fresh session." + ), + } +} + +/// If a session is active, require the session file to exist. +/// New sessions are only created by `hson explore start` — every other +/// path errors on an unknown session ID rather than silently auto-creating +/// (which would mask typos and lose the bias context of an existing session). +pub(crate) fn require_session_exists( + session_id: Option<&str>, +) -> anyhow::Result<()> { + if let Some(id) = session_id { + let path = session_file_path(id)?; + if !path.exists() { + anyhow::bail!( + "Session '{id}' not found. \ + Run `hson explore start` to create one." + ); + } + } + Ok(()) +} + +fn state_dir() -> anyhow::Result { + if let Some(dir) = env::var_os("XDG_STATE_HOME").filter(|v| !v.is_empty()) + { + return Ok(PathBuf::from(dir)); + } + match env::var_os("HOME").filter(|v| !v.is_empty()) { + Some(home) => Ok(PathBuf::from(home).join(".local").join("state")), + None => anyhow::bail!( + "cannot determine session state directory: \ + neither XDG_STATE_HOME nor HOME is set" + ), + } +} + +pub(crate) fn session_file_path(id: &str) -> anyhow::Result { + Ok(state_dir()? + .join("headson") + .join("sessions") + .join(format!("{id}.json"))) +} + +fn is_equals_form(arg: &str, flag: &str) -> bool { + arg.len() > flag.len() + && arg.starts_with(flag) + && arg.as_bytes()[flag.len()] == b'=' +} + +/// Remove session-control flags from a raw argv before it is written to the +/// session query log. Handles both `--flag value` and `--flag=value` forms. +pub(crate) fn strip_session_control_args(argv: &[String]) -> Vec { + let mut out = Vec::with_capacity(argv.len()); + let mut tokens = argv.iter(); + while let Some(token) = tokens.next() { + let is_bool_flag = STRIPPED_BOOL_FLAGS + .iter() + .any(|f| token == f || is_equals_form(token, f)); + if is_bool_flag + || STRIPPED_VALUE_FLAGS + .iter() + .any(|f| is_equals_form(token, f)) + { + continue; + } + if STRIPPED_VALUE_FLAGS.iter().any(|f| token == f) { + tokens.next(); // drop the flag's value token too + continue; + } + out.push(token.clone()); + } + out +} + +pub(crate) fn record_session( + id: &str, + shown_leaves: &[headson::BreadcrumbKey], + cwd: &str, + argv: &[String], + alpha: f64, + breadcrumb_cap: usize, +) { + let path = match session_file_path(id) { + Ok(p) => p, + Err(e) => { + eprintln!( + "warning: failed to record step for session '{id}': {e}" + ); + return; + } + }; + if let Err(e) = crate::session::io::record_step_atomic( + &path, + shown_leaves, + &crate::cli::timestamp::current_timestamp(), + cwd, + argv, + &crate::session::io::EvictionPolicy { + alpha, + breadcrumb_cap, + query_log_cap: QUERY_LOG_CAP, + }, + ) { + eprintln!("warning: failed to record step for session '{id}': {e}"); + } +} + +/// Lossily convert OS-level argv to `String`s. `std::env::args()` PANICS on +/// non-Unicode arguments (e.g. a filename with invalid UTF-8 bytes), which +/// would abort after the preview was already computed; degrading to U+FFFD +/// replacement characters in the query log is the right trade-off. +fn argv_to_string_lossy( + args: impl IntoIterator, +) -> Vec { + args.into_iter() + .map(|arg| arg.to_string_lossy().into_owned()) + .collect() +} + +pub(crate) fn maybe_record_session( + cli: &Cli, + session_id: Option<&str>, + from_stdin: bool, + shown_leaves: &[headson::BreadcrumbKey], +) { + let Some(id) = session_id else { return }; + if from_stdin || cli.no_record { + return; + } + let cwd = env::current_dir() + .unwrap_or_default() + .to_string_lossy() + .into_owned(); + let raw_argv = argv_to_string_lossy(env::args_os()); + let argv = strip_session_control_args(&raw_argv); + record_session( + id, + shown_leaves, + &cwd, + &argv, + cli.explore_decay, + cli.explore_memory, + ); +} + +#[cfg(test)] +mod tests { + use std::fs; + + use clap::Parser; + use serial_test::serial; + use tempfile::tempdir; + + use super::*; + use crate::cli::args::Cli; + use crate::cli::run::run; + + use crate::cli::test_helpers::IsolatedEnv; + + /// Pre-create an empty session file at the given ID under `state_dir`, + /// mimicking what `hson explore start` would do — needed because runtime + /// tracking now requires the session file to already exist. + fn pre_create_session(state_dir: &std::path::Path, id: &str) { + let dir = state_dir.join("headson").join("sessions"); + fs::create_dir_all(&dir).unwrap(); + let path = dir.join(format!("{id}.json")); + let session = crate::session::Session::new(id.into(), "lbl".into()); + crate::session::io::save_to_path(&session, &path).unwrap(); + } + + /// Step 31: When neither HSON_SESSION nor --session is set, running hson on + /// a file produces the same output as a baseline run and does NOT write a + /// session file anywhere under XDG_STATE_HOME. + #[test] + #[serial] + fn no_hson_session_env_output_unchanged() { + let dir = tempdir().unwrap(); + let state_dir = tempdir().unwrap(); + let path = dir.path().join("data.json"); + fs::write(&path, r#"{"x": 1}"#).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), None); + + let cli = Cli::parse_from(["hson", path.to_str().unwrap()]); + let (out, warnings) = + run(&cli).expect("run must succeed without session flag"); + + assert!( + !out.is_empty(), + "output must be non-empty; got empty string" + ); + assert!( + warnings.is_empty(), + "no warnings expected; got: {warnings:?}" + ); + + let sessions_dir = state_dir.path().join("headson").join("sessions"); + if sessions_dir.exists() { + let session_files: Vec<_> = fs::read_dir(&sessions_dir) + .unwrap() + .filter_map(std::result::Result::ok) + .filter(|e| e.file_name().to_string_lossy().ends_with(".json")) + .collect(); + assert!( + session_files.is_empty(), + "no session files must be written when HSON_SESSION is absent; \ + found: {session_files:?}" + ); + } + } + + /// Step 32: When HSON_SESSION= env var is set, running hson on a file + /// creates `$XDG_STATE_HOME/headson/sessions/.json`. + #[test] + #[serial] + fn hson_session_env_creates_session_file() { + let dir = tempdir().unwrap(); + let state_dir = tempdir().unwrap(); + let path = dir.path().join("data.json"); + fs::write(&path, r#"{"x": 1}"#).unwrap(); + + let session_id = "32000000-0000-0000-0000-000000000000"; + let _env = IsolatedEnv::new(state_dir.path(), Some(session_id)); + pre_create_session(state_dir.path(), session_id); + + let cli = Cli::parse_from(["hson", path.to_str().unwrap()]); + let result = run(&cli); + + result.expect("run must succeed with HSON_SESSION set"); + + let expected = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{session_id}.json")); + assert!( + expected.exists(), + "session file must be created at {expected:?} when HSON_SESSION is set" + ); + } + + /// Recorded breadcrumbs identify the input file by its resolved absolute + /// path; the path component is the in-file dot-path plus content hash, + /// with no filename embedded (issue #513 review). + #[test] + #[serial] + fn recorded_breadcrumbs_carry_absolute_file_and_inner_path() { + let dir = tempdir().unwrap(); + let state_dir = tempdir().unwrap(); + let path = dir.path().join("data.json"); + fs::write(&path, r#"{"x": 1}"#).unwrap(); + + let session_id = "3b000000-0000-0000-0000-000000000000"; + let _env = IsolatedEnv::new(state_dir.path(), Some(session_id)); + pre_create_session(state_dir.path(), session_id); + + let cli = Cli::parse_from(["hson", path.to_str().unwrap()]); + run(&cli).expect("run must succeed"); + + let session_file = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{session_id}.json")); + let session = + crate::session::io::load_from_path(&session_file).unwrap(); + let expected_file = path.canonicalize().unwrap(); + assert!(!session.breadcrumbs.is_empty(), "must record breadcrumbs"); + for crumb in &session.breadcrumbs { + assert_eq!( + crumb.file, + expected_file.to_string_lossy(), + "breadcrumb file must be the resolved absolute input path" + ); + assert!( + crumb.path.starts_with("x#"), + "path must be the in-file dot-path plus hash, with no \ + filename embedded; got: {:?}", + crumb.path + ); + } + } + + /// Step 33: Using `--session ` creates the session file. + #[test] + #[serial] + fn session_flag_creates_session_file() { + let dir = tempdir().unwrap(); + let state_dir = tempdir().unwrap(); + let path = dir.path().join("data.json"); + fs::write(&path, r#"{"x": 1}"#).unwrap(); + + let session_id = "33000000-0000-0000-0000-000000000000"; + let _env = IsolatedEnv::new(state_dir.path(), None); + pre_create_session(state_dir.path(), session_id); + + let cli = Cli::parse_from([ + "hson", + "--session", + session_id, + path.to_str().unwrap(), + ]); + let result = run(&cli); + + result.expect("run must succeed with --session flag"); + + let expected = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{session_id}.json")); + assert!( + expected.exists(), + "session file must be created at {expected:?} when --session is passed" + ); + } + + /// Step 34: `--session --no-record` does NOT mutate the session file. + #[test] + #[serial] + fn no_record_flag_suppresses_session_write() { + let dir = tempdir().unwrap(); + let state_dir = tempdir().unwrap(); + let path = dir.path().join("data.json"); + fs::write(&path, r#"{"x": 1}"#).unwrap(); + + let session_id = "34000000-0000-0000-0000-000000000000"; + let _env = IsolatedEnv::new(state_dir.path(), None); + pre_create_session(state_dir.path(), session_id); + + let cli = Cli::parse_from([ + "hson", + "--session", + session_id, + "--no-record", + path.to_str().unwrap(), + ]); + let result = run(&cli); + + result.expect("run must succeed with --session --no-record"); + + // Session file existed before run; with --no-record, run must not + // touch it — step_count and queries stay at their initial values. + let session_file = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{session_id}.json")); + let after = crate::session::io::load_from_path(&session_file).unwrap(); + assert_eq!( + after.step_count, 0, + "--no-record must not increment step_count" + ); + assert!( + after.queries.is_empty(), + "--no-record must not append to query log" + ); + assert!( + after.breadcrumbs.is_empty(), + "--no-record must not record breadcrumbs" + ); + } + + /// Step 35: Structural confirmation that stdin mode suppresses session writes. + #[test] + #[serial] + fn stdin_mode_no_session_write_active_session_id_resolves() { + let state_dir = tempdir().unwrap(); + let session_id = "35000000-0000-0000-0000-000000000000"; + + let _env = IsolatedEnv::new(state_dir.path(), Some(session_id)); + + let cli = Cli::parse_from(["hson"]); + let id = resolve_session_id(&cli) + .expect("valid HSON_SESSION must resolve without error"); + + assert_eq!( + id, + Some(session_id.to_string()), + "resolve_session_id must return Some(id) from HSON_SESSION env var" + ); + assert!( + !state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{session_id}.json")) + .exists(), + "session file must not be written when run() has not been called" + ); + } + + /// Regression (issue #513): passing `--session ` to a normal + /// `hson` invocation must NOT silently auto-create a session file at that + /// path. The user likely typo'd; auto-creating overwrites their intent and + /// loses the original session's bias context. The command must return an + /// error AND leave no session file on disk. + #[test] + #[serial] + fn unknown_session_id_in_run_errors_and_does_not_create_file() { + let dir = tempdir().unwrap(); + let state_dir = tempdir().unwrap(); + let path = dir.path().join("data.json"); + fs::write(&path, r#"{"x": 1}"#).unwrap(); + + let unknown_id = "11111111-2222-3333-4444-555555555555"; + let _env = IsolatedEnv::new(state_dir.path(), Some(unknown_id)); + + let cli = Cli::parse_from(["hson", path.to_str().unwrap()]); + let result = run(&cli); + + let expected = state_dir + .path() + .join("headson") + .join("sessions") + .join(format!("{unknown_id}.json")); + assert!( + result.is_err(), + "running with an unknown session ID must error; got: {result:?}" + ); + assert!( + !expected.exists(), + "running with an unknown session ID must not create the session \ + file at {expected:?}" + ); + } + + /// An EMPTY exported HSON_SESSION must behave exactly as if it were + /// unset: the run succeeds and no session file is written. + #[test] + #[serial] + fn empty_hson_session_env_is_treated_as_unset() { + let dir = tempdir().unwrap(); + let state_dir = tempdir().unwrap(); + let path = dir.path().join("data.json"); + fs::write(&path, r#"{"x": 1}"#).unwrap(); + + let _env = IsolatedEnv::new(state_dir.path(), Some("")); + + let cli = Cli::parse_from(["hson", path.to_str().unwrap()]); + let id = resolve_session_id(&cli) + .expect("empty HSON_SESSION must not be an error"); + assert_eq!(id, None, "empty HSON_SESSION must resolve to no session"); + + let (out, _) = run(&cli).expect("run must succeed"); + assert!(!out.is_empty(), "output must be non-empty"); + } + + /// Whitespace-only HSON_SESSION is also treated as unset. + #[test] + #[serial] + fn whitespace_hson_session_env_is_treated_as_unset() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), Some(" \t ")); + + let cli = Cli::parse_from(["hson"]); + let id = resolve_session_id(&cli) + .expect("whitespace-only HSON_SESSION must not be an error"); + assert_eq!(id, None); + } + + /// A non-empty, non-UUID HSON_SESSION must produce a clear error that + /// names HSON_SESSION (not the --session flag the user never typed). + #[test] + #[serial] + fn invalid_hson_session_env_errors_naming_the_env_var() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), Some("not-a-uuid")); + + let cli = Cli::parse_from(["hson"]); + let err = resolve_session_id(&cli) + .expect_err("non-UUID HSON_SESSION must be an error"); + + let msg = err.to_string(); + assert!( + msg.contains("HSON_SESSION"), + "error must name HSON_SESSION; got: {msg}" + ); + assert!( + !msg.contains("--session"), + "error must not blame the --session flag; got: {msg}" + ); + } + + /// An explicit --session flag takes precedence over HSON_SESSION, even + /// when the env value is garbage. + #[test] + #[serial] + fn session_flag_takes_precedence_over_env() { + let state_dir = tempdir().unwrap(); + let _env = IsolatedEnv::new(state_dir.path(), Some("not-a-uuid")); + + let flag_id = "ab000000-0000-0000-0000-000000000000"; + let cli = Cli::parse_from(["hson", "--session", flag_id]); + let id = resolve_session_id(&cli) + .expect("--session flag must win over an invalid env value"); + assert_eq!(id.as_deref(), Some(flag_id)); + } + + /// Non-UTF-8 argv must be lossily converted, never panic — a panic here + /// would discard the already-rendered preview (issue #513 review). + #[test] + #[cfg(unix)] + fn argv_to_string_lossy_replaces_invalid_utf8() { + use std::os::unix::ffi::OsStringExt; + + let bad = std::ffi::OsString::from_vec(b"bad\xff.json".to_vec()); + let argv = + argv_to_string_lossy(vec![std::ffi::OsString::from("hson"), bad]); + + assert_eq!(argv[0], "hson"); + assert_eq!( + argv[1], "bad\u{FFFD}.json", + "invalid UTF-8 bytes must degrade to U+FFFD, not panic" + ); + } + + /// Regression: record_session must evict stale breadcrumbs so session + /// files don't grow without bound across long-running explorations. The + /// cap comes from --explore-memory rather than a hard-coded constant. + #[test] + #[serial] + fn record_session_caps_breadcrumbs_when_limit_exceeded() { + let dir = tempdir().unwrap(); + let state_dir = dir.path(); + let session_id = "be000000-0000-0000-0000-000000000000"; + + let _env = IsolatedEnv::new(state_dir, None); + + // Pre-populate with 20 recent breadcrumbs — above the cap of 5. + let path = session_file_path(session_id).unwrap(); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut session = crate::session::Session::new( + session_id.to_string(), + "lbl".to_string(), + ); + for i in 0u64..20 { + session.record_breadcrumb("", &format!("k{i}#h{i}"), i + 1); + } + session.step_count = 20; + crate::session::io::save_to_path(&session, &path).unwrap(); + + // --explore-decay 1.0 disables decay-based pruning so only the + // --explore-memory cap is exercised here. + let cli = Cli::parse_from([ + "hson", + "--explore-decay", + "1.0", + "--explore-memory", + "5", + "input", + ]); + record_session( + session_id, + &[(String::new(), "new#newhash".to_string())], + "/cwd", + &[], + cli.explore_decay, + cli.explore_memory, + ); + + let final_session = crate::session::io::load_from_path(&path) + .expect("session file must exist after record_session"); + assert!( + final_session.breadcrumbs.len() <= cli.explore_memory, + "breadcrumbs must be capped at {} after record_session; got: {}", + cli.explore_memory, + final_session.breadcrumbs.len() + ); + } + + /// Regression: record_session must cap the query log to prevent unbounded growth. + #[test] + #[serial] + fn record_session_caps_query_log_when_limit_exceeded() { + let dir = tempdir().unwrap(); + let state_dir = dir.path(); + let session_id = "9c000000-0000-0000-0000-000000000000"; + + let _env = IsolatedEnv::new(state_dir, None); + + // Pre-populate with QUERY_LOG_CAP + 100 queries. + let path = session_file_path(session_id).unwrap(); + fs::create_dir_all(path.parent().unwrap()).unwrap(); + let mut session = crate::session::Session::new( + session_id.to_string(), + "lbl".to_string(), + ); + for _ in 0u64..(QUERY_LOG_CAP as u64 + 100) { + session.record_query("ts", "/cwd", &[]); + } + crate::session::io::save_to_path(&session, &path).unwrap(); + + record_session( + session_id, + &[], + "/cwd", + &[], + DEFAULT_ALPHA, + BREADCRUMB_CAP, + ); + + let final_session = crate::session::io::load_from_path(&path) + .expect("session file must exist after record_session"); + assert!( + final_session.queries.len() <= QUERY_LOG_CAP, + "query log must be capped at {}; got: {}", + QUERY_LOG_CAP, + final_session.queries.len() + ); + } + + fn to_argv(args: &[&str]) -> Vec { + args.iter().map(|s| (*s).to_string()).collect() + } + + #[test] + fn strip_removes_session_control_flags_space_form() { + let argv = to_argv(&[ + "hson", + "--session", + "be000000-0000-0000-0000-000000000000", + "--explore-decay", + "0.7", + "--explore-memory", + "42", + "--no-record", + "-n", + "5", + "data.json", + ]); + assert_eq!( + strip_session_control_args(&argv), + to_argv(&["hson", "-n", "5", "data.json"]) + ); + } + + #[test] + fn strip_removes_session_control_flags_equals_form() { + let argv = to_argv(&[ + "hson", + "--session=be000000-0000-0000-0000-000000000000", + "--explore-decay=0.7", + "--explore-memory=42", + "data.json", + ]); + assert_eq!( + strip_session_control_args(&argv), + to_argv(&["hson", "data.json"]) + ); + } + + #[test] + fn strip_keeps_unrelated_args_untouched() { + let argv = to_argv(&["hson", "--bytes", "200", "--tree", "src/"]); + assert_eq!(strip_session_control_args(&argv), argv); + } + + #[test] + fn strip_does_not_drop_prefix_lookalike_flags() { + // `--session-x` is not `--session`; only exact or `=`-joined forms + // are session-control flags. + let argv = to_argv(&["hson", "--session-x", "v", "data.json"]); + assert_eq!(strip_session_control_args(&argv), argv); + } + + #[test] + fn strip_handles_trailing_value_flag_without_value() { + let argv = to_argv(&["hson", "data.json", "--session"]); + assert_eq!( + strip_session_control_args(&argv), + to_argv(&["hson", "data.json"]) + ); + } + + struct StateEnvGuard { + old_state: Option, + old_home: Option, + } + + impl StateEnvGuard { + fn unset_all() -> Self { + let old_state = std::env::var_os("XDG_STATE_HOME"); + let old_home = std::env::var_os("HOME"); + unsafe { + std::env::remove_var("XDG_STATE_HOME"); + std::env::remove_var("HOME"); + } + Self { + old_state, + old_home, + } + } + } + + impl Drop for StateEnvGuard { + fn drop(&mut self) { + unsafe { + match &self.old_state { + Some(v) => std::env::set_var("XDG_STATE_HOME", v), + None => std::env::remove_var("XDG_STATE_HOME"), + } + match &self.old_home { + Some(v) => std::env::set_var("HOME", v), + None => std::env::remove_var("HOME"), + } + } + } + } + + /// With neither XDG_STATE_HOME nor HOME available, session_file_path + /// must return a clear error instead of silently building a relative + /// path like `.local/state/...` under the current directory. + #[test] + #[serial] + fn session_file_path_errors_when_no_state_dir_env() { + let _guard = StateEnvGuard::unset_all(); + + let result = session_file_path("be000000-0000-0000-0000-000000000000"); + + let err = result.expect_err( + "session_file_path must error when XDG_STATE_HOME and HOME are unset", + ); + let msg = err.to_string(); + assert!( + msg.contains("XDG_STATE_HOME") && msg.contains("HOME"), + "error must name the missing env vars; got: {msg}" + ); + } + + /// Empty-string env values must be treated the same as unset. + #[test] + #[serial] + fn session_file_path_errors_when_state_dir_env_empty() { + let _guard = StateEnvGuard::unset_all(); + unsafe { + std::env::set_var("XDG_STATE_HOME", ""); + std::env::set_var("HOME", ""); + } + + let result = session_file_path("be000000-0000-0000-0000-000000000000"); + + assert!( + result.is_err(), + "empty XDG_STATE_HOME and HOME must be treated as unset; got: {result:?}" + ); + } +} diff --git a/src/cli/test_helpers.rs b/src/cli/test_helpers.rs new file mode 100644 index 00000000..5911964f --- /dev/null +++ b/src/cli/test_helpers.rs @@ -0,0 +1,43 @@ +#![cfg(test)] +/// Shared test isolation helper for tests that mutate XDG_STATE_HOME and/or +/// HSON_SESSION environment variables. Restores prior values on drop. +pub(crate) struct IsolatedEnv { + old_state: Option, + old_session: Option, +} + +impl IsolatedEnv { + pub(crate) fn new( + state_dir: &std::path::Path, + session_id: Option<&str>, + ) -> Self { + let old_state = std::env::var("XDG_STATE_HOME").ok(); + let old_session = std::env::var("HSON_SESSION").ok(); + unsafe { + std::env::set_var("XDG_STATE_HOME", state_dir); + match session_id { + Some(id) => std::env::set_var("HSON_SESSION", id), + None => std::env::remove_var("HSON_SESSION"), + } + } + Self { + old_state, + old_session, + } + } +} + +impl Drop for IsolatedEnv { + fn drop(&mut self) { + unsafe { + match &self.old_state { + Some(v) => std::env::set_var("XDG_STATE_HOME", v), + None => std::env::remove_var("XDG_STATE_HOME"), + } + match &self.old_session { + Some(v) => std::env::set_var("HSON_SESSION", v), + None => std::env::remove_var("HSON_SESSION"), + } + } + } +} diff --git a/src/cli/timestamp.rs b/src/cli/timestamp.rs new file mode 100644 index 00000000..efb61a6a --- /dev/null +++ b/src/cli/timestamp.rs @@ -0,0 +1,94 @@ +//! Minimal ISO-8601 UTC timestamp formatting without a chrono dep. +//! +//! Lives here (and not in `session_middleware`) because the date math is +//! self-contained and the only consumer is the session query log writer. + +fn is_leap_year(y: u64) -> bool { + y % 400 == 0 || (y % 4 == 0 && y % 100 != 0) +} + +fn year_from_unix_secs(secs: u64) -> (u64, u64) { + let (mut y, mut rem) = (1970u64, secs); + loop { + let sy = if is_leap_year(y) { 366 } else { 365 } * 86_400; + if rem < sy { + break; + } + rem -= sy; + y += 1; + } + (y, rem) +} + +fn month_day_from_year_secs(year: u64, mut rem: u64) -> (u64, u64) { + let month_days: [u64; 12] = [ + 31, + if is_leap_year(year) { 29 } else { 28 }, + 31, + 30, + 31, + 30, + 31, + 31, + 30, + 31, + 30, + 31, + ]; + let mut mo = 1u64; + for &d in &month_days { + let sm = d * 86_400; + if rem < sm { + break; + } + rem -= sm; + mo += 1; + } + (mo, rem / 86_400 + 1) +} + +fn format_timestamp(secs: u64) -> String { + let (y, rem) = year_from_unix_secs(secs); + let (mo, day) = month_day_from_year_secs(y, rem); + let time_rem = rem % 86_400; + let (h, m, s) = (time_rem / 3600, time_rem % 3600 / 60, time_rem % 60); + format!("{y:04}-{mo:02}-{day:02}T{h:02}:{m:02}:{s:02}Z") +} + +pub(crate) fn current_timestamp() -> String { + use std::time::{SystemTime, UNIX_EPOCH}; + let secs = SystemTime::now() + .duration_since(UNIX_EPOCH) + .map(|d| d.as_secs()) + .unwrap_or(0); + format_timestamp(secs) +} + +#[cfg(test)] +mod tests { + use super::*; + + // Expected strings verified against `date -u -d @`. + + #[test] + fn epoch_formats_as_1970_01_01() { + assert_eq!(format_timestamp(0), "1970-01-01T00:00:00Z"); + } + + #[test] + fn known_recent_timestamp_formats_correctly() { + assert_eq!(format_timestamp(1_781_094_896), "2026-06-10T12:34:56Z"); + } + + #[test] + fn leap_year_feb_29_is_produced() { + assert_eq!(format_timestamp(1_709_164_800), "2024-02-29T00:00:00Z"); + assert_eq!(format_timestamp(1_709_251_199), "2024-02-29T23:59:59Z"); + } + + #[test] + fn end_of_year_rollover_is_exact() { + assert_eq!(format_timestamp(1_704_067_199), "2023-12-31T23:59:59Z"); + assert_eq!(format_timestamp(1_704_067_200), "2024-01-01T00:00:00Z"); + } +} diff --git a/src/ingest/fileset.rs b/src/ingest/fileset.rs index caffcc39..98b85a4b 100644 --- a/src/ingest/fileset.rs +++ b/src/ingest/fileset.rs @@ -168,7 +168,7 @@ pub(crate) fn build_fileset_root( suppressed, } in entries.drain(..) { - let child_root = append_subtree(&mut arena, child); + let child_root = arena.append(child); if let Some(node) = arena.nodes.get_mut(child_root) { node.fileset_suppressed = suppressed; } @@ -191,46 +191,3 @@ pub(crate) fn build_fileset_root( } arena } - -#[allow( - clippy::cognitive_complexity, - reason = "Tree merge touches multiple parallel arrays and offsets; easier to follow inline" -)] -fn append_subtree(dest: &mut JsonTreeArena, src: JsonTreeArena) -> usize { - let node_offset = dest.nodes.len(); - let child_offset = dest.children.len(); - let obj_key_offset = dest.obj_keys.len(); - let arr_idx_offset = dest.arr_indices.len(); - let root_id = src.root_id; - let JsonTreeArena { - nodes, - children, - obj_keys, - arr_indices, - code_lines, - .. - } = src; - - dest.nodes.extend(nodes); - for node in dest.nodes.iter_mut().skip(node_offset) { - if node.children_len > 0 { - node.children_start += child_offset; - } - if node.obj_keys_len > 0 { - node.obj_keys_start += obj_key_offset; - } - if node.arr_indices_len > 0 { - node.arr_indices_start += arr_idx_offset; - } - } - - dest.children - .extend(children.into_iter().map(|child| child + node_offset)); - dest.obj_keys.extend(obj_keys); - dest.arr_indices.extend(arr_indices); - for (arena_idx, lines) in code_lines { - dest.code_lines.insert(arena_idx + node_offset, lines); - } - - node_offset + root_id -} diff --git a/src/ingest/formats/json/mod.rs b/src/ingest/formats/json/mod.rs index 974975d1..6b938787 100644 --- a/src/ingest/formats/json/mod.rs +++ b/src/ingest/formats/json/mod.rs @@ -8,6 +8,9 @@ use serde::de::DeserializeSeed; use crate::PriorityConfig; use crate::utils::tree_arena::JsonTreeArena as TreeArena; +#[allow(dead_code, reason = "used only in non-test fn parse_jsonl_one")] +type ChunkResult = (TreeArena, Vec<(usize, usize)>); + #[cfg(test)] pub(crate) fn build_json_tree_arena( input: &str, @@ -94,11 +97,14 @@ pub(crate) fn jsonl_line_offsets(text: &str) -> Vec<(usize, usize)> { pub fn parse_jsonl_one( bytes: &[u8], cfg: &PriorityConfig, - must_include: impl Fn(usize) -> bool, + must_include: impl Fn(usize) -> bool + Sync, ) -> Result { use crate::ingest::sampling::{ ArraySamplerKind, choose_indices, merge_required, }; + use crate::order::NodeKind; + use crate::utils::tree_arena::JsonTreeNode; + use rayon::prelude::*; let text = std::str::from_utf8(bytes) .map_err(|e| anyhow::anyhow!("JSONL input is not valid UTF-8: {e}"))?; @@ -109,37 +115,95 @@ pub fn parse_jsonl_one( let sampled = choose_indices(sampler_kind, total, cfg.array_max_items); let kept_indices = merge_required(sampled, total, &must_include); - let builder = JsonTreeBuilder::new(cfg.array_max_items, sampler_kind); - let root_id = builder.push_default(); - let mut child_ids: Vec = Vec::with_capacity(kept_indices.len()); - let mut line_numbers: Vec = Vec::with_capacity(kept_indices.len()); - - for &sampled_idx in &kept_indices { - let (byte_start, line_num) = line_offsets[sampled_idx]; - let line = &text[byte_start..]; - let line = line.split('\n').next().unwrap_or("").trim_end(); - let mut line_bytes = line.as_bytes().to_vec(); - let mut de = simd_json::Deserializer::from_slice(&mut line_bytes) - .map_err(|e| anyhow::anyhow!("JSONL line {line_num}: {e}"))?; - let seed = builder.seed(); - let child_id: usize = seed - .deserialize(&mut de) - .map_err(|e| anyhow::anyhow!("JSONL line {line_num}: {e}"))?; - child_ids.push(child_id); - line_numbers.push(line_num); + let array_cap = cfg.array_max_items; + + // Parse kept lines in parallel, chunked to reduce per-task overhead. + // Each chunk shares a single JsonTreeBuilder, producing one arena per + // chunk instead of one per line. + let num_chunks = rayon::current_num_threads().max(1); + let chunk_size = (kept_indices.len() + num_chunks - 1) / num_chunks.max(1); + + let per_chunk: Vec = kept_indices + .par_chunks(chunk_size.max(1)) + .map(|chunk| { + let builder = JsonTreeBuilder::new(array_cap, sampler_kind); + let mut roots: Vec<(usize, usize)> = + Vec::with_capacity(chunk.len()); + let mut buf: Vec = Vec::new(); + for &idx in chunk { + let (byte_start, line_num) = line_offsets[idx]; + let raw = &text[byte_start..]; + let raw = raw.split('\n').next().unwrap_or("").trim_end(); + let raw_bytes = raw.as_bytes(); + buf.clear(); + buf.extend_from_slice(raw_bytes); + let mut de = simd_json::Deserializer::from_slice(&mut buf) + .map_err(|e| { + anyhow::anyhow!("JSONL line {line_num}: {e}") + })?; + let seed = builder.seed(); + let root: usize = seed.deserialize(&mut de).map_err(|e| { + anyhow::anyhow!("JSONL line {line_num}: {e}") + })?; + roots.push((root, line_num)); + } + Ok((builder.finish(), roots)) + }) + .collect::>>()?; + + // Sequential merge: combine chunk arenas into one, extracting each + // line's root node via its offset-adjusted ID. + let mut arena = TreeArena::default(); + let root_id = arena.nodes.len(); + arena.nodes.push(JsonTreeNode::default()); + + let kept = kept_indices.len(); + let mut child_ids: Vec = Vec::with_capacity(kept); + let mut line_numbers: Vec = Vec::with_capacity(kept); + + for (chunk_arena, roots) in per_chunk { + let base = arena.nodes.len(); + // arena.append shifts every node index in chunk_arena by `base`. + // chunk_root is the builder's synthetic container root — not needed. + // Each JSONL line's root is orig_root (index within chunk_arena), + // so its global ID after the shift is base + orig_root. + let _ = arena.append(chunk_arena); + for (orig_root, line_num) in roots { + child_ids.push(base + orig_root); + line_numbers.push(line_num); + } } - let kept = child_ids.len(); - builder.finish_array(root_id, kept, total, child_ids, line_numbers); - - let mut arena = builder.finish(); - arena.root_id = root_id; + // Detect contiguous indices to skip storing arr_indices. + // NOTE: `jsonl_line_offsets` yields 1-based line numbers while `i` is + // 0-based, so `ln == i` never holds and this branch is only taken for + // empty inputs (`all` on an empty iterator is true). If the intent was + // "no sampling gaps", the check would be `ln == i + 1`. Left unchanged + // to preserve behavior: arr_indices are always stored when `kept > 0`. + let contiguous = line_numbers.len() == kept + && line_numbers.iter().enumerate().all(|(i, &ln)| ln == i); + + let children_start = arena.children.len(); + arena.children.extend(&child_ids); + + let (arr_start, arr_len) = if kept == 0 || contiguous { + (0, 0) + } else { + let start = arena.arr_indices.len(); + arena.arr_indices.extend(&line_numbers); + (start, line_numbers.len()) + }; - if let Some(node) = arena.nodes.get_mut(root_id) { - node.array_len = Some(total); - node.is_jsonl_root = true; - } + let root = &mut arena.nodes[root_id]; + root.kind = NodeKind::Array; + root.children_start = children_start; + root.children_len = kept; + root.array_len = Some(total); + root.arr_indices_start = arr_start; + root.arr_indices_len = arr_len; + root.is_jsonl_root = true; + arena.root_id = root_id; Ok(arena) } diff --git a/src/ingest/mod.rs b/src/ingest/mod.rs index 6180f222..43a25642 100644 --- a/src/ingest/mod.rs +++ b/src/ingest/mod.rs @@ -34,11 +34,11 @@ pub(crate) fn grep_adjusted_cfg( grep: &GrepConfig, ) -> PriorityConfig { if grep.has_strong() { - let mut c = *cfg; + let mut c = cfg.clone(); c.array_max_items = usize::MAX; c } else { - *cfg + cfg.clone() } } @@ -50,7 +50,7 @@ pub(crate) fn grep_adjusted_cfg( pub(crate) fn jsonl_grep_predicate( bytes: &[u8], grep: &GrepConfig, -) -> Box bool> { +) -> Box bool + Sync> { let Some(re) = grep.patterns.strong() else { return Box::new(|_| false); }; diff --git a/src/lib.rs b/src/lib.rs index 3478916b..35316cc0 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -24,6 +24,7 @@ pub mod budget; mod debug; mod grep; mod ingest; +pub mod node_path; mod order; mod pruner; mod serialization; @@ -35,6 +36,7 @@ pub use grep::{ pub use ingest::fileset::{FilesetInput, FilesetInputKind}; pub use ingest::format::Format; pub use order::types::{ArrayBias, ArraySamplerStrategy}; +pub use order::types::{Breadcrumb, ExploreContext}; pub use order::{ DEFAULT_SAFETY_CAP, NodeId, NodeKind, PriorityConfig, PriorityOrder, RankedNode, build_order, @@ -42,7 +44,14 @@ pub use order::{ pub use utils::extensions; pub use utils::templates::map_json_template_for_style; -pub use pruner::budget::find_largest_render_under_budgets; +pub use node_path::{ + BreadcrumbKey, NodeFiles, compute_merkle_hashes, leaf_breadcrumb_key, + resolve_breadcrumb_file, +}; +pub use order::types::novelty_penalty; +pub use pruner::budget::{ + BudgetSearchResult, find_largest_render_under_budgets, +}; pub use prunist::{Budget, BudgetKind, Budgets}; pub use serialization::color::resolve_color_enabled; pub use serialization::types::{ @@ -60,6 +69,11 @@ pub struct RenderOutput { pub text: String, pub warnings: Vec, pub match_summary: Option, + /// `(file, path)` pairs for leaf nodes in the rendered output. + /// Used by CLI session middleware to record breadcrumbs. `file` is the + /// input file's resolved absolute path (or `""` when unknown); `path` is + /// the in-file dot-path plus a content-hash suffix. + pub shown_leaves: Vec, } #[derive(Copy, Clone, Debug)] @@ -87,6 +101,8 @@ pub fn headson( arena, mut warnings, } = crate::ingest::ingest_into_arena(input, priority_cfg, grep)?; + // Explore penalties (when active) are applied inside build_order, before + // fileset interleaving, so round-robin fairness survives the re-sort. let mut order_build = order::build_order(&arena, priority_cfg)?; if order_build.safety_cap_hit { warnings.push(format!( @@ -94,16 +110,28 @@ pub fn headson( priority_cfg.safety_cap )); } - let (text, match_summary) = find_largest_render_under_budgets( + let search = find_largest_render_under_budgets( &mut order_build, config, grep, budgets, ); + // Leaf recording (and its Merkle hashing) only matters when a session is + // active; without one, skip the hash pass entirely. + let shown_leaves = if let Some(explore) = priority_cfg.explore.as_ref() { + node_path::collect_shown_leaves( + &order_build, + &search, + explore.file.as_deref(), + ) + } else { + Vec::new() + }; Ok(RenderOutput { - text, + text: search.text, warnings, - match_summary, + match_summary: search.match_summary, + shown_leaves, }) } @@ -369,4 +397,515 @@ mod tests { summary.shown, summary.hidden, ); } + + // ── Step 25: find_largest_render_under_budgets returns a meaningful top_k ─ + + /// Build a `PriorityOrder` from a small JSON object for use in top_k tests. + fn make_order_for_top_k() -> PriorityOrder { + let input = InputKind::Json( + br#"{"a": 1, "b": 2, "c": 3, "d": 4, "e": 5}"#.to_vec(), + ); + let priority_cfg = PriorityConfig::new(usize::MAX, usize::MAX); + let grep_cfg = GrepConfig::default(); + let ingest_out = + crate::ingest::ingest_into_arena(input, &priority_cfg, &grep_cfg) + .expect("ingest must succeed"); + order::build_order(&ingest_out.arena, &priority_cfg) + .expect("build_order must succeed") + } + + #[test] + fn find_largest_render_returns_nonzero_top_k_under_tight_budget() { + // A 15-byte budget is far too small to render all 5 key-value pairs of + // {"a":1,"b":2,"c":3,"d":4,"e":5} (full render ≈ 44 bytes), so the + // budget search must stop before including all nodes. + let mut order = make_order_for_top_k(); + let total_nodes = order.total_nodes; + let tight_budgets = Budgets { + global: Some(Budget { + kind: BudgetKind::Bytes, + cap: 15, + }), + per_slot: None, + }; + let grep_cfg = GrepConfig::default(); + + let search = find_largest_render_under_budgets( + &mut order, + &test_render_config(), + &grep_cfg, + tight_budgets, + ); + let top_k = search.top_k; + + assert!( + top_k > 0, + "top_k must be > 0 (at least one node was selected); got {top_k}" + ); + assert!( + top_k < total_nodes, + "tight budget must not select all {total_nodes} nodes; top_k={top_k}" + ); + } + + // ── ExploreContext integration: deprioritize seen nodes via PriorityConfig ─ + + /// Compute the composite breadcrumb key (`"dot_path#hash"`) for a leaf at + /// `dot_path` in a single-file JSON input. + fn composite_key_for(json: &[u8], dot_path: &str) -> String { + let input = InputKind::Json(json.to_vec()); + let prio = PriorityConfig::new(usize::MAX, usize::MAX); + let grep = GrepConfig::default(); + let ingest_out = crate::ingest::ingest_into_arena(input, &prio, &grep) + .expect("ingest"); + let order = + order::build_order(&ingest_out.arena, &prio).expect("order"); + let hashes = node_path::compute_merkle_hashes(&order); + let files = node_path::NodeFiles::for_order(&order, None); + order + .by_priority + .iter() + .find_map(|&node_id| { + let (_, path) = node_path::leaf_breadcrumb_key( + &order, node_id, &hashes, &files, + )?; + let prefix = path.split_once('#').map(|(p, _)| p)?; + (prefix == dot_path).then_some(path) + }) + .unwrap_or_else(|| panic!("no leaf at dot_path {dot_path:?}")) + } + + /// When `headson()` is called with an `ExploreContext` that records a + /// breadcrumb for the 'a' key in `{"a": 1, "b": 2}`, under a tight global + /// byte budget (20 bytes), the rendered output must contain 'b' but not 'a'. + #[test] + fn explore_context_deprioritizes_seen_node_under_tight_budget() { + let crumb = Breadcrumb { + file: "".to_string(), + path: composite_key_for(br#"{"a": 1, "b": 2}"#, "a"), + count: 1, + last_step: 1, + }; + let ctx = ExploreContext { + breadcrumbs: vec![crumb], + current_step: 2, + alpha: 0.5, + file: None, + }; + let mut prio = PriorityConfig::new(usize::MAX, usize::MAX); + prio.explore = Some(ctx); + + let tight_budgets = Budgets { + global: Some(Budget { + kind: BudgetKind::Bytes, + cap: 20, + }), + per_slot: None, + }; + + let result = headson( + InputKind::Json(br#"{"a": 1, "b": 2}"#.to_vec()), + &test_render_config(), + &prio, + &GrepConfig::default(), + tight_budgets, + ) + .expect("headson should succeed"); + + assert!( + !result.text.contains("a:") + && !result.text.contains("a :") + && !result.text.contains("\"a\""), + "penalized key 'a' must NOT appear in output under tight budget; got: {:?}", + result.text + ); + assert!( + result.text.contains('b'), + "un-penalized key 'b' must appear in output; got: {:?}", + result.text + ); + } + + /// Baseline: with `explore = None` and the same tight budget, 'a' appears + /// (it has higher priority than 'b' under fair object-key ordering). + #[test] + fn explore_context_none_does_not_affect_output() { + let mut prio = PriorityConfig::new(usize::MAX, usize::MAX); + prio.explore = None; + + let tight_budgets = Budgets { + global: Some(Budget { + kind: BudgetKind::Bytes, + cap: 20, + }), + per_slot: None, + }; + + let result = headson( + InputKind::Json(br#"{"a": 1, "b": 2}"#.to_vec()), + &test_render_config(), + &prio, + &GrepConfig::default(), + tight_budgets, + ) + .expect("headson should succeed"); + + assert!( + result.text.contains('a'), + "without explore penalty, 'a' (first key) must appear in output under tight budget; got: {:?}", + result.text + ); + } + + // ── PENALTY_SCALE regression: array rotation under explore ──────────── + + /// 30 distinct numeric array items; values `1000 + i` make each index + /// directly greppable in the rendered text. + fn rotation_array_json() -> Vec { + let items: Vec = + (0..30).map(|i| (1000 + i).to_string()).collect(); + format!("[{}]", items.join(", ")).into_bytes() + } + + /// Render `rotation_array_json()` with an explore context holding the + /// given breadcrumbs. + fn render_rotation_array( + breadcrumbs: Vec, + current_step: u64, + budgets: Budgets, + ) -> RenderOutput { + let mut prio = PriorityConfig::new(usize::MAX, usize::MAX); + prio.explore = Some(ExploreContext { + breadcrumbs, + current_step, + alpha: 0.5, + file: None, + }); + headson( + InputKind::Json(rotation_array_json()), + &test_render_config(), + &prio, + &GrepConfig::default(), + budgets, + ) + .expect("headson should succeed") + } + + /// Indices `i` whose value `1000 + i` appears in the rendered text. + fn rendered_indices(text: &str) -> Vec { + (0..30) + .filter(|i| text.contains(&(1000 + i).to_string())) + .collect() + } + + /// Regression for the PENALTY_SCALE tuning (issue #513): array siblings + /// differ by `d^3 * ARRAY_INDEX_CUBIC_WEIGHT`, so an under-scaled penalty + /// can only break exact ties (object keys) and never rotates arrays. + /// After recording breadcrumbs for everything shown in a first + /// tight-budget render, a second render must surface array items that the + /// first one did not, and the very first item must yield its slot. + #[test] + fn explore_context_rotates_array_items_under_tight_budget() { + let tight = Budgets { + global: Some(Budget { + kind: BudgetKind::Bytes, + cap: 60, + }), + per_slot: None, + }; + + let first = render_rotation_array(vec![], 1, tight); + let crumbs: Vec = first + .shown_leaves + .iter() + .map(|(file, path)| Breadcrumb { + file: file.clone(), + path: path.clone(), + count: 1, + last_step: 1, + }) + .collect(); + assert!( + !crumbs.is_empty(), + "first render must record shown leaves; got text: {:?}", + first.text + ); + + let second = render_rotation_array(crumbs, 2, tight); + let first_idx = rendered_indices(&first.text); + let second_idx = rendered_indices(&second.text); + + assert!( + second_idx.iter().any(|i| !first_idx.contains(i)), + "second render must surface array items unseen in the first; \ + first: {first_idx:?}, second: {second_idx:?}" + ); + assert!( + !second_idx.contains(&0), + "head item (index 0) was just seen and must yield its slot; \ + second render showed {second_idx:?}" + ); + } + + /// Soft guarantee: the same breadcrumbs under a loose budget must not + /// exclude anything — the penalty reorders, it never filters. + #[test] + fn explore_penalty_is_soft_under_loose_budget() { + let tight = Budgets { + global: Some(Budget { + kind: BudgetKind::Bytes, + cap: 60, + }), + per_slot: None, + }; + let first = render_rotation_array(vec![], 1, tight); + let crumbs: Vec = first + .shown_leaves + .iter() + .map(|(file, path)| Breadcrumb { + file: file.clone(), + path: path.clone(), + count: 1, + last_step: 1, + }) + .collect(); + + let loose = render_rotation_array(crumbs, 2, Budgets::default()); + let shown = rendered_indices(&loose.text); + assert_eq!( + shown, + (0..30).collect::>(), + "loose budget must still show every item, seen or not" + ); + } + + /// With no explore context, shown-leaf collection (and its hashing) is + /// skipped entirely: `shown_leaves` must be empty. + #[test] + fn shown_leaves_empty_when_explore_is_none() { + let prio = PriorityConfig::new(usize::MAX, usize::MAX); + let result = headson( + InputKind::Json(br#"{"a": 1, "b": 2}"#.to_vec()), + &test_render_config(), + &prio, + &GrepConfig::default(), + Budgets::default(), + ) + .expect("headson should succeed"); + assert!( + result.shown_leaves.is_empty(), + "shown_leaves must be empty without explore; got {:?}", + result.shown_leaves + ); + } + + /// An active session with no breadcrumbs yet (first invocation) must + /// still record shown leaves, while skipping the penalty work. + #[test] + fn shown_leaves_recorded_when_explore_has_empty_breadcrumbs() { + let mut prio = PriorityConfig::new(usize::MAX, usize::MAX); + prio.explore = Some(ExploreContext { + breadcrumbs: vec![], + current_step: 0, + alpha: 0.5, + file: None, + }); + let result = headson( + InputKind::Json(br#"{"a": 1, "b": 2}"#.to_vec()), + &test_render_config(), + &prio, + &GrepConfig::default(), + Budgets::default(), + ) + .expect("headson should succeed"); + let paths: Vec<&str> = result + .shown_leaves + .iter() + .map(|(_, p)| p.as_str()) + .collect(); + assert!( + paths.iter().any(|p| p.starts_with("a#")) + && paths.iter().any(|p| p.starts_with("b#")), + "both leaves must be recorded as shown; got {paths:?}" + ); + } + + /// Fileset selection orders can pick a string's LeafPart without its + /// parent SplittableLeaf in the top-k prefix (the parent renders as a + /// reinserted ancestor). Those strings count as shown: under a tight + /// per-slot byte budget, leaves from every rendered file must be + /// recorded. + /// 30 keys with string values long enough to be truncated under a tight + /// per-slot byte budget. + fn flat_string_object_json() -> String { + let entries: Vec = (0..30) + .map(|i| { + format!("\"key_{i:02}\": \"value_number_{i:02}_padding\"") + }) + .collect(); + format!("{{{}}}", entries.join(",")) + } + + #[test] + fn fileset_truncated_strings_recorded_in_shown_leaves() { + let flat = flat_string_object_json(); + let inputs = vec![ + FilesetInput { + name: "flat.json".into(), + bytes: flat.clone().into_bytes(), + kind: FilesetInputKind::Json, + }, + FilesetInput { + name: "other.json".into(), + bytes: flat.into_bytes(), + kind: FilesetInputKind::Json, + }, + ]; + let mut prio = PriorityConfig::new(usize::MAX, usize::MAX); + prio.explore = Some(ExploreContext { + breadcrumbs: vec![], + current_step: 0, + alpha: 0.5, + file: None, + }); + let budgets = Budgets { + global: None, + per_slot: Some(Budget { + kind: BudgetKind::Bytes, + cap: 400, + }), + }; + let result = headson( + InputKind::Fileset(inputs), + &test_render_config(), + &prio, + &GrepConfig::default(), + budgets, + ) + .expect("headson should succeed"); + assert!( + result.text.contains("key_00"), + "render must show truncated string leaves; got {:?}", + result.text + ); + let files: std::collections::HashSet<&str> = result + .shown_leaves + .iter() + .map(|(file, _)| file.as_str()) + .collect(); + assert_eq!( + files.len(), + 2, + "shown leaves must cover both fileset files; got {:?}", + result.shown_leaves + ); + assert!( + result + .shown_leaves + .iter() + .any(|(_, path)| path.starts_with("key_00#")), + "truncated string leaf must be recorded; got {:?}", + result.shown_leaves + ); + } + + /// Strong grep matches forced into the render outside the top-k prefix + /// count as seen: their breadcrumb keys must land in `shown_leaves`. + #[test] + fn strong_grep_must_keep_leaves_recorded_in_shown_leaves() { + let input = br#"{"alpha": "needle one", "beta": "no match here", "gamma": "needle two"}"#; + let grep_cfg = build_grep_config( + Some("needle"), + None, + GrepShow::Matching, + false, + true, + ) + .expect("valid grep pattern"); + let mut prio = PriorityConfig::new(usize::MAX, usize::MAX); + prio.explore = Some(ExploreContext { + breadcrumbs: vec![], + current_step: 0, + alpha: 0.5, + file: None, + }); + // 1-line global budget: matches are only present because strong grep + // forces them in, outside the top-k prefix. + let budgets = Budgets { + global: Some(Budget { + kind: BudgetKind::Lines, + cap: 1, + }), + per_slot: None, + }; + + let result = headson( + InputKind::Json(input.to_vec()), + &test_render_config(), + &prio, + &grep_cfg, + budgets, + ) + .expect("headson should succeed"); + + let paths: Vec<&str> = result + .shown_leaves + .iter() + .map(|(_, p)| p.as_str()) + .collect(); + assert!( + paths.iter().any(|p| p.starts_with("alpha#")), + "grep-forced leaf 'alpha' must be recorded as shown; got {paths:?}" + ); + assert!( + paths.iter().any(|p| p.starts_with("gamma#")), + "grep-forced leaf 'gamma' must be recorded as shown; got {paths:?}" + ); + } + + // ── Step 26: top_k slice contains at least one leaf node ────────────────── + + #[test] + fn top_k_slice_contains_at_least_one_leaf_node() { + // The top_k slice from by_priority should include real leaf nodes + // (AtomicLeaf or SplittableLeaf) — not just ancestor scaffolding. + // 30 bytes comfortably fits root + 1-2 leaves (~16 bytes each) but + // not all 5 leaves (~44 bytes total), so top_k is neither 0 nor max. + let mut order = make_order_for_top_k(); + let tight_budgets = Budgets { + global: Some(Budget { + kind: BudgetKind::Bytes, + cap: 30, + }), + per_slot: None, + }; + let grep_cfg = GrepConfig::default(); + + let search = find_largest_render_under_budgets( + &mut order, + &test_render_config(), + &grep_cfg, + tight_budgets, + ); + let top_k = search.top_k; + + let has_leaf = order.by_priority[..top_k].iter().any(|node_id| { + matches!( + order.nodes[node_id.0], + RankedNode::AtomicLeaf { .. } + | RankedNode::SplittableLeaf { .. } + ) + }); + + assert!( + has_leaf, + "by_priority[..{}] must contain at least one AtomicLeaf or \ + SplittableLeaf; nodes in slice: {:?}", + top_k, + order.by_priority[..top_k] + .iter() + .map(|id| &order.nodes[id.0]) + .collect::>(), + ); + } } diff --git a/src/main.rs b/src/main.rs index 5bc849a9..6a24e3e5 100644 --- a/src/main.rs +++ b/src/main.rs @@ -2,7 +2,16 @@ clippy::multiple_crate_versions, reason = "Dependency graph pulls distinct versions (e.g., yaml-rust2)." )] +#![cfg_attr( + test, + allow( + clippy::cognitive_complexity, + clippy::float_cmp, + reason = "tests may be structurally complex or assert exact float values" + ) +)] mod cli; +mod session; mod sorting; use anyhow::Result; @@ -20,6 +29,19 @@ fn main() -> Result<()> { return Ok(()); } + if let Some(crate::cli::args::TopSubcommand::Explore(ref explore)) = + cli.subcommand + { + let output = + crate::cli::explore::run_subcommand(&explore.command, &cli)?; + // Some subcommands (e.g. `explore clear`) succeed with nothing to + // say; printing would emit a stray blank line. + if !output.is_empty() { + println!("{output}"); + } + return Ok(()); + } + let (output, warnings) = crate::cli::run::run(&cli)?; println!("{output}"); diff --git a/src/node_path.rs b/src/node_path.rs new file mode 100644 index 00000000..35487189 --- /dev/null +++ b/src/node_path.rs @@ -0,0 +1,740 @@ +use std::path::{Component, Path, PathBuf}; + +use crate::RankedNode; +use crate::order::{NodeId, ObjectType, PriorityOrder}; + +/// Composite breadcrumb key: `(file, "dot_path#hex_hash")`. +/// `file` is the input file's resolved absolute path (see +/// [`resolve_breadcrumb_file`]), or `""` when no file identity is known +/// (stdin, or library callers without an input path). The dot-path is the +/// structural address *inside* the file — never the filename — so the same +/// file produces the same key whether rendered as a single input, inside a +/// fileset, or from a different working directory. +pub type BreadcrumbKey = (String, String); + +/// Resolve an input path to the canonical absolute form used as the +/// breadcrumb `file` component, so the same file yields the same key +/// regardless of cwd or path spelling (`a.json`, `./a.json`, `../x/a.json`). +/// +/// Canonicalizes when possible (resolving symlinks); otherwise falls back to +/// joining onto the current directory and lexically dropping `.`/`..` +/// components. Returns `""` (unknown identity) for an empty name or when a +/// relative name cannot be anchored because the current directory is +/// unavailable — never a relative path, which could not match a +/// canonically-recorded key. +pub fn resolve_breadcrumb_file(name: &str) -> String { + if name.is_empty() { + return String::new(); + } + let path = Path::new(name); + if let Ok(canonical) = path.canonicalize() { + return canonical.to_string_lossy().into_owned(); + } + let joined = if path.is_absolute() { + path.to_path_buf() + } else { + let Ok(cwd) = std::env::current_dir() else { + return String::new(); + }; + cwd.join(path) + }; + lexically_normalized(&joined).to_string_lossy().into_owned() +} + +pub fn lexically_normalized(path: &Path) -> PathBuf { + let mut out = PathBuf::new(); + let mut has_root = false; + for comp in path.components() { + match comp { + Component::Prefix(prefix) => out.push(prefix.as_os_str()), + Component::RootDir => { + out.push(comp.as_os_str()); + has_root = true; + } + Component::CurDir => {} + Component::ParentDir => { + if !out.pop() && !has_root { + out.push(comp.as_os_str()); + } + } + Component::Normal(part) => out.push(part), + } + } + out +} + +/// Maps PQ nodes to the resolved absolute path of the input file they came +/// from, for use as the breadcrumb `file` component. +/// +/// Fileset slots resolve their as-typed input names (the fileset root's +/// child keys) via [`resolve_breadcrumb_file`]; nodes outside any fileset +/// slot fall back to the single-input path (threaded from the CLI through +/// `ExploreContext::file`), or `""` when unknown. +pub struct NodeFiles { + /// Per-PQ-node index into `paths`; `None` falls back to `single`. + slot_of: Vec>, + /// Resolved absolute path per fileset render slot. + paths: Vec, + /// Resolved absolute path of a single (non-fileset) input. + single: Option, +} + +impl NodeFiles { + pub fn for_order(order: &PriorityOrder, single: Option<&str>) -> Self { + let render_slots = order.fileset_render_slots().unwrap_or(&[]); + let mut slot_of: Vec> = vec![None; order.nodes.len()]; + let mut paths: Vec = Vec::with_capacity(render_slots.len()); + for slot in render_slots { + let name = order + .nodes + .get(slot.id.0) + .and_then(RankedNode::key_in_object) + .unwrap_or(""); + let path_idx = paths.len(); + paths.push(resolve_breadcrumb_file(name)); + mark_subtree(order, slot.id, path_idx, &mut slot_of); + } + Self { + slot_of, + paths, + single: single.map(str::to_string), + } + } + + fn file_for(&self, node_id: NodeId) -> &str { + match self.slot_of.get(node_id.0).copied().flatten() { + Some(idx) => self.paths.get(idx).map_or("", String::as_str), + None => self.single.as_deref().unwrap_or(""), + } + } +} + +/// Assign `path_idx` to every node in the subtree rooted at `root` that has +/// not been claimed by an earlier fileset slot. +fn mark_subtree( + order: &PriorityOrder, + root: NodeId, + path_idx: usize, + slot_of: &mut [Option], +) { + let mut stack = vec![root]; + while let Some(id) = stack.pop() { + let Some(entry @ None) = slot_of.get_mut(id.0) else { + continue; + }; + *entry = Some(path_idx); + if let Some(children) = order.children.get(id.0) { + stack.extend(children.iter().copied()); + } + } +} + +const FNV1A_INIT: u64 = 14_695_981_039_346_656_037; +const FNV1A_PRIME: u64 = 1_099_511_628_211; + +// LeafPart is a synthetic rendering node excluded from content identity. +const EXCLUDED_NODE_HASH: u64 = 0; + +fn fnv1a_update(mut h: u64, data: &[u8]) -> u64 { + for &b in data { + h ^= b as u64; + h = h.wrapping_mul(FNV1A_PRIME); + } + h +} + +fn fnv1a(data: &[u8]) -> u64 { + fnv1a_update(FNV1A_INIT, data) +} + +fn build_json_path(order: &PriorityOrder, node_id: NodeId) -> String { + let mut parts: Vec = Vec::new(); + let mut cursor = node_id; + while let Some(parent) = order.parent.get(cursor.0).and_then(|p| *p) { + // A fileset root's child keys are filenames, not structural address: + // file identity lives in the breadcrumb's `file` component instead, + // so the same file keys identically across invocation styles. + let parent_is_fileset_root = matches!( + order.object_type.get(parent.0), + Some(ObjectType::Fileset) + ); + if !parent_is_fileset_root + && let Some(part) = path_component(order, cursor) + { + parts.push(part); + } + cursor = parent; + } + parts.reverse(); + parts.join(".") +} + +/// The dot-path segment contributed by `cursor`: its object key, or its +/// original index within a parent array. +fn path_component(order: &PriorityOrder, cursor: NodeId) -> Option { + if let Some(key) = order + .nodes + .get(cursor.0) + .and_then(RankedNode::key_in_object) + { + return Some(key.to_string()); + } + order + .index_in_parent_array + .get(cursor.0) + .and_then(|x| *x) + .map(|idx| idx.to_string()) +} + +fn node_hash(order: &PriorityOrder, id: usize, hashes: &[u64]) -> u64 { + match order.nodes.get(id) { + Some(RankedNode::AtomicLeaf { token, .. }) => fnv1a(token.as_bytes()), + Some(RankedNode::SplittableLeaf { value, .. }) => { + fnv1a(value.as_bytes()) + } + None | Some(RankedNode::LeafPart { .. }) => EXCLUDED_NODE_HASH, + Some(RankedNode::Object { .. } | RankedNode::Array { .. }) => { + let mut h = FNV1A_INIT; + if let Some(children) = order.children.get(id) { + for &child_id in children { + if let Some(key) = order + .nodes + .get(child_id.0) + .and_then(|n| n.key_in_object()) + { + h = fnv1a_update(h, key.as_bytes()); + } + h = fnv1a_update( + h, + &hashes + .get(child_id.0) + .copied() + .unwrap_or(EXCLUDED_NODE_HASH) + .to_le_bytes(), + ); + } + } + h + } + } +} + +/// Compute a stable FNV-1a Merkle hash for every node in `order`. +/// +/// Hashes are built bottom-up: +/// - `AtomicLeaf` / `SplittableLeaf`: hash of token/value bytes. +/// - `Object`: hash of (key_bytes ++ child_hash) for each child in order. +/// - `Array`: hash of child hashes in order. +/// - `LeafPart`: 0 (synthetic split-rendering node; excluded from breadcrumbs). +/// +/// The returned `Vec` is indexed by PQ node id (`NodeId.0`). +/// Output is deterministic across process restarts. +#[allow( + clippy::cognitive_complexity, + reason = "post-order tree traversal inherently requires nested control flow" +)] +pub fn compute_merkle_hashes(order: &PriorityOrder) -> Vec { + let n = order.nodes.len(); + let mut hashes = vec![0u64; n]; + if n == 0 { + return hashes; + } + // Root is always PQ id 0 by construction in build_order. + let mut stack: Vec<(usize, bool)> = Vec::with_capacity(n); + stack.push((0, false)); + while let Some((id, processed)) = stack.pop() { + if processed { + hashes[id] = node_hash(order, id, &hashes); + } else { + stack.push((id, true)); + if let Some(children) = order.children.get(id) { + for &child in children.iter().rev() { + stack.push((child.0, false)); + } + } + } + } + hashes +} + +/// Returns `(file, path)` for a leaf node, or `None` for structural nodes. +/// +/// `file` is the resolved absolute path of the input file the node belongs +/// to (per-slot for filesets, the single-input path otherwise), or `""` when +/// no file identity is known (stdin, library callers without a path). +/// +/// `path` is `"dot.path#<16 hex digits>"`: the structural address *inside* +/// the file combined with the FNV-1a Merkle hash of the subtree at that node. +/// The composite key is stable across restarts and invocation styles. A +/// content change produces a new hash (no match); reverting the change +/// restores the original hash (penalty re-activates). +pub fn leaf_breadcrumb_key( + order: &PriorityOrder, + node_id: NodeId, + hashes: &[u64], + files: &NodeFiles, +) -> Option { + match order.nodes.get(node_id.0)? { + RankedNode::Array { .. } + | RankedNode::Object { .. } + | RankedNode::LeafPart { .. } => None, + RankedNode::AtomicLeaf { .. } | RankedNode::SplittableLeaf { .. } => { + let dot_path = build_json_path(order, node_id); + let hash = + hashes.get(node_id.0).copied().unwrap_or(EXCLUDED_NODE_HASH); + Some(( + files.file_for(node_id).to_string(), + format!("{dot_path}#{hash:016x}"), + )) + } + } +} + +/// The node that carries breadcrumb identity for a selected node: a +/// `LeafPart` resolves to its parent `SplittableLeaf`, everything else to +/// itself. +fn breadcrumb_carrier(order: &PriorityOrder, node_id: NodeId) -> NodeId { + if matches!( + order.nodes.get(node_id.0), + Some(RankedNode::LeafPart { .. }) + ) && let Some(parent) = order.parent.get(node_id.0).copied().flatten() + { + return parent; + } + node_id +} + +/// Collect breadcrumb keys for the leaves actually selected by the budget +/// search: the top-k prefix of the ordering the search indexed into (the +/// per-slot `selection_order` when present, `by_priority` otherwise), plus +/// any strong-grep must-keep nodes forced into the render outside that +/// prefix. Selected `LeafPart` nodes count as their parent string being +/// shown. +/// +/// Reuses the Merkle hash table stored by explore penalty matching when +/// available so the whole pipeline performs at most one full hash pass. +/// +/// `single_file` is the resolved absolute path of a single (non-fileset) +/// input, used as the `file` component for nodes outside any fileset slot. +pub(crate) fn collect_shown_leaves( + order: &PriorityOrder, + search: &crate::pruner::budget::BudgetSearchResult, + single_file: Option<&str>, +) -> Vec { + use std::borrow::Cow; + use std::collections::HashSet; + + let hashes: Cow<'_, [u64]> = order.merkle_hashes.as_deref().map_or_else( + || Cow::Owned(compute_merkle_hashes(order)), + Cow::Borrowed, + ); + let files = NodeFiles::for_order(order, single_file); + let base: &[NodeId] = search + .selection_order + .as_deref() + .unwrap_or(&order.by_priority); + let bound = search.top_k.min(base.len()); + let mut seen: HashSet = HashSet::with_capacity(bound); + let mut out = Vec::new(); + for &selected in base[..bound].iter().chain(&search.grep_must_keep) { + // A selected LeafPart renders its parent string (the SplittableLeaf + // is reinserted as an ancestor), so the parent carries the breadcrumb + // identity. Fileset selection orders can include parts without their + // parent in the top-k prefix. + let node_id = breadcrumb_carrier(order, selected); + if !seen.insert(node_id) { + continue; + } + if let Some(key) = leaf_breadcrumb_key(order, node_id, &hashes, &files) + { + out.push(key); + } + } + out +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ingest::{parse_json_one, parse_text_one_with_mode}; + use crate::order::PriorityConfig; + use crate::order::build_order; + + fn make_order(json: &[u8]) -> PriorityOrder { + let cfg = PriorityConfig::new(usize::MAX, usize::MAX); + let arena = + parse_json_one(json.to_vec(), &cfg).expect("parse must succeed"); + build_order(&arena, &cfg).expect("build_order must succeed") + } + + fn no_files(order: &PriorityOrder) -> NodeFiles { + NodeFiles::for_order(order, None) + } + + // A. compute_merkle_hashes is stable across two separate build_order calls + // for the same JSON bytes. + #[test] + fn merkle_hashes_stable_across_builds() { + let json = b"{\"x\": 42, \"y\": \"hello\"}"; + let order1 = make_order(json); + let order2 = make_order(json); + let hashes1 = compute_merkle_hashes(&order1); + let hashes2 = compute_merkle_hashes(&order2); + assert_eq!( + hashes1, hashes2, + "Merkle hashes must be identical across two builds of the same input" + ); + } + + // B. Root-node hash differs when a leaf value changes. + #[test] + fn merkle_hash_changes_when_leaf_value_changes() { + let order_a = make_order(b"{\"a\": 1}"); + let order_b = make_order(b"{\"a\": 2}"); + let hashes_a = compute_merkle_hashes(&order_a); + let hashes_b = compute_merkle_hashes(&order_b); + // Root is node 0 in PriorityOrder convention. + assert_ne!( + hashes_a[crate::order::types::ROOT_PQ_ID], + hashes_b[crate::order::types::ROOT_PQ_ID], + "Root hash must differ when leaf value changes" + ); + } + + // C. The hash of a leaf that did NOT change is unaffected by a sibling change. + #[test] + fn merkle_hash_sibling_unchanged_when_other_sibling_changes() { + // {"a": 1, "b": "hello"} vs {"a": 1, "b": "world"} + // The leaf for value 1 (key "a") must hash identically in both. + let order_hello = make_order(b"{\"a\": 1, \"b\": \"hello\"}"); + let order_world = make_order(b"{\"a\": 1, \"b\": \"world\"}"); + let hashes_hello = compute_merkle_hashes(&order_hello); + let hashes_world = compute_merkle_hashes(&order_world); + + // Find the node id for the leaf value of key "a" (AtomicLeaf with token "1") + // in each order. + let find_a_leaf_id = |order: &PriorityOrder| -> usize { + order.nodes.iter().position(|n| { + matches!(n, RankedNode::AtomicLeaf { token, key_in_object, .. } + if token == "1" && key_in_object.as_deref() == Some("a")) + }).expect("must find leaf for key 'a' with value 1") + }; + + let id_hello = find_a_leaf_id(&order_hello); + let id_world = find_a_leaf_id(&order_world); + + assert_eq!( + hashes_hello[id_hello], hashes_world[id_world], + "Hash for unchanged leaf ('a': 1) must be the same regardless of sibling change" + ); + } + + // D. The composite key format for a leaf contains '#' separating dot_path + // from a 16-hex-char hash string. + #[test] + fn composite_key_format_contains_hash_separator() { + let json = b"{\"name\": \"alice\"}"; + let order = make_order(json); + let hashes = compute_merkle_hashes(&order); + + // Find the SplittableLeaf for value "alice" + let alice_id = order.nodes.iter().position(|n| { + matches!(n, RankedNode::SplittableLeaf { value, key_in_object, .. } + if value == "alice" && key_in_object.as_deref() == Some("name")) + }).expect("must find 'alice' leaf"); + + let result = leaf_breadcrumb_key( + &order, + NodeId(alice_id), + &hashes, + &no_files(&order), + ); + let (_, path) = + result.expect("leaf_breadcrumb_key must return Some for a leaf"); + + let parts: Vec<&str> = path.splitn(2, '#').collect(); + assert_eq!( + parts.len(), + 2, + "path must contain exactly one '#' separator; got: {path:?}" + ); + assert_eq!( + parts[0], "name", + "dot_path before '#' must be 'name'; got: {path:?}" + ); + assert_eq!( + parts[1].len(), + 16, + "hex hash after '#' must be 16 chars; got: {path:?}" + ); + assert!( + parts[1].chars().all(|c| c.is_ascii_hexdigit()), + "hash part must be hex digits; got: {path:?}" + ); + } + + // D2. Without any file identity (stdin, bare library use), the file + // component is "". + #[test] + fn file_component_empty_without_identity() { + let order = make_order(b"{\"k\": \"v\"}"); + let hashes = compute_merkle_hashes(&order); + let leaf_id = order + .nodes + .iter() + .position(|n| matches!(n, RankedNode::SplittableLeaf { .. })) + .expect("must find a leaf"); + let (file, _) = leaf_breadcrumb_key( + &order, + NodeId(leaf_id), + &hashes, + &no_files(&order), + ) + .expect("leaf must produce a key"); + assert_eq!(file, "", "file must be empty without file identity"); + } + + // E. The composite key is stable across two separate builds of the same input. + #[test] + fn composite_key_stable_across_builds() { + let json = b"{\"k\": \"v\"}"; + let order1 = make_order(json); + let order2 = make_order(json); + let hashes1 = compute_merkle_hashes(&order1); + let hashes2 = compute_merkle_hashes(&order2); + + // Find the leaf in each order. + let find_leaf = |order: &PriorityOrder| -> NodeId { + NodeId(order.nodes.iter().position(|n| { + matches!(n, RankedNode::SplittableLeaf { value, .. } if value == "v") + }).expect("must find leaf 'v'")) + }; + + let key1 = leaf_breadcrumb_key( + &order1, + find_leaf(&order1), + &hashes1, + &no_files(&order1), + ); + let key2 = leaf_breadcrumb_key( + &order2, + find_leaf(&order2), + &hashes2, + &no_files(&order2), + ); + assert_eq!( + key1, key2, + "composite key must be identical across two builds of the same input" + ); + } + + // F. Composite key for unchanged leaf is stable when a sibling leaf changes. + #[test] + fn composite_key_stable_when_sibling_changes() { + let order_hello = make_order(b"{\"a\": 1, \"b\": \"hello\"}"); + let order_world = make_order(b"{\"a\": 1, \"b\": \"world\"}"); + let hashes_hello = compute_merkle_hashes(&order_hello); + let hashes_world = compute_merkle_hashes(&order_world); + + let find_a = |order: &PriorityOrder| -> NodeId { + NodeId(order.nodes.iter().position(|n| { + matches!(n, RankedNode::AtomicLeaf { token, key_in_object, .. } + if token == "1" && key_in_object.as_deref() == Some("a")) + }).expect("must find AtomicLeaf for key 'a'")) + }; + + let key_hello = leaf_breadcrumb_key( + &order_hello, + find_a(&order_hello), + &hashes_hello, + &no_files(&order_hello), + ); + let key_world = leaf_breadcrumb_key( + &order_world, + find_a(&order_world), + &hashes_world, + &no_files(&order_world), + ); + assert_eq!( + key_hello, key_world, + "composite key for unchanged leaf must be stable when a sibling changes" + ); + } + + // G. leaf_breadcrumb_key returns None for all Array and Object structural nodes. + #[test] + fn structural_nodes_return_none() { + // {"a": [1, 2]} — contains a root Object and one Array child. + let order = make_order(b"{\"a\": [1, 2]}"); + let hashes = compute_merkle_hashes(&order); + let files = no_files(&order); + + for (idx, node) in order.nodes.iter().enumerate() { + if matches!( + node, + RankedNode::Array { .. } | RankedNode::Object { .. } + ) { + let result = + leaf_breadcrumb_key(&order, NodeId(idx), &hashes, &files); + assert!( + result.is_none(), + "leaf_breadcrumb_key must return None for structural node at index {idx}: {node:?}" + ); + } + } + } + + // H. Code mode: AtomicLeaf composite key path contains '#' and the dot_path + // part (before '#') is NOT a standalone hex string (old behavior was to + // return only a hash with no dot-path prefix). + #[test] + #[allow( + clippy::cognitive_complexity, + reason = "test iterates nodes and checks multiple properties" + )] + fn code_mode_no_longer_special_cased() { + let code = b"fn foo() {\n let x = 1;\n}\n"; + let cfg = PriorityConfig::new(usize::MAX, usize::MAX); + let arena = parse_text_one_with_mode(code.to_vec(), &cfg, true) + .expect("parse must succeed"); + let order = + build_order(&arena, &cfg).expect("build_order must succeed"); + let hashes = compute_merkle_hashes(&order); + let files = no_files(&order); + + let mut found_leaf = false; + for (idx, node) in order.nodes.iter().enumerate() { + if matches!(node, RankedNode::AtomicLeaf { .. }) { + if let Some((_, path)) = + leaf_breadcrumb_key(&order, NodeId(idx), &hashes, &files) + { + found_leaf = true; + assert!( + path.contains('#'), + "code-mode leaf path must contain '#'; got: {path:?}" + ); + let dot_path = path.split('#').next().unwrap_or(""); + // Old behavior was a bare hex hash with no dot_path prefix. + // The dot_path must NOT be a purely hex string (or must be non-empty + // with non-hex characters like digits-only index or a line label). + // Specifically, it must not look like a standalone 16-hex-char hash. + let is_standalone_hex_hash = dot_path.len() >= 8 + && dot_path.chars().all(|c| c.is_ascii_hexdigit()); + assert!( + !is_standalone_hex_hash, + "dot_path before '#' must NOT be a standalone hex hash; got dot_path={dot_path:?} in path={path:?}" + ); + } + } + } + assert!( + found_leaf, + "must have found at least one AtomicLeaf in code input" + ); + } + + fn make_fileset_order( + files: Vec<(&str, &[u8])>, + ) -> (PriorityOrder, Vec) { + use crate::ingest::fileset::{FilesetInput, FilesetInputKind}; + let cfg = PriorityConfig::new(usize::MAX, usize::MAX); + let inputs = files + .into_iter() + .map(|(name, bytes)| FilesetInput { + name: name.to_string(), + bytes: bytes.to_vec(), + kind: FilesetInputKind::Json, + }) + .collect(); + let out = crate::ingest::ingest_into_arena( + crate::InputKind::Fileset(inputs), + &cfg, + &crate::GrepConfig::default(), + ) + .expect("fileset ingest must succeed"); + let order = + build_order(&out.arena, &cfg).expect("build_order must succeed"); + let hashes = compute_merkle_hashes(&order); + (order, hashes) + } + + fn keys_of(order: &PriorityOrder, hashes: &[u64]) -> Vec { + let files = no_files(order); + (0..order.nodes.len()) + .filter_map(|idx| { + leaf_breadcrumb_key(order, NodeId(idx), hashes, &files) + }) + .collect() + } + + // I. Fileset keys carry the resolved absolute file path and an inner + // dot-path that does NOT embed the filename. + #[test] + fn fileset_keys_use_resolved_file_and_inner_dot_path() { + let (order, hashes) = + make_fileset_order(vec![("f1.json", br#"{"x": 1}"#)]); + let keys = keys_of(&order, &hashes); + let (file, path) = keys + .iter() + .find(|(_, p)| p.starts_with("x#")) + .expect("must find leaf at inner path 'x'"); + assert_eq!( + file, + &resolve_breadcrumb_file("f1.json"), + "file must be the resolved absolute input path" + ); + assert!( + Path::new(file).is_absolute(), + "file must be absolute; got {file:?}" + ); + assert!( + !path.contains("f1.json"), + "inner dot-path must not embed the filename; got {path:?}" + ); + } + + // J. Two fileset files with identical content produce keys that differ + // only in the file component — same inner path, same hash. + #[test] + fn identical_files_share_paths_but_not_file_identity() { + let content: &[u8] = br#"{"version": "1.0"}"#; + let (order, hashes) = + make_fileset_order(vec![("a.json", content), ("b.json", content)]); + let keys = keys_of(&order, &hashes); + let version_keys: Vec<&BreadcrumbKey> = keys + .iter() + .filter(|(_, p)| p.starts_with("version#")) + .collect(); + assert_eq!(version_keys.len(), 2, "one 'version' leaf per file"); + assert_eq!( + version_keys[0].1, version_keys[1].1, + "identical content must hash identically" + ); + assert_ne!( + version_keys[0].0, version_keys[1].0, + "different files must have distinct file identity" + ); + } + + // K. resolve_breadcrumb_file is spelling-independent: `./x` and `x` + // resolve identically, `..` components are removed, and the result is + // absolute even for paths that do not exist. + #[test] + fn resolve_breadcrumb_file_normalizes_spellings() { + let plain = resolve_breadcrumb_file("no_such_dir/x.json"); + assert_eq!(resolve_breadcrumb_file("./no_such_dir/x.json"), plain); + assert_eq!( + resolve_breadcrumb_file("no_such_dir/sub/../x.json"), + plain + ); + assert!( + Path::new(&plain).is_absolute(), + "fallback resolution must produce an absolute path; got {plain:?}" + ); + } + + #[test] + fn resolve_breadcrumb_file_maps_empty_name_to_unknown_identity() { + assert_eq!(resolve_breadcrumb_file(""), ""); + } +} diff --git a/src/order/build.rs b/src/order/build.rs index 729f6283..d8ab57cf 100644 --- a/src/order/build.rs +++ b/src/order/build.rs @@ -816,7 +816,7 @@ pub fn build_order( } } - let fileset_render_slots = if arena.is_fileset { + let (fileset_root_ids, fileset_render_slots) = if arena.is_fileset { let root = &arena.nodes[arena.root_id]; let mut ids: Vec = Vec::with_capacity(root.children_len); let mut slots: Vec = @@ -833,10 +833,9 @@ pub fn build_order( slots.push(FilesetRenderSlot { id, suppressed }); } } - interleave_fileset_priority(&mut order, &node_slots, &ids); - Some(slots) + (Some(ids), Some(slots)) } else { - None + (None, None) }; let total = next_pq_id; @@ -853,7 +852,7 @@ pub fn build_order( } } } - Ok(PriorityOrder { + let mut result = PriorityOrder { metrics, nodes, scores, @@ -866,7 +865,23 @@ pub fn build_order( code_lines, fileset_render_slots, safety_cap_hit, - }) + merkle_hashes: None, + }; + // Penalty must precede interleaving: its re-sort is by raw score, which + // would silently undo the round-robin fairness if applied afterwards. + if let Some(ctx) = config.explore.as_ref() + && !ctx.breadcrumbs.is_empty() + { + super::explore::apply_explore_penalty(&mut result, ctx); + } + if let Some(ids) = fileset_root_ids { + interleave_fileset_priority( + &mut result.by_priority, + &node_slots, + &ids, + ); + } + Ok(result) } #[allow( diff --git a/src/order/explore.rs b/src/order/explore.rs new file mode 100644 index 00000000..297671dd --- /dev/null +++ b/src/order/explore.rs @@ -0,0 +1,332 @@ +use std::collections::HashMap; + +use crate::node_path; +use crate::order::types::{ + Breadcrumb, ExploreContext, NodeId, PriorityOrder, RankedNode, + novelty_penalty, +}; + +/// Scales the raw novelty penalty (`ln(1 + count) * alpha^steps_ago`) into +/// `u128` score space. Score anchors: object/string siblings differ by ~1; +/// array siblings by `d^3 * 1e12` where `d` is distance to the nearest +/// sampling anchor. At 1e15 a leaf seen once on the previous step (scaled +/// ≈ 3.47e14) outranks unseen siblings up to `d ≈ 7`, shrinking to `d ≈ 4` +/// after three steps at alpha=0.5. The penalty reorders, never excludes. +const PENALTY_SCALE: f64 = 1_000_000_000_000_000.0; + +/// Apply the explore novelty penalty to previously-seen leaves and re-sort +/// `by_priority` by the adjusted scores. +/// +/// Must run BEFORE fileset interleaving (`build_order` guarantees this): the +/// re-sort restores raw score order, which would otherwise destroy the +/// round-robin fairness applied by `interleave_fileset_priority`. +/// +/// The Merkle hash table computed for breadcrumb matching is stored on +/// `order.merkle_hashes` so shown-leaf collection can reuse it without a +/// second full-tree hashing pass. +pub(crate) fn apply_explore_penalty( + order: &mut PriorityOrder, + ctx: &ExploreContext, +) { + if ctx.breadcrumbs.is_empty() { + return; + } + let hashes = node_path::compute_merkle_hashes(order); + let files = node_path::NodeFiles::for_order(order, ctx.file.as_deref()); + let by_key: HashMap<(&str, &str), &Breadcrumb> = ctx + .breadcrumbs + .iter() + .map(|b| ((b.file.as_str(), b.path.as_str()), b)) + .collect(); + let penalties: Vec<(NodeId, u128)> = order + .by_priority + .iter() + .filter_map(|&node_id| { + let (file, path) = node_path::leaf_breadcrumb_key( + order, node_id, &hashes, &files, + )?; + let bc = by_key.get(&(file.as_str(), path.as_str()))?; + let steps_ago = ctx.current_step.saturating_sub(bc.last_step); + let penalty = novelty_penalty(bc.count, steps_ago, ctx.alpha); + (penalty > 0.0) + .then_some((node_id, (penalty * PENALTY_SCALE) as u128)) + }) + .collect(); + if !penalties.is_empty() { + // Phase 1: bump leaf scores. + let mut leaf_penalties: Vec = vec![0; order.total_nodes]; + for &(node_id, delta) in &penalties { + leaf_penalties[node_id.0] = delta; + bump_score(order, node_id, delta); + } + // Phase 4: propagate mean child penalty upward through structural nodes + // so that heavily-seen subtrees (e.g. Cargo.lock) lose budget priority + // relative to less-explored peers, not just individual leaves. + propagate_penalties_upward(order, &leaf_penalties); + order.by_priority.sort_by_key(|id| order.scores[id.0]); + } + order.merkle_hashes = Some(hashes); +} + +/// Propagate leaf penalties upward through the priority tree. +/// +/// For each structural node (Array/Object), applies a penalty equal to the +/// mean of its direct children's penalties. This makes heavily-explored +/// subtrees lose budget priority relative to untouched peers, not just +/// individual leaves. One pass over all nodes suffices because the tree is +/// acyclic and we only read leaf penalties (not yet-accumulated parent ones). +fn propagate_penalties_upward( + order: &mut PriorityOrder, + leaf_penalties: &[u128], +) { + let (child_sum, child_cnt) = + accumulate_child_penalties(order, leaf_penalties); + apply_mean_penalties(order, &child_sum, &child_cnt); +} + +fn accumulate_child_penalties( + order: &PriorityOrder, + leaf_penalties: &[u128], +) -> (Vec, Vec) { + let n = order.total_nodes; + let mut child_sum: Vec = vec![0; n]; + let mut child_cnt: Vec = vec![0; n]; + for (pq_idx, &penalty) in leaf_penalties.iter().enumerate().take(n) { + if penalty == 0 { + continue; + } + if let Some(parent_id) = order.parent[pq_idx] { + child_sum[parent_id.0] = + child_sum[parent_id.0].saturating_add(penalty); + child_cnt[parent_id.0] += 1; + } + } + (child_sum, child_cnt) +} + +fn apply_mean_penalties( + order: &mut PriorityOrder, + child_sum: &[u128], + child_cnt: &[u32], +) { + for (pq_idx, (&sum, &cnt)) in + child_sum.iter().zip(child_cnt.iter()).enumerate() + { + if cnt == 0 { + continue; + } + let mean = sum / u128::from(cnt); + if mean > 0 { + order.scores[pq_idx] = order.scores[pq_idx].saturating_add(mean); + } + } +} + +fn bump_score(order: &mut PriorityOrder, node_id: NodeId, delta: u128) { + order.scores[node_id.0] = order.scores[node_id.0].saturating_add(delta); + // Carry the penalty onto the LeafPart children of a penalized + // SplittableLeaf so the parts cannot sort ahead of their parent; + // otherwise a part inside the top-k prefix would render the string while + // the parent leaf goes unrecorded as shown. + if matches!(order.nodes[node_id.0], RankedNode::SplittableLeaf { .. }) { + for i in 0..order.children[node_id.0].len() { + let child = order.children[node_id.0][i]; + order.scores[child.0] = + order.scores[child.0].saturating_add(delta); + } + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::ingest::parse_json_one; + use crate::order::{PriorityConfig, build_order}; + + fn make_order(json: &[u8]) -> PriorityOrder { + let cfg = PriorityConfig::new(usize::MAX, usize::MAX); + let arena = parse_json_one(json.to_vec(), &cfg).expect("parse"); + build_order(&arena, &cfg).expect("build_order") + } + + fn key_for(order: &PriorityOrder, dot_path: &str) -> (String, String) { + let hashes = node_path::compute_merkle_hashes(order); + let files = node_path::NodeFiles::for_order(order, None); + order + .by_priority + .iter() + .find_map(|&node_id| { + let key = node_path::leaf_breadcrumb_key( + order, node_id, &hashes, &files, + )?; + let prefix = key.1.split_once('#').map(|(p, _)| p)?; + (prefix == dot_path).then_some(key) + }) + .unwrap_or_else(|| panic!("no leaf at dot_path {dot_path:?}")) + } + + #[test] + fn empty_breadcrumbs_leave_order_and_hashes_untouched() { + let mut order = make_order(b"{\"a\": 1, \"b\": 2}"); + let before = order.by_priority.clone(); + let ctx = ExploreContext { + breadcrumbs: vec![], + current_step: 3, + alpha: 0.5, + file: None, + }; + apply_explore_penalty(&mut order, &ctx); + assert_eq!(order.by_priority, before); + assert!(order.merkle_hashes.is_none()); + } + + #[test] + fn penalized_splittable_leaf_stays_before_its_leaf_parts() { + // Two string values; penalize "alpha" so its SplittableLeaf moves + // back — its LeafPart children must move back with it. + let json = b"{\"x\": \"alpha\", \"y\": \"beta\"}"; + let mut order = make_order(json); + let (file, path) = key_for(&order, "x"); + let ctx = ExploreContext { + breadcrumbs: vec![Breadcrumb { + file, + path, + count: 5, + last_step: 1, + }], + current_step: 2, + alpha: 0.9, + file: None, + }; + apply_explore_penalty(&mut order, &ctx); + + let leaf_idx = order + .nodes + .iter() + .position(|n| { + matches!(n, RankedNode::SplittableLeaf { value, .. } if value == "alpha") + }) + .expect("alpha leaf"); + let pos_of = |id: usize| { + order + .by_priority + .iter() + .position(|n| n.0 == id) + .expect("node present in by_priority") + }; + let leaf_pos = pos_of(leaf_idx); + for child in &order.children[leaf_idx] { + assert!( + pos_of(child.0) > leaf_pos, + "LeafPart {child:?} sorted before its penalized parent" + ); + } + assert!( + order.merkle_hashes.is_some(), + "hash table must be stored for reuse by shown-leaf collection" + ); + } + + #[test] + fn breadcrumb_for_other_file_does_not_match_identical_content() { + // The breadcrumb's path matches leaf "x" exactly (same content hash), + // but it was recorded for a different file — no penalty may apply. + let json = b"{\"x\": \"alpha\", \"y\": \"beta\"}"; + let mut order = make_order(json); + let before = order.by_priority.clone(); + let (_, path) = key_for(&order, "x"); + let ctx = ExploreContext { + breadcrumbs: vec![Breadcrumb { + file: "/abs/other.json".to_string(), + path, + count: 5, + last_step: 1, + }], + current_step: 2, + alpha: 0.9, + file: Some("/abs/this.json".to_string()), + }; + apply_explore_penalty(&mut order, &ctx); + assert_eq!( + order.by_priority, before, + "a breadcrumb from another file must not penalize this file" + ); + } + + /// Phase 4: when all leaves under a top-level key "a" are penalized, the + /// Object node for "a" accumulates a mean penalty and sorts AFTER the + /// unpenalized "b" Object node. + /// + /// Without Phase 4, both Object nodes have the same low base score, so + /// "a" keeps its original position ahead of "b". With Phase 4, "a" gets + /// a propagated penalty and moves behind "b" in by_priority. + #[test] + fn phase4_penalized_section_ranks_below_unpenalized_sibling() { + // Two top-level objects. Penalize all leaves under "a". + let json = br#"{"a": {"x": 1, "y": 2}, "b": {"p": 3, "q": 4}}"#; + let base_order = make_order(json); + + // Collect breadcrumb keys for all leaves under "a". + let hashes = node_path::compute_merkle_hashes(&base_order); + let files = node_path::NodeFiles::for_order(&base_order, None); + let a_breadcrumbs: Vec = base_order + .by_priority + .iter() + .filter_map(|&id| { + let (file, path) = node_path::leaf_breadcrumb_key( + &base_order, + id, + &hashes, + &files, + )?; + // Keys under "a" start with "a." in their dot-path. + let dot_path = path.split_once('#').map(|(p, _)| p)?; + (dot_path.starts_with("a.")).then_some(Breadcrumb { + file, + path, + count: 3, + last_step: 1, + }) + }) + .collect(); + assert!(!a_breadcrumbs.is_empty(), "must find leaves under 'a'"); + + let mut cfg = PriorityConfig::new(usize::MAX, usize::MAX); + cfg.explore = Some(ExploreContext { + breadcrumbs: a_breadcrumbs, + current_step: 2, + alpha: 0.5, + file: None, + }); + + // Build via ingest so the explore penalty (including Phase 4) runs. + use crate::InputKind; + use crate::grep::GrepConfig; + use crate::ingest::ingest_into_arena; + let arena = ingest_into_arena( + InputKind::Json(json.to_vec()), + &cfg, + &GrepConfig::default(), + ) + .unwrap() + .arena; + let order = build_order(&arena, &cfg).unwrap(); + + // Find positions of the "a" and "b" Object nodes in by_priority. + let pos_of_obj = |key: &str| { + order.by_priority.iter().position(|id| { + matches!(&order.nodes[id.0], + RankedNode::Object { key_in_object: Some(k), .. } if k == key) + }) + }; + let a_pos = pos_of_obj("a").expect("Object 'a' in by_priority"); + let b_pos = pos_of_obj("b").expect("Object 'b' in by_priority"); + + assert!( + b_pos < a_pos, + "Object 'b' (pos {b_pos}) must rank ahead of penalized Object 'a' \ + (pos {a_pos}) — Phase 4 must propagate the leaf penalty to the \ + parent Object" + ); + } +} diff --git a/src/order/mod.rs b/src/order/mod.rs index 2e4ecee9..1f3b1def 100644 --- a/src/order/mod.rs +++ b/src/order/mod.rs @@ -1,4 +1,5 @@ pub mod build; +pub(crate) mod explore; pub mod scoring; pub mod types; diff --git a/src/order/types.rs b/src/order/types.rs index e1729ed5..a0e1f3c3 100644 --- a/src/order/types.rs +++ b/src/order/types.rs @@ -1,9 +1,40 @@ use std::collections::HashMap; use std::sync::Arc; +use serde::{Deserialize, Serialize}; + use super::scoring::DEFAULT_SAFETY_CAP; -#[derive(Copy, Clone, Debug)] +#[derive(Clone, Debug, Serialize, Deserialize, PartialEq)] +pub struct Breadcrumb { + pub file: String, + pub path: String, + pub count: u64, + pub last_step: u64, +} + +/// Novelty penalty for a previously-seen node: `ln(1 + count) * alpha^steps_ago`. +/// +/// `steps_ago` is clamped before the `i32` conversion: a raw `as i32` cast of +/// a large `u64` could wrap negative, turning the decay into a huge +/// amplification for `alpha < 1`. +pub fn novelty_penalty(count: u64, steps_ago: u64, alpha: f64) -> f64 { + let steps = i32::try_from(steps_ago).unwrap_or(i32::MAX); + (1.0 + count as f64).ln() * alpha.powi(steps) +} + +#[derive(Clone, Debug)] +pub struct ExploreContext { + pub breadcrumbs: Vec, + pub current_step: u64, + pub alpha: f64, + /// Resolved absolute path of a single (non-fileset) input file, used as + /// the breadcrumb `file` component. `None` for stdin and fileset inputs + /// (fileset slots resolve their own per-file paths from input names). + pub file: Option, +} + +#[derive(Clone, Debug)] pub struct PriorityConfig { pub max_string_graphemes: usize, pub array_max_items: usize, @@ -19,6 +50,8 @@ pub struct PriorityConfig { /// Hard ceiling on priority queue nodes to prevent degenerate inputs /// from exhausting memory/time. Default is 2,000,000. pub safety_cap: usize, + /// Optional explore context for deprioritizing previously-seen nodes. + pub explore: Option, } impl PriorityConfig { @@ -31,6 +64,7 @@ impl PriorityConfig { array_sampler: ArraySamplerStrategy::Default, line_budget_only: false, safety_cap: DEFAULT_SAFETY_CAP, + explore: None, } } @@ -58,6 +92,7 @@ impl PriorityConfig { array_sampler, line_budget_only, safety_cap: DEFAULT_SAFETY_CAP, + explore: None, } } } @@ -189,6 +224,10 @@ pub struct PriorityOrder { pub fileset_render_slots: Option>, /// True if the priority queue expansion hit the safety cap. pub safety_cap_hit: bool, + /// FNV-1a Merkle hashes indexed by PQ node id, populated when explore + /// penalty matching computed them, so shown-leaf collection can reuse + /// the table instead of running a second full-tree hash pass. + pub merkle_hashes: Option>, } #[derive(Copy, Clone, Debug)] diff --git a/src/pruner/budget.rs b/src/pruner/budget.rs index 588cc1dc..cf05a0f9 100644 --- a/src/pruner/budget.rs +++ b/src/pruner/budget.rs @@ -51,25 +51,55 @@ fn match_summary_zero_shown( }) } +/// Outcome of the budget search: rendered text plus the data needed to +/// reconstruct exactly which nodes were selected. +#[derive(Debug)] +pub struct BudgetSearchResult { + pub text: String, + pub match_summary: Option, + /// Number of selected nodes. Indexes into `selection_order` when present, + /// otherwise into `order.by_priority`. + pub top_k: usize, + /// Custom selection ordering used when per-slot caps are active; in that + /// mode `by_priority[..top_k]` is NOT the selected set. + pub selection_order: Option>, + /// Nodes force-included by strong grep, possibly outside the top-k prefix. + pub grep_must_keep: Vec, +} + +impl BudgetSearchResult { + fn empty(match_summary: Option) -> Self { + Self { + text: String::new(), + match_summary, + top_k: 0, + selection_order: None, + grep_must_keep: Vec::new(), + } + } +} + pub fn find_largest_render_under_budgets( order_build: &mut PriorityOrder, config: &RenderConfig, grep: &GrepConfig, budgets: Budgets, -) -> (String, Option) { +) -> BudgetSearchResult { let total = order_build.total_nodes; if total == 0 { let summary = grep.patterns.is_active().then_some(MatchSummary { shown: 0, hidden: 0, }); - return (String::new(), summary); + return BudgetSearchResult::empty(summary); } let root_is_fileset = is_fileset_root(order_build); let mut grep_state = compute_grep_state(order_build, grep); if strong_fileset_grep_without_matches(grep, &grep_state, root_is_fileset) { - return (String::new(), match_summary_zero_shown(&grep_state)); + return BudgetSearchResult::empty(match_summary_zero_shown( + &grep_state, + )); } filter_fileset_without_matches( order_build, @@ -140,7 +170,9 @@ pub fn find_largest_render_under_budgets( selection, &finalize_ctx, ) - .unwrap_or_else(|| (String::new(), match_summary_zero_shown(&grep_state))) + .unwrap_or_else(|| { + BudgetSearchResult::empty(match_summary_zero_shown(&grep_state)) + }) } struct FinalizeContext<'a> { @@ -159,7 +191,7 @@ fn finalize_render_from_selection( config: &RenderConfig, selection: PruningResult, ctx: &FinalizeContext<'_>, -) -> Option<(String, Option)> { +) -> Option { let PruningResult { top_k: k_opt, mut inclusion_flags, @@ -194,6 +226,9 @@ fn finalize_render_from_selection( &mut inclusion_flags, render_set_id, ); + // Strong-grep matches count as "seen" for explore sessions even though + // they sit outside the top-k prefix; surface them for leaf recording. + let grep_must_keep = grep_must_keep_nodes(ctx.grep, ctx.grep_state); if per_slot_caps_active && !config.count_fileset_headers_in_budgets { ensure_fileset_headers_for_empty_slots( order_build, @@ -238,7 +273,13 @@ fn finalize_render_from_selection( ); let summary = compute_match_summary(ctx.grep_state, &inclusion_flags, render_set_id); - Some((text, summary)) + Some(BudgetSearchResult { + text, + match_summary: summary, + top_k: k, + selection_order, + grep_must_keep, + }) } fn strong_fileset_grep_without_matches( @@ -265,6 +306,27 @@ fn is_strong_grep(grep: &GrepConfig, state: &Option) -> bool { grep.has_strong() && grep.force_strong_inclusion && count > 0 } +/// Node ids force-included by `include_strong_grep_must_keep`; empty when +/// strong grep is not forcing inclusion. +fn grep_must_keep_nodes( + grep: &GrepConfig, + state: &Option, +) -> Vec { + if !is_strong_grep(grep, state) { + return Vec::new(); + } + state + .as_ref() + .map(|s| { + s.guaranteed_nodes + .iter() + .enumerate() + .filter_map(|(idx, keep)| keep.then_some(NodeId(idx))) + .collect() + }) + .unwrap_or_default() +} + fn apply_selection( order_build: &PriorityOrder, selection_order: Option<&[NodeId]>, diff --git a/src/serialization/tests.rs b/src/serialization/tests.rs index c71814e2..ea228bd1 100644 --- a/src/serialization/tests.rs +++ b/src/serialization/tests.rs @@ -113,6 +113,7 @@ fn array_omitted_markers_pseudo_head_and_tail() { array_sampler: crate::ArraySamplerStrategy::Default, line_budget_only: false, safety_cap: crate::DEFAULT_SAFETY_CAP, + explore: None, }; let arena = crate::ingest::formats::json::build_json_tree_arena( "[1,2,3]", &cfg_prio, @@ -151,6 +152,7 @@ fn array_omitted_markers_js_head_and_tail() { array_sampler: crate::ArraySamplerStrategy::Default, line_budget_only: false, safety_cap: crate::DEFAULT_SAFETY_CAP, + explore: None, }; let arena = crate::ingest::formats::json::build_json_tree_arena( "[1,2,3]", &cfg_prio, @@ -189,6 +191,7 @@ fn array_omitted_markers_yaml_head_and_tail() { array_sampler: crate::ArraySamplerStrategy::Default, line_budget_only: false, safety_cap: crate::DEFAULT_SAFETY_CAP, + explore: None, }; let arena = crate::ingest::formats::json::build_json_tree_arena( "[1,2,3]", &cfg_prio, @@ -527,6 +530,7 @@ fn force_child_hooks_removed() { code_lines: HashMap::new(), fileset_render_slots: None, safety_cap_hit: false, + merkle_hashes: None, }; let mut flags = Vec::new(); let render_id = 1u32; diff --git a/src/session/io.rs b/src/session/io.rs new file mode 100644 index 00000000..f98135d6 --- /dev/null +++ b/src/session/io.rs @@ -0,0 +1,384 @@ +use std::fs; +use std::io; +use std::path::{Path, PathBuf}; +use std::sync::atomic::{AtomicU64, Ordering}; + +use super::types::Session; + +/// Counter for unique-per-process tmp filenames in `save_to_path`. +static TMP_SEQ: AtomicU64 = AtomicU64::new(0); + +pub fn save_to_path(session: &Session, path: &Path) -> Result<(), io::Error> { + let json = serde_json::to_string(session) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e))?; + // Each writer gets a unique tmp filename so concurrent invocations on the + // same destination don't race on the rename target. + let pid = std::process::id(); + let seq = TMP_SEQ.fetch_add(1, Ordering::Relaxed); + let tmp_path = path.with_extension(format!("tmp.{pid}.{seq}")); + fs::write(&tmp_path, &json)?; + fs::rename(&tmp_path, path) +} + +pub fn load_from_path(path: &Path) -> Result { + let data = fs::read_to_string(path)?; + serde_json::from_str(&data) + .map_err(|e| io::Error::new(io::ErrorKind::InvalidData, e)) +} + +/// RAII guard for a sibling-file advisory lock acquired via `O_CREAT|O_EXCL`. +/// Released on drop. Stale locks (from crashed processes) require manual +/// cleanup — acceptable trade-off for zero deps and a CLI use case. +pub struct SessionLock { + path: PathBuf, +} + +impl Drop for SessionLock { + fn drop(&mut self) { + let _ = fs::remove_file(&self.path); + } +} + +/// Acquire the advisory lock that serializes all read-modify-write cycles on +/// a session file (`record_step_atomic`, `explore clear`). +pub fn acquire_session_lock(session_path: &Path) -> io::Result { + let lock_path = session_path.with_extension("lock"); + if let Some(parent) = lock_path.parent() { + fs::create_dir_all(parent)?; + } + let max_attempts: u32 = 100; // ~1s at 10ms per attempt + for _ in 0..max_attempts { + match fs::OpenOptions::new() + .write(true) + .create_new(true) + .open(&lock_path) + { + Ok(_) => return Ok(SessionLock { path: lock_path }), + Err(e) if e.kind() == io::ErrorKind::AlreadyExists => { + std::thread::sleep(std::time::Duration::from_millis(10)); + } + Err(e) => return Err(e), + } + } + Err(io::Error::new( + io::ErrorKind::TimedOut, + format!( + "could not acquire session lock at {} after {max_attempts} \ + attempts; remove the file if you're sure no other hson is running", + lock_path.display() + ), + )) +} + +/// Eviction policy for `record_step_atomic`. +pub struct EvictionPolicy { + pub alpha: f64, + pub breadcrumb_cap: usize, + pub query_log_cap: usize, +} + +/// Atomically record one step against the session at `path`: +/// 1. Acquire a sibling-file lock (serializes concurrent writers). +/// 2. Read current state from disk (so step_count is fresh). +/// 3. Append the query (which assigns the next step number). +/// 4. Record breadcrumbs at that step. +/// 5. Evict and cap, then atomic-rename write. +/// 6. Release the lock. +/// +/// If the on-disk file is corrupt or missing, return an error rather than +/// silently overwriting it. +pub fn record_step_atomic( + path: &Path, + new_breadcrumbs: &[(String, String)], + timestamp: &str, + cwd: &str, + argv: &[String], + policy: &EvictionPolicy, +) -> Result<(), io::Error> { + let _lock = acquire_session_lock(path)?; + let mut session = load_from_path(path).map_err(|e| { + io::Error::new( + e.kind(), + format!( + "session file {} is unreadable ({e}); refusing to overwrite \ + — delete the file or run `hson explore start` to create a \ + fresh session", + path.display() + ), + ) + })?; + + session.record_query(timestamp, cwd, argv); + let new_step = session.step_count; + for (file, p) in new_breadcrumbs { + session.record_breadcrumb(file, p, new_step); + } + + session.evict(new_step, policy.alpha, policy.breadcrumb_cap); + if session.queries.len() > policy.query_log_cap { + let excess = session.queries.len() - policy.query_log_cap; + session.queries.drain(0..excess); + } + save_to_path(&session, path) +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::session::{QueryEntry, Session}; + use headson::Breadcrumb; + use tempfile::tempdir; + + #[test] + fn record_step_atomic_preserves_breadcrumbs_across_calls() { + let dir = tempdir().unwrap(); + let path = dir.path().join("session.json"); + save_to_path(&Session::new("sid".into(), "lbl".into()), &path) + .unwrap(); + + record_step_atomic( + &path, + &[("a.json".into(), "x".into())], + "ts1", + "/", + &[], + &EvictionPolicy { + alpha: 0.5, + breadcrumb_cap: 500, + query_log_cap: 1000, + }, + ) + .unwrap(); + record_step_atomic( + &path, + &[("b.json".into(), "y".into())], + "ts2", + "/", + &[], + &EvictionPolicy { + alpha: 0.5, + breadcrumb_cap: 500, + query_log_cap: 1000, + }, + ) + .unwrap(); + + let on_disk = load_from_path(&path).unwrap(); + let has_a = on_disk + .breadcrumbs + .iter() + .any(|b| b.file == "a.json" && b.path == "x"); + let has_b = on_disk + .breadcrumbs + .iter() + .any(|b| b.file == "b.json" && b.path == "y"); + assert!(has_a, "breadcrumb (a.json, x) missing after second record"); + assert!(has_b, "breadcrumb (b.json, y) missing after second record"); + } + + #[test] + fn concurrent_record_sessions_preserve_distinct_steps() { + use std::collections::HashSet; + let dir = tempdir().unwrap(); + let path = dir.path().join("session.json"); + save_to_path(&Session::new("id".into(), "lbl".into()), &path).unwrap(); + + const N: usize = 10; + std::thread::scope(|s| { + for i in 0..N { + let path = path.clone(); + s.spawn(move || { + let bc = vec![("f".into(), format!("p{i}"))]; + record_step_atomic( + &path, + &bc, + "ts", + "/cwd", + &[], + &EvictionPolicy { + alpha: 0.5, + breadcrumb_cap: 500, + query_log_cap: 1000, + }, + ) + .unwrap(); + }); + } + }); + + let final_session = load_from_path(&path).unwrap(); + assert_eq!( + final_session.step_count, N as u64, + "step_count should equal number of concurrent writers" + ); + assert_eq!( + final_session.queries.len(), + N, + "queries log should have one entry per concurrent writer" + ); + let distinct_steps: HashSet = + final_session.queries.iter().map(|q| q.step).collect(); + assert_eq!( + distinct_steps.len(), + N, + "every query should have a distinct step value" + ); + } + + #[test] + fn save_to_path_leaves_no_tmp_sibling_after_success() { + let dir = tempdir().unwrap(); + let path = dir.path().join("session.json"); + let session = Session::new("id".to_string(), "lbl".to_string()); + + save_to_path(&session, &path).unwrap(); + + let tmp_files: Vec<_> = std::fs::read_dir(dir.path()) + .unwrap() + .filter_map(std::result::Result::ok) + .filter(|e| e.file_name().to_string_lossy().contains(".tmp")) + .collect(); + assert!( + tmp_files.is_empty(), + "unexpected .tmp files: {:?}", + tmp_files + .iter() + .map(std::fs::DirEntry::file_name) + .collect::>() + ); + } + + #[test] + fn save_and_load_round_trips_all_fields() { + let dir = tempdir().unwrap(); + let path = dir.path().join("session.json"); + + let mut session = + Session::new("test-id".to_string(), "test-label".to_string()); + session.step_count = 3; + session.breadcrumbs.push(Breadcrumb { + file: "a.json".to_string(), + path: "users.0.name".to_string(), + count: 2, + last_step: 2, + }); + session.queries.push(QueryEntry { + step: 1, + timestamp: "2026-01-01T00:00:00Z".to_string(), + cwd: "/home/user".to_string(), + argv: vec!["src/".to_string()], + }); + + save_to_path(&session, &path).unwrap(); + let loaded = load_from_path(&path).unwrap(); + + assert_eq!(loaded.id, session.id); + assert_eq!(loaded.label, session.label); + assert_eq!(loaded.step_count, session.step_count); + assert_eq!(loaded.breadcrumbs.len(), 1); + assert_eq!(loaded.breadcrumbs[0].file, "a.json"); + assert_eq!(loaded.breadcrumbs[0].path, "users.0.name"); + assert_eq!(loaded.breadcrumbs[0].count, 2); + assert_eq!(loaded.breadcrumbs[0].last_step, 2); + assert_eq!(loaded.queries.len(), 1); + assert_eq!(loaded.queries[0].step, 1); + assert_eq!(loaded.queries[0].cwd, "/home/user"); + } + + #[test] + fn corrupt_session_file_is_not_silently_overwritten() { + let dir = tempdir().unwrap(); + let path = dir.path().join("session.json"); + + let garbage = b"\x00\x01garbage{not-json,\""; + std::fs::write(&path, garbage).unwrap(); + + let result = record_step_atomic( + &path, + &[], + "ts", + "/cwd", + &[], + &EvictionPolicy { + alpha: 0.5, + breadcrumb_cap: 500, + query_log_cap: 1000, + }, + ); + + assert!( + result.is_err(), + "recording against a corrupt session file must error; got: {result:?}" + ); + let on_disk_after = std::fs::read(&path).unwrap(); + assert_eq!( + on_disk_after, + garbage, + "corrupt session file must not be modified; on-disk bytes \ + changed to: {:?}", + String::from_utf8_lossy(&on_disk_after) + ); + } + + #[test] + fn unreadable_session_error_carries_path_and_remedy() { + let dir = tempdir().unwrap(); + let path = dir.path().join("session.json"); + std::fs::write(&path, b"not json").unwrap(); + + let err = record_step_atomic( + &path, + &[], + "ts", + "/cwd", + &[], + &EvictionPolicy { + alpha: 0.5, + breadcrumb_cap: 500, + query_log_cap: 1000, + }, + ) + .unwrap_err(); + + let msg = err.to_string(); + assert!( + msg.contains(&path.display().to_string()), + "error message should name the session file path: {msg}" + ); + assert!( + msg.contains("refusing to overwrite"), + "error message should explain what happened: {msg}" + ); + assert!( + msg.contains("hson explore start"), + "error message should suggest a remedy: {msg}" + ); + } + + #[test] + fn record_step_atomic_accepts_legacy_file_without_version_field() { + let dir = tempdir().unwrap(); + let path = dir.path().join("session.json"); + let legacy = r#"{"id":"sid","label":"lbl","step_count":0,"breadcrumbs":[],"queries":[]}"#; + std::fs::write(&path, legacy).unwrap(); + + record_step_atomic( + &path, + &[("a.json".into(), "x".into())], + "ts", + "/", + &[], + &EvictionPolicy { + alpha: 0.5, + breadcrumb_cap: 500, + query_log_cap: 1000, + }, + ) + .unwrap(); + + let on_disk = load_from_path(&path).unwrap(); + assert_eq!(on_disk.version, 1); + assert_eq!(on_disk.step_count, 1); + assert_eq!(on_disk.breadcrumbs.len(), 1); + } +} diff --git a/src/session/mod.rs b/src/session/mod.rs new file mode 100644 index 00000000..a4cccda3 --- /dev/null +++ b/src/session/mod.rs @@ -0,0 +1,8 @@ +pub mod io; +mod types; + +pub use types::Session; + +// QueryEntry is only referenced in test code. +#[cfg(test)] +pub use types::QueryEntry; diff --git a/src/session/types.rs b/src/session/types.rs new file mode 100644 index 00000000..e14e96c2 --- /dev/null +++ b/src/session/types.rs @@ -0,0 +1,310 @@ +use serde::{Deserialize, Serialize}; + +const DECAY_EPSILON: f64 = 0.001; +const SCHEMA_VERSION: u64 = 1; + +pub use headson::Breadcrumb; + +// Files written before the version field existed must keep loading. +fn default_schema_version() -> u64 { + SCHEMA_VERSION +} + +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct QueryEntry { + pub step: u64, + pub timestamp: String, + pub cwd: String, + pub argv: Vec, +} + +// Every field defaults so that adding fields later never makes existing +// session files unreadable (record_step_atomic refuses to overwrite +// unreadable files, which would otherwise wedge a session permanently). +#[derive(Debug, Clone, Serialize, Deserialize, PartialEq)] +pub struct Session { + #[serde(default = "default_schema_version")] + pub version: u64, + #[serde(default)] + pub id: String, + #[serde(default)] + pub label: String, + #[serde(default)] + pub step_count: u64, + #[serde(default)] + pub breadcrumbs: Vec, + #[serde(default)] + pub queries: Vec, +} + +impl Session { + pub fn new(id: String, label: String) -> Self { + Self { + version: SCHEMA_VERSION, + id, + label, + step_count: 0, + breadcrumbs: vec![], + queries: vec![], + } + } + + pub fn record_breadcrumb(&mut self, file: &str, path: &str, step: u64) { + if let Some(entry) = self + .breadcrumbs + .iter_mut() + .find(|b| b.file == file && b.path == path) + { + entry.count += 1; + entry.last_step = step; + } else { + self.breadcrumbs.push(Breadcrumb { + file: file.to_string(), + path: path.to_string(), + count: 1, + last_step: step, + }); + } + } + + #[cfg(test)] + pub fn penalty_for( + &self, + file: &str, + path: &str, + current_step: u64, + alpha: f64, + ) -> f64 { + match self + .breadcrumbs + .iter() + .find(|b| b.file == file && b.path == path) + { + None => 0.0, + Some(b) => { + let steps_ago = current_step + .saturating_sub(b.last_step) + .min(i32::MAX as u64) + as i32; + (1.0 + b.count as f64).ln() * alpha.powi(steps_ago) + } + } + } + + pub fn evict(&mut self, current_step: u64, alpha: f64, cap: usize) { + // Epsilon prune: drop entries whose decay factor is below DECAY_EPSILON + self.breadcrumbs.retain(|b| { + let steps_ago = current_step + .saturating_sub(b.last_step) + .min(i32::MAX as u64) as i32; + alpha.powi(steps_ago) >= DECAY_EPSILON + }); + // Cap: keep only the `cap` most recently seen entries + if self.breadcrumbs.len() > cap { + self.breadcrumbs + .sort_by_key(|b| std::cmp::Reverse(b.last_step)); + self.breadcrumbs.truncate(cap); + } + } + + pub fn clear(&mut self) { + self.breadcrumbs = vec![]; + self.step_count = 0; + } + + pub fn record_query( + &mut self, + timestamp: &str, + cwd: &str, + argv: &[String], + ) { + self.step_count += 1; + self.queries.push(QueryEntry { + step: self.step_count, + timestamp: timestamp.to_string(), + cwd: cwd.to_string(), + argv: argv.to_vec(), + }); + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn step_count_increments_on_record_query() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + assert_eq!(session.step_count, 0); + session.record_query("t1", "/", &[]); + session.record_query("t2", "/", &[]); + assert_eq!(session.step_count, 2); + } + + #[test] + fn record_query_stores_correct_fields() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + session.record_query( + "2026-01-01T00:00:00Z", + "/home/user", + &["src/".into(), "-C".into(), "8000".into()], + ); + assert_eq!(session.queries.len(), 1); + let entry = &session.queries[0]; + assert_eq!(entry.step, 1); + assert_eq!(entry.timestamp, "2026-01-01T00:00:00Z"); + assert_eq!(entry.cwd, "/home/user"); + assert_eq!(entry.argv, vec!["src/", "-C", "8000"]); + } + + #[test] + fn clear_zeroes_breadcrumbs_and_step_count_preserving_the_rest() { + let mut session = Session::new("x".to_string(), "lbl".to_string()); + session.step_count = 5; + session.record_breadcrumb("a.json", "x", 1); + session.record_breadcrumb("b.json", "y", 2); + session.record_breadcrumb("c.json", "z", 3); + session.queries.push(QueryEntry { + step: 1, + timestamp: "t1".to_string(), + cwd: "/".to_string(), + argv: vec![], + }); + session.queries.push(QueryEntry { + step: 2, + timestamp: "t2".to_string(), + cwd: "/".to_string(), + argv: vec![], + }); + session.clear(); + assert!(session.breadcrumbs.is_empty()); + assert_eq!(session.step_count, 0); + assert_eq!(session.queries.len(), 2); + assert_eq!(session.queries[0].timestamp, "t1"); + assert_eq!(session.queries[1].timestamp, "t2"); + assert_eq!(session.label, "lbl"); + assert_eq!(session.id, "x"); + } + + #[test] + fn cap_eviction_truncates_to_limit_retaining_most_recent() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + // Add 5 breadcrumbs all above epsilon threshold at current_step=6, alpha=0.5 + // last_steps: 5, 4, 3, 2, 1 — all recent enough + for (i, last_step) in [5u64, 4, 3, 2, 1].iter().enumerate() { + session.record_breadcrumb( + "file.json", + &format!("key.{i}"), + *last_step, + ); + } + session.evict(6, 0.5, 3); + assert_eq!(session.breadcrumbs.len(), 3); + // The 3 retained should have the highest last_step values: 5, 4, 3 + let mut retained_steps: Vec = + session.breadcrumbs.iter().map(|b| b.last_step).collect(); + retained_steps.sort(); + assert_eq!(retained_steps, vec![3, 4, 5]); + } + + #[test] + fn epsilon_prune_drops_fully_decayed_entries() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + session.record_breadcrumb("file.json", "old.path", 0); + session.evict(50, 0.5, 10000); + assert!(session.breadcrumbs.is_empty()); + } + + #[test] + fn more_recently_seen_node_has_higher_penalty() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + session.record_breadcrumb("file.json", "key_a", 4); + session.record_breadcrumb("file.json", "key_b", 1); + let p_a = session.penalty_for("file.json", "key_a", 5, 0.5); + let p_b = session.penalty_for("file.json", "key_b", 5, 0.5); + assert!(p_a > p_b); + } + + #[test] + fn penalty_formula_produces_correct_value() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + session.record_breadcrumb("file.json", "a.b", 3); + let p = session.penalty_for("file.json", "a.b", 5, 0.5); + let expected = f64::ln(2.0) * 0.5_f64.powi(2); + assert!((p - expected).abs() < 1e-12); + } + + #[test] + fn penalty_for_unseen_node_is_zero() { + let session = Session::new("id".to_string(), "lbl".to_string()); + let p = session.penalty_for("file.json", "missing.path", 5, 0.5); + assert_eq!(p, 0.0); + } + + #[test] + fn record_breadcrumb_increments_existing_entry() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + session.record_breadcrumb("file.json", "a.b", 1); + session.record_breadcrumb("file.json", "a.b", 2); + assert_eq!(session.breadcrumbs.len(), 1); + assert_eq!(session.breadcrumbs[0].count, 2); + assert_eq!(session.breadcrumbs[0].last_step, 2); + } + + #[test] + fn record_breadcrumb_creates_new_entry() { + let mut session = Session::new("id".to_string(), "lbl".to_string()); + session.record_breadcrumb("file.json", "users.0.name", 1); + assert_eq!(session.breadcrumbs.len(), 1); + assert_eq!(session.breadcrumbs[0].count, 1); + assert_eq!(session.breadcrumbs[0].last_step, 1); + } + + #[test] + fn session_round_trips_through_serde() { + let session = Session::new("s1".to_string(), "lbl".to_string()); + let json = serde_json::to_string(&session).unwrap(); + let deserialized: Session = serde_json::from_str(&json).unwrap(); + assert_eq!(deserialized.version, 1); + assert_eq!(deserialized.id, "s1"); + assert_eq!(deserialized.label, "lbl"); + assert_eq!(deserialized.step_count, 0); + assert!(deserialized.breadcrumbs.is_empty()); + assert!(deserialized.queries.is_empty()); + } + + #[test] + fn legacy_session_json_without_version_field_still_loads() { + // Written by code that predates the version field. + let legacy = r#"{ + "id": "legacy-id", + "label": "legacy label", + "step_count": 7, + "breadcrumbs": [ + {"file": "a.json", "path": "x.y", "count": 2, "last_step": 6} + ], + "queries": [ + {"step": 1, "timestamp": "t", "cwd": "/", "argv": ["src/"]} + ] + }"#; + let session: Session = serde_json::from_str(legacy).unwrap(); + assert_eq!(session.version, 1); + assert_eq!(session.id, "legacy-id"); + assert_eq!(session.label, "legacy label"); + assert_eq!(session.step_count, 7); + assert_eq!(session.breadcrumbs.len(), 1); + assert_eq!(session.queries.len(), 1); + } + + #[test] + fn penalty_and_evict_tolerate_last_step_ahead_of_current_step() { + // last_step > current_step can happen if a session file is hand-edited + // or written by a newer step counter; must not panic on subtraction. + let mut session = Session::new("id".to_string(), "lbl".to_string()); + session.record_breadcrumb("file.json", "a.b", 10); + let p = session.penalty_for("file.json", "a.b", 5, 0.5); + assert!((p - f64::ln(2.0)).abs() < 1e-12); // steps_ago saturates to 0 + session.evict(5, 0.5, 10); + assert_eq!(session.breadcrumbs.len(), 1); + } +} diff --git a/src/utils/tree_arena.rs b/src/utils/tree_arena.rs index 1f5158d4..d61424e1 100644 --- a/src/utils/tree_arena.rs +++ b/src/utils/tree_arena.rs @@ -19,6 +19,53 @@ pub struct JsonTreeArena { pub code_lines: HashMap>>, } +impl JsonTreeArena { + /// Append all nodes from `src` into `self`, adjusting internal offsets. + /// Returns the new node ID of `src`'s root within `self`. + #[allow( + clippy::cognitive_complexity, + reason = "Tree merge touches multiple parallel arrays and offsets; easier to follow inline" + )] + pub fn append(&mut self, src: JsonTreeArena) -> usize { + let node_offset = self.nodes.len(); + let child_offset = self.children.len(); + let obj_key_offset = self.obj_keys.len(); + let arr_idx_offset = self.arr_indices.len(); + let root_id = src.root_id; + let JsonTreeArena { + nodes, + children, + obj_keys, + arr_indices, + code_lines, + .. + } = src; + + self.nodes.extend(nodes); + for node in self.nodes.iter_mut().skip(node_offset) { + if node.children_len > 0 { + node.children_start += child_offset; + } + if node.obj_keys_len > 0 { + node.obj_keys_start += obj_key_offset; + } + if node.arr_indices_len > 0 { + node.arr_indices_start += arr_idx_offset; + } + } + + self.children + .extend(children.into_iter().map(|child| child + node_offset)); + self.obj_keys.extend(obj_keys); + self.arr_indices.extend(arr_indices); + for (arena_idx, lines) in code_lines { + self.code_lines.insert(arena_idx + node_offset, lines); + } + + node_offset + root_id + } +} + #[derive(Debug, Clone)] pub struct JsonTreeNode { pub kind: NodeKind, diff --git a/tests/explore_session.rs b/tests/explore_session.rs new file mode 100644 index 00000000..a8ab3663 --- /dev/null +++ b/tests/explore_session.rs @@ -0,0 +1,364 @@ +//! Explore-session behaviour of the library pipeline: fileset round-robin +//! fairness must survive penalty application, shown-leaf recording must +//! reflect the actually-selected node set, and breadcrumb identity must be +//! per file (resolved absolute path + in-file dot-path). + +use headson::{ + Breadcrumb, Budget, BudgetKind, Budgets, ColorMode, ExploreContext, + FilesetInput, FilesetInputKind, GrepConfig, InputKind, OutputTemplate, + PriorityConfig, RenderConfig, RenderOutput, Style, headson, + resolve_breadcrumb_file, +}; + +fn render_config() -> RenderConfig { + RenderConfig { + template: OutputTemplate::Pseudo, + indent_unit: " ".to_string(), + space: " ".to_string(), + newline: "\n".to_string(), + color_mode: ColorMode::Off, + color_enabled: false, + style: Style::Default, + prefer_tail_arrays: false, + string_free_prefix_graphemes: None, + debug: false, + primary_source_name: None, + show_fileset_headers: false, + fileset_tree: false, + count_fileset_headers_in_budgets: false, + grep_highlight: None, + } +} + +const FILE_A: &[u8] = br#"{"a1": 1, "a2": 2, "a3": 3}"#; +const FILE_B: &[u8] = br#"{"b1": 1, "b2": 2, "b3": 3}"#; + +fn two_file_fileset() -> InputKind { + InputKind::Fileset(vec![ + FilesetInput { + name: "a.json".to_string(), + bytes: FILE_A.to_vec(), + kind: FilesetInputKind::Json, + }, + FilesetInput { + name: "b.json".to_string(), + bytes: FILE_B.to_vec(), + kind: FilesetInputKind::Json, + }, + ]) +} + +fn run_input( + input: InputKind, + explore: Option, + budgets: Budgets, +) -> RenderOutput { + let mut prio = PriorityConfig::new(usize::MAX, usize::MAX); + prio.explore = explore; + headson( + input, + &render_config(), + &prio, + &GrepConfig::default(), + budgets, + ) + .expect("headson must succeed") +} + +fn run_fileset( + explore: Option, + budgets: Budgets, +) -> RenderOutput { + run_input(two_file_fileset(), explore, budgets) +} + +fn fresh_session(file: Option) -> ExploreContext { + ExploreContext { + breadcrumbs: vec![], + current_step: 0, + alpha: 0.5, + file, + } +} + +fn tight_line_budget() -> Budgets { + Budgets { + global: Some(Budget { + kind: BudgetKind::Lines, + cap: 8, + }), + per_slot: None, + } +} + +/// Shown-leaf key for the leaf whose in-file dot-path equals `dot_path`, +/// captured by running the fileset with a fresh (breadcrumb-free) session. +fn captured_key_for(dot_path: &str) -> (String, String) { + let out = run_fileset(Some(fresh_session(None)), Budgets::default()); + out.shown_leaves + .iter() + .find_map(|(file, path)| { + let prefix = path.split_once('#').map(|(p, _)| p)?; + (prefix == dot_path).then(|| (file.clone(), path.clone())) + }) + .unwrap_or_else(|| { + panic!("no shown leaf at {dot_path:?}; got {:?}", out.shown_leaves) + }) +} + +/// A session whose breadcrumbs match nothing must not perturb the fileset +/// round-robin interleave: output is byte-identical to the no-session run. +#[test] +fn fileset_interleave_unchanged_by_non_matching_breadcrumb() { + let baseline = run_fileset(None, tight_line_budget()); + let with_session = run_fileset( + Some(ExploreContext { + breadcrumbs: vec![Breadcrumb { + file: "/nonexistent/zz.json".to_string(), + path: "zz.nonexistent#0000000000000000".to_string(), + count: 3, + last_step: 1, + }], + current_step: 2, + alpha: 0.5, + file: None, + }), + tight_line_budget(), + ); + assert_eq!( + baseline.text, with_session.text, + "non-matching breadcrumbs must leave fileset output byte-identical" + ); +} + +/// An active session with zero breadcrumbs (first invocation) must also +/// leave the output byte-identical to the no-session run. +#[test] +fn fileset_interleave_unchanged_by_empty_breadcrumbs() { + let baseline = run_fileset(None, tight_line_budget()); + let with_session = run_fileset( + Some(ExploreContext { + breadcrumbs: vec![], + current_step: 1, + alpha: 0.5, + file: None, + }), + tight_line_budget(), + ); + assert_eq!( + baseline.text, with_session.text, + "empty breadcrumbs must leave fileset output byte-identical" + ); +} + +/// With a matching breadcrumb penalizing one leaf in file a, the round-robin +/// interleave must still alternate between files: both files keep content in +/// the output, the penalized leaf yields to its unpenalized siblings. +#[test] +fn fileset_interleave_survives_matching_breadcrumb() { + let (file, path) = captured_key_for("a1"); + assert_eq!( + file, + resolve_breadcrumb_file("a.json"), + "fileset leaf must carry the resolved absolute path of its file" + ); + let out = run_fileset( + Some(ExploreContext { + breadcrumbs: vec![Breadcrumb { + file, + path, + count: 5, + last_step: 1, + }], + current_step: 2, + alpha: 0.9, + file: None, + }), + tight_line_budget(), + ); + assert!( + out.text.contains("b1"), + "file b content must survive penalty re-sort; got: {:?}", + out.text + ); + assert!( + out.text.contains("a2") || out.text.contains("a3"), + "unpenalized file a content must still render; got: {:?}", + out.text + ); + assert!( + !out.text.contains("a1"), + "penalized leaf must yield under a tight budget; got: {:?}", + out.text + ); +} + +/// Under per-slot caps the budget search selects from a custom round-robin +/// ordering, not `by_priority`. Recorded shown leaves must agree exactly +/// with the leaves visible in the rendered text. +#[test] +fn per_slot_caps_record_exactly_the_rendered_leaves() { + // Cap 4 fits exactly one leaf per file ({, key, omission, }); the custom + // per-slot selection order diverges from by_priority here, so recording + // from by_priority[..top_k] would miss file b's rendered leaf. + let budgets = Budgets { + global: None, + per_slot: Some(Budget { + kind: BudgetKind::Lines, + cap: 4, + }), + }; + let out = run_fileset(Some(fresh_session(None)), budgets); + let recorded: Vec<(&str, &str)> = out + .shown_leaves + .iter() + .filter_map(|(file, path)| { + path.split_once('#').map(|(p, _)| (file.as_str(), p)) + }) + .collect(); + for key in ["a1", "a2", "a3", "b1", "b2", "b3"] { + let rendered = out.text.contains(key); + let file = resolve_breadcrumb_file(&format!( + "{}.json", + key.chars().next().map(String::from).unwrap_or_default() + )); + let was_recorded = recorded.contains(&(file.as_str(), key)); + assert_eq!( + rendered, was_recorded, + "leaf {key:?}: rendered={rendered} but recorded={was_recorded}\n\ + text: {:?}\nrecorded: {recorded:?}", + out.text + ); + } + assert!( + !out.shown_leaves.is_empty(), + "per-slot run must record at least one shown leaf" + ); +} + +// ── Per-file breadcrumb identity ─────────────────────────────────────────── + +const SHARED_CONTENT: &[u8] = br#"{"a": 1, "b": 2}"#; + +fn tight_byte_budget() -> Budgets { + Budgets { + global: Some(Budget { + kind: BudgetKind::Bytes, + cap: 20, + }), + per_slot: None, + } +} + +/// Regression (issue #513 review): breadcrumbs recorded while looking at one +/// file must not penalize an identical-valued leaf in a *different* file on +/// the user's first look at it — and the same breadcrumbs must still +/// penalize the file they were recorded for. +#[test] +fn breadcrumbs_do_not_cross_penalize_identical_files() { + let file_a = "/abs/fixtures/a.json".to_string(); + let file_b = "/abs/fixtures/b.json".to_string(); + + let first = run_input( + InputKind::Json(SHARED_CONTENT.to_vec()), + Some(fresh_session(Some(file_a.clone()))), + tight_byte_budget(), + ); + assert!( + first.shown_leaves.iter().all(|(file, _)| file == &file_a), + "single-file leaves must carry the context file; got {:?}", + first.shown_leaves + ); + let crumbs: Vec = first + .shown_leaves + .iter() + .map(|(file, path)| Breadcrumb { + file: file.clone(), + path: path.clone(), + count: 1, + last_step: 1, + }) + .collect(); + assert!(!crumbs.is_empty(), "first render must record shown leaves"); + + // First look at identically-structured file B: output must match a + // breadcrumb-free render exactly — no cross-file penalty. + let b_fresh = run_input( + InputKind::Json(SHARED_CONTENT.to_vec()), + Some(fresh_session(Some(file_b.clone()))), + tight_byte_budget(), + ); + let b_with_a_crumbs = run_input( + InputKind::Json(SHARED_CONTENT.to_vec()), + Some(ExploreContext { + breadcrumbs: crumbs.clone(), + current_step: 2, + alpha: 0.5, + file: Some(file_b), + }), + tight_byte_budget(), + ); + assert_eq!( + b_fresh.text, b_with_a_crumbs.text, + "breadcrumbs from file A must not affect the first look at file B" + ); + + // Sanity: the same breadcrumbs DO penalize file A itself. + let a_again = run_input( + InputKind::Json(SHARED_CONTENT.to_vec()), + Some(ExploreContext { + breadcrumbs: crumbs, + current_step: 2, + alpha: 0.5, + file: Some(file_a), + }), + tight_byte_budget(), + ); + assert_ne!( + b_fresh.text, a_again.text, + "breadcrumbs must still penalize the file they were recorded for" + ); +} + +/// The same file produces identical breadcrumb keys whether rendered as a +/// single input or inside a fileset, and regardless of relative path +/// spelling — penalty continuity must survive `hson a.json` vs `hson .`. +#[test] +fn keys_stable_across_single_file_and_fileset_invocations() { + let bytes = br#"{"k1": "v1", "k2": 42}"#; + let name = "stable_fixture.json"; + + let single = run_input( + InputKind::Json(bytes.to_vec()), + Some(fresh_session(Some(resolve_breadcrumb_file(name)))), + Budgets::default(), + ); + let fileset_keys_for = |spelled: &str| { + let out = run_input( + InputKind::Fileset(vec![FilesetInput { + name: spelled.to_string(), + bytes: bytes.to_vec(), + kind: FilesetInputKind::Json, + }]), + Some(fresh_session(None)), + Budgets::default(), + ); + let mut keys = out.shown_leaves; + keys.sort(); + keys + }; + + let mut single_keys = single.shown_leaves; + single_keys.sort(); + assert!(!single_keys.is_empty(), "must record shown leaves"); + assert_eq!( + single_keys, + fileset_keys_for(name), + "single-file and fileset invocations must produce identical keys" + ); + assert_eq!( + single_keys, + fileset_keys_for(&format!("./{name}")), + "a different relative spelling must produce identical keys" + ); +} diff --git a/tests/fileset_tree.rs b/tests/fileset_tree.rs index bbec708b..12e8e73b 100644 --- a/tests/fileset_tree.rs +++ b/tests/fileset_tree.rs @@ -631,6 +631,7 @@ fn tree_omitted_folders_render_in_input_order() { array_sampler: headson::ArraySamplerStrategy::Default, line_budget_only: true, safety_cap: headson::DEFAULT_SAFETY_CAP, + explore: None, }; let grep_cfg = headson::GrepConfig::default(); let budgets = headson::Budgets { diff --git a/tests/multi_input_dir.rs b/tests/multi_input_dir.rs index bfc13ad9..697fed69 100644 --- a/tests/multi_input_dir.rs +++ b/tests/multi_input_dir.rs @@ -37,7 +37,7 @@ fn directory_inputs_are_ignored_and_reported() { let err_t = err.trim_end(); assert!( - err_t.ends_with(&format!("Ignored directory: {sub_s}")), + err_t.contains(&format!("Ignored directory: {sub_s}")), "stderr should end with directory ignore notice. stderr: {err_t}" ); }