diff --git a/CLAUDE.md b/CLAUDE.md index e14b943a..b6c19eb2 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -43,10 +43,10 @@ Source of truth lives under `resources/` (YAML + a few text blobs). `genscripts/` regenerates per-consumer artifacts under `rust/data/` (e.g. `rust/data/names/stopwords.json`, `rust/data/territories/data.jsonl`, -`rust/data/names/person_names.txt`) — these are committed. Large -blobs (person-names corpus, territories JSONL) get zstd-compressed -at crate-build time by `rust/build.rs` and embedded into the binary -via `include_bytes!`. +`rust/data/names/person_names.txt`) — these are committed. All but +the few-KB tables get zstd-compressed by `rust/build.rs` and embedded +via `include_bytes!`; build-only inputs are decoded into caller-owned +values, never stashed in a static. The **Python-side** `rigour/data/` directory is mostly drained: the wordlist / stopword / ordinals / names / territories / org-types diff --git a/genscripts/generate_text.py b/genscripts/generate_text.py index d5c7ef79..325dadcc 100644 --- a/genscripts/generate_text.py +++ b/genscripts/generate_text.py @@ -19,7 +19,7 @@ def generate_ordinals() -> None: """Emit `rust/data/text/ordinals.json` — array of `{number, forms}` records, sorted by number. Consumed by the Rust-side `ordinals_dict()` accessor (Python consumers) and by the Rust - tagger build path via `include_str!`.""" + tagger build path.""" ordinals_path = RESOURCES_PATH / "text" / "ordinals.yml" with open(ordinals_path, "r", encoding="utf-8") as ufh: ordinals_mapping: dict[str, dict[int, list[str]]] = yaml.safe_load(ufh.read()) diff --git a/rust/Cargo.toml b/rust/Cargo.toml index 77041685..8fb1b73f 100644 --- a/rust/Cargo.toml +++ b/rust/Cargo.toml @@ -66,18 +66,15 @@ aho-corasick = "1" # LazyLock. regex = "1" -# Serde + serde_json for reading committed JSON data artifacts like -# rust/data/names/org_types.json inside a LazyLock at first use. +# Serde + serde_json for reading the committed JSON data artifacts +# under rust/data/ (org types, symbols, ordinals, stopwords, …). serde = { version = "1", features = ["derive"] } serde_json = "1" -# zstd — compile-time compression of the person-names corpus -# (`rust/data/names/person_names.txt`) into compressed bytes baked -# into the binary via OUT_DIR. Runtime decodes on first access in -# `names::person_names`. See `build.rs` for the compression step -# and `plans/arch-rust-core.md` for the three-tier data-embedding -# strategy. Listed in both [dependencies] (runtime decode) and -# [build-dependencies] (build.rs uses the same crate for encoding). +# zstd — compile-time compression of the larger committed data files +# (see the FILES table in `build.rs`); consumers `include_bytes!` the +# OUT_DIR artifact and decode on use. Listed in both [dependencies] +# (decode) and [build-dependencies] (build.rs encodes). zstd = "0.13" [build-dependencies] diff --git a/rust/build.rs b/rust/build.rs index ff8bb5e3..b08cb123 100644 --- a/rust/build.rs +++ b/rust/build.rs @@ -3,12 +3,6 @@ // UTF-8) for diffability; `build.rs` compresses each into `OUT_DIR` // and the corresponding Rust module picks it up via `include_bytes!`. // -// Current files: -// - `data/names/person_names.txt` (~8.1 MB → ~2.7 MB) -// - `data/names/symbols.json` (~85 KiB → ~12 KiB) -// - `data/names/org_types.json` (~125 KiB → ~15 KiB) -// - `data/territories/data.jsonl` (~783 KiB → ~214 KiB) -// // All source files are committed, so a missing one means a broken // checkout — fail the build rather than embed an empty blob that // would ship as a silently non-functional wheel (empty tagger, zero @@ -49,6 +43,18 @@ const FILES: &[Compress] = &[ missing: "rust/data/names/org_types.json not found. Run \ `make build-names` to regenerate.", }, + Compress { + src: "data/text/ordinals.json", + dst: "ordinals.json.zst", + missing: "rust/data/text/ordinals.json not found. Run \ + `make build-text` to regenerate.", + }, + Compress { + src: "data/addresses/forms.json", + dst: "address_forms.json.zst", + missing: "rust/data/addresses/forms.json not found. Run \ + `make build-addresses` to regenerate.", + }, ]; fn main() { diff --git a/rust/src/addresses/tagger.rs b/rust/src/addresses/tagger.rs index ef9c8970..cadad369 100644 --- a/rust/src/addresses/tagger.rs +++ b/rust/src/addresses/tagger.rs @@ -126,15 +126,16 @@ impl AddressTagger { } } -const FORMS_JSON: &str = include_str!("../../data/addresses/forms.json"); +const FORMS_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/address_forms.json.zst")); fn build_tagger() -> AddressTagger { let mut b = Builder::new(); // Keyword forms: every alias and the canonical key itself map // to the canonical form, so "blvd" in text tags as blvd too. + let forms_json = zstd::decode_all(FORMS_ZST).expect("zstd decode address_forms.json.zst"); let forms: BTreeMap> = - serde_json::from_str(FORMS_JSON).expect("rust/data/addresses/forms.json parses"); + serde_json::from_slice(&forms_json).expect("forms.json parses"); for (canonical, aliases) in &forms { b.add_keyword(canonical, canonical); for alias in aliases { diff --git a/rust/src/names/org_types.rs b/rust/src/names/org_types.rs index 93c405b5..119a7879 100644 --- a/rust/src/names/org_types.rs +++ b/rust/src/names/org_types.rs @@ -55,10 +55,12 @@ pub(crate) struct OrgTypeSpec { const ORG_TYPES_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/org_types.json.zst")); -pub(crate) static ORG_TYPE_SPECS: LazyLock> = LazyLock::new(|| { +/// Decode the spec list into a fresh, caller-owned Vec. Every reader +/// is a replacer/tagger build cached upstream — do not stash this. +pub(crate) fn org_type_specs() -> Vec { let bytes = zstd::decode_all(ORG_TYPES_ZST).expect("zstd decode org_types.json.zst"); serde_json::from_slice(&bytes).expect("org_types.json parses") -}); +} /// Selects which mapping (alias → target) the Replacer is built from. #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)] @@ -124,7 +126,7 @@ fn build_compare(flags: Normalize, cleanup: Cleanup) -> Replacer { let norm = norm_fn(flags, cleanup); let mut mapping: HashMap = HashMap::new(); - for spec in ORG_TYPE_SPECS.iter() { + for spec in org_type_specs() { let display_norm = spec.display.as_deref().and_then(&norm); // `compare: ""` means "remove this org type" — an intentional // empty target. Absent `compare` falls back to display. @@ -163,7 +165,7 @@ fn build_generic(flags: Normalize, cleanup: Cleanup) -> Replacer { let norm = norm_fn(flags, cleanup); let mut mapping: HashMap = HashMap::new(); - for spec in ORG_TYPE_SPECS.iter() { + for spec in org_type_specs() { let Some(generic_norm) = spec.generic.as_deref().and_then(&norm) else { continue; }; @@ -205,7 +207,7 @@ fn build_display(flags: Normalize, cleanup: Cleanup) -> Replacer { let mut seen_targets: HashMap = HashMap::new(); let mut clashes: HashSet = HashSet::new(); - for spec in ORG_TYPE_SPECS.iter() { + for spec in org_type_specs() { if spec.display.as_deref().and_then(&norm).is_none() { continue; // display doesn't survive key normalisation } diff --git a/rust/src/names/symbols.rs b/rust/src/names/symbols.rs index d9878513..81a919d9 100644 --- a/rust/src/names/symbols.rs +++ b/rust/src/names/symbols.rs @@ -6,13 +6,12 @@ // the tagger) and the aliases become needles in the AC automaton. // // The JSON is indented on disk for reviewability; build.rs -// zstd-compresses it into OUT_DIR and this module decodes on first -// use. Internal to the Rust crate — no PyO3 surface. Consumed by -// `names::tagger::build_{org,person}_tagger`. +// zstd-compresses it into OUT_DIR. No resident static: the only +// consumers, `names::tagger::build_{org,person}_tagger`, are cached +// in `TAGGER_CACHE`. Internal to the crate — no PyO3 surface. use serde::Deserialize; use std::collections::HashMap; -use std::sync::LazyLock; const SYMBOLS_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/symbols.json.zst")); @@ -25,13 +24,10 @@ pub struct NameSymbols { pub person_name_parts: HashMap>, } -static DATA: LazyLock = LazyLock::new(|| { +/// Decode the symbol tables into a fresh, caller-owned value. +pub fn data() -> NameSymbols { let bytes = zstd::decode_all(SYMBOLS_ZST).expect("zstd decode symbols.json.zst"); serde_json::from_slice(&bytes).expect("symbols.json parses") -}); - -pub fn data() -> &'static NameSymbols { - &DATA } #[cfg(test)] diff --git a/rust/src/names/tagger.rs b/rust/src/names/tagger.rs index c267e51b..57951d01 100644 --- a/rust/src/names/tagger.rs +++ b/rust/src/names/tagger.rs @@ -169,7 +169,7 @@ fn build_org_tagger(flags: Normalize) -> Tagger { // Org types → ORG_CLASS symbols keyed on the `generic` field. let mut class_syms: HashMap = HashMap::new(); - for spec in org_types::ORG_TYPE_SPECS.iter() { + for spec in org_types::org_type_specs() { let Some(generic) = spec.generic.as_deref() else { continue; }; diff --git a/rust/src/territories.rs b/rust/src/territories.rs index 29ef3ba0..8c84d8db 100644 --- a/rust/src/territories.rs +++ b/rust/src/territories.rs @@ -6,17 +6,13 @@ // names_strong, names_weak, ...}`. Authoritative emission is // `genscripts/generate_territories.py::update_data`. // -// The JSONL ships as plain UTF-8 in git (~783 KiB, diff-friendly when -// the generator regenerates) and gets zstd-compressed at crate-build -// time by `build.rs` (~214 KiB). +// The JSONL ships as plain UTF-8 in git (diff-friendly when the +// generator regenerates) and is zstd-compressed by `build.rs`. // // No static `LazyLock` cache — each `decompressed()` call -// returns a fresh owned `String`. Both consumers are one-shot: -// Python's `rigour.territories.*` reads via `@cache`-decorated index -// builders (so exactly one FFI hop per process), and the Rust tagger -// walks the lines once per `(TaggerKind, flags, cleanup)` cache miss. -// A persistent Rust-side copy would just duplicate what's already in -// Python's cached PyString / the tagger's AC automaton. +// returns a fresh owned `String`; all consumers are one-shot reads +// behind their own caches (Python's `@cache`-decorated index +// builders, the Rust tagger builds). use serde::Deserialize; diff --git a/rust/src/text/ordinals.rs b/rust/src/text/ordinals.rs index a61416ab..0627d626 100644 --- a/rust/src/text/ordinals.rs +++ b/rust/src/text/ordinals.rs @@ -4,13 +4,12 @@ // `rigour.addresses.normalize` and the Rust tagger build path. // // The JSON on disk is an array of `{number, forms}` records (see -// `genscripts/generate_text.py::generate_ordinals`); we deserialise -// to a Vec and materialise the HashMap on each accessor call since -// Python takes ownership. +// `genscripts/generate_text.py::generate_ordinals`), zstd-compressed +// into OUT_DIR by `build.rs`. No resident static: every consumer +// caches downstream (tagger builds, the Python dict). use serde::Deserialize; use std::collections::HashMap; -use std::sync::LazyLock; #[derive(Debug, Deserialize)] pub struct OrdinalSpec { @@ -18,21 +17,22 @@ pub struct OrdinalSpec { pub forms: Vec, } -const JSON: &str = include_str!("../../data/text/ordinals.json"); +const ORDINALS_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ordinals.json.zst")); -static DATA: LazyLock> = - LazyLock::new(|| serde_json::from_str(JSON).expect("rust/data/text/ordinals.json parses")); +/// Decode the spec list into a fresh, caller-owned Vec — do not +/// stash the result in a static. +pub fn ordinals() -> Vec { + let bytes = zstd::decode_all(ORDINALS_ZST).expect("zstd decode ordinals.json.zst"); + serde_json::from_slice(&bytes).expect("ordinals.json parses") +} /// Ordinals as a `{number: [forms...]}` map — matches the Python /// consumer's `ORDINALS.items()` iteration pattern. pub fn ordinals_dict() -> HashMap> { - DATA.iter().map(|o| (o.number, o.forms.clone())).collect() -} - -/// Pure-Rust accessor for the raw spec list. Used by the Rust -/// tagger build path, which iterates without needing a HashMap. -pub fn ordinals() -> &'static [OrdinalSpec] { - &DATA + ordinals() + .into_iter() + .map(|o| (o.number, o.forms)) + .collect() } #[cfg(test)]