Compress ordinals/forms data, drop build-only data retention

CLAUDE.md

@@ -43,10 +43,10 @@ Source of truth lives under `resources/` (YAML + a few text blobs).
 `genscripts/` regenerates per-consumer artifacts under `rust/data/`
 (e.g. `rust/data/names/stopwords.json`,
 `rust/data/territories/data.jsonl`,
-`rust/data/names/person_names.txt`) — these are committed. Large
-blobs (person-names corpus, territories JSONL) get zstd-compressed
-at crate-build time by `rust/build.rs` and embedded into the binary
-via `include_bytes!`.
+`rust/data/names/person_names.txt`) — these are committed. All but
+the few-KB tables get zstd-compressed by `rust/build.rs` and embedded
+via `include_bytes!`; build-only inputs are decoded into caller-owned
+values, never stashed in a static.
 
 The **Python-side** `rigour/data/` directory is mostly drained: the
 wordlist / stopword / ordinals / names / territories / org-types

genscripts/generate_text.py

@@ -19,7 +19,7 @@ def generate_ordinals() -> None:
     """Emit `rust/data/text/ordinals.json` — array of `{number, forms}`
     records, sorted by number. Consumed by the Rust-side
     `ordinals_dict()` accessor (Python consumers) and by the Rust
-    tagger build path via `include_str!`."""
+    tagger build path."""
     ordinals_path = RESOURCES_PATH / "text" / "ordinals.yml"
     with open(ordinals_path, "r", encoding="utf-8") as ufh:
         ordinals_mapping: dict[str, dict[int, list[str]]] = yaml.safe_load(ufh.read())

rust/Cargo.toml

@@ -66,18 +66,15 @@ aho-corasick = "1"
 # LazyLock.
 regex = "1"
 
-# Serde + serde_json for reading committed JSON data artifacts like
-# rust/data/names/org_types.json inside a LazyLock at first use.
+# Serde + serde_json for reading the committed JSON data artifacts
+# under rust/data/ (org types, symbols, ordinals, stopwords, …).
 serde = { version = "1", features = ["derive"] }
 serde_json = "1"
 
-# zstd — compile-time compression of the person-names corpus
-# (`rust/data/names/person_names.txt`) into compressed bytes baked
-# into the binary via OUT_DIR. Runtime decodes on first access in
-# `names::person_names`. See `build.rs` for the compression step
-# and `plans/arch-rust-core.md` for the three-tier data-embedding
-# strategy. Listed in both [dependencies] (runtime decode) and
-# [build-dependencies] (build.rs uses the same crate for encoding).
+# zstd — compile-time compression of the larger committed data files
+# (see the FILES table in `build.rs`); consumers `include_bytes!` the
+# OUT_DIR artifact and decode on use. Listed in both [dependencies]
+# (decode) and [build-dependencies] (build.rs encodes).
 zstd = "0.13"
 
 [build-dependencies]

rust/build.rs

@@ -3,12 +3,6 @@
 // UTF-8) for diffability; `build.rs` compresses each into `OUT_DIR`
 // and the corresponding Rust module picks it up via `include_bytes!`.
 //
-// Current files:
-//   - `data/names/person_names.txt`        (~8.1 MB → ~2.7 MB)
-//   - `data/names/symbols.json`            (~85 KiB → ~12 KiB)
-//   - `data/names/org_types.json`          (~125 KiB → ~15 KiB)
-//   - `data/territories/data.jsonl`        (~783 KiB → ~214 KiB)
-//
 // All source files are committed, so a missing one means a broken
 // checkout — fail the build rather than embed an empty blob that
 // would ship as a silently non-functional wheel (empty tagger, zero
@@ -49,6 +43,18 @@ const FILES: &[Compress] = &[
         missing: "rust/data/names/org_types.json not found. Run \
                   `make build-names` to regenerate.",
     },
+    Compress {
+        src: "data/text/ordinals.json",
+        dst: "ordinals.json.zst",
+        missing: "rust/data/text/ordinals.json not found. Run \
+                  `make build-text` to regenerate.",
+    },
+    Compress {
+        src: "data/addresses/forms.json",
+        dst: "address_forms.json.zst",
+        missing: "rust/data/addresses/forms.json not found. Run \
+                  `make build-addresses` to regenerate.",
+    },
 ];
 
 fn main() {

rust/src/addresses/tagger.rs

@@ -126,15 +126,16 @@ impl AddressTagger {
     }
 }
 
-const FORMS_JSON: &str = include_str!("../../data/addresses/forms.json");
+const FORMS_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/address_forms.json.zst"));
 
 fn build_tagger() -> AddressTagger {
     let mut b = Builder::new();
 
     // Keyword forms: every alias and the canonical key itself map
     // to the canonical form, so "blvd" in text tags as blvd too.
+    let forms_json = zstd::decode_all(FORMS_ZST).expect("zstd decode address_forms.json.zst");
     let forms: BTreeMap<String, Vec<String>> =
-        serde_json::from_str(FORMS_JSON).expect("rust/data/addresses/forms.json parses");
+        serde_json::from_slice(&forms_json).expect("forms.json parses");
     for (canonical, aliases) in &forms {
         b.add_keyword(canonical, canonical);
         for alias in aliases {

rust/src/names/org_types.rs

@@ -55,10 +55,12 @@ pub(crate) struct OrgTypeSpec {
 
 const ORG_TYPES_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/org_types.json.zst"));
 
-pub(crate) static ORG_TYPE_SPECS: LazyLock<Vec<OrgTypeSpec>> = LazyLock::new(|| {
+/// Decode the spec list into a fresh, caller-owned Vec. Every reader
+/// is a replacer/tagger build cached upstream — do not stash this.
+pub(crate) fn org_type_specs() -> Vec<OrgTypeSpec> {
     let bytes = zstd::decode_all(ORG_TYPES_ZST).expect("zstd decode org_types.json.zst");
     serde_json::from_slice(&bytes).expect("org_types.json parses")
-});
+}
 
 /// Selects which mapping (alias → target) the Replacer is built from.
 #[derive(Clone, Copy, Debug, PartialEq, Eq, Hash)]
@@ -124,7 +126,7 @@ fn build_compare(flags: Normalize, cleanup: Cleanup) -> Replacer {
     let norm = norm_fn(flags, cleanup);
     let mut mapping: HashMap<String, String> = HashMap::new();
 
-    for spec in ORG_TYPE_SPECS.iter() {
+    for spec in org_type_specs() {
         let display_norm = spec.display.as_deref().and_then(&norm);
         // `compare: ""` means "remove this org type" — an intentional
         // empty target. Absent `compare` falls back to display.
@@ -163,7 +165,7 @@ fn build_generic(flags: Normalize, cleanup: Cleanup) -> Replacer {
     let norm = norm_fn(flags, cleanup);
     let mut mapping: HashMap<String, String> = HashMap::new();
 
-    for spec in ORG_TYPE_SPECS.iter() {
+    for spec in org_type_specs() {
         let Some(generic_norm) = spec.generic.as_deref().and_then(&norm) else {
             continue;
         };
@@ -205,7 +207,7 @@ fn build_display(flags: Normalize, cleanup: Cleanup) -> Replacer {
     let mut seen_targets: HashMap<String, String> = HashMap::new();
     let mut clashes: HashSet<String> = HashSet::new();
 
-    for spec in ORG_TYPE_SPECS.iter() {
+    for spec in org_type_specs() {
         if spec.display.as_deref().and_then(&norm).is_none() {
             continue; // display doesn't survive key normalisation
         }

rust/src/names/symbols.rs

@@ -6,13 +6,12 @@
 // the tagger) and the aliases become needles in the AC automaton.
 //
 // The JSON is indented on disk for reviewability; build.rs
-// zstd-compresses it into OUT_DIR and this module decodes on first
-// use. Internal to the Rust crate — no PyO3 surface. Consumed by
-// `names::tagger::build_{org,person}_tagger`.
+// zstd-compresses it into OUT_DIR. No resident static: the only
+// consumers, `names::tagger::build_{org,person}_tagger`, are cached
+// in `TAGGER_CACHE`. Internal to the crate — no PyO3 surface.
 
 use serde::Deserialize;
 use std::collections::HashMap;
-use std::sync::LazyLock;
 
 const SYMBOLS_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/symbols.json.zst"));
 
@@ -25,13 +24,10 @@ pub struct NameSymbols {
     pub person_name_parts: HashMap<String, Vec<String>>,
 }
 
-static DATA: LazyLock<NameSymbols> = LazyLock::new(|| {
+/// Decode the symbol tables into a fresh, caller-owned value.
+pub fn data() -> NameSymbols {
     let bytes = zstd::decode_all(SYMBOLS_ZST).expect("zstd decode symbols.json.zst");
     serde_json::from_slice(&bytes).expect("symbols.json parses")
-});
-
-pub fn data() -> &'static NameSymbols {
-    &DATA
 }
 
 #[cfg(test)]

rust/src/names/tagger.rs

@@ -169,7 +169,7 @@ fn build_org_tagger(flags: Normalize) -> Tagger {
 
     // Org types → ORG_CLASS symbols keyed on the `generic` field.
     let mut class_syms: HashMap<String, Symbol> = HashMap::new();
-    for spec in org_types::ORG_TYPE_SPECS.iter() {
+    for spec in org_types::org_type_specs() {
         let Some(generic) = spec.generic.as_deref() else {
             continue;
         };

rust/src/territories.rs

@@ -6,17 +6,13 @@
 // names_strong, names_weak, ...}`. Authoritative emission is
 // `genscripts/generate_territories.py::update_data`.
 //
-// The JSONL ships as plain UTF-8 in git (~783 KiB, diff-friendly when
-// the generator regenerates) and gets zstd-compressed at crate-build
-// time by `build.rs` (~214 KiB).
+// The JSONL ships as plain UTF-8 in git (diff-friendly when the
+// generator regenerates) and is zstd-compressed by `build.rs`.
 //
 // No static `LazyLock<String>` cache — each `decompressed()` call
-// returns a fresh owned `String`. Both consumers are one-shot:
-// Python's `rigour.territories.*` reads via `@cache`-decorated index
-// builders (so exactly one FFI hop per process), and the Rust tagger
-// walks the lines once per `(TaggerKind, flags, cleanup)` cache miss.
-// A persistent Rust-side copy would just duplicate what's already in
-// Python's cached PyString / the tagger's AC automaton.
+// returns a fresh owned `String`; all consumers are one-shot reads
+// behind their own caches (Python's `@cache`-decorated index
+// builders, the Rust tagger builds).
 
 use serde::Deserialize;
 

rust/src/text/ordinals.rs

@@ -4,35 +4,35 @@
 // `rigour.addresses.normalize` and the Rust tagger build path.
 //
 // The JSON on disk is an array of `{number, forms}` records (see
-// `genscripts/generate_text.py::generate_ordinals`); we deserialise
-// to a Vec and materialise the HashMap on each accessor call since
-// Python takes ownership.
+// `genscripts/generate_text.py::generate_ordinals`), zstd-compressed
+// into OUT_DIR by `build.rs`. No resident static: every consumer
+// caches downstream (tagger builds, the Python dict).
 
 use serde::Deserialize;
 use std::collections::HashMap;
-use std::sync::LazyLock;
 
 #[derive(Debug, Deserialize)]
 pub struct OrdinalSpec {
     pub number: u32,
     pub forms: Vec<String>,
 }
 
-const JSON: &str = include_str!("../../data/text/ordinals.json");
+const ORDINALS_ZST: &[u8] = include_bytes!(concat!(env!("OUT_DIR"), "/ordinals.json.zst"));
 
-static DATA: LazyLock<Vec<OrdinalSpec>> =
-    LazyLock::new(|| serde_json::from_str(JSON).expect("rust/data/text/ordinals.json parses"));
+/// Decode the spec list into a fresh, caller-owned Vec — do not
+/// stash the result in a static.
+pub fn ordinals() -> Vec<OrdinalSpec> {
+    let bytes = zstd::decode_all(ORDINALS_ZST).expect("zstd decode ordinals.json.zst");
+    serde_json::from_slice(&bytes).expect("ordinals.json parses")
+}
 
 /// Ordinals as a `{number: [forms...]}` map — matches the Python
 /// consumer's `ORDINALS.items()` iteration pattern.
 pub fn ordinals_dict() -> HashMap<u32, Vec<String>> {
-    DATA.iter().map(|o| (o.number, o.forms.clone())).collect()
-}
-
-/// Pure-Rust accessor for the raw spec list. Used by the Rust
-/// tagger build path, which iterates without needing a HashMap.
-pub fn ordinals() -> &'static [OrdinalSpec] {
-    &DATA
+    ordinals()
+        .into_iter()
+        .map(|o| (o.number, o.forms))
+        .collect()
 }
 
 #[cfg(test)]