Deterministic symbol table on both x86 and arm64

src/builder.rs

@@ -843,26 +843,12 @@ impl CompressorBuilder {
 /// A candidate for inclusion in a symbol table.
 ///
 /// This is really only useful for the `optimize` step of training.
-#[derive(Copy, Clone, Debug)]
+#[derive(Copy, Clone, Debug, Eq, PartialEq)]
 struct Candidate {
     gain: usize,
     symbol: Symbol,
 }
 
-impl Candidate {
-    fn comparable_form(&self) -> (usize, usize) {
-        (self.gain, self.symbol.len())
-    }
-}
-
-impl Eq for Candidate {}
-
-impl PartialEq<Self> for Candidate {
-    fn eq(&self, other: &Self) -> bool {
-        self.comparable_form().eq(&other.comparable_form())
-    }
-}
-
 impl PartialOrd<Self> for Candidate {
     fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
         Some(self.cmp(other))
@@ -871,16 +857,57 @@ impl PartialOrd<Self> for Candidate {
 
 impl Ord for Candidate {
     fn cmp(&self, other: &Self) -> Ordering {
-        let self_ord = (self.gain, self.symbol.len());
-        let other_ord = (other.gain, other.symbol.len());
-
-        self_ord.cmp(&other_ord)
+        // The content tie-breaker makes training deterministic regardless of hash-map iteration
+        // order, which varies by architecture with hashbrown's SIMD group width.
+        (self.gain, self.symbol.len(), self.symbol.to_u64()).cmp(&(
+            other.gain,
+            other.symbol.len(),
+            other.symbol.to_u64(),
+        ))
     }
 }
 
 #[cfg(test)]
 mod test {
-    use crate::{Compressor, ESCAPE_CODE, builder::CodesBitmap};
+    use super::Candidate;
+    use crate::{Compressor, ESCAPE_CODE, Symbol, builder::CodesBitmap};
+    use std::collections::BinaryHeap;
+
+    #[test]
+    fn test_candidate_heap_order_is_insertion_order_independent() {
+        let candidates = [
+            Candidate {
+                gain: 10,
+                symbol: Symbol::from_slice(b"aa\0\0\0\0\0\0"),
+            },
+            Candidate {
+                gain: 10,
+                symbol: Symbol::from_slice(b"bb\0\0\0\0\0\0"),
+            },
+            Candidate {
+                gain: 10,
+                symbol: Symbol::from_slice(b"cc\0\0\0\0\0\0"),
+            },
+            Candidate {
+                gain: 10,
+                symbol: Symbol::from_slice(b"dd\0\0\0\0\0\0"),
+            },
+        ];
+        let insertion_orders = [[0, 1, 2, 3], [1, 2, 3, 0], [2, 3, 0, 1], [3, 2, 1, 0]];
+
+        let pop_sequences: Vec<Vec<Symbol>> = insertion_orders
+            .into_iter()
+            .map(|order| {
+                let mut heap = BinaryHeap::new();
+                heap.extend(order.map(|index| candidates[index]));
+                std::iter::from_fn(|| heap.pop().map(|candidate| candidate.symbol)).collect()
+            })
+            .collect();
+
+        for sequence in &pop_sequences[1..] {
+            assert_eq!(sequence, &pop_sequences[0]);
+        }
+    }
 
     #[test]
     fn test_builder() {

tests/correctness.rs

@@ -23,6 +23,50 @@ const fn scaled(full: usize, under_miri: usize) -> usize {
     if cfg!(miri) { under_miri } else { full }
 }
 
+fn fnv1a64(bytes: impl IntoIterator<Item = u8>) -> u64 {
+    let mut hash = 0xcbf29ce484222325;
+    for byte in bytes {
+        hash ^= u64::from(byte);
+        hash = hash.wrapping_mul(0x100000001b3);
+    }
+    hash
+}
+
+fn training_golden(input: &str) -> (usize, u64, usize, u64) {
+    let trained = Compressor::train(&vec![input.as_bytes()]);
+    let table_fingerprint = fnv1a64(
+        trained
+            .symbol_table()
+            .iter()
+            .flat_map(|symbol| symbol.to_u64().to_le_bytes())
+            .chain(trained.symbol_lengths().iter().copied()),
+    );
+    let compressed = trained.compress(input.as_bytes());
+
+    (
+        trained.n_symbols(),
+        table_fingerprint,
+        compressed.len(),
+        fnv1a64(compressed),
+    )
+}
+
+// Full-corpus training is prohibitively slow under Miri.
+#[cfg_attr(miri, ignore)]
+#[test]
+fn test_training_is_cross_architecture_deterministic() {
+    // These goldens guard against x86_64 and aarch64 hashbrown iteration-order differences.
+    // When training intentionally changes, regenerate them by temporarily printing these tuples.
+    assert_eq!(
+        training_golden(DECLARATION),
+        (243, 2302118744919910234, 3736, 16602696328334332958),
+    );
+    assert_eq!(
+        training_golden(ART_OF_WAR),
+        (239, 11323366151389446290, 4744, 10727487692352854482),
+    );
+}
+
 #[test]
 fn test_basic() {
     // Roundtrip the declaration