From ed10b6602db5ae710a4eacfac671d8174f1ff29e Mon Sep 17 00:00:00 2001 From: XiangpengHao Date: Tue, 1 Sep 2026 00:08:53 -0400 Subject: [PATCH 1/2] update --- src/builder.rs | 51 ++++++++++++++++++++++++++++++++++++++------ tests/correctness.rs | 44 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 88 insertions(+), 7 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 556cf4c..06f3524 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -850,8 +850,10 @@ struct Candidate { } impl Candidate { - fn comparable_form(&self) -> (usize, usize) { - (self.gain, self.symbol.len()) + // The content tie-breaker makes training deterministic regardless of hash-map iteration order, + // which varies by architecture with hashbrown's SIMD group width. + fn comparable_form(&self) -> (usize, usize, u64) { + (self.gain, self.symbol.len(), self.symbol.to_u64()) } } @@ -871,16 +873,51 @@ impl PartialOrd for Candidate { impl Ord for Candidate { fn cmp(&self, other: &Self) -> Ordering { - let self_ord = (self.gain, self.symbol.len()); - let other_ord = (other.gain, other.symbol.len()); - - self_ord.cmp(&other_ord) + self.comparable_form().cmp(&other.comparable_form()) } } #[cfg(test)] mod test { - use crate::{Compressor, ESCAPE_CODE, builder::CodesBitmap}; + use super::Candidate; + use crate::{Compressor, ESCAPE_CODE, Symbol, builder::CodesBitmap}; + use std::collections::BinaryHeap; + + #[test] + fn test_candidate_heap_order_is_insertion_order_independent() { + let candidates = [ + Candidate { + gain: 10, + symbol: Symbol::from_slice(b"aa\0\0\0\0\0\0"), + }, + Candidate { + gain: 10, + symbol: Symbol::from_slice(b"bb\0\0\0\0\0\0"), + }, + Candidate { + gain: 10, + symbol: Symbol::from_slice(b"cc\0\0\0\0\0\0"), + }, + Candidate { + gain: 10, + symbol: Symbol::from_slice(b"dd\0\0\0\0\0\0"), + }, + ]; + let insertion_orders = [[0, 1, 2, 3], [1, 2, 3, 0], [2, 3, 0, 1], [3, 2, 1, 0]]; + + let pop_sequences: Vec> = insertion_orders + .into_iter() + .map(|order| { + let mut heap = BinaryHeap::new(); + heap.extend(order.map(|index| candidates[index])); + std::iter::from_fn(|| heap.pop().map(|candidate| candidate.symbol)).collect() + }) + .collect(); + + for sequence in &pop_sequences[1..] { + assert_eq!(sequence, &pop_sequences[0]); + } + } #[test] fn test_builder() { diff --git a/tests/correctness.rs b/tests/correctness.rs index 5442858..cd88e4c 100644 --- a/tests/correctness.rs +++ b/tests/correctness.rs @@ -23,6 +23,50 @@ const fn scaled(full: usize, under_miri: usize) -> usize { if cfg!(miri) { under_miri } else { full } } +fn fnv1a64(bytes: impl IntoIterator) -> u64 { + let mut hash = 0xcbf29ce484222325; + for byte in bytes { + hash ^= u64::from(byte); + hash = hash.wrapping_mul(0x100000001b3); + } + hash +} + +fn training_golden(input: &str) -> (usize, u64, usize, u64) { + let trained = Compressor::train(&vec![input.as_bytes()]); + let table_fingerprint = fnv1a64( + trained + .symbol_table() + .iter() + .flat_map(|symbol| symbol.to_u64().to_le_bytes()) + .chain(trained.symbol_lengths().iter().copied()), + ); + let compressed = trained.compress(input.as_bytes()); + + ( + trained.n_symbols(), + table_fingerprint, + compressed.len(), + fnv1a64(compressed), + ) +} + +// Full-corpus training is prohibitively slow under Miri. +#[cfg_attr(miri, ignore)] +#[test] +fn test_training_is_cross_architecture_deterministic() { + // These goldens guard against x86_64 and aarch64 hashbrown iteration-order differences. + // When training intentionally changes, regenerate them by temporarily printing these tuples. + assert_eq!( + training_golden(DECLARATION), + (243, 2302118744919910234, 3736, 16602696328334332958), + ); + assert_eq!( + training_golden(ART_OF_WAR), + (239, 11323366151389446290, 4744, 10727487692352854482), + ); +} + #[test] fn test_basic() { // Roundtrip the declaration From a3eef9415b6f2047041bd0740a944859455ec9c1 Mon Sep 17 00:00:00 2001 From: XiangpengHao Date: Tue, 1 Sep 2026 21:56:10 -0400 Subject: [PATCH 2/2] fix --- src/builder.rs | 26 ++++++++------------------ 1 file changed, 8 insertions(+), 18 deletions(-) diff --git a/src/builder.rs b/src/builder.rs index 06f3524..a5b9353 100644 --- a/src/builder.rs +++ b/src/builder.rs @@ -843,28 +843,12 @@ impl CompressorBuilder { /// A candidate for inclusion in a symbol table. /// /// This is really only useful for the `optimize` step of training. -#[derive(Copy, Clone, Debug)] +#[derive(Copy, Clone, Debug, Eq, PartialEq)] struct Candidate { gain: usize, symbol: Symbol, } -impl Candidate { - // The content tie-breaker makes training deterministic regardless of hash-map iteration order, - // which varies by architecture with hashbrown's SIMD group width. - fn comparable_form(&self) -> (usize, usize, u64) { - (self.gain, self.symbol.len(), self.symbol.to_u64()) - } -} - -impl Eq for Candidate {} - -impl PartialEq for Candidate { - fn eq(&self, other: &Self) -> bool { - self.comparable_form().eq(&other.comparable_form()) - } -} - impl PartialOrd for Candidate { fn partial_cmp(&self, other: &Self) -> Option { Some(self.cmp(other)) @@ -873,7 +857,13 @@ impl PartialOrd for Candidate { impl Ord for Candidate { fn cmp(&self, other: &Self) -> Ordering { - self.comparable_form().cmp(&other.comparable_form()) + // The content tie-breaker makes training deterministic regardless of hash-map iteration + // order, which varies by architecture with hashbrown's SIMD group width. + (self.gain, self.symbol.len(), self.symbol.to_u64()).cmp(&( + other.gain, + other.symbol.len(), + other.symbol.to_u64(), + )) } }