diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a8ee18c..7c80ab0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -37,6 +37,11 @@ jobs: - run: cargo build - run: cargo test - run: cargo test --release + - name: Test SIMD formatting path + run: cargo test --release + if: matrix.rust == 'stable' + env: + RUSTFLAGS: -Dwarnings -C target-feature=+sse4.1,+lzcnt - run: cargo build --no-default-features - run: cargo test --tests --no-default-features - run: cargo test --tests --no-default-features --release diff --git a/Cargo.toml b/Cargo.toml index d8bd99a..5980cee 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -17,11 +17,16 @@ no-panic = { version = "0.1", optional = true } [target.'cfg(not(miri))'.dev-dependencies] criterion = { version = "0.8", default-features = false } +gungraun = "=0.18.2" [[bench]] name = "bench" harness = false +[[bench]] +name = "gungraun" +harness = false + [package.metadata.docs.rs] targets = ["x86_64-unknown-linux-gnu"] rustdoc-args = [ diff --git a/README.md b/README.md index c539202..728989f 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,28 @@ library. ![performance](https://raw.githubusercontent.com/dtolnay/itoa/master/itoa-benchmark.png) +This repository also contains Criterion and Gungraun microbenchmarks. Run the +Gungraun benchmarks with: + +```console +cargo install --version 0.18.2 gungraun-runner +cargo bench --bench gungraun +``` + +Gungraun requires Valgrind to be installed. +On x86-64, the SIMD-optimized path is gated on the `sse4.1` and `lzcnt` +target features. Enable exactly those to benchmark it: + +```console +RUSTFLAGS="-C target-feature=+sse4.1,+lzcnt" cargo bench --bench gungraun +``` + +Do not use `-C target-cpu=native` here. On AVX-512-capable CPUs (for example +AMD Zen 4/5) it lets the compiler autovectorize the digit conversion into +EVEX-encoded AVX-512 instructions, which Valgrind cannot decode and which make +Gungraun abort with `SIGILL`. The `+sse4.1,+lzcnt` feature set exercises the +same optimized code without emitting instructions Valgrind rejects. +
#### License diff --git a/benches/gungraun.rs b/benches/gungraun.rs new file mode 100644 index 0000000..d38f5a2 --- /dev/null +++ b/benches/gungraun.rs @@ -0,0 +1,80 @@ +use gungraun::prelude::*; +use std::hint; +use std::io::Write; + +macro_rules! benchmarks { + ( + $itoa:ident, + $std_fmt:ident, + $setup_std_fmt:ident, + $ty:ty, + $($id:ident => $value:expr),+ $(,)? + ) => { + fn $setup_std_fmt(value: $ty) -> (Vec, $ty) { + ( + Vec::with_capacity(<$ty as itoa::Integer>::MAX_STR_LEN), + value, + ) + } + + #[library_benchmark] + $( + #[bench::$id($value)] + )+ + fn $itoa(int: $ty) -> usize { + let mut buf = itoa::Buffer::new(); + let formatted = buf.format(hint::black_box(int)); + hint::black_box(formatted).len() + } + + #[library_benchmark] + $( + #[bench::$id(args = ($value), setup = $setup_std_fmt)] + )+ + fn $std_fmt((mut buf, int): (Vec, $ty)) -> usize { + write!(&mut buf, "{}", hint::black_box(int)).unwrap(); + hint::black_box(buf.as_slice()).len() + } + }; +} + +benchmarks!( + itoa_u64, + std_fmt_u64, + setup_std_fmt_u64, + u64, + zero => 0, + half => u64::from(u32::MAX), + max => u64::MAX, +); + +benchmarks!( + itoa_i16, + std_fmt_i16, + setup_std_fmt_i16, + i16, + zero => 0, + min => i16::MIN, +); + +benchmarks!( + itoa_u128, + std_fmt_u128, + setup_std_fmt_u128, + u128, + zero => 0, + max => u128::MAX, +); + +library_benchmark_group!( + name = benches; + benchmarks = + itoa_u64, + std_fmt_u64, + itoa_i16, + std_fmt_i16, + itoa_u128, + std_fmt_u128 +); + +main!(library_benchmark_groups = benches); diff --git a/src/lib.rs b/src/lib.rs index 74cd9b4..0434850 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -167,12 +167,21 @@ macro_rules! impl_Integer { #[cfg_attr(feature = "no-panic", no_panic)] fn write(self, buf: &mut Self::Buffer) -> &str { let mut offset = Self::MAX_STR_LEN - $Unsigned::MAX_STR_LEN; - offset += Unsigned::fmt( - self.unsigned_abs(), - (&mut buf[offset..]).try_into().unwrap(), - ); + // SAFETY: `offset == Self::MAX_STR_LEN - $Unsigned::MAX_STR_LEN`, + // so the `$Unsigned::MAX_STR_LEN` bytes starting at `offset` are + // exactly the tail of `buf` and form a valid unsigned buffer. + let unsigned_buf = unsafe { + &mut *buf + .as_mut_ptr() + .add(offset) + .cast::<<$Unsigned as private::Sealed>::Buffer>() + }; + offset += Unsigned::fmt(self.unsigned_abs(), unsigned_buf); if self < 0 { offset -= 1; + // SAFETY: `offset` indexes the byte immediately before the + // digits, which is within `buf`. + unsafe { assert_unchecked(offset < buf.len()) }; buf[offset].write(b'-'); } // SAFETY: Starting from `offset`, all elements of the slice have been set. @@ -236,6 +245,77 @@ fn divmod100(value: u32) -> (u32, u32) { (div, value - div * 100) } +/// Informs the optimizer that `cond` always holds, so that it can discard the +/// bounds checks and panic paths that depend on it. +/// +/// This is [`core::hint::assert_unchecked`], which is newer than this crate's +/// minimum supported Rust version. +/// +/// # Safety +/// +/// `cond` must be true. +#[inline(always)] +unsafe fn assert_unchecked(cond: bool) { + debug_assert!(cond); + if !cond { + // SAFETY: The caller guarantees that `cond` holds. + unsafe { hint::unreachable_unchecked() }; + } +} + +/// Writes `pair` as exactly two decimal digits into `buf[offset..offset + 2]`, +/// zero padded (for example 7 becomes `"07"`). +/// +/// # Safety +/// +/// `pair` must be below 100, and both `offset` and `offset + 1` must be valid +/// indices into `buf`. +#[inline(always)] +unsafe fn write_pair(buf: &mut [MaybeUninit], offset: usize, pair: u32) { + // SAFETY: These are this function's caller-provided invariants. They let + // the indexing below compile without bounds checks. Each index is asserted + // individually because `offset + n < buf.len()` alone would also hold for + // an `offset` whose addition wraps around. + unsafe { + assert_unchecked(pair < 100); + assert_unchecked(offset + 0 < buf.len()); + assert_unchecked(offset + 1 < buf.len()); + } + + let pair = pair as usize; + buf[offset + 0].write(DECIMAL_PAIRS.0[pair * 2 + 0]); + buf[offset + 1].write(DECIMAL_PAIRS.0[pair * 2 + 1]); +} + +/// Writes `quad` as exactly four decimal digits into `buf[offset..offset + 4]`, +/// zero padded (for example 42 becomes `"0042"`). +/// +/// # Safety +/// +/// `quad` must be below 10_000, and `offset..offset + 4` must all be valid +/// indices into `buf`. +#[inline(always)] +unsafe fn write_quad(buf: &mut [MaybeUninit], offset: usize, quad: u32) { + // SAFETY: These are this function's caller-provided invariants. + unsafe { + assert_unchecked(quad < 10_000); + assert_unchecked(offset + 0 < buf.len()); + assert_unchecked(offset + 1 < buf.len()); + assert_unchecked(offset + 2 < buf.len()); + assert_unchecked(offset + 3 < buf.len()); + } + + let (pair1, pair2) = divmod100(quad); + // SAFETY: `quad` is below 10_000, so both halves are below 100. + unsafe { assert_unchecked(pair1 < 100 && pair2 < 100) }; + + // SAFETY: Both pairs are below 100 and all four indices are in bounds. + unsafe { + write_pair(buf, offset + 0, pair1); + write_pair(buf, offset + 2, pair2); + } +} + /// This function converts a slice of ascii characters into a `&str` starting /// from `offset`. /// @@ -263,36 +343,35 @@ macro_rules! impl_Unsigned { fn fmt(self, buf: &mut Self::Buffer) -> usize { // Count the number of bytes in buf that are not initialized. let mut offset = buf.len(); + if self == 0 { + offset -= 1; + // SAFETY: Every integer buffer has room for at least one digit. + unsafe { assert_unchecked(offset < buf.len()) }; + buf[offset].write(b'0'); + return offset; + } // Consume the least-significant decimals from a working copy. let mut remain = self; // Format per four digits from the lookup table. - // Four digits need a 16-bit $Unsigned or wider. + // Four digits need a 16-bit $Unsigned or wider. The fallible + // conversions cannot panic: for `u8` the `size_of` guard makes + // the loop dead, and `unwrap_or` avoids a panic path there; for + // wider types the constants convert successfully. while mem::size_of::() > 1 - && remain - > 999 - .try_into() - .expect("branch is not hit for types that cannot fit 999 (u8)") + && remain > 999.try_into().unwrap_or(Self::MAX) { offset -= 4; // pull two pairs - let scale: Self = 1_00_00 - .try_into() - .expect("branch is not hit for types that cannot fit 1E4 (u8)"); + let scale: Self = 1_00_00.try_into().unwrap_or(1); let quad = remain % scale; remain /= scale; - let (pair1, pair2) = divmod100(quad as u32); - unsafe { - buf[offset + 0] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 0)); - buf[offset + 1] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 1)); - buf[offset + 2] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 0)); - buf[offset + 3] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 1)); - } + // SAFETY: `quad` is a remainder modulo 10_000. Every four + // digits written consume four of the `MAX_STR_LEN` bytes + // this type's buffer is sized for, so `offset` was just + // decremented into bounds and never drops below zero. + unsafe { write_quad(buf.as_mut_slice(), offset, quad as u32) }; } // Format per two digits from the lookup table. @@ -301,21 +380,19 @@ macro_rules! impl_Unsigned { let (last, pair) = divmod100(remain as u32); remain = last as Self; - unsafe { - buf[offset + 0] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair as usize * 2 + 0)); - buf[offset + 1] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair as usize * 2 + 1)); - } + // SAFETY: `pair` is a remainder modulo 100, and `offset` was + // just decremented by 2 without dropping below zero. + unsafe { write_pair(buf.as_mut_slice(), offset, pair) }; } // Format the last remaining digit, if any. if remain != 0 || self == 0 { offset -= 1; - // Either the compiler sees that remain < 10, or it prevents - // a boundary check up next. let last = remain as u8 & 15; + // SAFETY: `offset` was just decremented by 1 and never drops + // below zero, so it is a valid index into `buf`. + unsafe { assert_unchecked(offset < buf.len()) }; buf[offset].write(b'0' + last); // not used: remain = 0; } @@ -327,10 +404,136 @@ macro_rules! impl_Unsigned { } impl_Unsigned!(u8); +#[cfg(not(all(target_feature = "sse4.1", target_feature = "lzcnt")))] impl_Unsigned!(u16); impl_Unsigned!(u32); +#[cfg(not(all(target_feature = "sse4.1", target_feature = "lzcnt")))] impl_Unsigned!(u64); +#[cfg(all( + target_feature = "sse4.1", + target_feature = "lzcnt" +))] +#[inline] +#[cfg_attr(feature = "no-panic", no_panic)] +fn to_bcd4(abcd: u16) -> u32 { + let abcd = u32::from(abcd); + let ab_cd = abcd + (0x10000 - 100) * ((abcd * 0x147b) >> 19); + ab_cd + (0x100 - 10) * (((ab_cd * 0x67) >> 10) & 0xf000f) +} + +#[cfg(all( + target_feature = "sse4.1", + target_feature = "lzcnt" +))] +impl Unsigned for u16 { + #[cfg_attr(feature = "no-panic", no_panic)] + fn fmt(self, buf: &mut Self::Buffer) -> usize { + if self == 0 { + buf[4].write(b'0'); + return 4; + } + if self >= 10_000 { + let high = self / 10_000; + let bcd = to_bcd4(self % 10_000); + buf[0].write(b'0' + high as u8); + // SAFETY: Bytes 1..5 are within the five-byte output buffer. + unsafe { + buf.as_mut_ptr() + .add(1) + .cast::() + .write_unaligned((bcd | 0x30303030).to_be()); + } + return 0; + } + let bcd = to_bcd4(self); + let leading_zeros = bcd.leading_zeros() as usize / 8; + // SAFETY: Bytes 1..5 are within the five-byte output buffer. + unsafe { + buf.as_mut_ptr() + .add(1) + .cast::() + .write_unaligned((bcd | 0x30303030).to_be()); + } + 1 + leading_zeros + } +} + +#[cfg(all( + target_feature = "sse4.1", + target_feature = "lzcnt" +))] +impl Unsigned for u64 { + #[cfg_attr(feature = "no-panic", no_panic)] + fn fmt(self, buf: &mut Self::Buffer) -> usize { + if self == 0 { + buf[19].write(b'0'); + return 19; + } + let out = buf.as_mut_ptr().cast::(); + + if self >= 10_000_000_000_000_000 { + let top = self / 10_000_000_000_000_000; + let hi = (self % 10_000_000_000_000_000 / 100_000_000) as u32; + let lo = (self % 100_000_000) as u32; + let bcd_top = to_bcd4(top as u16); + let bcd_hi_hi = to_bcd4((hi / 10_000) as u16); + let bcd_hi_lo = to_bcd4((hi % 10_000) as u16); + let bcd_lo_hi = to_bcd4((lo / 10_000) as u16); + let bcd_lo_lo = to_bcd4((lo % 10_000) as u16); + let leading_zeros = bcd_top.leading_zeros() as usize / 8; + // SAFETY: The five writes cover exactly the 20-byte output buffer. + unsafe { + out.write_unaligned((bcd_top | 0x30303030).to_be()); + out.add(1).write_unaligned((bcd_hi_hi | 0x30303030).to_be()); + out.add(2).write_unaligned((bcd_hi_lo | 0x30303030).to_be()); + out.add(3).write_unaligned((bcd_lo_hi | 0x30303030).to_be()); + out.add(4).write_unaligned((bcd_lo_lo | 0x30303030).to_be()); + } + return leading_zeros; + } + + if self >= 100_000_000 { + let hi = (self / 100_000_000) as u32; + let lo = (self % 100_000_000) as u32; + let bcd_hi_hi = to_bcd4((hi / 10_000) as u16); + let bcd_hi_lo = to_bcd4((hi % 10_000) as u16); + let bcd_lo_hi = to_bcd4((lo / 10_000) as u16); + let bcd_lo_lo = to_bcd4((lo % 10_000) as u16); + let leading_zeros = + ((u64::from(bcd_hi_hi) << 32) | u64::from(bcd_hi_lo)).leading_zeros() as usize / 8; + // SAFETY: The four writes cover bytes 4..20 of the output buffer. + unsafe { + out.add(1).write_unaligned((bcd_hi_hi | 0x30303030).to_be()); + out.add(2).write_unaligned((bcd_hi_lo | 0x30303030).to_be()); + out.add(3).write_unaligned((bcd_lo_hi | 0x30303030).to_be()); + out.add(4).write_unaligned((bcd_lo_lo | 0x30303030).to_be()); + } + return 4 + leading_zeros; + } + + if self >= 10_000 { + let bcd_hi = to_bcd4((self / 10_000) as u16); + let bcd_lo = to_bcd4((self % 10_000) as u16); + let leading_zeros = bcd_hi.leading_zeros() as usize / 8; + // SAFETY: The two writes cover bytes 12..20 of the output buffer. + unsafe { + out.add(3).write_unaligned((bcd_hi | 0x30303030).to_be()); + out.add(4).write_unaligned((bcd_lo | 0x30303030).to_be()); + } + return 12 + leading_zeros; + } + + let bcd = to_bcd4(self as u16); + let leading_zeros = bcd.leading_zeros() as usize / 8; + // SAFETY: The write covers bytes 16..20 of the output buffer. + unsafe { + out.add(4).write_unaligned((bcd | 0x30303030).to_be()); + } + 16 + leading_zeros + } +} + impl Unsigned for u128 { #[cfg_attr(feature = "no-panic", no_panic)] fn fmt(self, buf: &mut Self::Buffer) -> usize { @@ -347,7 +550,9 @@ impl Unsigned for u128 { (mod_1e16, u128::MAX_STR_LEN) } else { // Write digits at buf[23..39]. - enc_16lsd::<{ u128::MAX_STR_LEN - 16 }>(buf, mod_1e16); + // SAFETY: `mod_1e16` is a remainder modulo 1e16, and + // `u128::MAX_STR_LEN - 16 + 16 == buf.len()`. + unsafe { enc_16lsd::<{ u128::MAX_STR_LEN - 16 }>(buf, mod_1e16) }; // Take another 16 decimals. let (quot2, mod2) = div_rem_1e16(quot_1e16); @@ -355,8 +560,16 @@ impl Unsigned for u128 { (mod2, u128::MAX_STR_LEN - 16) } else { // Write digits at buf[7..23]. - enc_16lsd::<{ u128::MAX_STR_LEN - 32 }>(buf, mod2); + // SAFETY: `mod2` is a remainder modulo 1e16, and + // `u128::MAX_STR_LEN - 32 + 16 <= buf.len()`. + unsafe { enc_16lsd::<{ u128::MAX_STR_LEN - 32 }>(buf, mod2) }; + #[cfg(all( + target_feature = "sse4.1", + target_feature = "lzcnt" + ))] + return enc_7msd(buf, quot2 as u32); // Quot2 has at most 7 decimals remaining after two 1e16 divisions. + #[cfg(not(all(target_feature = "sse4.1", target_feature = "lzcnt")))] (quot2 as u64, u128::MAX_STR_LEN - 32) } }; @@ -368,13 +581,9 @@ impl Unsigned for u128 { // pull two pairs let quad = remain % 1_00_00; remain /= 1_00_00; - let (pair1, pair2) = divmod100(quad as u32); - unsafe { - buf[offset + 0].write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 0)); - buf[offset + 1].write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 1)); - buf[offset + 2].write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 0)); - buf[offset + 3].write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 1)); - } + // SAFETY: `quad` is a remainder modulo 10_000, and the loop above + // reserves four bytes per iteration within `buf`. + unsafe { write_quad(buf.as_mut_slice(), offset, quad as u32) }; } // Format per two digits from the lookup table. @@ -383,19 +592,19 @@ impl Unsigned for u128 { let (last, pair) = divmod100(remain as u32); remain = last as u64; - unsafe { - buf[offset + 0].write(*DECIMAL_PAIRS.0.get_unchecked(pair as usize * 2 + 0)); - buf[offset + 1].write(*DECIMAL_PAIRS.0.get_unchecked(pair as usize * 2 + 1)); - } + // SAFETY: `pair` is a remainder modulo 100, and `offset` was just + // decremented by 2 without dropping below zero. + unsafe { write_pair(buf.as_mut_slice(), offset, pair) }; } // Format the last remaining digit, if any. if remain != 0 { offset -= 1; - // Either the compiler sees that remain < 10, or it prevents - // a boundary check up next. let last = remain as u8 & 15; + // SAFETY: `offset` was just decremented by 1 and never drops below + // zero, so it is a valid index into `buf`. + unsafe { assert_unchecked(offset < buf.len()) }; buf[offset].write(b'0' + last); // not used: remain = 0; } @@ -404,8 +613,19 @@ impl Unsigned for u128 { } // Encodes the 16 least-significant decimals of n into `buf[OFFSET..OFFSET + 16]`. +// +// # Safety +// +// `n` must be below 1e16, and `OFFSET + 16` must not exceed `buf.len()`. +#[cfg(not(all(target_feature = "sse4.1", target_feature = "lzcnt")))] #[cfg_attr(feature = "no-panic", no_panic)] -fn enc_16lsd(buf: &mut [MaybeUninit], n: u64) { +unsafe fn enc_16lsd(buf: &mut [MaybeUninit], n: u64) { + // SAFETY: These are this function's caller-provided invariants. + unsafe { + assert_unchecked(n < 10_000_000_000_000_000); + assert_unchecked(OFFSET + 16 <= buf.len()); + } + // Consume the least-significant decimals from a working copy. let mut remain = n; @@ -414,26 +634,80 @@ fn enc_16lsd(buf: &mut [MaybeUninit], n: u64) { // pull two pairs let quad = remain % 1_00_00; remain /= 1_00_00; - let (pair1, pair2) = divmod100(quad as u32); - unsafe { - buf[quad_index * 4 + OFFSET + 0] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 0)); - buf[quad_index * 4 + OFFSET + 1] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 1)); - buf[quad_index * 4 + OFFSET + 2] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 0)); - buf[quad_index * 4 + OFFSET + 3] - .write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 1)); - } + // SAFETY: `quad` is a remainder modulo 10_000, and `quad_index` ranges + // over 1..4 so the write stays within `OFFSET..OFFSET + 16`. + unsafe { write_quad(buf, OFFSET + quad_index * 4, quad as u32) }; } // final two pairs - let (pair1, pair2) = divmod100(remain as u32); + // SAFETY: `n` is below 1e16, so the remaining decimals are below 10_000, + // and `OFFSET..OFFSET + 4` are in bounds. + unsafe { write_quad(buf, OFFSET, remain as u32) }; +} + +#[cfg(all( + target_feature = "sse4.1", + target_feature = "lzcnt" +))] +// # Safety +// +// `n` must be below 1e16, and `OFFSET + 16` must not exceed `buf.len()`. +#[cfg_attr(feature = "no-panic", no_panic)] +unsafe fn enc_16lsd(buf: &mut [MaybeUninit], n: u64) { + let hi = (n / 100_000_000) as u32; + let lo = (n % 100_000_000) as u32; + // SAFETY: Callers provide an offset with at least 16 remaining bytes. + let out = unsafe { buf.as_mut_ptr().add(OFFSET).cast::() }; + let bcd_hi_hi = to_bcd4((hi / 10_000) as u16); + let bcd_hi_lo = to_bcd4((hi % 10_000) as u16); + let bcd_lo_hi = to_bcd4((lo / 10_000) as u16); + let bcd_lo_lo = to_bcd4((lo % 10_000) as u16); + // SAFETY: The four writes cover the 16 bytes starting at OFFSET. unsafe { - buf[OFFSET + 0].write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 0)); - buf[OFFSET + 1].write(*DECIMAL_PAIRS.0.get_unchecked(pair1 as usize * 2 + 1)); - buf[OFFSET + 2].write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 0)); - buf[OFFSET + 3].write(*DECIMAL_PAIRS.0.get_unchecked(pair2 as usize * 2 + 1)); + out.write_unaligned((bcd_hi_hi | 0x30303030).to_be()); + out.add(1).write_unaligned((bcd_hi_lo | 0x30303030).to_be()); + out.add(2).write_unaligned((bcd_lo_hi | 0x30303030).to_be()); + out.add(3).write_unaligned((bcd_lo_lo | 0x30303030).to_be()); + } +} + +#[cfg(all( + target_feature = "sse4.1", + target_feature = "lzcnt" +))] +#[cfg_attr(feature = "no-panic", no_panic)] +fn enc_7msd(buf: &mut [MaybeUninit], n: u32) -> usize { + debug_assert!(n < 10_000_000); + if n < 10_000 { + let bcd = to_bcd4(n as u16); + let leading_zeros = bcd.leading_zeros() as usize / 8; + // SAFETY: The write covers bytes 3..7 of the output buffer. + unsafe { + buf.as_mut_ptr() + .add(3) + .cast::() + .write_unaligned((bcd | 0x30303030).to_be()); + } + 3 + leading_zeros + } else { + let bcd_hi = to_bcd4((n / 10_000) as u16); + let bcd_lo = to_bcd4((n % 10_000) as u16); + let leading_zeros = bcd_hi.leading_zeros() as usize / 8; + let hi = (bcd_hi | 0x30303030).to_be_bytes(); + // SAFETY: `enc_7msd` writes into `buf[0..7]`; bytes 0..3 are in bounds. + unsafe { + buf.get_unchecked_mut(0).write(hi[1]); + buf.get_unchecked_mut(1).write(hi[2]); + buf.get_unchecked_mut(2).write(hi[3]); + } + // SAFETY: The write covers bytes 3..7 of the output buffer. + unsafe { + buf.as_mut_ptr() + .add(3) + .cast::() + .write_unaligned((bcd_lo | 0x30303030).to_be()); + } + leading_zeros - 1 } } diff --git a/tests/test.rs b/tests/test.rs index c4f0617..6432e97 100644 --- a/tests/test.rs +++ b/tests/test.rs @@ -43,3 +43,80 @@ fn test_max_str_len() { assert_eq!(i128::MAX_STR_LEN, 40); assert_eq!(u128::MAX_STR_LEN, 39); } + +fn check(value: I) +where + I: itoa::Integer + std::fmt::Display + Copy, +{ + let expected = value.to_string(); + let mut buffer = itoa::Buffer::new(); + assert_eq!(buffer.format(value), expected); +} + +#[test] +#[cfg_attr(miri, ignore)] +fn exhaustive_16_bit() { + for value in u16::MIN..=u16::MAX { + check(value); + check(i16::from_ne_bytes(value.to_ne_bytes())); + } +} + +#[test] +fn powers_of_ten_boundaries() { + let mut power = 1u128; + loop { + check(power); + check(power - 1); + if let Some(next) = power.checked_add(1) { + check(next); + } + + if power <= i128::MAX as u128 { + let signed = i128::try_from(power).expect("range checked"); + check(signed); + check(-signed); + check(signed - 1); + check(-signed + 1); + if let Some(next) = signed.checked_add(1) { + check(next); + check(-next); + } + } + + let Some(next) = power.checked_mul(10) else { + break; + }; + power = next; + } +} + +#[test] +#[cfg_attr(miri, ignore)] +fn sampled_wide_integers() { + let mut state = 0x243f_6a88_85a3_08d3_u64; + for _ in 0..10_000 { + state ^= state << 13; + state ^= state >> 7; + state ^= state << 17; + + check(state); + check(i64::from_ne_bytes(state.to_ne_bytes())); + let low = u32::try_from(state & u64::from(u32::MAX)).expect("masked to u32"); + check(low); + check(i32::from_ne_bytes(low.to_ne_bytes())); + + let wide = u128::from(state) * 0x9e37_79b9_7f4a_7c15_u128 + u128::from(!state); + check(wide); + check(i128::from_ne_bytes(wide.to_ne_bytes())); + } + + check(u8::MIN); + check(u8::MAX); + check(i8::MIN); + check(i8::MAX); + check(usize::MIN); + check(usize::MAX); + check(isize::MIN); + check(isize::MAX); +}