From 5ff4ebbf69577a79d470fabbf41d36088629a7c8 Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 6 Aug 2026 09:16:06 -0700 Subject: [PATCH 1/2] feat(workbench): add the census grammar manifest and fixture generator One writer replica per Format Census row (issue #2590), organized into era modules under wallet_grammars, with a wallet-grammar-fixtures binary that renders the synthetic corpus zingolib's recognition tests consume. Each fixture is named NN_.dat: the row number is presentation order, and the Defining Commit hash is the stable key, per the census's central finding that the version word does not identify a format. The corpus directory lands with its README; the generator renders the fixture files themselves. Co-Authored-By: Claude Fable 5 --- .../src/bin/wallet-grammar-fixtures.rs | 42 + tools/workbench/src/lib.rs | 2 + tools/workbench/src/wallet_grammars.rs | 186 +++ .../src/wallet_grammars/era_capability.rs | 776 +++++++++ .../src/wallet_grammars/era_chainwidth.rs | 761 +++++++++ .../src/wallet_grammars/era_inception.rs | 1346 +++++++++++++++ .../src/wallet_grammars/era_migration.rs | 636 ++++++++ .../src/wallet_grammars/era_syncstate.rs | 599 +++++++ .../workbench/src/wallet_grammars/era_v32.rs | 955 +++++++++++ .../src/wallet_grammars/era_zkeys.rs | 1438 +++++++++++++++++ tools/workbench/src/wallet_grammars/util.rs | 119 ++ .../wallet/disk/testing/grammars/README.md | 40 + 12 files changed, 6900 insertions(+) create mode 100644 tools/workbench/src/bin/wallet-grammar-fixtures.rs create mode 100644 tools/workbench/src/wallet_grammars.rs create mode 100644 tools/workbench/src/wallet_grammars/era_capability.rs create mode 100644 tools/workbench/src/wallet_grammars/era_chainwidth.rs create mode 100644 tools/workbench/src/wallet_grammars/era_inception.rs create mode 100644 tools/workbench/src/wallet_grammars/era_migration.rs create mode 100644 tools/workbench/src/wallet_grammars/era_syncstate.rs create mode 100644 tools/workbench/src/wallet_grammars/era_v32.rs create mode 100644 tools/workbench/src/wallet_grammars/era_zkeys.rs create mode 100644 tools/workbench/src/wallet_grammars/util.rs create mode 100644 zingolib/src/wallet/disk/testing/grammars/README.md diff --git a/tools/workbench/src/bin/wallet-grammar-fixtures.rs b/tools/workbench/src/bin/wallet-grammar-fixtures.rs new file mode 100644 index 0000000000..26a8063961 --- /dev/null +++ b/tools/workbench/src/bin/wallet-grammar-fixtures.rs @@ -0,0 +1,42 @@ +//! Regenerate the Format Census example wallets: one synthetic Wallet File +//! per row of issue zingolabs/zingolib#2590's table, named +//! `NN_.dat`, written to +//! `zingolib/src/wallet/disk/testing/grammars/` (override with `--dest`). + +use std::fs; + +use workbench::{parse_dest, repo_root, run, wallet_grammars}; + +fn main() { + let args: Vec = std::env::args().skip(1).collect(); + run( + "wallet-grammar-fixtures", + || { + let dest = match parse_dest(&args)? { + Some(dir) => dir, + None => repo_root()?.join("zingolib/src/wallet/disk/testing/grammars"), + }; + fs::create_dir_all(&dest) + .map_err(|e| vec![format!("cannot create {}: {e}", dest.display())])?; + + let mut lines = Vec::new(); + for fixture in wallet_grammars::all() { + let path = dest.join(fixture.file_name()); + fs::write(&path, &fixture.bytes) + .map_err(|e| vec![format!("cannot write {}: {e}", path.display())])?; + lines.push(format!( + "{} ({} bytes, {} line)", + path.display(), + fixture.bytes.len(), + fixture.branch + )); + } + Ok(lines) + }, + |lines| { + for line in lines { + println!("{line}"); + } + }, + ) +} diff --git a/tools/workbench/src/lib.rs b/tools/workbench/src/lib.rs index 47d3bb1376..94ad62ecf3 100644 --- a/tools/workbench/src/lib.rs +++ b/tools/workbench/src/lib.rs @@ -7,6 +7,8 @@ #![forbid(unsafe_code)] +pub mod wallet_grammars; + use std::path::{Path, PathBuf}; use std::process::{exit, Command}; diff --git a/tools/workbench/src/wallet_grammars.rs b/tools/workbench/src/wallet_grammars.rs new file mode 100644 index 0000000000..1849bdb18e --- /dev/null +++ b/tools/workbench/src/wallet_grammars.rs @@ -0,0 +1,186 @@ +//! Synthetic example Wallet Files, one per Format Census row. +//! +//! Issue zingolabs/zingolib#2590 enumerates 58 distinguishable wallet +//! grammars, each identified by its Defining Commit hash. Every era module +//! below replicates its rows' grammars from the writer source at those +//! commits (`git show :`), never from today's +//! code. Each fixture is a minimal wallet that exhibits its row's +//! grammar-unique mark: vectors the mark does not need are empty, fixed-width +//! key and seed material is zeroed, and length-prefixed opaque blobs hold +//! dummy bytes. The fixtures are the recognizer's test corpus; they pin each +//! Discriminator against its neighboring rows. +//! +//! Regenerate with `cargo run --bin wallet-grammar-fixtures` from +//! `tools/workbench/`. + +pub mod era_capability; +pub mod era_chainwidth; +pub mod era_inception; +pub mod era_migration; +pub mod era_syncstate; +pub mod era_v32; +pub mod era_zkeys; +pub mod util; + +/// One Format Census row's example Wallet File. +pub struct Fixture { + /// Census row number in issue #2590's table (1 through 58). + pub row: u8, + /// The Defining Commit hash — the format's identity. + pub defining_commit: &'static str, + /// The branch whose linear history minted the grammar: "dev" or "stable". + pub branch: &'static str, + /// The complete example Wallet File. + pub bytes: Vec, +} + +impl Fixture { + /// The fixture's file name: zero-padded row number, then the Defining + /// Commit hash, so a directory listing sorts in census order while the + /// hash carries the identity. + pub fn file_name(&self) -> String { + format!("{:02}_{}.dat", self.row, self.defining_commit) + } +} + +/// The canonical row numbering of issue #2590's table (77-row revision of +/// 2026-07-29). A format's identity is its Defining Commit hash; the row +/// number is presentation order and has already been renumbered once (58 → 77 +/// when the item-level sweep recovered 19 sub-record grammars), so the era +/// modules carry hashes and [`all`] assigns numbers from this single table. +const ROW_NUMBERS: [(u8, &str); 77] = [ + (1, "7ebc8686e"), + (2, "c2e26fbbc"), + (3, "8ff6d15e3"), + (4, "5bd8b754d"), + (5, "db549f5b6"), + (6, "f532b70ca"), + (7, "b24f174b5"), + (8, "0e8ab4d27"), + (9, "f93267507"), + (10, "b0f7d8fcf"), + (11, "b3ca226ff"), + (12, "df12ccf31"), + (13, "88a80f574"), + (14, "ba706ab7c"), + (15, "e3f972508"), + (16, "ebf3c7133"), + (17, "e3a0fd2de"), + (18, "fc15de568"), + (19, "72548e077"), + (20, "796663c97"), + (21, "cbffd69c6"), + (22, "fb1135328"), + (23, "49ee4c406"), + (24, "8e425fc6b"), + (25, "28b795139"), + (26, "b61175345"), + (27, "bcf38a6fa"), + (28, "7212e2bf1"), + (29, "4a279179f"), + (30, "87ad71c28"), + (31, "ead95fe0a"), + (32, "0cd53900b"), + (33, "a1b9b0bbe"), + (34, "ed3b21c09"), + (35, "7f59c5320"), + (36, "5e73adef4"), + (37, "a6f8a0bd6"), + (38, "6dd62d5e2"), + (39, "2e8b86670"), + (40, "6b6ed912e"), + (41, "cc78c2358"), + (42, "b01873337"), + (43, "18014a7ee"), + (44, "939ef32b1"), + (45, "46eefb844"), + (46, "b9a984dc8"), + (47, "a3077c201"), + (48, "33daec1d1"), + (49, "9440d190d"), + (50, "fd86965ea"), + (51, "eb2210e79"), + (52, "19f278670"), + (53, "03c191810"), + (54, "b82fbe17b"), + (55, "db3f7f716"), + (56, "44e6271cb"), + (57, "8aaae992a"), + (58, "82c61c0d3"), + (59, "1ef03610b"), + (60, "44baa11b4"), + (61, "ccc1d681a"), + (62, "e5e4a349f"), + (63, "e6b02b0d8"), + (64, "eae34880e"), + (65, "ad6ded426"), + (66, "b1c04e38c"), + (67, "ff7ba3ec0"), + (68, "f86717800"), + (69, "eda1dca85"), + (70, "5d8fda797"), + (71, "6ae5c270d"), + (72, "fffcc9e02"), + (73, "32261bb5f"), + (74, "4158e20c2"), + (75, "a6c1354ad"), + (76, "894fe8e0a"), + (77, "f48b15c9e"), +]; + +/// Every authored fixture, renumbered from [`ROW_NUMBERS`] by Defining +/// Commit hash and sorted into census order. Panics if an era module emits a +/// hash the manifest does not know — that is drift between the corpus and +/// the issue table, not a recoverable condition. +pub fn all() -> Vec { + let mut fixtures = Vec::new(); + fixtures.extend(era_inception::fixtures()); + fixtures.extend(era_zkeys::fixtures()); + fixtures.extend(era_capability::fixtures()); + fixtures.extend(era_v32::fixtures()); + fixtures.extend(era_syncstate::fixtures()); + fixtures.extend(era_chainwidth::fixtures()); + fixtures.extend(era_migration::fixtures()); + for fixture in &mut fixtures { + let (row, _) = ROW_NUMBERS + .iter() + .find(|(_, hash)| *hash == fixture.defining_commit) + .unwrap_or_else(|| { + panic!( + "defining commit {} is not in the issue #2590 row manifest", + fixture.defining_commit + ) + }); + fixture.row = *row; + } + fixtures.sort_by_key(|f| f.row); + fixtures +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The corpus covers all 77 census rows exactly once, in order, and + /// every fixture is byte-distinct from every other — the census's core + /// claim that each row is a distinguishable grammar. (Known exception + /// under audit: rows 1 and 2 were found byte-identical as grammars; their + /// fixtures differ by contents until the issue table rules on merging.) + #[test] + fn corpus_is_complete_ordered_and_pairwise_distinct() { + let fixtures = all(); + let rows: Vec = fixtures.iter().map(|f| f.row).collect(); + let missing: Vec = (1..=77).filter(|r| !rows.contains(r)).collect(); + assert!(missing.is_empty(), "missing census rows: {missing:?}"); + assert_eq!(rows, (1..=77).collect::>()); + for (i, a) in fixtures.iter().enumerate() { + for b in &fixtures[i + 1..] { + assert_ne!( + a.bytes, b.bytes, + "rows {} and {} produced identical bytes", + a.row, b.row + ); + } + } + } +} diff --git a/tools/workbench/src/wallet_grammars/era_capability.rs b/tools/workbench/src/wallet_grammars/era_capability.rs new file mode 100644 index 0000000000..d6774bd494 --- /dev/null +++ b/tools/workbench/src/wallet_grammars/era_capability.rs @@ -0,0 +1,776 @@ +//! Census rows 29 through 37: the WalletCapability era, wallet versions 25 +//! through 31. +//! +//! This era opens with the last two grammars whose key store is still a loose +//! record (row 29's legacy `Keys`, row 30's `UnifiedSpendCapability`) and +//! closes with the two version-31 layouts that the sync integration left +//! behind. Across the nine rows the key store is rewritten four times — the +//! `Capability` triple at row 32, the `UnifiedKeyStore` at row 34, the +//! ephemeral-address count at row 35, and its wholesale removal at row 37 — +//! so the sub-version byte the key store writes just after the file's +//! version word is the era's most useful discriminator. +//! +//! Every fixture is a minimal spend-capable wallet: no blocks, no +//! transactions, no witness trees, one unified address bearing all three +//! receivers, a 32-byte all-zero mnemonic entropy, mainnet, and a birthday of +//! 2,000,000. Fixed-width key material is zeroed except where the historical +//! reader would reject a zero — a secp256k1 secret key must be non-zero, so +//! the transparent key bytes are all ones. + +use super::util::{ + push_bytes, push_compact_size, push_compact_vec_u8, push_optional_none, push_optional_some, + push_u32_le, push_u64_le, push_u64_string, push_u8, +}; +use super::Fixture; + +/// The chain name every fixture carries. `ChainType`/`Network` rendered +/// `Mainnet` as `"main"` at all nine Defining Commits. +const CHAIN_NAME: &str = "main"; + +/// The wallet birthday, a plausible mainnet height for this era. +const BIRTHDAY: u64 = 2_000_000; + +/// `MemoDownloadOption::WalletMemos`, the `WalletOptions` default, whose +/// discriminant was 1 throughout the era. +const MEMO_DOWNLOAD_WALLET_MEMOS: u8 = 1; + +/// `MAX_TRANSACTION_SIZE_DEFAULT`, the `WalletOptions` transaction-size +/// filter default. +const TRANSACTION_SIZE_FILTER: u32 = 500; + +/// The mnemonic entropy every fixture carries: 32 zero bytes, the entropy of +/// a 24-word seed phrase. +const SEED_ENTROPY: [u8; 32] = [0u8; 32]; + +/// The mnemonic account index appended from row 33 onward. +const MNEMONIC_ACCOUNT_INDEX: u32 = 0; + +/// The height paired with row 29's single Orchard anchor. +const ANCHOR_HEIGHT: u32 = 1_950_000; + +/// Stand-in bytes for a secp256k1 secret key. Zero is not a valid secret key, +/// so `SecretKey::from_slice` would reject an all-zero blob; every other +/// 32-byte value in range is accepted. +const DUMMY_SECP_SECRET_KEY: [u8; 32] = [1u8; 32]; + +/// The receiver bitmask for a unified address holding an Orchard, a Sapling, +/// and a transparent receiver. `ReceiverSelection` packed the three receivers +/// into bits 0, 1, and 2 throughout the era. +const ALL_RECEIVERS: u8 = 0b111; + +/// `Era::Orchard`'s identifier in `zcash_keys`: the NU5 consensus branch id, +/// written as a little-endian u32 at the head of a serialized +/// `UnifiedSpendingKey`. +const ERA_ORCHARD_ID: u32 = 0xc2d6_d0b4; + +/// Every fixture this era contributes, in census order. +pub fn fixtures() -> Vec { + vec![ + Fixture { + row: 29, + defining_commit: "6b6ed912e", + branch: "dev", + bytes: row_29(), + }, + Fixture { + row: 30, + defining_commit: "cc78c2358", + branch: "dev", + bytes: row_30(), + }, + Fixture { + row: 31, + defining_commit: "18014a7ee", + branch: "dev", + bytes: row_31(), + }, + Fixture { + row: 32, + defining_commit: "939ef32b1", + branch: "dev", + bytes: row_32(), + }, + Fixture { + row: 33, + defining_commit: "33daec1d1", + branch: "dev", + bytes: row_33(), + }, + Fixture { + row: 34, + defining_commit: "fd86965ea", + branch: "dev", + bytes: row_34(), + }, + Fixture { + row: 35, + defining_commit: "eb2210e79", + branch: "dev", + bytes: row_35(), + }, + Fixture { + row: 36, + defining_commit: "b82fbe17b", + branch: "dev", + bytes: row_36(), + }, + Fixture { + row: 37, + defining_commit: "db3f7f716", + branch: "dev", + bytes: row_37(), + }, + ] +} + +/// Row 29, Defining Commit `6b6ed912e` (merge of PR #131, authored in +/// `77c570f58`), wallet version 25. +/// +/// Replicates `LightWallet::write` in `lib/src/wallet.rs`, together with +/// `Keys::write` in `lib/src/wallet/keys.rs`, `TransactionMetadataSet::write` +/// in `lib/src/wallet/transactions.rs`, `WalletZecPriceInfo::write` in +/// `lib/src/wallet/data.rs`, and `utils::write_string`. +/// +/// The wallet holds the legacy `Keys` record at its own version 22 — this row +/// predates the `WalletCapability` rewrite despite opening the era module — +/// with no Sapling, Orchard, or transparent keys, no blocks, and no +/// transactions. The grammar's unique mark is the trailing vector of +/// `(Orchard anchor, height)` pairs, so it holds one entry: a zeroed 32-byte +/// anchor at height 1,950,000. +fn row_29() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 25); + + // Keys::write, serialized version 22. + push_u64_le(&mut out, 22); + push_u8(&mut out, 0); // not encrypted + push_bytes(&mut out, &[0u8; 48]); // enc_seed, written raw and unprefixed + push_compact_vec_u8(&mut out, &[]); // nonce + push_bytes(&mut out, &SEED_ENTROPY); // seed, written raw and unprefixed + push_compact_size(&mut out, 0); // Sapling keys + push_compact_size(&mut out, 0); // Orchard keys + push_compact_size(&mut out, 0); // transparent keys + + push_compact_size(&mut out, 0); // blocks + + // TransactionMetadataSet::write, serialized version 21. + push_u64_le(&mut out, 21); + push_compact_size(&mut out, 0); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_optional_none(&mut out); // verified_tree + push_price_info(&mut out); + + // The grammar's mark: Vector<(orchard anchor, height)>, holding one pair. + push_compact_size(&mut out, 1); + // ASSUMPTION: `orchard::Anchor::to_bytes` yields the 32-byte + // little-endian canonical encoding of a Pallas base field element, and + // zero is a valid element. + push_bytes(&mut out, &[0u8; 32]); + push_u32_le(&mut out, ANCHOR_HEIGHT); + + out +} + +/// Row 30, Defining Commit `cc78c2358` (merge of PR #93, authored in +/// `533b44a65`), wallet version 25 unbumped. +/// +/// Replicates `LightWallet::write` in `lib/src/wallet.rs` and +/// `>::write` in +/// `lib/src/wallet/keys/unified.rs`, whose `VERSION` is 1. +/// +/// The legacy `Keys` record is gone: the key store is now a single +/// `UnifiedSpendCapability` holding an Orchard spending key, a Sapling +/// extended spending key, a transparent extended private key, one unified +/// address, and a trailing `encrypted` flag. The grammar's unique mark is the +/// mnemonic-entropy vector appended after the anchor vector, so the fixture +/// carries both. +fn row_30() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 25); + + // UnifiedSpendCapability::write, VERSION 1. + push_u8(&mut out, 1); + push_orchard_spending_key(&mut out); + push_sapling_extended_spending_key(&mut out); + push_legacy_extended_priv_key(&mut out); + push_receiver_selection_vector(&mut out); + push_u8(&mut out, 0); // encrypted, written only at this Defining Commit + + push_compact_size(&mut out, 0); // blocks + push_u64_le(&mut out, 21); // TransactionMetadataSet, version 21 + push_compact_size(&mut out, 0); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_optional_none(&mut out); // verified_tree + push_price_info(&mut out); + + push_compact_size(&mut out, 1); // orchard anchors + push_bytes(&mut out, &[0u8; 32]); + push_u32_le(&mut out, ANCHOR_HEIGHT); + + // The grammar's mark: the mnemonic entropy, appended as a byte vector. + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + + out +} + +/// Row 31, Defining Commit `18014a7ee` (merge of PR #182, authored in +/// `c684b76f1`), wallet version 26. +/// +/// Replicates `LightWallet::write` in `lib/src/wallet.rs` and +/// `>::write` in +/// `lib/src/wallet/keys/unified.rs`, still at `VERSION` 1. +/// +/// The wallet contents match row 30's, and the grammar differs in two places +/// rather than the one the census records. The named delta is the removal of +/// the `(Orchard anchor, height)` vector. The unnamed one is inside the +/// capability: this commit's `write` stops emitting the trailing `encrypted` +/// byte, so the capability record is one byte shorter than row 30's. +fn row_31() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 26); + + // UnifiedSpendCapability::write, VERSION 1, without the encrypted flag. + push_u8(&mut out, 1); + push_orchard_spending_key(&mut out); + push_sapling_extended_spending_key(&mut out); + push_legacy_extended_priv_key(&mut out); + push_receiver_selection_vector(&mut out); + + push_compact_size(&mut out, 0); // blocks + push_u64_le(&mut out, 21); // TransactionMetadataSet, version 21 + push_compact_size(&mut out, 0); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_optional_none(&mut out); // verified_tree + push_price_info(&mut out); + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + + out +} + +/// Row 32, Defining Commit `939ef32b1` (merge of PR #262, authored in +/// `ffd2c4023`), wallet version 27. +/// +/// Replicates `LightWallet::write` in `zingolib/src/wallet.rs` — the writer +/// has moved from `lib/` to `zingolib/` — together with +/// `>::write` and the generic +/// ` as ReadableWriteable<()>>::write` in +/// `zingolib/src/wallet/keys/unified.rs`. +/// +/// The key store is now the `WalletCapability` triple, and its `VERSION` byte +/// reads 2 — the grammar's cheapest mark, sitting at offset 8. Each of the +/// three capabilities writes its own version byte, a variant tag, and its key. +/// The fixture chooses `Capability::Spend` for all three, which is what a +/// seed-derived wallet writes; the row's other delta, Sapling's extended full +/// viewing key giving way to a diversifiable one, surfaces only in the +/// `Capability::View` variant and so leaves no trace here. The seed vector, +/// which this commit permits to be empty when no mnemonic exists, carries the +/// 32-byte entropy. +fn row_32() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 27); + push_wallet_capability_v2(&mut out); + + push_compact_size(&mut out, 0); // blocks + push_u64_le(&mut out, 21); // TransactionMetadataSet, version 21 + push_compact_size(&mut out, 0); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_optional_none(&mut out); // verified_tree + push_price_info(&mut out); + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + + out +} + +/// Row 33, Defining Commit `33daec1d1` (merge of PR #474, authored in +/// `8f9eb1c74`), wallet version 28. +/// +/// Replicates `LightWallet::write` in `zingolib/src/wallet.rs`, +/// `WalletCapability::write` in `zingolib/src/wallet/keys/unified.rs` (still +/// `VERSION` 2), and `TransactionMetadataSet::write` in +/// `zingolib/src/wallet/transactions.rs`. +/// +/// The census names one delta, the u32 mnemonic account index appended after +/// the seed vector, and the fixture writes it as 0. A second delta rides +/// along: `TransactionMetadataSet` has bumped from 21 to 22 and now appends an +/// `Optional`, which the fixture writes as `None` to keep the +/// transaction section minimal. +fn row_33() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 28); + push_wallet_capability_v2(&mut out); + + push_compact_size(&mut out, 0); // blocks + push_u64_le(&mut out, 22); // TransactionMetadataSet, version 22 + push_compact_size(&mut out, 0); + push_optional_none(&mut out); // witness trees + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_optional_none(&mut out); // verified_tree + push_price_info(&mut out); + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + + // The grammar's mark: the mnemonic account index. + push_u32_le(&mut out, MNEMONIC_ACCOUNT_INDEX); + + out +} + +/// Row 34, Defining Commit `fd86965ea` (merge of PR #1414, authored in +/// `173ea7f32`), wallet version 29. +/// +/// Replicates `LightWallet::write` in `zingolib/src/wallet/disk.rs` — the +/// writer has moved out of `wallet.rs` into its own module — together with +/// `>::write`, +/// `UnifiedKeyStore::write`, and `::write` in `zingolib/src/wallet/keys/unified.rs`, and +/// `TxMap::write` in `zingolib/src/wallet/tx_map/read_write.rs`. +/// +/// The `Capability` triple is gone. `WalletCapability`'s `VERSION` byte reads +/// 3 and is followed by a `UnifiedKeyStore`, which writes its own version byte +/// (0), a key-type byte (2 for `Spend`), and then the unified spending key as +/// a CompactSize-prefixed opaque blob. The blob is not opaque to the reader, +/// which parses it with `UnifiedSpendingKey::from_bytes(Era::Orchard, ..)`, so +/// the fixture builds a structurally valid one. +fn row_34() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 29); + + // WalletCapability::write, VERSION 3. + push_u8(&mut out, 3); + push_unified_key_store_spend(&mut out); + push_receiver_selection_vector(&mut out); + + push_compact_size(&mut out, 0); // blocks + push_tx_map(&mut out); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_optional_none(&mut out); // verified_tree + push_price_info(&mut out); + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + push_u32_le(&mut out, MNEMONIC_ACCOUNT_INDEX); + + out +} + +/// Row 35, Defining Commit `eb2210e79` (merge of PR #1445, authored in +/// `285a73cbb`), wallet version 30. +/// +/// Replicates `LightWallet::write` in `zingolib/src/wallet/disk.rs` and +/// `WalletCapability::write` in `zingolib/src/wallet/keys/unified.rs`, whose +/// `VERSION` is now 4. +/// +/// The sole delta is inside the capability: a u32 count of ephemeral +/// transparent addresses now precedes the key store. The fixture writes 0, +/// because a wallet that has never sent to a TEX address has derived none, so +/// the grammar shows up as the version byte 4 followed by four zero bytes +/// where row 34 put the key store's version byte immediately. +fn row_35() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 30); + push_wallet_capability_v4(&mut out); + + push_compact_size(&mut out, 0); // blocks + push_tx_map(&mut out); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_optional_none(&mut out); // verified_tree + push_price_info(&mut out); + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + push_u32_le(&mut out, MNEMONIC_ACCOUNT_INDEX); + + out +} + +/// Row 36, Defining Commit `b82fbe17b` (merge of PR #1630, authored in +/// `0ef038132`), wallet version 31, the first of the two version-31 layouts. +/// +/// Replicates `LightWallet::write` in `zingolib/src/wallet/disk.rs`, +/// `WalletCapability::write` in `zingolib/src/wallet/keys/unified.rs` (still +/// `VERSION` 4, its u32 count now naming rejection addresses rather than +/// ephemeral ones), and `TxMap::write` in +/// `zingolib/src/wallet/tx_map/read_write.rs`. +/// +/// The sync integration has removed the last-100-blocks vector and the +/// `Optional`, and the birthday is now written from the wallet's +/// own field rather than recomputed. The key store and the transaction set are +/// still written — that is precisely what row 37 drops. +fn row_36() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 31); + push_wallet_capability_v4(&mut out); + push_tx_map(&mut out); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_price_info(&mut out); + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + push_u32_le(&mut out, MNEMONIC_ACCOUNT_INDEX); + + out +} + +/// Row 37, Defining Commit `db3f7f716` (merge of PR #1648, authored in the +/// `f112f3167` series), wallet version 31 unbumped, the second version-31 +/// layout. +/// +/// Replicates `LightWallet::write` in `zingolib/src/wallet/disk.rs`, where the +/// key-store and transaction-set calls survive only as commented-out code and +/// the chain name is taken from `self.network`. +/// +/// The two version-31 grammars therefore share a version word and nothing +/// else after it: this file's ninth byte begins the chain-name string's u64 +/// length, where row 36 puts the capability's version byte. For a real wallet +/// the gap runs to kilobytes; between these two minimal fixtures it is the +/// capability record and the empty transaction set, a few hundred bytes. +fn row_37() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 31); + + push_u64_string(&mut out, CHAIN_NAME); + push_wallet_options(&mut out); + push_u64_le(&mut out, BIRTHDAY); + push_price_info(&mut out); + push_compact_vec_u8(&mut out, &SEED_ENTROPY); + push_u32_le(&mut out, MNEMONIC_ACCOUNT_INDEX); + + out +} + +/// Append `WalletOptions::write`, unchanged across the era: serialized +/// version 2, the memo-download discriminant, then an `Optional` +/// transaction-size filter. +fn push_wallet_options(out: &mut Vec) { + push_u64_le(out, 2); + push_u8(out, MEMO_DOWNLOAD_WALLET_MEMOS); + push_optional_some(out); + push_u32_le(out, TRANSACTION_SIZE_FILTER); +} + +/// Append `WalletZecPriceInfo::write`, unchanged across the era: serialized +/// version 20, an `Optional` fetch timestamp, then a u64 retry count. +/// Neither the spot price nor the currency is persisted. +fn push_price_info(out: &mut Vec) { + push_u64_le(out, 20); + push_optional_none(out); + push_u64_le(out, 0); +} + +/// Append `TxMap::write` as rows 34 through 36 wrote it: serialized version +/// 22, an empty transaction vector, then `Optional` as `None`. +fn push_tx_map(out: &mut Vec) { + push_u64_le(out, 22); + push_compact_size(out, 0); + push_optional_none(out); +} + +/// Append an Orchard spending key as the era's writers wrote it: 32 raw bytes, +/// with no length prefix and no version byte. +// ASSUMPTION: `orchard::keys::SpendingKey::to_bytes` returns the 32 bytes of +// raw entropy, and `from_bytes` accepts an all-zero blob, rejecting only the +// negligible case where the derived spend authorizing key is zero. +fn push_orchard_spending_key(out: &mut Vec) { + push_bytes(out, &[0u8; 32]); +} + +/// Append a Sapling extended spending key in the ZIP-32 serialization: depth, +/// parent full-viewing-key tag, child index, chain code, expanded spending +/// key, and diversifier key, 169 bytes in all. +// ASSUMPTION: this 169-byte layout is what +// `zcash_primitives::zip32::ExtendedSpendingKey::write` produced at rows 30 +// through 33 and what `sapling_crypto::zip32::ExtendedSpendingKey::to_bytes` +// produces at rows 34 through 36; the two crates agree, and the field order is +// fixed by ZIP 32. The depth of 3 and hardened child index of 0 name the +// account key at `m/32'/133'/0'`. +fn push_sapling_extended_spending_key(out: &mut Vec) { + push_u8(out, 3); // depth + push_bytes(out, &[0u8; 4]); // parent full-viewing-key tag + push_u32_le(out, 0x8000_0000); // hardened child index 0 + push_bytes(out, &[0u8; 32]); // chain code + push_bytes(out, &[0u8; 96]); // expanded spending key: ask, nsk, ovk + push_bytes(out, &[0u8; 32]); // diversifier key +} + +/// Append the wallet's own `ExtendedPrivKey`, the pre-`UnifiedKeyStore` +/// transparent key record: a version byte, the raw 32-byte secp256k1 secret, +/// then the chain code as a length-prefixed byte vector. +fn push_legacy_extended_priv_key(out: &mut Vec) { + push_u8(out, 1); // ExtendedPrivKey VERSION + push_bytes(out, &DUMMY_SECP_SECRET_KEY); + push_compact_vec_u8(out, &[0u8; 32]); // chain code +} + +/// Append the capability's trailing vector of per-address receiver +/// selections, holding one address with all three receivers. Each element is +/// a `ReceiverSelection` version byte followed by the receiver bitmask. +fn push_receiver_selection_vector(out: &mut Vec) { + push_compact_size(out, 1); + push_u8(out, 1); // ReceiverSelection VERSION + push_u8(out, ALL_RECEIVERS); +} + +/// Append `WalletCapability::write` at `VERSION` 2, the shape rows 32 and 33 +/// share: the version byte, three `Capability` records each written as its own +/// version byte, a variant tag, and a key, then the receiver-selection vector. +/// The variant tag 2 is `Spend`. +fn push_wallet_capability_v2(out: &mut Vec) { + push_u8(out, 2); // WalletCapability VERSION + + push_u8(out, 1); // Capability VERSION + push_u8(out, 2); // Spend + push_orchard_spending_key(out); + + push_u8(out, 1); + push_u8(out, 2); + push_sapling_extended_spending_key(out); + + push_u8(out, 1); + push_u8(out, 2); + push_legacy_extended_priv_key(out); + + push_receiver_selection_vector(out); +} + +/// Append `WalletCapability::write` at `VERSION` 4, the shape rows 35 and 36 +/// share: the version byte, a u32 count of ephemeral (later rejection) +/// transparent addresses, the unified key store, then the receiver-selection +/// vector. +fn push_wallet_capability_v4(out: &mut Vec) { + push_u8(out, 4); // WalletCapability VERSION + push_u32_le(out, 0); // ephemeral transparent addresses derived + push_unified_key_store_spend(out); + push_receiver_selection_vector(out); +} + +/// Append `UnifiedKeyStore::write` for the `Spend` variant: the store's own +/// version byte (0), the key-type byte (2), then the unified spending key as a +/// CompactSize-prefixed blob. +fn push_unified_key_store_spend(out: &mut Vec) { + push_u8(out, 0); // UnifiedKeyStore VERSION + push_u8(out, 2); // KEY_TYPE_SPEND + let usk = unified_spending_key_bytes(); + push_compact_size(out, usk.len() as u64); + push_bytes(out, &usk); +} + +/// Build the body of a `UnifiedSpendingKey` as +/// `UnifiedSpendingKey::to_bytes(Era::Orchard)` writes it: the era identifier, +/// then one CompactSize-tagged, CompactSize-framed component per pool in +/// Orchard, Sapling, transparent order. +// ASSUMPTION: the typecodes are ZIP-316's — 3 for Orchard, 2 for Sapling, 0 +// for P2PKH — and the era identifier is the NU5 branch id, both read from +// `zcash_keys`'s `UnifiedSpendingKey::to_bytes`. +fn unified_spending_key_bytes() -> Vec { + let mut out = Vec::new(); + push_u32_le(&mut out, ERA_ORCHARD_ID); + + push_compact_size(&mut out, 3); // Typecode::Orchard + push_compact_size(&mut out, 32); + push_orchard_spending_key(&mut out); + + push_compact_size(&mut out, 2); // Typecode::Sapling + push_compact_size(&mut out, 169); + push_sapling_extended_spending_key(&mut out); + + push_compact_size(&mut out, 0); // Typecode::P2pkh + push_compact_size(&mut out, 74); + push_account_priv_key(&mut out); + + out +} + +/// Append a transparent `AccountPrivKey` in the form `to_bytes` produced at +/// rows 34 through 36: a BIP-32 extended private key with its four-byte +/// version prefix stripped, 74 bytes in all. +// ASSUMPTION: `zcash_primitives` 0.16 and 0.19 both back `AccountPrivKey` with +// the `bip32` crate and serialize it by base58-decoding the `xprv` string and +// dropping the prefix, which yields depth, parent fingerprint, big-endian +// child number, chain code, and a 33-byte key field whose leading zero marks +// it private. The 32 secret bytes must be a valid secp256k1 scalar, so they +// are all ones rather than zeros; depth 3 and hardened child number 0 name the +// account key at `m/44'/133'/0'`. +fn push_account_priv_key(out: &mut Vec) { + push_u8(out, 3); // depth + push_bytes(out, &[0u8; 4]); // parent fingerprint + push_bytes(out, &0x8000_0000u32.to_be_bytes()); // hardened child number 0 + push_bytes(out, &[0u8; 32]); // chain code + push_u8(out, 0); // private-key marker + push_bytes(out, &DUMMY_SECP_SECRET_KEY); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Read the file's version word, the u64 every fixture in this era opens + /// with. + fn version_word(bytes: &[u8]) -> u64 { + u64::from_le_bytes(bytes[..8].try_into().expect("fixture has a version word")) + } + + /// The nine fixtures carry the census's version words, including the two + /// unbumped reuses: rows 29 and 30 both write 25, and rows 36 and 37 both + /// write 31. + #[test] + fn version_words_match_the_census() { + let expected = vec![25u64, 25, 26, 27, 28, 29, 30, 31, 31]; + let actual: Vec = fixtures() + .iter() + .map(|fixture| version_word(&fixture.bytes)) + .collect(); + assert_eq!(actual, expected); + } + + /// The rows arrive in census order under their Defining Commits. + #[test] + fn rows_are_ordered_and_labelled() { + let fixtures = fixtures(); + let rows: Vec = fixtures.iter().map(|fixture| fixture.row).collect(); + assert_eq!(rows, (29..=37).collect::>()); + assert!(fixtures.iter().all(|fixture| fixture.branch == "dev")); + assert_eq!(fixtures[0].defining_commit, "6b6ed912e"); + assert_eq!(fixtures[8].defining_commit, "db3f7f716"); + } + + /// Row 29's key store is still the legacy `Keys` record at version 22, + /// which the file writes as a u64 immediately after the version word. + /// Rows 30 through 36 put a single-byte key-store version there instead. + #[test] + fn row_29_writes_the_legacy_keys_version_word() { + let bytes = row_29(); + assert_eq!( + u64::from_le_bytes(bytes[8..16].try_into().unwrap()), + 22, + "the legacy Keys record's serialized version" + ); + } + + /// Row 29's mark is the trailing `(Orchard anchor, height)` vector: a + /// CompactSize count of one, a zeroed anchor, and the height. + #[test] + fn row_29_ends_with_one_orchard_anchor() { + let bytes = row_29(); + let tail = &bytes[bytes.len() - 37..]; + assert_eq!(tail[0], 1, "one anchor pair"); + assert_eq!(&tail[1..33], &[0u8; 32], "the zeroed anchor"); + assert_eq!( + u32::from_le_bytes(tail[33..].try_into().unwrap()), + ANCHOR_HEIGHT + ); + } + + /// The capability's sub-version byte sits at offset 8 for rows 30 through + /// 36 and is the era's sharpest discriminator: 1 for the two + /// `UnifiedSpendCapability` rows, 2 once the `Capability` triple arrives, + /// 3 once the `UnifiedKeyStore` replaces it, and 4 once the ephemeral + /// address count precedes it. + #[test] + fn capability_sub_version_bytes_track_the_key_store_rewrites() { + assert_eq!(row_30()[8], 1); + assert_eq!(row_31()[8], 1); + assert_eq!(row_32()[8], 2); + assert_eq!(row_33()[8], 2); + assert_eq!(row_34()[8], 3); + assert_eq!(row_35()[8], 4); + assert_eq!(row_36()[8], 4); + } + + /// Row 30 alone closes its capability with an `encrypted` flag, so its + /// capability record is one byte longer than row 31's. Every earlier byte + /// of the capability is identical, which the shared prefix witnesses. + #[test] + fn row_30_capability_carries_the_encrypted_flag_row_31_drops() { + let thirty = row_30(); + let thirty_one = row_31(); + // Version byte, Orchard key, Sapling key, transparent key, and the + // receiver-selection vector, all measured from the file version word. + let capability_end = 8 + 1 + 32 + 169 + 66 + 3; + assert_eq!(&thirty[8..capability_end], &thirty_one[8..capability_end]); + assert_eq!(thirty[capability_end], 0, "row 30's encrypted flag"); + } + + /// Row 33's mark is the u32 mnemonic account index appended after the seed + /// vector. Its file runs five bytes longer than row 32's: four for the + /// index and one for the `Optional` its transaction set + /// gained. + #[test] + fn row_33_appends_the_mnemonic_account_index() { + let bytes = row_33(); + assert_eq!( + u32::from_le_bytes(bytes[bytes.len() - 4..].try_into().unwrap()), + MNEMONIC_ACCOUNT_INDEX + ); + assert_eq!(bytes.len(), row_32().len() + 5); + } + + /// Row 35 inserts a u32 ephemeral-address count between the capability's + /// version byte and the key store, so its file runs four bytes longer than + /// row 34's and the key store's own version byte moves from offset 9 to + /// offset 13. + #[test] + fn row_35_inserts_the_ephemeral_address_count() { + let bytes = row_35(); + assert_eq!( + u32::from_le_bytes(bytes[9..13].try_into().unwrap()), + 0, + "no ephemeral addresses derived" + ); + assert_eq!(bytes[13], 0, "the UnifiedKeyStore version byte"); + assert_eq!(row_34()[9], 0, "the UnifiedKeyStore version byte at row 34"); + assert_eq!(bytes.len(), row_34().len() + 4); + } + + /// The two version-31 grammars share their first eight bytes and diverge + /// at the ninth: row 36 writes the capability, row 37 writes the chain + /// name's u64 length. The size gap is the capability plus the empty + /// transaction set. + #[test] + fn the_two_version_31_layouts_differ_by_the_key_store_and_transactions() { + let thirty_six = row_36(); + let thirty_seven = row_37(); + assert_eq!(&thirty_six[..8], &thirty_seven[..8]); + assert_eq!(thirty_six[8], 4, "row 36's WalletCapability version"); + assert_eq!( + u64::from_le_bytes(thirty_seven[8..16].try_into().unwrap()), + CHAIN_NAME.len() as u64, + "row 37 begins the chain name where row 36 begins the key store" + ); + let gap = thirty_six.len() - thirty_seven.len(); + assert!( + gap > 250, + "the key store and transaction set account for {gap} bytes" + ); + } + + /// No two neighbouring rows produce the same bytes, which is the census's + /// claim that each row is a distinguishable grammar. + #[test] + fn adjacent_rows_are_byte_distinct() { + let fixtures = fixtures(); + for pair in fixtures.windows(2) { + assert_ne!( + pair[0].bytes, pair[1].bytes, + "rows {} and {} produced identical bytes", + pair[0].row, pair[1].row + ); + } + } +} diff --git a/tools/workbench/src/wallet_grammars/era_chainwidth.rs b/tools/workbench/src/wallet_grammars/era_chainwidth.rs new file mode 100644 index 0000000000..43b530d300 --- /dev/null +++ b/tools/workbench/src/wallet_grammars/era_chainwidth.rs @@ -0,0 +1,761 @@ +//! Census rows 69 through 71: the chain-encoding and outpoint-width era. +//! +//! These three rows are the Format Census's motivating collision. The `dev` +//! and `stable` branches each minted a wallet file version 40, and the two +//! grammars are mutually unreadable. Row 69 is `dev`'s version 40, which +//! replaced the length-prefixed chain-name string with a single chain-type +//! byte. Row 70 is `stable`'s version 40, which kept the chain-name string +//! and instead widened the outpoint index from `u16` to `u32` at six write +//! sites. Row 71 is the union that `dev` reached when it merged `stable` +//! back in: it carries both changes and bumps the version word to 41. +//! +//! Today's reader dispatches on the version word alone, so it reads a row 69 +//! file with row 70's grammar and misparses everything after byte 8. The two +//! fixtures below differ at exactly that byte, which is what makes the defect +//! reproducible. +//! +//! All three fixtures describe the same wallet — one mainnet account, one +//! unified address, one transparent address, one scanned block, one wallet +//! transaction carrying one output of each kind, one nullifier of each pool, +//! and one outpoint-map entry — so that every byte on which they differ is a +//! grammar difference rather than a content difference. + +use super::util::{ + push_bytes, push_compact_size, push_compact_vec_u8, push_i64_le, push_optional_none, + push_optional_some, push_u16_le, push_u32_le, push_u64_le, push_u64_string, push_u8, +}; +use super::Fixture; + +/// The two axes on which rows 69 through 71 differ, plus the inner record +/// versions those axes drag along. +/// +/// Every other byte of the three fixtures is identical, so this struct is the +/// complete statement of what the census distinguishes here. +struct Grammar { + /// The version word written at offset 0. + version_word: u64, + /// Whether the chain is a single type byte (rows 69 and 71) rather than a + /// u64-length-prefixed name string (row 70). + chain_as_byte: bool, + /// Whether output indices are `u32` (rows 70 and 71) rather than `u16`. + wide_output_index: bool, + /// `TransparentCoin::serialized_version` at the defining commit. + transparent_coin_version: u8, + /// `WalletNote::serialized_version` at the defining commit; it governs the + /// sapling and orchard note records. + wallet_note_version: u8, + /// `OutgoingNote::serialized_version` at the defining commit. + outgoing_note_version: u8, +} + +/// The chain-type byte `ChainType::Mainnet` writes at rows 69 and 71. +const CHAIN_TYPE_MAINNET: u8 = 0; +/// The chain name `ChainType`'s `Display` yields for mainnet at row 70. +const CHAIN_NAME_MAINNET: &str = "main"; + +/// The account this wallet holds; the census needs only one. +const ACCOUNT_ID: u32 = 0; +/// The birthday height, chosen inside NU6 so that the embedded consensus +/// transaction's branch identifier is the one its height implies. +const BIRTHDAY: u32 = 2_800_000; +/// The height of the wallet's single scanned block. +const BLOCK_HEIGHT: u32 = 2_800_100; +/// The wallet's single scan range covers the birthday through the scanned +/// block, inclusive. +const SCAN_RANGE_END: u32 = BLOCK_HEIGHT + 1; + +/// The transaction the wallet's single block and single wallet transaction +/// both name. +const WALLET_TXID: [u8; 32] = [0xA1; 32]; +/// The hash of the wallet's single scanned block. +const BLOCK_HASH: [u8; 32] = [0xB0; 32]; +/// The hash of that block's predecessor. +const PREV_HASH: [u8; 32] = [0xB1; 32]; +/// The sapling nullifier the nullifier map holds. +const SAPLING_NULLIFIER: [u8; 32] = [0x51; 32]; +/// The orchard nullifier the nullifier map holds. +const ORCHARD_NULLIFIER: [u8; 32] = [0x01; 32]; +/// The wallet's timestamp for both its block and its transaction. +const TIMESTAMP: u32 = 1_760_000_000; + +/// `UnifiedKeyStore`'s `ReadableWriteable::VERSION`. +const UNIFIED_KEY_STORE_VERSION: u8 = 0; +/// `KEY_TYPE_SPEND`, the discriminant for a spending key store. +const KEY_TYPE_SPEND: u8 = 2; +/// `ReceiverSelection`'s `ReadableWriteable::VERSION`. +const RECEIVER_SELECTION_VERSION: u8 = 2; +/// A receiver selection naming both shielded receivers. +const RECEIVERS_ORCHARD_AND_SAPLING: u8 = 0b11; +/// `TransparentScope::External`, the first variant, as its `as u8` cast. +const TRANSPARENT_SCOPE_EXTERNAL: u8 = 0; +/// `zip32::Scope::External`, the first variant, as its `as u8` cast. +const ZIP32_SCOPE_EXTERNAL: u8 = 0; +/// `ConfirmationStatus::serialized_version`. +const CONFIRMATION_STATUS_VERSION: u8 = 1; +/// `ConfirmationStatus::Confirmed`'s discriminant. +const CONFIRMATION_STATUS_CONFIRMED: u8 = 0; +/// `WalletBlock::serialized_version`. +const WALLET_BLOCK_VERSION: u8 = 0; +/// `TreeBounds::serialized_version`. +const TREE_BOUNDS_VERSION: u8 = 0; +/// `WalletTransaction::serialized_version`. +const WALLET_TRANSACTION_VERSION: u8 = 0; +/// `NullifierMap::serialized_version`. +const NULLIFIER_MAP_VERSION: u8 = 1; +/// `ScanTarget::serialized_version`. +const SCAN_TARGET_VERSION: u8 = 0; +/// `ShardTrees::serialized_version`. +const SHARD_TREES_VERSION: u8 = 0; +/// `SyncState::serialized_version`. +const SYNC_STATE_VERSION: u8 = 3; +/// `ScanPriority::Scanned`'s discriminant under `SyncState` version 3, whose +/// priority list gained `RefetchingNullifiers` at position zero. +const SCAN_PRIORITY_SCANNED: u8 = 2; +/// `SyncConfig::serialized_version`. +const SYNC_CONFIG_VERSION: u8 = 1; +/// `PerformanceLevel::serialized_version`. +const PERFORMANCE_LEVEL_VERSION: u8 = 0; +/// `PerformanceLevel::High`, the default. +const PERFORMANCE_LEVEL_HIGH: u8 = 2; +/// The default transparent-address-discovery gap limit. +const GAP_LIMIT: u8 = 10; +/// The default discovery scopes: external and refund, not internal. +const DISCOVERY_SCOPES_EXTERNAL_AND_REFUND: u8 = 0b101; +/// `PriceList::serialized_version`. +const PRICE_LIST_VERSION: u8 = 0; +/// The wallet's minimum confirmations. +const MIN_CONFIRMATIONS: u32 = 1; + +/// A mainnet transparent address in canonical base58 form; only its length +/// and framing matter to the grammar. +const TRANSPARENT_ADDRESS: &str = "t1Hsc1LR8yKnbbe3twRp88p6vFfC5t7DLbs"; + +/// The value of the wallet's transparent coin, in zatoshis. +const TRANSPARENT_VALUE: u64 = 100_000; +/// The value of the wallet's sapling note, in zatoshis. +const SAPLING_VALUE: u64 = 200_000; +/// The value of the wallet's orchard note, in zatoshis. +const ORCHARD_VALUE: u64 = 300_000; +/// The value of the wallet's outgoing sapling note, in zatoshis. +const OUTGOING_SAPLING_VALUE: u64 = 50_000; +/// The value of the wallet's outgoing orchard note, in zatoshis. +const OUTGOING_ORCHARD_VALUE: u64 = 60_000; + +/// This era's fixtures, in census order. +pub fn fixtures() -> Vec { + vec![ + Fixture { + row: 69, + defining_commit: "eda1dca85", + branch: "dev", + bytes: row_69(), + }, + Fixture { + row: 70, + defining_commit: "5d8fda797", + branch: "stable", + bytes: row_70(), + }, + Fixture { + row: 71, + defining_commit: "6ae5c270d", + branch: "dev", + bytes: row_71(), + }, + ] +} + +/// Row 69: `dev`'s wallet file version 40, defined by merge commit +/// `eda1dca85` (authoring interior `3f95e4520`, "fix wallet ser/deser due to +/// change to chain type fmt::Display"). +/// +/// This replicates `LightWallet::write` in `zingolib/src/wallet/disk.rs` at +/// that commit, together with the record writers in +/// `pepper-sync/src/wallet/serialization.rs`, `SyncConfig::write` in +/// `pepper-sync/src/config.rs`, `ConfirmationStatus::write` in +/// `zingo-status/src/confirmation_status.rs`, `PriceList::write` in +/// `zingo-price/src/lib.rs`, and the `ReadableWriteable` implementations for +/// `UnifiedKeyStore` and `ReceiverSelection` in +/// `zingolib/src/wallet/keys/unified.rs`. +/// +/// The grammar-unique mark sits at byte 8: where every earlier version wrote +/// a u64-length-prefixed chain name, this writer emits a single chain-type +/// byte, `0` for mainnet. Outpoint and output indices remain `u16` here. This +/// is the grammar today's reader misparses, because it dispatches on the +/// version word alone and version 40 also names row 70's incompatible +/// `stable` grammar. +/// +/// The fixture is a mainnet wallet holding one account, one unified address, +/// one transparent address, one scanned block, one wallet transaction with +/// one output of each of the five kinds, one nullifier per shielded pool, and +/// one outpoint-map entry. The outpoint-map entry is mandatory across this +/// era: its index width is the column that separates this row from rows 70 +/// and 71. +fn row_69() -> Vec { + wallet(&Grammar { + version_word: 40, + chain_as_byte: true, + wide_output_index: false, + transparent_coin_version: 0, + wallet_note_version: 1, + outgoing_note_version: 0, + }) +} + +/// Row 70: `stable`'s independent wallet file version 40, defined by merge +/// commit `5d8fda797` ("Merge PR #2360 fix_output_id_type", authoring +/// interior `83bbd10c9`). The grammar shipped under tag +/// `zingolib_nu6_2_for_zaino_v0.4.0` and was superseded on `stable` by +/// `0d6c997a6` on 2026-06-09. +/// +/// This replicates the same writers as row 69, read from `5d8fda797`'s tree. +/// Two differences distinguish it. The chain stays a u64-length-prefixed name +/// string, which `ChainType`'s `Display` renders as `"main"` for mainnet, so +/// bytes 8 through 15 are the length `4` rather than a chain-type byte. And +/// the output index widens from `u16` to `u32` at six write sites: the +/// outpoint map in `LightWallet::write`, and `TransparentCoin`, +/// `SaplingNote`, `OrchardNote`, `OutgoingSaplingNote` and +/// `OutgoingOrchardNote` in `pepper-sync/src/wallet/serialization.rs`. Each +/// widened record bumped its own inner version so that a reader can tell the +/// widths apart: `TransparentCoin` went from 0 to 1, `WalletNote` from 1 to +/// 2, and `OutgoingNote` from 0 to 1. +/// +/// The wallet contents match row 69 exactly, so every differing byte is a +/// grammar difference. +fn row_70() -> Vec { + wallet(&Grammar { + version_word: 40, + chain_as_byte: false, + wide_output_index: true, + transparent_coin_version: 1, + wallet_note_version: 2, + outgoing_note_version: 1, + }) +} + +/// Row 71: the union, wallet file version 41, defined by merge commit +/// `6ae5c270d` (authoring interior `344dc548d`, "solve merge conflicts with +/// stable"). +/// +/// This replicates the same writers as rows 69 and 70, read from +/// `6ae5c270d`'s tree. It carries `dev`'s chain-type byte and `stable`'s +/// `u32` output indices at once, and it bumps the version word to 41 so that +/// the union is distinguishable from either parent. The pepper-sync record +/// writers here are byte-for-byte identical to row 70's, so this fixture also +/// exhibits the bumped inner versions — `TransparentCoin` 1, `WalletNote` 2, +/// `OutgoingNote` 1 — which appear only inside a populated wallet +/// transaction. That is why the fixture populates one output of each kind +/// rather than leaving the transaction vector empty. +/// +/// The wallet contents match rows 69 and 70 exactly. +fn row_71() -> Vec { + wallet(&Grammar { + version_word: 41, + chain_as_byte: true, + wide_output_index: true, + transparent_coin_version: 1, + wallet_note_version: 2, + outgoing_note_version: 1, + }) +} + +/// Write the complete wallet file under `grammar`. +/// +/// The field order follows `LightWallet::write` at all three defining +/// commits, which agree on everything but the chain encoding and the index +/// widths. +fn wallet(grammar: &Grammar) -> Vec { + let mut out = Vec::new(); + + push_u64_le(&mut out, grammar.version_word); + + if grammar.chain_as_byte { + push_u8(&mut out, CHAIN_TYPE_MAINNET); + } else { + push_u64_string(&mut out, CHAIN_NAME_MAINNET); + } + + // The mnemonic's entropy, written as a byte vector; 32 bytes is the + // 24-word length. Seed material is zeroed throughout the corpus. + push_compact_vec_u8(&mut out, &[0u8; 32]); + + push_u32_le(&mut out, BIRTHDAY); + + // The unified key store: one spending-key entry for account zero. + push_compact_size(&mut out, 1); + push_u32_le(&mut out, ACCOUNT_ID); + push_u8(&mut out, UNIFIED_KEY_STORE_VERSION); + push_u8(&mut out, KEY_TYPE_SPEND); + push_unified_spending_key(&mut out); + + // The unified addresses: one address at index zero with both shielded + // receivers. + push_compact_size(&mut out, 1); + push_u32_le(&mut out, ACCOUNT_ID); + push_u32_le(&mut out, 0); + push_u8(&mut out, RECEIVER_SELECTION_VERSION); + push_u8(&mut out, RECEIVERS_ORCHARD_AND_SAPLING); + + // The transparent addresses: one external address at index zero. + push_compact_size(&mut out, 1); + push_u32_le(&mut out, ACCOUNT_ID); + push_u8(&mut out, TRANSPARENT_SCOPE_EXTERNAL); + push_u32_le(&mut out, 0); + + // The wallet blocks. + push_compact_size(&mut out, 1); + push_wallet_block(&mut out); + + // The wallet transactions. + push_compact_size(&mut out, 1); + push_wallet_transaction(&mut out, grammar); + + push_nullifier_map(&mut out); + push_outpoint_map(&mut out, grammar); + push_shard_trees(&mut out); + push_sync_state(&mut out); + push_sync_config(&mut out); + + push_u32_le(&mut out, MIN_CONFIRMATIONS); + + push_price_list(&mut out); + + out +} + +/// Append an output index in the width `grammar` selects. +/// +/// This is the single discriminating column of the era, and it appears at six +/// write sites: the outpoint map in `disk.rs`, and the five output records in +/// pepper-sync. +fn push_output_index(out: &mut Vec, grammar: &Grammar, index: u32) { + if grammar.wide_output_index { + push_u32_le(out, index); + } else { + push_u16_le(out, index as u16); + } +} + +/// Append a `UnifiedSpendingKey` as `ReadableWriteable for UnifiedSpendingKey` +/// writes it: a CompactSize length, then `UnifiedSpendingKey::to_bytes` under +/// the Orchard era. +/// +/// The key bytes themselves are zeroed, as the corpus does for all key +/// material; only their lengths shape the grammar. +fn push_unified_spending_key(out: &mut Vec) { + // ASSUMPTION: `UnifiedSpendingKey::to_bytes` is a dependency encoding + // (`zcash_keys`), read from the vendored 0.13.0 source rather than from + // the zingolib tree. It writes the era identifier, then a CompactSize + // typecode and a CompactSize-length-prefixed key for orchard, sapling and + // transparent in that order. + let mut usk = Vec::new(); + // ASSUMPTION: `Era::Orchard`'s identifier is the NU5 consensus branch id. + push_u32_le(&mut usk, 0xC2D6_D0B4); + // ASSUMPTION: ZIP 316 typecodes: orchard 3, sapling 2, P2PKH 0. + push_compact_size(&mut usk, 3); + push_compact_vec_u8(&mut usk, &[0u8; 32]); + push_compact_size(&mut usk, 2); + // ASSUMPTION: a sapling `ExtendedSpendingKey` serializes to 169 bytes. + push_compact_vec_u8(&mut usk, &[0u8; 169]); + push_compact_size(&mut usk, 0); + // ASSUMPTION: `AccountPrivKey::to_bytes` yields the 78-byte BIP 32 xprv + // encoding less its 4-byte prefix, so 74 bytes. + push_compact_vec_u8(&mut usk, &[0u8; 74]); + + push_compact_vec_u8(out, &usk); +} + +/// Append a `WalletBlock` as `WalletBlock::write` in +/// `pepper-sync/src/wallet/serialization.rs` writes it. The block names the +/// wallet's single transaction and carries the tree bounds that bracket it. +fn push_wallet_block(out: &mut Vec) { + push_u8(out, WALLET_BLOCK_VERSION); + push_u32_le(out, BLOCK_HEIGHT); + push_bytes(out, &BLOCK_HASH); + push_bytes(out, &PREV_HASH); + push_u32_le(out, TIMESTAMP); + push_compact_size(out, 1); + push_bytes(out, &WALLET_TXID); + + push_u8(out, TREE_BOUNDS_VERSION); + push_u32_le(out, 3_000_000); + push_u32_le(out, 3_000_001); + push_u32_le(out, 5_000_000); + push_u32_le(out, 5_000_002); +} + +/// Append a `WalletTransaction` as `WalletTransaction::write` writes it: the +/// record version, the transaction identifier, the confirmation status, the +/// whole consensus transaction, the wallet's timestamp, and then the five +/// output vectors whose index width this era changed. +fn push_wallet_transaction(out: &mut Vec, grammar: &Grammar) { + push_u8(out, WALLET_TRANSACTION_VERSION); + push_bytes(out, &WALLET_TXID); + + push_u8(out, CONFIRMATION_STATUS_VERSION); + push_u8(out, CONFIRMATION_STATUS_CONFIRMED); + push_u32_le(out, BLOCK_HEIGHT); + + push_consensus_transaction(out); + + push_u32_le(out, TIMESTAMP); + + push_compact_size(out, 1); + push_transparent_coin(out, grammar); + push_compact_size(out, 1); + push_sapling_note(out, grammar); + push_compact_size(out, 1); + push_orchard_note(out, grammar); + push_compact_size(out, 1); + push_outgoing_sapling_note(out, grammar); + push_compact_size(out, 1); + push_outgoing_orchard_note(out, grammar); +} + +/// Append the embedded consensus transaction. +/// +/// `WalletTransaction::write` calls `Transaction::write` with no length +/// prefix, so a reader must parse the consensus encoding to find where the +/// record resumes. The fixture therefore emits a well-formed but minimal +/// version 5 transaction: one transparent output, the one the wallet's +/// transparent coin claims, and no shielded bundles. Reproducing real sapling +/// and orchard bundles would mean synthesising zero-knowledge proofs, which a +/// std-only generator cannot do, and no census discriminator reads inside the +/// bundles. +fn push_consensus_transaction(out: &mut Vec) { + // ASSUMPTION: the whole of this function is a dependency encoding + // (`zcash_primitives::transaction::Transaction::write_v5` and + // `zcash_transparent`'s `TxOut` and `Script`), read from the vendored + // 0.28.0 and 0.8.0 sources. Rows 70 and 71 pin exactly those versions; + // row 69 pins zcash_primitives 0.26.4 and zcash_transparent 0.6.3, which + // are not vendored here, and this fixture assumes the version 5 layout is + // unchanged between them. + + // The header: the overwintered bit set over transaction version 5, then + // the version 5 version-group identifier. + push_u32_le(out, 0x8000_0000 | 5); + push_u32_le(out, 0x26A7_270A); + // ASSUMPTION: the NU6 consensus branch identifier, which is the one + // BLOCK_HEIGHT implies on mainnet. + push_u32_le(out, 0xC8E7_1055); + // The lock time and the expiry height. + push_u32_le(out, 0); + push_u32_le(out, BLOCK_HEIGHT + 40); + + // The transparent bundle: no inputs, one P2PKH output. + push_compact_size(out, 0); + push_compact_size(out, 1); + push_i64_le(out, TRANSPARENT_VALUE as i64); + push_compact_vec_u8(out, &p2pkh_script()); + + // No sapling bundle: an empty spend vector and an empty output vector. + push_compact_size(out, 0); + push_compact_size(out, 0); + + // No orchard bundle: an empty action vector. + push_compact_size(out, 0); +} + +/// The 25-byte P2PKH script `OP_DUP OP_HASH160 <20 bytes> OP_EQUALVERIFY +/// OP_CHECKSIG`, with a zeroed key hash. +fn p2pkh_script() -> Vec { + let mut script = vec![0x76, 0xA9, 0x14]; + script.extend_from_slice(&[0u8; 20]); + script.extend_from_slice(&[0x88, 0xAC]); + script +} + +/// Append a `TransparentCoin` as `TransparentCoin::write` writes it. Its +/// record version is 0 at row 69 and 1 at rows 70 and 71, and that version is +/// what tells a reader whether the output index that follows is two bytes or +/// four. +fn push_transparent_coin(out: &mut Vec, grammar: &Grammar) { + push_u8(out, grammar.transparent_coin_version); + push_bytes(out, &WALLET_TXID); + push_output_index(out, grammar, 0); + + push_u32_le(out, ACCOUNT_ID); + push_u8(out, TRANSPARENT_SCOPE_EXTERNAL); + push_u32_le(out, 0); + + // The address uses the historical u64-length string framing, which + // pepper-sync keeps in its own `write_string`. + push_u64_string(out, TRANSPARENT_ADDRESS); + push_compact_vec_u8(out, &p2pkh_script()); + push_u64_le(out, TRANSPARENT_VALUE); + push_optional_none(out); +} + +/// Append a `SaplingNote` as `SaplingNote::write` writes it. The record +/// version is `WalletNote::serialized_version`, 1 at row 69 and 2 at rows 70 +/// and 71. +/// +/// The note has a nullifier but no commitment-tree position, which is the +/// state of a note found by a scan whose shard trees have not yet been built. +/// The fixture's shard trees are correspondingly empty. +fn push_sapling_note(out: &mut Vec, grammar: &Grammar) { + push_u8(out, grammar.wallet_note_version); + push_bytes(out, &WALLET_TXID); + push_output_index(out, grammar, 0); + + push_u32_le(out, ACCOUNT_ID); + push_u8(out, ZIP32_SCOPE_EXTERNAL); + + // The sapling payment address is 43 bytes. + push_bytes(out, &[0u8; 43]); + push_u64_le(out, SAPLING_VALUE); + // An after-ZIP-212 rseed, then its 32 bytes. + push_u8(out, 1); + push_bytes(out, &[0u8; 32]); + + push_optional_some(out); + push_bytes(out, &SAPLING_NULLIFIER); + push_optional_none(out); + push_empty_memo(out); + push_optional_none(out); + + // The refetch-nullifier ranges, present from `WalletNote` version 1. + push_compact_size(out, 0); +} + +/// Append an `OrchardNote` as `OrchardNote::write` writes it. It shares +/// `WalletNote`'s record version with the sapling note and differs in +/// carrying a rho alongside its rseed. +fn push_orchard_note(out: &mut Vec, grammar: &Grammar) { + push_u8(out, grammar.wallet_note_version); + push_bytes(out, &WALLET_TXID); + push_output_index(out, grammar, 0); + + push_u32_le(out, ACCOUNT_ID); + push_u8(out, ZIP32_SCOPE_EXTERNAL); + + // The orchard raw address is 43 bytes. + push_bytes(out, &[0u8; 43]); + push_u64_le(out, ORCHARD_VALUE); + push_bytes(out, &[0u8; 32]); + push_bytes(out, &[0u8; 32]); + + push_optional_some(out); + push_bytes(out, &ORCHARD_NULLIFIER); + push_optional_none(out); + push_empty_memo(out); + push_optional_none(out); + + push_compact_size(out, 0); +} + +/// Append an `OutgoingSaplingNote` as `OutgoingSaplingNote::write` writes it. +/// Its record version is `OutgoingNote::serialized_version`, 0 at row 69 and +/// 1 at rows 70 and 71. +fn push_outgoing_sapling_note(out: &mut Vec, grammar: &Grammar) { + push_u8(out, grammar.outgoing_note_version); + push_bytes(out, &WALLET_TXID); + push_output_index(out, grammar, 0); + + push_u32_le(out, ACCOUNT_ID); + push_u8(out, ZIP32_SCOPE_EXTERNAL); + + push_bytes(out, &[0u8; 43]); + push_u64_le(out, OUTGOING_SAPLING_VALUE); + push_u8(out, 1); + push_bytes(out, &[0u8; 32]); + + push_empty_memo(out); + // No recorded full unified address for the recipient. + push_optional_none(out); +} + +/// Append an `OutgoingOrchardNote` as `OutgoingOrchardNote::write` writes it. +fn push_outgoing_orchard_note(out: &mut Vec, grammar: &Grammar) { + push_u8(out, grammar.outgoing_note_version); + push_bytes(out, &WALLET_TXID); + push_output_index(out, grammar, 0); + + push_u32_le(out, ACCOUNT_ID); + push_u8(out, ZIP32_SCOPE_EXTERNAL); + + push_bytes(out, &[0u8; 43]); + push_u64_le(out, OUTGOING_ORCHARD_VALUE); + push_bytes(out, &[0u8; 32]); + push_bytes(out, &[0u8; 32]); + + push_empty_memo(out); + push_optional_none(out); +} + +/// Append the 512-byte ZIP 302 encoding of an absent memo. +fn push_empty_memo(out: &mut Vec) { + // ASSUMPTION: `MemoBytes::empty` is a dependency encoding + // (`zcash_protocol`), read from the vendored 0.9.0 source: the marker + // byte 0xF6 followed by 511 zero bytes. + push_u8(out, 0xF6); + push_bytes(out, &[0u8; 511]); +} + +/// Append the `NullifierMap` as `NullifierMap::write` writes it: the record +/// version, then the sapling and orchard maps, each a vector of a raw 32-byte +/// nullifier followed by the scan target that will resolve it. +fn push_nullifier_map(out: &mut Vec) { + push_u8(out, NULLIFIER_MAP_VERSION); + + push_compact_size(out, 1); + push_bytes(out, &SAPLING_NULLIFIER); + push_scan_target(out); + + push_compact_size(out, 1); + push_bytes(out, &ORCHARD_NULLIFIER); + push_scan_target(out); +} + +/// Append the outpoint map as `LightWallet::write` writes it: a vector whose +/// elements are a raw transaction identifier, the output index, and a scan +/// target. +/// +/// This is the write site the census reads first. Row 69 emits a `u16` index +/// here and rows 70 and 71 emit a `u32`, and unlike the pepper-sync records +/// this one carries no inner version to announce the change. +fn push_outpoint_map(out: &mut Vec, grammar: &Grammar) { + push_compact_size(out, 1); + push_bytes(out, &WALLET_TXID); + push_output_index(out, grammar, 0); + push_scan_target(out); +} + +/// Append a `ScanTarget` as `ScanTarget::write` writes it. +fn push_scan_target(out: &mut Vec) { + push_u8(out, SCAN_TARGET_VERSION); + push_u32_le(out, BLOCK_HEIGHT); + push_bytes(out, &WALLET_TXID); + push_u8(out, 1); +} + +/// Append the `ShardTrees` as `ShardTrees::write` writes them: the record +/// version, then the sapling and orchard trees in turn. +fn push_shard_trees(out: &mut Vec) { + push_u8(out, SHARD_TREES_VERSION); + push_empty_shard_tree(out); + push_empty_shard_tree(out); +} + +/// Append one empty memory-backed shard tree: no shards, no checkpoints, and +/// an empty cap. +fn push_empty_shard_tree(out: &mut Vec) { + push_compact_size(out, 0); + push_compact_size(out, 0); + // ASSUMPTION: `write_shard` is a dependency encoding + // (`zcash_client_backend::serialization::shardtree`), read from the + // vendored 0.23.0 source: the serialization version 1, then the tree, + // whose empty form is the single nil tag 0. `MemoryShardStore::empty` + // starts with an empty cap, so that is what an unsynced wallet writes. + push_u8(out, 1); + push_u8(out, 0); +} + +/// Append the `SyncState` as `SyncState::write` writes it: the record +/// version, the scan ranges, the sapling and orchard shard ranges, and the +/// scan targets. +fn push_sync_state(out: &mut Vec) { + push_u8(out, SYNC_STATE_VERSION); + + push_compact_size(out, 1); + push_u32_le(out, BIRTHDAY); + push_u32_le(out, SCAN_RANGE_END); + push_u8(out, SCAN_PRIORITY_SCANNED); + + push_compact_size(out, 0); + push_compact_size(out, 0); + push_compact_size(out, 0); +} + +/// Append the `SyncConfig` as `SyncConfig::write` in +/// `pepper-sync/src/config.rs` writes it: the record version, the gap limit, +/// the discovery scope bitfield, and the nested performance level. +fn push_sync_config(out: &mut Vec) { + push_u8(out, SYNC_CONFIG_VERSION); + push_u8(out, GAP_LIMIT); + push_u8(out, DISCOVERY_SCOPES_EXTERNAL_AND_REFUND); + push_u8(out, PERFORMANCE_LEVEL_VERSION); + push_u8(out, PERFORMANCE_LEVEL_HIGH); +} + +/// Append the `PriceList` as `PriceList::write` in `zingo-price/src/lib.rs` +/// writes it. A wallet that has never fetched a price writes the record +/// version, two absent optionals, and an empty vector. +fn push_price_list(out: &mut Vec) { + push_u8(out, PRICE_LIST_VERSION); + push_optional_none(out); + push_optional_none(out); + push_compact_size(out, 0); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Read the version word a fixture writes at offset 0. + fn version_word(bytes: &[u8]) -> u64 { + u64::from_le_bytes(bytes[..8].try_into().expect("fixture has a version word")) + } + + #[test] + fn the_era_covers_rows_69_through_71_in_order() { + let rows: Vec = fixtures().iter().map(|fixture| fixture.row).collect(); + assert_eq!(rows, vec![69, 70, 71]); + } + + /// The two mutually unreadable version 40s and the version 41 union. + #[test] + fn version_words_are_40_40_and_41() { + assert_eq!(version_word(&row_69()), 40); + assert_eq!(version_word(&row_70()), 40); + assert_eq!(version_word(&row_71()), 41); + } + + /// The census's motivating defect in one assertion: the two version 40s + /// diverge at byte 8, where `dev` writes a chain-type byte and `stable` + /// writes the first byte of a u64 string length. + #[test] + fn the_two_version_40s_diverge_at_byte_8() { + let dev = row_69(); + let stable = row_70(); + + assert_eq!(dev[8], 0x00, "row 69 writes the mainnet chain-type byte"); + + let length = u64::from_le_bytes( + stable[8..16] + .try_into() + .expect("row 70 writes a u64 string length"), + ); + assert_eq!(length, CHAIN_NAME_MAINNET.len() as u64); + assert_eq!(&stable[16..20], CHAIN_NAME_MAINNET.as_bytes()); + } + + /// Row 71 keeps `dev`'s chain-type byte alongside `stable`'s widths. + #[test] + fn the_union_keeps_the_chain_type_byte() { + assert_eq!(row_71()[8], 0x00); + } + + /// The wide index costs two bytes at each of the six write sites, and + /// rows 69 and 71 share the chain-type byte, so the union is exactly + /// twelve bytes longer than row 69. + #[test] + fn widening_the_output_index_adds_two_bytes_at_six_sites() { + assert_eq!(row_71().len() - row_69().len(), 12); + } + + /// Each row is a distinguishable grammar, which is the census's claim. + #[test] + fn the_three_rows_are_pairwise_distinct() { + let fixtures = fixtures(); + for (index, earlier) in fixtures.iter().enumerate() { + for later in &fixtures[index + 1..] { + assert_ne!( + earlier.bytes, later.bytes, + "rows {} and {} produced identical bytes", + earlier.row, later.row + ); + } + } + } +} diff --git a/tools/workbench/src/wallet_grammars/era_inception.rs b/tools/workbench/src/wallet_grammars/era_inception.rs new file mode 100644 index 0000000000..5ce8eaf7cb --- /dev/null +++ b/tools/workbench/src/wallet_grammars/era_inception.rs @@ -0,0 +1,1346 @@ +//! Census rows 1 through 21: the writer's inception, on zecwallet-light-cli's +//! `dev` line between 2019-09-06 and 2020-05-09. +//! +//! Every fixture below is derived from the writer source at its row's Defining +//! Commit, read with `git show :`. The serializer moved four +//! times inside this era, so the replicated path differs by row: +//! `rust-lightclient/src/lightwallet.rs` for rows 1 through 7, +//! `src/lightwallet.rs` for rows 8 and 9, `src/lightwallet/mod.rs` with +//! `src/lightwallet/data.rs` for rows 10 through 13, and +//! `lib/src/lightwallet.rs` with `lib/src/lightwallet/data.rs` for rows 14 +//! through 21. +//! +//! The era's wallet is a single-account, unencrypted mainnet wallet holding +//! exactly one transaction. Its seed and key material are all-zero, its block +//! vector is empty except at row 1, its chain name is `main`, and its birthday +//! is 1000000. The one transaction is what makes the era legible: eight of its +//! twenty-one grammars change nothing but the `WalletTx`, `SaplingNoteData`, +//! or `Utxo` sub-records, and a wallet with no transactions would render those +//! eight rows byte-identical to their neighbors. Every fixture therefore +//! carries one transaction with one note, plus one UTXO and one +//! outgoing-metadata record once the grammar admits them. +//! +//! Two findings from this walk contradict the issue table, and both are +//! documented on the rows they affect: row 1's writer does emit an 8-byte +//! version word, which makes rows 1 and 2 byte-identical grammars, and row 20 +//! compresses with gzip, not zstd. + +use super::util::{ + push_bytes, push_compact_size, push_compact_vec_u8, push_i32_le, push_optional_none, + push_u32_le, push_u64_le, push_u64_string, push_u8, +}; +use super::Fixture; + +/// The wallet's seed, written raw as `[u8; 32]` from row 4 onward. +const SEED: [u8; 32] = [0u8; 32]; + +/// One `zip32::ExtendedSpendingKey`, all-zero. +/// +// ASSUMPTION: librustzcash is a path or git dependency at every commit in this +// era (`../../librustzcash/zcash_primitives` at row 4, then +// `github.com/adityapk00/librustzcash` rev `98f9bda32` by row 20), so its +// source is not available in this checkout. The 169-byte layout is taken from +// the crate's documented and stable `ExtendedSpendingKey::write`: `depth` u8, +// `parent_fvk_tag` 4 bytes, `child_index` u32 LE, `chain_code` 32 bytes, +// `ExpandedSpendingKey` (`ask` 32, `nsk` 32, `ovk` 32), and `dk` 32 bytes. +// Zeroing every field makes the record 169 zero bytes. A real `ask` and `nsk` +// would be Jubjub scalars, but the writer never validates on the way out and +// nothing in the census reads them back through the curve. +const EXTENDED_SPENDING_KEY: [u8; 169] = [0u8; 169]; + +/// One `zip32::ExtendedFullViewingKey`, all-zero. Every note record carries +/// one of these, and row 17 adds a wallet-level vector of them. +/// +// ASSUMPTION: the same dependency situation as `EXTENDED_SPENDING_KEY`. The +// 169-byte layout is `depth` u8, `parent_fvk_tag` 4 bytes, `child_index` u32 +// LE, `chain_code` 32 bytes, `FullViewingKey` (`ak` 32, `nk` 32, `ovk` 32), +// and `dk` 32 bytes. The two key records coincide in width, which is why the +// census separates them by position rather than by size. +const EXTENDED_FULL_VIEWING_KEY: [u8; 169] = [0u8; 169]; + +/// One transparent secret key, written as 32 raw bytes from row 7 onward. +/// +// ASSUMPTION: `secp256k1::SecretKey` derefs to its 32 big-endian scalar bytes, +// which is how the writer reaches them, through `&self.tkeys[0][..]`. +const TRANSPARENT_KEY: [u8; 32] = [0u8; 32]; + +/// The wallet's transparent address, written from row 18 onward at the wallet +/// level and from row 6 onward inside every `Utxo`. +/// +/// This is the genuine Zcash mainnet P2PKH address for a twenty-byte all-zero +/// HASH160: Base58Check over the `0x1CB8` prefix and that hash. +const TRANSPARENT_ADDRESS: &str = "t1Hsc1LR8yKnbbe3twRp88p6vFfC5t7DLbs"; + +/// The recipient recorded in the outgoing-metadata record that row 10 +/// introduces. `scan_full_tx` recovers outgoing sends through the outgoing +/// viewing key, so the recipient is always a Sapling address. +/// +// ASSUMPTION: this is the well-formed mainnet Sapling address (Bech32, HRP +// `zs`) over a 43-byte all-zero payload; its checksum verifies. The payload is +// not a valid diversifier and `pk_d` pair, but the writer stores the address +// as an opaque string and never decodes it. +const SAPLING_ADDRESS: &str = + "zs1qqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqqpq6d8g"; + +/// The chain name, written from row 11 onward through `utils::write_string`. +const CHAIN_NAME: &str = "main"; + +/// The wallet birthday, written from row 13 onward as a u64. +const BIRTHDAY: u64 = 1_000_000; + +/// The height carried by the single `BlockData` of row 1, by the wallet's one +/// transaction, and by the single `Utxo`. +const HEIGHT: i32 = 1_000_000; + +/// The transaction timestamp that row 16 inserts into `WalletTx`. This is +/// 2019-10-02T05:46:40Z, inside the window in which the field was added. +const DATETIME: u64 = 1_570_000_000; + +/// The value of the single `Utxo`, in zatoshi. +const UTXO_VALUE: u64 = 100_000; + +/// The value of the single `SaplingNoteData`, in zatoshi. +const NOTE_VALUE: u64 = 50_000; + +/// The value recorded in the outgoing-metadata record, in zatoshi. +const OUTGOING_VALUE: u64 = 25_000; + +/// The `enc_seed` field, `[u8; 48]`, written raw from row 19 onward. An +/// unencrypted wallet leaves it zeroed. +const ENC_SEED: [u8; 48] = [0u8; 48]; + +/// This era's fixtures, census rows 1 through 21 of the 77-row table. The row +/// numbers here are informational: `wallet_grammars::all` reassigns them from +/// the central manifest by Defining Commit hash. +pub fn fixtures() -> Vec { + let rows: [(u8, &'static str, Vec); 21] = [ + (1, "7ebc8686e", row_01()), + (2, "c2e26fbbc", row_02()), + (3, "8ff6d15e3", row_03()), + (4, "5bd8b754d", row_04()), + (5, "db549f5b6", row_05()), + (6, "f532b70ca", row_06()), + (7, "b24f174b5", row_07()), + (8, "0e8ab4d27", row_08()), + (9, "f93267507", row_09()), + (10, "b0f7d8fcf", row_10()), + (11, "b3ca226ff", row_11()), + (12, "df12ccf31", row_12()), + (13, "88a80f574", row_13()), + (14, "ba706ab7c", row_14()), + (15, "e3f972508", row_15()), + (16, "ebf3c7133", row_16()), + (17, "e3a0fd2de", row_17()), + (18, "fc15de568", row_18()), + (19, "72548e077", row_19()), + (20, "796663c97", row_20()), + (21, "cbffd69c6", row_21()), + ]; + rows.into_iter() + .map(|(row, defining_commit, bytes)| Fixture { + row, + defining_commit, + branch: "dev", + bytes, + }) + .collect() +} + +// --------------------------------------------------------------------------- +// The transaction sub-record, which eight of this era's rows are about +// --------------------------------------------------------------------------- + +/// The `WalletTx` and `SaplingNoteData` grammar in force at a given row. Each +/// field names the Defining Commit that introduced it, so a row's shape states +/// which sub-record commits that row stands after. +#[derive(Clone, Copy)] +struct TransactionShape { + /// What `WalletTx::serialized_version()` returns. + version: u64, + /// The 32-byte txid after `block`, the trailing + /// `total_shielded_value_spent`, and the note record's `is_change` byte. + /// All three arrive together at row 3 (`8ff6d15e3`). + txid_and_shielded_total: bool, + /// `total_transparent_value_spent`, introduced at row 5 (`db549f5b6`). + transparent_total: bool, + /// The in-transaction `Vector`, introduced at row 8 (`0e8ab4d27`). + /// `Some(true)` while `Utxo` still wrote its trailing unconfirmed-spent + /// `Optional`, `Some(false)` from row 9 (`f93267507`) onward. + utxos: Option, + /// The `Vector`, introduced at row 10 (`b0f7d8fcf`) + /// alongside the sub-version bump from 1 to 2. + outgoing_metadata: bool, + /// The trailing `full_tx_scanned` byte, introduced at row 12 + /// (`df12ccf31`) without a sub-version bump. + full_tx_scanned: bool, + /// The u64 `datetime` inserted after `block`, introduced at row 16 + /// (`ebf3c7133`) alongside the sub-version bump from 3 to 4. + datetime: bool, +} + +/// Rows 1 and 2: the born transaction record, a version word, the block +/// height, and the note vector. +const TX_BORN: TransactionShape = TransactionShape { + version: 1, + txid_and_shielded_total: false, + transparent_total: false, + utxos: None, + outgoing_metadata: false, + full_tx_scanned: false, + datetime: false, +}; + +/// Rows 3 and 4, after `8ff6d15e3`. +const TX_TXID: TransactionShape = TransactionShape { + txid_and_shielded_total: true, + ..TX_BORN +}; + +/// Rows 5 through 7, after `db549f5b6`. +const TX_TRANSPARENT_TOTAL: TransactionShape = TransactionShape { + transparent_total: true, + ..TX_TXID +}; + +/// Row 8, after `0e8ab4d27`, whose `Utxo` still writes two `Optional` spend +/// markers. +const TX_INLINE_UTXO: TransactionShape = TransactionShape { + utxos: Some(true), + ..TX_TRANSPARENT_TOTAL +}; + +/// Row 9, after `f93267507` trimmed the `Utxo`'s unconfirmed-spent `Optional`. +const TX_TRIMMED_UTXO: TransactionShape = TransactionShape { + utxos: Some(false), + ..TX_INLINE_UTXO +}; + +/// Rows 10 and 11, after `b0f7d8fcf`. +const TX_OUTGOING: TransactionShape = TransactionShape { + version: 2, + outgoing_metadata: true, + ..TX_TRIMMED_UTXO +}; + +/// Rows 12 through 14, after `df12ccf31`. +const TX_SCANNED: TransactionShape = TransactionShape { + full_tx_scanned: true, + ..TX_OUTGOING +}; + +/// Row 15, after `e3f972508` bumped the sub-version from 2 to 3 without +/// changing a byte of the record's shape. +const TX_SCANNED_V3: TransactionShape = TransactionShape { + version: 3, + ..TX_SCANNED +}; + +/// Rows 16 through 21, after `ebf3c7133`. +const TX_DATETIME: TransactionShape = TransactionShape { + version: 4, + datetime: true, + ..TX_SCANNED_V3 +}; + +/// Append the wallet's transaction map: a `Vector` of one tuple, the txid as +/// 32 raw bytes followed by the `WalletTx` record. The map's own txid is +/// written even after row 3 gave `WalletTx` a second copy of it. +fn push_transaction_map(out: &mut Vec, shape: TransactionShape) { + push_compact_size(out, 1); + push_bytes(out, &[0u8; 32]); + push_wallet_tx(out, shape); +} + +/// Append one `WalletTx` as `WalletTx::write` emits it under `shape`. +fn push_wallet_tx(out: &mut Vec, shape: TransactionShape) { + push_u64_le(out, shape.version); + push_i32_le(out, HEIGHT); + if shape.datetime { + push_u64_le(out, DATETIME); + } + if shape.txid_and_shielded_total { + push_bytes(out, &[0u8; 32]); + } + push_compact_size(out, 1); + push_sapling_note_data(out, shape.txid_and_shielded_total); + if let Some(unconfirmed_spent) = shape.utxos { + push_compact_size(out, 1); + push_utxo(out, unconfirmed_spent); + } + if shape.txid_and_shielded_total { + push_u64_le(out, 0); + } + if shape.transparent_total { + push_u64_le(out, 0); + } + if shape.outgoing_metadata { + push_compact_size(out, 1); + push_outgoing_tx_metadata(out); + } + if shape.full_tx_scanned { + push_u8(out, 0); + } +} + +/// Append one `SaplingNoteData` as `SaplingNoteData::write` emits it: the +/// sub-record version 1 as a u64, the account as a u64, the note's own +/// `ExtendedFullViewingKey`, the 11-byte diversifier, the note value as a u64, +/// the note randomness as 32 raw bytes, the witness vector, the 32-byte +/// nullifier, and two `Optional`s for the spending txid and the memo. The +/// `is_change` byte closes the record from row 3 onward. +/// +/// The note is unspent with no memo and no witnesses, so all three of those +/// fields collapse to a single byte each. The sub-record version stays at 1 +/// across this whole era. +fn push_sapling_note_data(out: &mut Vec, is_change: bool) { + push_u64_le(out, 1); + push_u64_le(out, 0); + push_bytes(out, &EXTENDED_FULL_VIEWING_KEY); + push_bytes(out, &[0u8; 11]); + push_u64_le(out, NOTE_VALUE); + push_bytes(out, &[0u8; 32]); + push_compact_size(out, 0); + push_bytes(out, &[0u8; 32]); + push_optional_none(out); + push_optional_none(out); + if is_change { + push_u8(out, 0); + } +} + +/// Append one `Utxo` as `Utxo::write` emits it: the sub-record version 1 as a +/// u64, the address as a u32 little-endian byte length followed by its ASCII, +/// the txid as 32 raw bytes, the output index and the value as u64s, the +/// height as an i32, the script as a `Vector`, and the spend markers. The +/// address length is the era's third length discipline: neither a CompactSize +/// nor `write_string`'s u64, but a bare u32 that only `Utxo` uses. +/// +/// `unconfirmed_spent` selects the trailing `Optional` that `f93267507` +/// removed at row 9. +fn push_utxo(out: &mut Vec, unconfirmed_spent: bool) { + push_u64_le(out, 1); + push_u32_le(out, TRANSPARENT_ADDRESS.len() as u32); + push_bytes(out, TRANSPARENT_ADDRESS.as_bytes()); + push_bytes(out, &[0u8; 32]); + push_u64_le(out, 0); + push_u64_le(out, UTXO_VALUE); + push_i32_le(out, HEIGHT); + push_compact_vec_u8(out, &p2pkh_script()); + push_optional_none(out); + if unconfirmed_spent { + push_optional_none(out); + } +} + +/// Append one `OutgoingTxMetadata` as `OutgoingTxMetadata::write` emits it: +/// the address in the `write_string` discipline of a u64 length and its UTF-8, +/// the value as a u64, and the memo as 512 raw bytes with neither a length nor +/// an `Optional` to frame it. +fn push_outgoing_tx_metadata(out: &mut Vec) { + push_u64_string(out, SAPLING_ADDRESS); + push_u64_le(out, OUTGOING_VALUE); + push_bytes(out, &[0u8; 512]); +} + +// --------------------------------------------------------------------------- +// Other sub-record writers +// --------------------------------------------------------------------------- + +/// Append one `BlockData` as `BlockData::write` emits it: the height as an +/// i32, the block hash as 32 raw bytes, the Sapling `CommitmentTree`, and the +/// literal end tag 11 as a u64. The shape holds unchanged across this era. +fn push_block_data(out: &mut Vec, height: i32) { + push_i32_le(out, height); + push_bytes(out, &[0u8; 32]); + push_empty_commitment_tree(out); + push_u64_le(out, 11); +} + +/// Append an empty `CommitmentTree`. +/// +// ASSUMPTION: librustzcash's source is not in this checkout, so the encoding +// comes from the crate's documented `CommitmentTree::write`: `Optional +// left`, `Optional right`, then `Vector> parents`. A tree +// that has absorbed no notes writes three zero bytes, a None, a None, and an +// empty vector. +fn push_empty_commitment_tree(out: &mut Vec) { + push_optional_none(out); + push_optional_none(out); + push_compact_size(out, 0); +} + +/// The standard 25-byte P2PKH `scriptPubKey` for the all-zero HASH160 that +/// [`TRANSPARENT_ADDRESS`] encodes: `OP_DUP`, `OP_HASH160`, twenty pushed +/// bytes, `OP_EQUALVERIFY`, `OP_CHECKSIG`. +fn p2pkh_script() -> Vec { + let mut script = vec![0x76, 0xA9, 0x14]; + script.extend_from_slice(&[0u8; 20]); + script.extend_from_slice(&[0x88, 0xAC]); + script +} + +/// Append the `Vector` holding this era's single key. +fn push_spending_keys(out: &mut Vec) { + push_compact_size(out, 1); + push_bytes(out, &EXTENDED_SPENDING_KEY); +} + +/// Append the `Vector` that row 17 introduces. +fn push_full_viewing_keys(out: &mut Vec) { + push_compact_size(out, 1); + push_bytes(out, &EXTENDED_FULL_VIEWING_KEY); +} + +/// Append the `Vector` that row 14 introduces, each key +/// written as its 32 raw bytes. +fn push_transparent_key_vector(out: &mut Vec) { + push_compact_size(out, 1); + push_bytes(out, &TRANSPARENT_KEY); +} + +/// Append the `Vector` of transparent addresses that row 18 +/// introduces. The vector uses `Vector`'s CompactSize count while each element +/// uses `write_string`'s u64 length, so the two disciplines meet inside one +/// field. +fn push_transparent_address_vector(out: &mut Vec) { + push_compact_size(out, 1); + push_u64_string(out, TRANSPARENT_ADDRESS); +} + +/// Append the standalone `Vector` that rows 6 and 7 carry at the wallet +/// level, holding one UTXO. +fn push_standalone_utxo_vector(out: &mut Vec) { + push_compact_size(out, 1); + push_utxo(out, true); +} + +// --------------------------------------------------------------------------- +// Rows +// --------------------------------------------------------------------------- + +/// Row 1, Defining Commit `7ebc8686e` ("Save and Read wallet", 2019-09-06), +/// replicating `LightWallet::write`, `BlockData::write`, `WalletTx::write`, +/// and `SaplingNoteData::write` in `rust-lightclient/src/lightwallet.rs`. The +/// writer emits a u64 version word, the `Vector`, and the +/// transaction map as a `Vector` of txid and `WalletTx` tuples. No key +/// material reaches disk at all: the reader re-derives the wallet's keys from +/// a hard-coded `[1; 32]`. +/// +/// The wallet holds one block at height 1000000 with a zero hash and an empty +/// commitment tree, and one transaction with one note. +/// +/// DISCREPANCY: the issue table records row 1 as carrying no version prefix, +/// with the file beginning at a `Vector` length. The source disagrees. This +/// commit's `LightWallet::write` opens with +/// `writer.write_u64::(1)?`, and its parent `81b6b52ba` has no +/// `LightWallet::write` at all, so no earlier writer omitted the word. Row 2's +/// commit `c2e26fbbc` only replaces that literal with a call to a new +/// `serialized_version()` that returns 1, which changes no byte. Rows 1 and 2 +/// are therefore one grammar, and no discriminator can separate them. These +/// two fixtures differ only in wallet contents: row 1 carries the block that +/// row 2 omits. +fn row_01() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_compact_size(&mut out, 1); + push_block_data(&mut out, HEIGHT); + push_transaction_map(&mut out, TX_BORN); + out +} + +/// Row 2, Defining Commit `c2e26fbbc` ("Cleanup", 2019-09-06), replicating +/// `LightWallet::write` in `rust-lightclient/src/lightwallet.rs`. The commit +/// introduces `LightWallet::serialized_version()`, +/// `WalletTx::serialized_version()`, and +/// `SaplingNoteData::serialized_version()`, each returning the literal its +/// writer already emitted. +/// +/// The wallet holds no blocks and one transaction. See [`row_01`] for the +/// finding that this grammar is byte-identical to row 1's. +fn row_02() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_BORN); + out +} + +/// Row 3, Defining Commit `8ff6d15e3` ("Add more fields to sapling note +/// data", 2019-09-06), replicating `SaplingNoteData::write` and +/// `WalletTx::write` in `rust-lightclient/src/lightwallet.rs`. The wallet's +/// own writer is untouched: every new byte lands inside the transaction. +/// `SaplingNoteData` gains a trailing `is_change` u8, and `WalletTx` inserts +/// its own copy of the 32-byte txid after the block height and appends +/// `total_shielded_value_spent` as a u64. +/// +/// The wallet is row 2's, so those three fields are the whole difference. +fn row_03() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_TXID); + out +} + +/// Row 4, Defining Commit `5bd8b754d` ("Derive addresses from seed", +/// 2019-09-07), replicating `LightWallet::write` in +/// `rust-lightclient/src/lightwallet.rs`. The writer gains the raw 32-byte +/// seed and a `Vector` between the version word and the +/// block vector, so the wallet's own key material reaches disk for the first +/// time. +/// +/// The wallet is row 3's plus the zeroed seed and one all-zero spending key. +/// The key vector carries one element because the row's mark is that vector's +/// presence. +fn row_04() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_TXID); + out +} + +/// Row 5, Defining Commit `db549f5b6` ("scan tx for transparent inputs", +/// 2019-09-13), replicating `WalletTx::write` in +/// `rust-lightclient/src/lightwallet.rs`. The commit teaches the scanner to +/// recognise the wallet's own transparent inputs and records what it finds as +/// `total_transparent_value_spent`, a u64 appended after +/// `total_shielded_value_spent`. The wallet's own writer is untouched. +/// +/// The wallet is row 4's, eight bytes longer inside the transaction. +fn row_05() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_TRANSPARENT_TOTAL); + out +} + +/// Row 6, Defining Commit `f532b70ca` ("Add transparent support for +/// transactions", 2019-09-13), replicating `LightWallet::write` and +/// `Utxo::write` in `rust-lightclient/src/lightwallet.rs`. A standalone +/// `Vector` is inserted between the spending keys and the block vector. +/// +/// The wallet holds one UTXO so the new vector is visibly present: the mainnet +/// address of [`TRANSPARENT_ADDRESS`], a zero txid, output index 0, 100000 +/// zatoshi at height 1000000, the standard P2PKH script, and neither of the +/// two spend markers set. +fn row_06() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_standalone_utxo_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_TRANSPARENT_TOTAL); + out +} + +/// Row 7, Defining Commit `b24f174b5` ("Separate t / z balance", 2019-09-16), +/// replicating `LightWallet::write` in `rust-lightclient/src/lightwallet.rs`. +/// The writer appends `self.tkeys[0]` as 32 raw, unprefixed bytes between the +/// spending-key vector and the UTXO vector. There is no count and no length, +/// so the reader recovers the field by position alone. +/// +/// The wallet is row 6's, plus the zeroed transparent key. +fn row_07() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_bytes(&mut out, &TRANSPARENT_KEY); + push_standalone_utxo_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_TRANSPARENT_TOTAL); + out +} + +/// Row 8, Defining Commit `0e8ab4d27` ("Read UTXOs from walletTx", +/// 2019-09-17), replicating `LightWallet::write` and `WalletTx::write` in +/// `src/lightwallet.rs`, the path the serializer moved to at this commit. The +/// standalone `Vector` is deleted from the wallet, and the same commit +/// adds a `Vector` inside `WalletTx::write` after the note vector, so +/// UTXOs now travel with their transaction. +/// +/// The wallet is row 7's with its one UTXO relocated, which is what makes the +/// move legible: the same 127-byte record appears at a new offset rather than +/// vanishing. +fn row_08() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_bytes(&mut out, &TRANSPARENT_KEY); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_INLINE_UTXO); + out +} + +/// Row 9, Defining Commit `f93267507` ("Don't write unconfirmed fields to +/// disk", 2019-09-17), replicating `Utxo::write` in `src/lightwallet.rs`. The +/// trailing `Optional` that recorded an unconfirmed spend is dropped +/// from both the reader and the writer, on the reasoning that a restarted +/// wallet should not be bound to a transaction that may expire. The wallet's +/// own writer is untouched, and the note record keeps its own spend +/// `Optional`. +/// +/// The wallet is row 8's, one byte shorter inside its UTXO. +fn row_09() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_bytes(&mut out, &TRANSPARENT_KEY); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_TRIMMED_UTXO); + out +} + +/// Row 10, Defining Commit `b0f7d8fcf` ("Write outgoing metadata", +/// 2019-09-19), replicating `OutgoingTxMetadata::write` and `WalletTx::write` +/// in `src/lightwallet/data.rs`, the file the sub-records moved to when +/// `lightwallet` became a directory. `WalletTx::serialized_version()` moves +/// from 1 to 2 and a `Vector` is appended after the two +/// value-spent totals. Each element writes its address in the `write_string` +/// discipline, its value as a u64, and its memo as 512 raw bytes. +/// +/// The transaction carries one outgoing record so the new vector is visibly +/// present, naming the Sapling recipient of [`SAPLING_ADDRESS`]. The reader +/// gates the vector on the sub-version rather than on emptiness, so an empty +/// vector would still mark the field, but only a populated one exercises +/// `OutgoingTxMetadata::write`. +fn row_10() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 1); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_bytes(&mut out, &TRANSPARENT_KEY); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_OUTGOING); + out +} + +/// Row 11, Defining Commit `b3ca226ff` ("Write chain name to wallet", +/// 2019-09-24), replicating `LightWallet::write` in `src/lightwallet/mod.rs` +/// and `utils::write_string` in `src/utils.rs`. The version word moves to 2 +/// and the chain name is appended after the transaction map, framed by the +/// `write_string` pair this commit introduces: a u64 little-endian byte length +/// followed by the UTF-8 bytes. +/// +/// The wallet is row 10's on mainnet, so the file now ends +/// `04 00 00 00 00 00 00 00 6D 61 69 6E`. +fn row_11() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 2); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_bytes(&mut out, &TRANSPARENT_KEY); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_OUTGOING); + push_u64_string(&mut out, CHAIN_NAME); + out +} + +/// Row 12, Defining Commit `df12ccf31` ("Explicitly mark Txs as fully +/// scanned", 2019-09-25), replicating `WalletTx::write` in +/// `src/lightwallet/data.rs`. A `full_tx_scanned` u8 closes the transaction +/// record while the sub-version stands still at 2, so the reader's +/// `match version { 1 => false, _ => read_u8() }` cannot tell the two +/// version-2 layouts apart and a file written just before this commit +/// desynchronises. The wallet's own writer is untouched. +/// +/// The wallet is row 11's, one byte longer inside the transaction. +fn row_12() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 2); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_bytes(&mut out, &TRANSPARENT_KEY); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_SCANNED); + push_u64_string(&mut out, CHAIN_NAME); + out +} + +/// Row 13, Defining Commit `88a80f574` (a merge of `dev`, 2019-09-27, whose +/// authoring interior commit is `f78c3fa48`, "Add wallet birthday"), +/// replicating `LightWallet::write` in `src/lightwallet/mod.rs` as the merge's +/// own tree carries it. The birthday is appended after the chain name as a +/// u64, and the version word stays at 2. +/// +/// The wallet is row 12's with a birthday of 1000000. +fn row_13() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 2); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_bytes(&mut out, &TRANSPARENT_KEY); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_SCANNED); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + out +} + +/// Row 14, Defining Commit `ba706ab7c` ("Write vector tkeys", 2019-10-01), +/// replicating `LightWallet::write` in `src/lightwallet/mod.rs`. The single +/// unprefixed 32-byte transparent key of rows 7 through 13 becomes a +/// `Vector`, so a CompactSize count now precedes the key +/// material. The version word stays at 2. +/// +/// The wallet is row 13's, its one transparent key now inside the vector, +/// which makes the file exactly one byte longer. +fn row_14() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 2); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_transparent_key_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_SCANNED); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + out +} + +/// Row 15, Defining Commit `e3f972508` ("Update serialization versions", +/// 2019-10-01), replicating `LightWallet::write` in `src/lightwallet/mod.rs` +/// and `WalletTx::write` in `src/lightwallet/data.rs`. The commit writes no +/// new bytes: `LightWallet::serialized_version()` moves from 2 to 3, +/// `WalletTx::serialized_version()` from 2 to 3, and the reader's version-1 +/// branches are dropped. +/// +/// The wallet is row 14's. The two fixtures differ at exactly two offsets, the +/// wallet's version word and the transaction's, which is the whole of this +/// row's discriminating evidence. +fn row_15() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 3); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_transparent_key_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_SCANNED_V3); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + out +} + +/// Row 16, Defining Commit `ebf3c7133` ("Add datetime to transactions", +/// 2019-10-18), replicating `WalletTx::write` in +/// `lib/src/lightwallet/data.rs`, the path the sub-records moved to when the +/// crate split into `lib` and `cli`. `WalletTx::serialized_version()` moves +/// from 3 to 4 and a u64 timestamp is inserted between the block height and +/// the txid. The wallet's own writer is untouched and its version word stays +/// at 3. +/// +/// The wallet is row 15's, eight bytes longer inside the transaction. +fn row_16() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 3); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_transparent_key_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_DATETIME); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + out +} + +/// Row 17, Defining Commit `e3a0fd2de` ("Support for wallet encryption", +/// 2019-10-18), replicating `LightWallet::write` in `lib/src/lightwallet.rs`. +/// The version word moves to 4, a `locked` u8 is inserted immediately after +/// it, and a `Vector` is inserted after the spending +/// keys. +/// +/// The wallet is unlocked, so the flag is zero, and it holds one all-zero full +/// viewing key. +fn row_17() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 4); + push_u8(&mut out, 0); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_full_viewing_keys(&mut out); + push_transparent_key_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_DATETIME); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + out +} + +/// Row 18, Defining Commit `fc15de568` ("Support mutable wallets", +/// 2019-10-19), replicating `LightWallet::write` in `lib/src/lightwallet.rs`. +/// A `Vector` of transparent addresses is appended after the +/// transparent keys, and the version word stays at 4. +/// +/// The wallet is row 17's plus the one address that its transparent key +/// controls. +fn row_18() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 4); + push_u8(&mut out, 0); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_full_viewing_keys(&mut out); + push_transparent_key_vector(&mut out); + push_transparent_address_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_DATETIME); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + out +} + +/// Row 19, Defining Commit `72548e077` ("Add lock/unlock API", 2019-10-19), +/// replicating `LightWallet::write` in `lib/src/lightwallet.rs`. The encrypted +/// seed and its nonce are inserted between the `locked` flag and the plaintext +/// seed: `enc_seed` is the `[u8; 48]` field written raw with `write_all`, and +/// `nonce` is a `Vector`. The version word stays at 4. +/// +/// The wallet is unlocked, which is what makes it minimal here. `enc_seed` +/// holds its 48 zero bytes and `nonce` is empty, exactly as an unencrypted +/// wallet writes them, and the plaintext seed still follows. +fn row_19() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 4); + push_bytes(&mut out, &encryption_era_body()); + out +} + +/// Row 20, Defining Commit `796663c97` ("Gzip the output", 2020-04-12), +/// replicating `LightWallet::write` in `lib/src/lightwallet.rs`. The version +/// word moves to 5 and is the last plaintext byte of the file: the writer +/// wraps the remaining output in `AutoFinishUnchecked::new(Encoder::new(out))`, +/// so the whole body becomes one compressed frame. +/// +/// The body is byte for byte row 19's, because no grammar change landed on the +/// line between 2019-10-19 and 2020-04-12. The two rows differ only in the +/// version word and the compression. +/// +/// DISCREPANCY: the issue table records a zstd frame here. The source says +/// gzip. The commit adds `libflate = "0.1"` to `lib/Cargo.toml` and imports +/// `libflate::{gzip::{Decoder, Encoder}, finish::AutoFinishUnchecked}`, so the +/// magic at offset 8 is `1F 8B`, not zstd's `28 B5 2F FD`. The commit message +/// is accurate and the table's note is not. +fn row_20() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 5); + push_bytes(&mut out, ROW_20_GZIP_FRAME); + out +} + +/// Row 21, Defining Commit `cbffd69c6` ("Undo the gzip encoding", +/// 2020-05-09), replicating `LightWallet::write` in `lib/src/lightwallet.rs`. +/// The encoder is removed and the version word moves to 6. The reader keeps a +/// `version != 5` guard so that the one compressed generation stays readable. +/// +/// The wallet is row 19's and row 20's, written in the clear, so this fixture +/// differs from row 19's only in the version word and holds exactly what row +/// 20's frame decompresses to. +fn row_21() -> Vec { + let mut out = Vec::new(); + push_u64_le(&mut out, 6); + push_bytes(&mut out, &encryption_era_body()); + out +} + +/// Everything after the version word for rows 19, 20, and 21, whose writers +/// agree byte for byte: the `locked` flag (renamed `encrypted` by row 20), the +/// raw `enc_seed`, the nonce vector, the raw seed, the spending keys, the full +/// viewing keys, the transparent keys, the transparent addresses, the block +/// vector, the transaction map, the chain name, and the birthday. +fn encryption_era_body() -> Vec { + let mut out = Vec::new(); + push_u8(&mut out, 0); + push_bytes(&mut out, &ENC_SEED); + push_compact_size(&mut out, 0); + push_bytes(&mut out, &SEED); + push_spending_keys(&mut out); + push_full_viewing_keys(&mut out); + push_transparent_key_vector(&mut out); + push_transparent_address_vector(&mut out); + push_compact_size(&mut out, 0); + push_transaction_map(&mut out, TX_DATETIME); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + out +} + +/// The gzip frame that row 20 carries after its version word: the bytes of +/// [`encryption_era_body`] compressed once, ahead of time, and embedded here. +/// +/// The workbench crate is deliberately std-only, so it cannot compress at +/// runtime. The frame was produced by piping the body through the system gzip, +/// with no intermediate file: +/// +/// ```text +/// cargo test --lib wallet_grammars::era_inception::tests::dump_row_20_body \ +/// -- --nocapture --ignored \ +/// | rg '^ROW20BODY ' | cut -d' ' -f2 \ +/// | python3 -c "import sys, subprocess; \ +/// body = bytes.fromhex(sys.stdin.read().strip()); \ +/// print(subprocess.run(['gzip', '-n', '-c'], input=body, \ +/// capture_output=True).stdout.hex())" +/// ``` +/// +/// The tool was GNU `gzip 1.14-modified` on Arch Linux, run on 2026-07-29 at +/// its default compression level, with `-n` so that neither a file name nor a +/// modification time enters the header. +/// +// ASSUMPTION: the deflate bit stream is gzip's, not libflate 0.1's. Deflate +// output is not canonical, so two conforming compressors agree on a frame's +// framing but not on its interior, and libflate 0.1 is unavailable to a +// std-only crate. The header does match what libflate 0.1 emitted: its default +// `gzip::Header` carries modification time 0, an unknown compression level +// (XFL 0), and OS 3 (Unix), which is what `gzip -n` writes on this host, the +// ten bytes `1F 8B 08 00 00 00 00 00 00 03`. The trailer is bound to the +// derived body by `the_row_20_frame_wraps_the_derived_body`, which recomputes +// the CRC-32 and the input length gzip recorded. A recognizer keying on the +// magic, the header, or the trailer therefore sees what the historical writer +// produced; only the compressed interior is a stand-in. +const ROW_20_GZIP_FRAME: &[u8] = &[ + 0x1F, 0x8B, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03, 0x63, 0x60, 0xA0, 0x36, 0x60, 0xA4, + 0xBA, 0x89, 0xB4, 0x02, 0xC3, 0xC8, 0xA5, 0x8C, 0xCA, 0x50, 0x46, 0x89, 0xA1, 0x47, 0x71, 0xB2, + 0xA1, 0x4F, 0x90, 0x45, 0xA5, 0x77, 0x5E, 0x52, 0x52, 0xAA, 0x71, 0x49, 0x79, 0x50, 0x81, 0x85, + 0x45, 0x81, 0x59, 0x99, 0x5B, 0x9A, 0xB3, 0x69, 0x89, 0xB9, 0x8B, 0x4F, 0x52, 0x31, 0x61, 0xD3, + 0x58, 0xA0, 0xB4, 0x83, 0x13, 0x3F, 0x43, 0x83, 0xCF, 0x94, 0x58, 0xA2, 0x9C, 0x38, 0x74, 0x42, + 0x13, 0x37, 0x08, 0x38, 0x4C, 0x3D, 0xB3, 0xE0, 0x01, 0x02, 0x8A, 0x1A, 0x62, 0xA2, 0x85, 0x48, + 0xB0, 0xA0, 0x0D, 0x62, 0x30, 0x28, 0x72, 0x24, 0xCB, 0x56, 0x8A, 0x60, 0x53, 0xD3, 0xB1, 0x06, + 0xD3, 0x39, 0x7E, 0x50, 0x46, 0x55, 0xB1, 0x61, 0x21, 0x35, 0x40, 0x41, 0xA1, 0x59, 0x8A, 0x45, + 0xFA, 0x8A, 0x44, 0x92, 0x02, 0x65, 0x14, 0x0C, 0x57, 0x00, 0x2B, 0x33, 0x72, 0x13, 0x33, 0xF3, + 0x40, 0x49, 0x13, 0x04, 0x00, 0x47, 0xDA, 0xA2, 0x50, 0x5D, 0x06, 0x00, 0x00, +]; + +#[cfg(test)] +mod tests { + use super::*; + + /// The little-endian u64 at offset 0. + fn version_word(bytes: &[u8]) -> u64 { + u64::from_le_bytes(bytes[..8].try_into().expect("fixture has a version word")) + } + + /// Whether `haystack` contains `needle` anywhere. + fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack + .windows(needle.len()) + .any(|window| window == needle) + } + + /// The bytes of one `WalletTx` under `shape`, for length arithmetic. + fn wallet_tx(shape: TransactionShape) -> Vec { + let mut out = Vec::new(); + push_wallet_tx(&mut out, shape); + out + } + + /// CRC-32 as gzip computes it, over the reflected IEEE polynomial + /// `0xEDB88320`, so a test can bind the embedded frame's trailer to the + /// body this module derives. + fn crc32(data: &[u8]) -> u32 { + let mut crc = 0xFFFF_FFFFu32; + for byte in data { + crc ^= u32::from(*byte); + for _ in 0..8 { + let mask = (crc & 1).wrapping_neg(); + crc = (crc >> 1) ^ (0xEDB8_8320 & mask); + } + } + !crc + } + + /// The issue table claims row 1 writes no version word. It does. This test + /// records the finding rather than the claim: both rows open with the + /// 8-byte little-endian 1, `c2e26fbbc` changed no byte, and so the two + /// fixtures are separated only by their wallet contents. + #[test] + fn rows_01_and_02_are_one_grammar_and_both_open_with_version_one() { + let (row_1, row_2) = (row_01(), row_02()); + assert_eq!(version_word(&row_1), 1); + assert_eq!(version_word(&row_2), 1); + assert_eq!(row_1.len(), row_2.len() + 47); + assert_eq!(row_1[8], 1); + assert_eq!(row_2[8], 0); + assert_ne!(row_1, row_2); + } + + /// Row 1's one block reaches disk: the block vector's count is 1, and the + /// block closes with `BlockData::write`'s literal end tag 11. Everything + /// after it is row 2's transaction map. + #[test] + fn row_01_carries_one_block_with_its_end_tag() { + let bytes = row_01(); + assert_eq!(&bytes[9..13], &HEIGHT.to_le_bytes()); + assert_eq!(&bytes[45..48], &[0, 0, 0]); + assert_eq!(&bytes[48..56], &11u64.to_le_bytes()); + assert_eq!(&bytes[56..], &row_02()[9..]); + } + + /// Every row's version word is the one its Defining Commit's + /// `LightWallet::serialized_version()` returns. + #[test] + fn version_words_match_the_defining_commits() { + let expected = [ + (row_01(), 1), + (row_02(), 1), + (row_03(), 1), + (row_04(), 1), + (row_05(), 1), + (row_06(), 1), + (row_07(), 1), + (row_08(), 1), + (row_09(), 1), + (row_10(), 1), + (row_11(), 2), + (row_12(), 2), + (row_13(), 2), + (row_14(), 2), + (row_15(), 3), + (row_16(), 3), + (row_17(), 4), + (row_18(), 4), + (row_19(), 4), + (row_20(), 5), + (row_21(), 6), + ]; + for (index, (bytes, version)) in expected.iter().enumerate() { + assert_eq!(version_word(bytes), *version, "row {}", index + 1); + } + } + + /// Row 3's mark: the transaction record gains its own txid and a shielded + /// total, and the note record gains `is_change`. The wallet's own bytes do + /// not move, so the whole delta is 41 bytes inside the transaction. + #[test] + fn row_03_grows_the_transaction_and_note_records() { + let (row_2, row_3) = (row_02(), row_03()); + assert_eq!(&row_3[..9], &row_2[..9]); + assert_eq!(row_3.len(), row_2.len() + 32 + 8 + 1); + assert_eq!( + wallet_tx(TX_TXID).len(), + wallet_tx(TX_BORN).len() + 32 + 8 + 1 + ); + let inner_txid = 8 + 1 + 32 + 8 + 4; + assert_eq!(&row_3[inner_txid..inner_txid + 32], &[0u8; 32]); + } + + /// Row 4's mark: the raw seed sits immediately after the version word, and + /// the spending-key vector's single 169-byte element follows it. + #[test] + fn row_04_carries_the_seed_then_one_spending_key() { + let (row_3, row_4) = (row_03(), row_04()); + assert_eq!(&row_4[8..40], &SEED); + assert_eq!(row_4[40], 1); + assert_eq!(&row_4[41..210], &EXTENDED_SPENDING_KEY); + assert_eq!(row_4.len(), row_3.len() + 32 + 1 + 169); + } + + /// Row 5's mark: a second value-spent total closes the transaction record, + /// so the file is exactly eight bytes longer than row 4's. + #[test] + fn row_05_appends_the_transparent_value_spent_total() { + let (row_4, row_5) = (row_04(), row_05()); + assert_eq!(row_5.len(), row_4.len() + 8); + assert_eq!( + wallet_tx(TX_TRANSPARENT_TOTAL).len(), + wallet_tx(TX_TXID).len() + 8 + ); + assert_eq!(&row_5[row_5.len() - 16..], &[0u8; 16]); + } + + /// Row 6's mark: a standalone UTXO vector holds one element, whose + /// sub-record version is 1 and whose address is visible as ASCII. + #[test] + fn row_06_carries_one_standalone_utxo() { + let (row_5, row_6) = (row_05(), row_06()); + let count = 8 + 32 + 1 + 169; + assert_eq!(row_6[count], 1); + assert_eq!(&row_6[count + 1..count + 9], &1u64.to_le_bytes()); + assert!(contains(&row_6, TRANSPARENT_ADDRESS.as_bytes())); + assert!(!contains(&row_5, TRANSPARENT_ADDRESS.as_bytes())); + assert_eq!(row_6.len(), row_5.len() + 1 + 127); + } + + /// Row 7's mark: 32 unprefixed transparent-key bytes sit between the + /// spending-key vector and the UTXO vector, making the file exactly 32 + /// bytes longer than row 6's. + #[test] + fn row_07_inserts_a_bare_transparent_key() { + let (row_6, row_7) = (row_06(), row_07()); + let split = 8 + 32 + 1 + 169; + assert_eq!(row_7.len(), row_6.len() + 32); + assert_eq!(&row_7[..split], &row_6[..split]); + assert_eq!(&row_7[split..split + 32], &TRANSPARENT_KEY); + assert_eq!(&row_7[split + 32..], &row_6[split..]); + } + + /// Row 8's mark: the standalone UTXO vector is gone from the wallet and + /// the same 127-byte record reappears inside the transaction, so the file + /// keeps its length while the address moves far to the right. + #[test] + fn row_08_moves_the_utxo_into_the_transaction() { + let (row_7, row_8) = (row_07(), row_08()); + assert_eq!(row_8.len(), row_7.len()); + assert_ne!(row_8, row_7); + assert!(contains(&row_8, TRANSPARENT_ADDRESS.as_bytes())); + let utxo_vector_count = 8 + 32 + 1 + 169 + 32; + assert_eq!(row_7[utxo_vector_count], 1); + assert_eq!(row_8[utxo_vector_count], 0); + assert_eq!( + wallet_tx(TX_INLINE_UTXO).len(), + wallet_tx(TX_TRANSPARENT_TOTAL).len() + 1 + 127 + ); + } + + /// Row 9's mark: the UTXO record loses its trailing unconfirmed-spent + /// `Optional`, so the file is exactly one byte shorter than row 8's. + #[test] + fn row_09_trims_the_utxo_unconfirmed_spend_marker() { + let (row_8, row_9) = (row_08(), row_09()); + assert_eq!(row_9.len(), row_8.len() - 1); + let mut trimmed = Vec::new(); + push_utxo(&mut trimmed, false); + let mut full = Vec::new(); + push_utxo(&mut full, true); + assert_eq!(trimmed.len(), 126); + assert_eq!(full.len(), 127); + + // The trimmed record is a prefix of the untrimmed one, so distinguish + // them by the whole transaction map rather than by the UTXO alone. + let mut map_with_full = Vec::new(); + push_transaction_map(&mut map_with_full, TX_INLINE_UTXO); + let mut map_with_trimmed = Vec::new(); + push_transaction_map(&mut map_with_trimmed, TX_TRIMMED_UTXO); + assert!(contains(&row_8, &map_with_full)); + assert!(contains(&row_9, &map_with_trimmed)); + assert!(!contains(&row_9, &map_with_full)); + } + + /// Row 10's marks: the transaction sub-version moves from 1 to 2 and a + /// populated outgoing-metadata vector closes the record, carrying its + /// Sapling recipient and a 512-byte memo. + #[test] + fn row_10_bumps_the_transaction_and_adds_outgoing_metadata() { + let (row_9, row_10) = (row_09(), row_10()); + let mut metadata = Vec::new(); + push_outgoing_tx_metadata(&mut metadata); + assert_eq!(metadata.len(), 8 + 78 + 8 + 512); + assert!(contains(&row_10, SAPLING_ADDRESS.as_bytes())); + assert!(!contains(&row_9, SAPLING_ADDRESS.as_bytes())); + assert_eq!(row_10.len(), row_9.len() + 1 + metadata.len()); + let tx_version = 8 + 32 + 1 + 169 + 32 + 1 + 1 + 32; + assert_eq!(&row_10[tx_version..tx_version + 8], &2u64.to_le_bytes()); + assert_eq!(&row_9[tx_version..tx_version + 8], &1u64.to_le_bytes()); + } + + /// Row 11's mark: the chain name closes the file in `write_string` + /// discipline, a u64 length followed by ASCII, and everything before it is + /// row 10's under the new version word. + #[test] + fn row_11_appends_the_chain_name_string() { + let (row_10, row_11) = (row_10(), row_11()); + let body = row_11.len() - 12; + assert_eq!(&row_11[body..body + 8], &4u64.to_le_bytes()); + assert_eq!(&row_11[body + 8..], b"main"); + assert_eq!(&row_11[8..body], &row_10[8..]); + } + + /// Row 12's mark: a `full_tx_scanned` byte closes the transaction record + /// while its sub-version stands still at 2, which is why a version-2 file + /// from before this commit desynchronises its reader. + #[test] + fn row_12_appends_full_tx_scanned_without_a_sub_version_bump() { + let (row_11, row_12) = (row_11(), row_12()); + assert_eq!(row_12.len(), row_11.len() + 1); + assert_eq!(TX_SCANNED.version, TX_OUTGOING.version); + assert_eq!( + wallet_tx(TX_SCANNED).len(), + wallet_tx(TX_OUTGOING).len() + 1 + ); + } + + /// Row 13's mark: the birthday closes the file as a u64, and everything + /// before it is row 12's. + #[test] + fn row_13_appends_the_birthday() { + let (row_12, row_13) = (row_12(), row_13()); + assert_eq!(row_13.len(), row_12.len() + 8); + assert_eq!(&row_13[..row_12.len()], &row_12[..]); + assert_eq!(&row_13[row_12.len()..], &BIRTHDAY.to_le_bytes()); + } + + /// Row 14's mark: a CompactSize count of 1 now precedes the transparent + /// key material, which lengthens the file by exactly that one byte. + #[test] + fn row_14_prefixes_the_transparent_keys_with_a_count() { + let (row_13, row_14) = (row_13(), row_14()); + let split = 8 + 32 + 1 + 169; + assert_eq!(row_14.len(), row_13.len() + 1); + assert_eq!(&row_14[..split], &row_13[..split]); + assert_eq!(row_14[split], 1); + assert_eq!(&row_14[split + 1..split + 33], &TRANSPARENT_KEY); + } + + /// Row 15's mark is two version words and nothing else: the wallet's moves + /// from 2 to 3 and the transaction's from 2 to 3, with every other byte + /// unchanged from row 14's. + #[test] + fn row_15_changes_only_the_two_version_words() { + let (row_14, row_15) = (row_14(), row_15()); + assert_eq!(row_15.len(), row_14.len()); + let differing: Vec = (0..row_15.len()) + .filter(|index| row_15[*index] != row_14[*index]) + .collect(); + let tx_version = 8 + 32 + 1 + 169 + 1 + 32 + 1 + 1 + 32; + assert_eq!(differing, vec![0, tx_version]); + assert_eq!(row_15[0], 3); + assert_eq!(row_15[tx_version], 3); + } + + /// Row 16's marks: the transaction sub-version moves from 3 to 4 and a u64 + /// timestamp is inserted between the block height and the txid. + #[test] + fn row_16_inserts_the_transaction_datetime() { + let (row_15, row_16) = (row_15(), row_16()); + assert_eq!(row_16.len(), row_15.len() + 8); + assert_eq!(TX_DATETIME.version, 4); + let tx_version = 8 + 32 + 1 + 169 + 1 + 32 + 1 + 1 + 32; + assert_eq!(&row_16[tx_version..tx_version + 8], &4u64.to_le_bytes()); + let datetime = tx_version + 8 + 4; + assert_eq!(&row_16[datetime..datetime + 8], &DATETIME.to_le_bytes()); + } + + /// Row 17's marks: a `locked` flag directly after the version word, and a + /// full-viewing-key vector after the spending keys. + #[test] + fn row_17_adds_the_locked_flag_and_the_viewing_keys() { + let (row_16, row_17) = (row_16(), row_17()); + assert_eq!(row_17[8], 0); + assert_eq!(&row_17[9..41], &SEED); + let fvk_count = 9 + 32 + 1 + 169; + assert_eq!(row_17[fvk_count], 1); + assert_eq!( + &row_17[fvk_count + 1..fvk_count + 170], + &EXTENDED_FULL_VIEWING_KEY + ); + assert_eq!(row_17.len(), row_16.len() + 1 + 1 + 169); + } + + /// Row 18's mark: a vector of transparent-address strings, whose one + /// element carries its own u64 length inside the CompactSize-counted + /// vector. + #[test] + fn row_18_appends_the_transparent_address_vector() { + let (row_17, row_18) = (row_17(), row_18()); + let mut vector = Vec::new(); + push_transparent_address_vector(&mut vector); + assert_eq!(vector.len(), 44); + assert!(contains(&row_18, &vector)); + assert_eq!(row_18.len(), row_17.len() + 44); + } + + /// Row 19's mark: the 48-byte `enc_seed` and the nonce vector are inserted + /// between the `locked` flag and the plaintext seed. + #[test] + fn row_19_inserts_the_encrypted_seed_and_nonce() { + let (row_18, row_19) = (row_18(), row_19()); + assert_eq!(row_19[8], 0); + assert_eq!(&row_19[9..57], &ENC_SEED); + assert_eq!(row_19[57], 0); + assert_eq!(&row_19[58..90], &SEED); + assert_eq!(row_19.len(), row_18.len() + 49); + } + + /// Row 20's mark: a gzip frame begins at offset 8, immediately after the + /// plaintext version word 5. The table's zstd magic is nowhere in the + /// file. + #[test] + fn row_20_starts_a_gzip_frame_at_offset_eight() { + let bytes = row_20(); + assert_eq!(version_word(&bytes), 5); + assert_eq!(&bytes[8..10], &[0x1F, 0x8B]); + assert_eq!( + &bytes[10..18], + &[0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x03] + ); + assert!(!contains(&bytes, &[0x28, 0xB5, 0x2F, 0xFD])); + assert!(bytes.len() < row_21().len()); + } + + /// The embedded frame really wraps the body this module derives: gzip's + /// trailer records the uncompressed CRC-32 and length, and both match + /// [`encryption_era_body`]. This is what ties the pre-computed constant + /// back to the writer source, since a std-only crate cannot decompress it. + #[test] + fn the_row_20_frame_wraps_the_derived_body() { + let body = encryption_era_body(); + let trailer = &ROW_20_GZIP_FRAME[ROW_20_GZIP_FRAME.len() - 8..]; + assert_eq!( + u32::from_le_bytes(trailer[..4].try_into().expect("four CRC bytes")), + crc32(&body) + ); + assert_eq!( + u32::from_le_bytes(trailer[4..].try_into().expect("four length bytes")), + body.len() as u32 + ); + } + + /// Row 21's mark: the frame is gone and the body is plaintext again. The + /// fixture is row 19's under a different version word, and its body is + /// what row 20's frame decompresses to. + #[test] + fn row_21_restores_the_plaintext_body() { + let bytes = row_21(); + assert_eq!(version_word(&bytes), 6); + assert_ne!(&bytes[8..10], &[0x1F, 0x8B]); + assert_eq!(&bytes[8..], &encryption_era_body()[..]); + assert_eq!(&bytes[8..], &row_19()[8..]); + } + + /// No two rows in this era collapse to the same bytes, which is the + /// corpus-wide distinctness claim restricted to the rows this module owns. + #[test] + fn all_rows_are_pairwise_distinct() { + let fixtures = fixtures(); + for (index, a) in fixtures.iter().enumerate() { + for b in &fixtures[index + 1..] { + assert_ne!( + a.bytes, b.bytes, + "rows {} and {} produced identical bytes", + a.row, b.row + ); + } + } + } + + /// The era contributes rows 1 through 21 of the 77-row table, in order, + /// each tagged with its Defining Commit and with `dev`. + #[test] + fn the_era_covers_rows_one_through_twenty_one() { + let fixtures = fixtures(); + let rows: Vec = fixtures.iter().map(|f| f.row).collect(); + assert_eq!(rows, (1..=21).collect::>()); + assert!(fixtures.iter().all(|f| f.branch == "dev")); + assert!(fixtures.iter().all(|f| f.defining_commit.len() == 9)); + } + + /// Prints the exact bytes row 20 compresses, so that the embedded frame + /// can be regenerated by the pipeline documented on [`ROW_20_GZIP_FRAME`]. + /// It is ignored by default because it asserts nothing. + #[test] + #[ignore = "regeneration aid: prints the row 20 body for the gzip pipeline"] + fn dump_row_20_body() { + let hex: String = encryption_era_body() + .iter() + .map(|byte| format!("{byte:02x}")) + .collect(); + println!("ROW20BODY {hex}"); + } +} diff --git a/tools/workbench/src/wallet_grammars/era_migration.rs b/tools/workbench/src/wallet_grammars/era_migration.rs new file mode 100644 index 0000000000..481b156b8a --- /dev/null +++ b/tools/workbench/src/wallet_grammars/era_migration.rs @@ -0,0 +1,636 @@ +//! Census rows 72 through 77: the version-42 era and the one-day version 43. +//! +//! Six Defining Commits share one wallet-file skeleton. Five of them write the +//! version word 42 and one writes 43, so the version word alone identifies +//! almost nothing here; the grammar deltas live in a single settings byte and +//! in the migration section's private `INNER_VERSION`, which climbed 1 → 2 → +//! 3 → 4 in twelve days while `LightWallet::serialized_version()` never moved. +//! +//! The skeleton is `LightWallet::write` in `zingolib/src/wallet/disk.rs`. Its +//! sub-writers in this era are `pepper_sync::wallet::serialization` +//! (`NullifierMap`, `ScanTarget`, `ShardTrees`, `SyncState`), +//! `pepper_sync::config` (`SyncConfig`, `PerformanceLevel`), +//! `zingo_price::PriceList`, the `ReadableWriteable` implementations for +//! `UnifiedKeyStore` and `ReceiverSelection` in +//! `zingolib/src/wallet/keys/unified.rs`, and +//! `zingolib/src/wallet/migration/store.rs`. Every one of those files except +//! `store.rs` is byte-identical across all six commits, which is exactly why +//! this era's rows are so hard to tell apart: the diff from `fffcc9e02` to +//! `f48b15c9e` touches no write path outside `disk.rs` and `store.rs`. +//! +//! Every fixture here writes the migration section as `Some` with one part, +//! because that section is where five of the six rows differ. Every fixture +//! also carries one outpoint-map entry, whose output index is a u32 in this +//! era, continuing the width the preceding era's row 71 established. +//! +//! The migration parameter *values* are a single synthetic set held constant +//! across all six fixtures. The values are not part of the grammar — only the +//! field list is — and holding them fixed makes the byte differences between +//! rows isolate the grammar deltas rather than the drifting provisional ZIP +//! 318 constants. The real `MigrationParams::provisional` moved twice inside +//! this era, changing both its denomination ladder and its bucket modulus, +//! without altering a single field width. + +use super::util::{ + push_bytes, push_compact_size, push_optional_none, push_optional_some, push_u32_le, + push_u64_le, push_u8, +}; +use super::Fixture; + +/// The wallet's birthday, a plausible mainnet height for this era. +const BIRTHDAY: u32 = 3_000_000; + +/// The transaction whose output the outpoint map points at. +const OUTPOINT_TXID: [u8; 32] = [0x11; 32]; + +/// The transaction the outpoint map's `ScanTarget` names. +const SCAN_TARGET_TXID: [u8; 32] = [0x22; 32]; + +/// The transaction holding the migration part's bound note. +const BOUND_NOTE_TXID: [u8; 32] = [0x33; 32]; + +/// The bound note's nullifier. +const BOUND_NOTE_NULLIFIER: [u8; 32] = [0x44; 32]; + +/// The bound note's commitment. +const BOUND_NOTE_COMMITMENT: [u8; 32] = [0x55; 32]; + +/// The height at which the outpoint map's scan target sits. +const SCAN_TARGET_HEIGHT: u32 = 3_012_345; + +/// The migration part's bucket, a boundary height divided by the modulus. +const BUCKET_INDEX: u64 = 20_920; + +/// The migration part's anchor bucket, one bucket earlier than its own. +const ANCHOR_BUCKET: u64 = 20_919; + +/// The boundary height the migration part targets, `BUCKET_INDEX` times 144. +const TARGET_HEIGHT: u32 = 3_012_480; + +/// The migration part's expiry, the target height plus the era's provisional +/// delta of one bucket length and the standard forty-block margin. +const EXPIRY_HEIGHT: u32 = 3_012_664; + +/// The moment the user consented to the migration plan, as a Unix timestamp. +const CONSENTED_AT: u64 = 1_752_000_000; + +/// This era's fixtures, in census order. +pub fn fixtures() -> Vec { + vec![ + Fixture { + row: 72, + defining_commit: "fffcc9e02", + branch: "dev", + bytes: row_72(), + }, + Fixture { + row: 73, + defining_commit: "32261bb5f", + branch: "dev", + bytes: row_73(), + }, + Fixture { + row: 74, + defining_commit: "4158e20c2", + branch: "dev", + bytes: row_74(), + }, + Fixture { + row: 75, + defining_commit: "a6c1354ad", + branch: "dev", + bytes: row_75(), + }, + Fixture { + row: 76, + defining_commit: "894fe8e0a", + branch: "dev", + bytes: row_76(), + }, + Fixture { + row: 77, + defining_commit: "f48b15c9e", + branch: "dev", + bytes: row_77(), + }, + ] +} + +/// Row 53, version 42 layout A, defined by merge `fffcc9e02` (PR #2428, +/// authored by `643e5eea8` and `bf3ebdd19`). This replicates +/// `LightWallet::write` in `zingolib/src/wallet/disk.rs` at that commit +/// together with `crate::wallet::migration::store::write` and its `write_part` +/// helper, whose `INNER_VERSION` is 1. +/// +/// The grammar-unique mark is the `allow_v6_transactions` u8 written between +/// `min_confirmations` and the price list. No other grammar in the census +/// carries that byte: the setting was removed one day later and never +/// restored. +/// +/// The wallet is a mainnet wallet holding a zeroed mnemonic seed, one account +/// whose key store is the `Empty` variant, no addresses, no blocks and no +/// transactions, one scanned range, one outpoint-map entry, empty shard trees, +/// and a migration in the `PartsScheduled` phase holding one assigned part. +/// Every other fixture in this era carries the same contents, so the six files +/// differ only where their grammars do. +fn row_72() -> Vec { + let mut out = Vec::new(); + push_prefix(&mut out, 42, true); + push_optional_some(&mut out); + push_migration_section(&mut out, 1); + out +} + +/// Row 54, version 43, defined by `32261bb5f` ("feat!: remove the dead +/// allow_v6_transactions setting"). This replicates the same +/// `LightWallet::write` and the same `migration/store.rs` writer at +/// `INNER_VERSION` 1. The commit deleted one `write_u8` call from `disk.rs` +/// and raised `serialized_version()` from 42 to 43. +/// +/// The grammar-unique mark is the version word 43, which dev wrote for roughly +/// one day on 2026-07-14 before `4158e20c2` renumbered it back. The wallet's +/// contents match row 72's exactly, so the two fixtures differ only where +/// their grammars do. +fn row_73() -> Vec { + let mut out = Vec::new(); + push_prefix(&mut out, 43, false); + push_optional_some(&mut out); + push_migration_section(&mut out, 1); + out +} + +/// Row 55, version 42 reused, defined by `4158e20c2` ("fix: keep the wallet +/// file at version 42; the removed byte never shipped"). The commit changed +/// `serialized_version()` from 43 back to 42 and dropped the reader's +/// consume-and-discard branch for the removed byte. The written grammar is +/// otherwise identical to row 73's, so these two fixtures differ at the +/// version word alone. +/// +/// Against row 72 the difference is the absent `allow_v6_transactions` byte, +/// one byte in the middle of the settings section, with the same version word +/// on both files. Telling those two apart requires parsing the tail, which is +/// what the dual-parse reader introduced at `a6c1354ad` does by anchoring the +/// price list and the migration section at end of file. +fn row_74() -> Vec { + let mut out = Vec::new(); + push_prefix(&mut out, 42, false); + push_optional_some(&mut out); + push_migration_section(&mut out, 1); + out +} + +/// Row 56, version 42, defined by merge `a6c1354ad` (authored by `a2f4e2f08`). +/// This replicates `migration/store.rs::write` at `INNER_VERSION` 2, which +/// appends a `MigrationMode` u8 after the parts vector: 0 for `Scheduled`, 1 +/// for `Immediate`. Nothing in `disk.rs` changed, so the file differs from row +/// 55 by the inner version byte and one trailing byte inside the migration +/// section. The fixture writes the `Scheduled` mode, the reading that same +/// commit's reader assigns to any version-1 blob. +fn row_75() -> Vec { + let mut out = Vec::new(); + push_prefix(&mut out, 42, false); + push_optional_some(&mut out); + push_migration_section(&mut out, 2); + out +} + +/// Row 57, version 42, defined by `894fe8e0a` ("chore: align"). This +/// replicates `migration/store.rs::write` at `INNER_VERSION` 3, which removed +/// the params' `expiry_delta` u32: the canonical expiry became the fixed ZIP +/// 318 formula rather than a stored parameter. The neighbouring rename of +/// `dust_floor` to `max_residual_value` keeps the same u64 at the same offset +/// and so writes no different byte. The params record is therefore exactly +/// four bytes shorter than row 75's. +fn row_76() -> Vec { + let mut out = Vec::new(); + push_prefix(&mut out, 42, false); + push_optional_some(&mut out); + push_migration_section(&mut out, 3); + out +} + +/// Row 58, version 42, defined by `f48b15c9e` ("chore: mobile initial batch +/// fix"). This replicates `migration/store.rs::write_part` at `INNER_VERSION` +/// 4, which appends an `Optional` `anchor_bucket` after each part's +/// `bucket_index`: the anchor became a bucket of its own rather than the +/// broadcast window's boundary (ADR 0018). This is the grammar dev writes +/// today. The fixture's part carries an anchor one bucket earlier than the +/// part's own, the arrangement the change exists to allow. +fn row_77() -> Vec { + let mut out = Vec::new(); + push_prefix(&mut out, 42, false); + push_optional_some(&mut out); + push_migration_section(&mut out, 4); + out +} + +/// Everything `LightWallet::write` emits before the optional migration +/// section: the header, the key material, the sync structures, the settings +/// and the price list. `allow_v6_transactions` says whether to emit the u8 +/// that only row 72's grammar carries. +fn push_prefix(out: &mut Vec, version_word: u64, allow_v6_transactions: bool) { + push_header_through_min_confirmations(out, version_word); + if allow_v6_transactions { + // `writer.write_u8(u8::from(self.wallet_settings.allow_v6_transactions))`. + // The setting is false, the value a fresh wallet held. + push_u8(out, 0); + } + push_price_list(out); +} + +/// The wallet file from its version word through `min_confirmations`, the last +/// field before the byte that distinguishes row 72. +fn push_header_through_min_confirmations(out: &mut Vec, version_word: u64) { + push_u64_le(out, version_word); + + // The chain type as a u8, the encoding minted at row 69: 0 mainnet, + // 1 testnet, 2 regtest. + push_u8(out, 0); + + // The mnemonic entropy as a `Vector`. Thirty-two zero bytes stand in + // for a twenty-four-word seed; key material in this corpus is zeroed. + push_compact_size(out, 32); + push_bytes(out, &[0u8; 32]); + + // The birthday as a u32, the width in force since row 56. + push_u32_le(out, BIRTHDAY); + + // `Vector<(account u32, UnifiedKeyStore)>`. One account, whose key store + // is the `Empty` variant: a version byte of 0 followed by the key-type + // discriminant `KEY_TYPE_EMPTY`, also 0. Writing the spend or view variant + // would embed a `UnifiedSpendingKey` or `UnifiedFullViewingKey`, neither + // of which this era changed. + push_compact_size(out, 1); + push_u32_le(out, 0); + push_u8(out, 0); + push_u8(out, 0); + + // `Vector<(account u32, address index u32, ReceiverSelection)>`, the + // unified addresses, and `Vector<(account u32, scope u8, address index + // u32)>`, the transparent addresses. Both empty: neither is needed to + // exhibit this era's marks, and the reader regenerates each stored + // address from its account's key store, which the `Empty` variant cannot + // do. Leaving them empty keeps every fixture in this era loadable, not + // merely well-formed. + push_compact_size(out, 0); + push_compact_size(out, 0); + + // `Vector` and `Vector`, both empty. + push_compact_size(out, 0); + push_compact_size(out, 0); + + push_nullifier_map(out); + push_outpoint_map(out); + push_shard_trees(out); + push_sync_state(out); + push_sync_config(out); + + // `min_confirmations`, a `NonZeroU32` written as a u32. Three is the value + // the reader substitutes for files predating the field. + push_u32_le(out, 3); +} + +/// `NullifierMap::write` in `pepper-sync/src/wallet/serialization.rs`. Its +/// version is 2 throughout this era, the value that added the Ironwood map +/// beside the Sapling and Orchard ones. All three maps are empty. +fn push_nullifier_map(out: &mut Vec) { + push_u8(out, 2); + push_compact_size(out, 0); + push_compact_size(out, 0); + push_compact_size(out, 0); +} + +/// The outpoint map, written inline by `LightWallet::write` as a `Vector` of +/// `(txid, output index u32, ScanTarget)`. The output index is a u32 here, the +/// width row 71 widened it to; `ScanTarget::write` contributes its own version +/// byte, 0 throughout this era. +fn push_outpoint_map(out: &mut Vec) { + push_compact_size(out, 1); + push_bytes(out, &OUTPOINT_TXID); + push_u32_le(out, 1); + + push_u8(out, 0); + push_u32_le(out, SCAN_TARGET_HEIGHT); + push_bytes(out, &SCAN_TARGET_TXID); + push_u8(out, 1); +} + +/// `ShardTrees::write`, whose version is 1 throughout this era. It writes the +/// Sapling, Orchard and Ironwood trees in turn, each as a vector of shards, a +/// vector of checkpoints, and a cap. All three trees are empty. +fn push_shard_trees(out: &mut Vec) { + push_u8(out, 1); + for _ in 0..3 { + push_compact_size(out, 0); + push_compact_size(out, 0); + push_empty_shard(out); + } +} + +/// The cap of an empty shard tree, written by +/// `zcash_client_backend::serialization::shardtree::write_shard`. +fn push_empty_shard(out: &mut Vec) { + // ASSUMPTION: `write_shard` emits its `SER_V1` tag of 1 and then walks the + // tree, emitting `NIL_TAG` of 0 for the empty node, so an empty cap is the + // two bytes `01 00`. Read from zcash_client_backend 0.23.0's + // `src/serialization/shardtree.rs`; this era's Cargo.lock pins that + // version from a librustzcash git revision rather than from crates.io, and + // the git tree was not consulted. + push_u8(out, 1); + push_u8(out, 0); +} + +/// `SyncState::write`, whose version is 4 throughout this era. One scanned +/// range covers the wallet's history; the Sapling, Orchard and Ironwood shard +/// ranges and the scan targets are all empty. +fn push_sync_state(out: &mut Vec) { + push_u8(out, 4); + + // `Vector<(start u32, end u32, priority u8)>`. Under version 4 the + // priority discriminants run RefetchingNullifiers 0, Scanning 1, Scanned + // 2, ScannedWithoutMapping 3, Historic 4, OpenAdjacent 5, FoundNote 6, + // ChainTip 7, Verify 8. This range is Scanned. + push_compact_size(out, 1); + push_u32_le(out, BIRTHDAY); + push_u32_le(out, SCAN_TARGET_HEIGHT + 1); + push_u8(out, 2); + + push_compact_size(out, 0); + push_compact_size(out, 0); + push_compact_size(out, 0); + push_compact_size(out, 0); +} + +/// `SyncConfig::write` in `pepper-sync/src/config.rs`, whose version is 1 +/// throughout this era. It writes the gap limit, a scope bitmask with external +/// at bit 0, internal at bit 1 and refund at bit 2, and then a +/// `PerformanceLevel` record carrying its own version byte of 0. +fn push_sync_config(out: &mut Vec) { + push_u8(out, 1); + push_u8(out, 10); + push_u8(out, 0b101); + push_u8(out, 0); + push_u8(out, 2); +} + +/// `PriceList::write` in `zingo-price/src/lib.rs`, whose version is 0 +/// throughout this era. The wallet has never fetched a price, so both +/// optionals are absent and the daily-price vector is empty. +fn push_price_list(out: &mut Vec) { + push_u8(out, 0); + push_optional_none(out); + push_optional_none(out); + push_compact_size(out, 0); +} + +/// The migration section, `zingolib/src/wallet/migration/store.rs::write`, at +/// the given `INNER_VERSION`. The four versions in this era share one field +/// order and differ in three places: version 2 appends the `MigrationMode` +/// byte after the parts vector, version 3 removes the params' `expiry_delta` +/// u32, and version 4 appends each part's `anchor_bucket`. +fn push_migration_section(out: &mut Vec, inner_version: u8) { + push_u8(out, inner_version); + + // The params record. Its values are this module's synthetic set, held + // constant across the era so the fixtures differ only where the grammars + // do. + push_u32_le(out, 1); + push_compact_size(out, 2); + push_u64_le(out, 100_000_000); + push_u64_le(out, 10_000_000); + push_u64_le(out, 100_000_000); + // `dust_floor` through version 2, renamed `max_residual_value` at version + // 3. The rename left the u64 at the same offset, so it writes no different + // byte. + push_u64_le(out, 10_000_000); + push_u64_le(out, 10_000); + push_u32_le(out, 144); + push_u32_le(out, 8); + push_u32_le(out, 6); + // `max_actions_per_split_tx`, a usize widened to u64 on the wire. + push_u64_le(out, 32); + if inner_version <= 2 { + // `expiry_delta`, removed at version 3. + push_u32_le(out, 184); + } + push_u64_le(out, 10_000); + + // The consent binding: two raw thirty-two byte hashes and a timestamp. The + // hashes are zeroed like the rest of this corpus's fixed-width material. + push_bytes(out, &[0u8; 32]); + push_bytes(out, &[0u8; 32]); + push_u64_le(out, CONSENTED_AT); + + // The signing strategy: 0 for LazyAtBoundary, 1 for PreSigned. + push_u8(out, 0); + + // The account index. + push_u32_le(out, 0); + + // The phase: 0 Planned, 1 NoteSplitting (with a round and a txid vector), + // 2 PartsScheduled, 3 Complete (with a residual). PartsScheduled carries + // no payload and matches a wallet holding a scheduled part. + push_u8(out, 2); + + // `Vector`, written by `write_part`. + push_compact_size(out, 1); + push_part(out, inner_version); + + if inner_version >= 2 { + // The `MigrationMode`: 0 Scheduled, 1 Immediate. Version 1 predates + // the byte, and its reader defaults such a blob to Scheduled. + push_u8(out, 0); + } +} + +/// One `PartRecord`, written by `write_part` in `migration/store.rs`. The part +/// is bound to a note, placed in a bucket, given a target height and an expiry, +/// and assigned but not yet signed, so it carries neither a signed blob nor a +/// transaction id nor a boundary witness. +fn push_part(out: &mut Vec, inner_version: u8) { + push_u32_le(out, 0); + push_u64_le(out, 100_000_000); + + // `Optional`: the note's txid and u32 output index, then its + // nullifier and commitment as raw thirty-two byte fields. + push_optional_some(out); + push_bytes(out, &BOUND_NOTE_TXID); + push_u32_le(out, 0); + push_bytes(out, &BOUND_NOTE_NULLIFIER); + push_bytes(out, &BOUND_NOTE_COMMITMENT); + + // `Optional` bucket index. + push_optional_some(out); + push_u64_le(out, BUCKET_INDEX); + + if inner_version >= 4 { + // `Optional` anchor bucket, appended at version 4. + push_optional_some(out); + push_u64_le(out, ANCHOR_BUCKET); + } + + // `Optional` target height. + push_optional_some(out); + push_u32_le(out, TARGET_HEIGHT); + + // The part state: 0 Bound, 1 Assigned, 2 Signed, 3 Broadcast, 4 Confirmed + // (with a u32 height), 5 Expired, 6 Invalidated. + push_u8(out, 1); + + // `Optional`, absent: nothing has been broadcast. + push_optional_none(out); + + // `Optional` expiry height. + push_optional_some(out); + push_u32_le(out, EXPIRY_HEIGHT); + + // `Optional>` signed blob, absent under the lazy strategy. + push_optional_none(out); + + // `Optional`, absent for the same reason. + push_optional_none(out); + + // The attempt counter. + push_u8(out, 0); +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The offset at which row 72 writes its `allow_v6_transactions` byte, + /// derived by writing the file up to that point. + fn allow_v6_offset() -> usize { + let mut out = Vec::new(); + push_header_through_min_confirmations(&mut out, 42); + out.len() + } + + /// The offset at which a row writes the migration section's inner version + /// byte: everything before the section, plus the `Optional` Some marker. + fn inner_version_offset(allow_v6_transactions: bool) -> usize { + let mut out = Vec::new(); + push_prefix(&mut out, 42, allow_v6_transactions); + out.len() + 1 + } + + /// The version words the six writers emit. Five rows share 42 and one + /// writes the 43 that dev carried for a day, so the word identifies a row + /// only in that single case. + #[test] + fn version_words_run_forty_two_forty_three_then_forty_two() { + let words: Vec = fixtures() + .iter() + .map(|fixture| { + let mut word = [0u8; 8]; + word.copy_from_slice(&fixture.bytes[..8]); + u64::from_le_bytes(word) + }) + .collect(); + assert_eq!(words, vec![42, 43, 42, 42, 42, 42]); + } + + /// Rows 54 and 55 are the census's purest version-word-only pair: + /// `4158e20c2` changed the literal 43 back to 42 and nothing else in the + /// writer. + #[test] + fn rows_73_and_74_differ_only_in_the_version_word() { + let earlier = row_73(); + let later = row_74(); + assert_eq!(earlier.len(), later.len()); + assert_eq!(earlier[8..], later[8..]); + assert_ne!(earlier[..8], later[..8]); + } + + /// Row 53's grammar-unique mark is one `allow_v6_transactions` byte between + /// `min_confirmations` and the price list. Removing it turns row 72 into + /// row 74 exactly, which is what leaves the two version-42 layouts + /// distinguishable only by parsing the tail. + #[test] + fn row_72_carries_the_allow_v6_byte_and_row_74_does_not() { + let layout_a = row_72(); + let reused = row_74(); + let offset = allow_v6_offset(); + + assert_eq!(layout_a.len(), reused.len() + 1); + assert_eq!(layout_a[offset], 0); + + let mut stripped = layout_a.clone(); + stripped.remove(offset); + assert_eq!(stripped, reused); + } + + /// The migration section's private `INNER_VERSION` is this era's real + /// discriminator: it moved 1 → 2 → 3 → 4 while the wallet version word + /// stayed at 42. + #[test] + fn migration_inner_versions_climb_one_two_three_four() { + assert_eq!(row_72()[inner_version_offset(true)], 1); + + let rows = [ + (row_73(), 1u8), + (row_74(), 1), + (row_75(), 2), + (row_76(), 3), + (row_77(), 4), + ]; + for (bytes, inner_version) in rows { + assert_eq!(bytes[inner_version_offset(false)], inner_version); + } + } + + /// Version 3 removed the params' `expiry_delta` u32 and added nothing, so + /// row 76 is exactly four bytes shorter than row 75. + #[test] + fn dropping_expiry_delta_shortens_the_params_record_by_four() { + assert_eq!(row_75().len(), row_76().len() + 4); + } + + /// Version 4 appended one present `Optional` to the single part, so + /// row 77 is nine bytes longer than row 76. + #[test] + fn adding_an_anchor_bucket_lengthens_each_part_by_nine() { + assert_eq!(row_77().len(), row_76().len() + 9); + } + + /// Neighbouring rows are distinct files. The census claims every row is a + /// grammar of its own, and adjacent rows are the hardest pairs. + #[test] + fn adjacent_rows_are_byte_distinct() { + let fixtures = fixtures(); + for pair in fixtures.windows(2) { + assert_ne!( + pair[0].bytes, pair[1].bytes, + "rows {} and {} produced identical bytes", + pair[0].row, pair[1].row + ); + } + } + + /// The era covers census rows 72 through 77, every one of them minted on + /// dev's first-parent line. + #[test] + fn fixtures_carry_their_census_identities() { + let fixtures = fixtures(); + let rows: Vec = fixtures.iter().map(|fixture| fixture.row).collect(); + assert_eq!(rows, vec![72, 73, 74, 75, 76, 77]); + + let commits: Vec<&str> = fixtures + .iter() + .map(|fixture| fixture.defining_commit) + .collect(); + assert_eq!( + commits, + vec![ + "fffcc9e02", + "32261bb5f", + "4158e20c2", + "a6c1354ad", + "894fe8e0a", + "f48b15c9e" + ] + ); + + assert!(fixtures.iter().all(|fixture| fixture.branch == "dev")); + } +} diff --git a/tools/workbench/src/wallet_grammars/era_syncstate.rs b/tools/workbench/src/wallet_grammars/era_syncstate.rs new file mode 100644 index 0000000000..b390201277 --- /dev/null +++ b/tools/workbench/src/wallet_grammars/era_syncstate.rs @@ -0,0 +1,599 @@ +//! Census rows 66 through 68: the sync-state era, where the wallet-file +//! version word stands still at 39 while three sub-records move underneath it. +//! +//! Every grammar in this era shares one top-level layout, replicated from +//! `LightWallet::write` in `zingolib/src/wallet/disk.rs` at each Defining +//! Commit. The three rows differ only at four byte positions, all of them +//! inside records the top-level writer delegates to. +//! +//! Row 66 (`b1c04e38c`) writes the `SyncState` inner version byte 2 and, with +//! it, the `ScanPriority` numbering that inserts `ScannedWithoutMapping` at 2. +//! Row 67 (`ff7ba3ec0`) writes the `SyncState` inner version byte 3, the +//! `ScanPriority` numbering that puts `RefetchingNullifiers` at 0, and a +//! `WalletNote` inner version byte 1 whose record gains a trailing +//! refetch-nullifier `Vector<(u32, u32)>`. Row 68 (`f86717800`) keeps row 67's +//! sync and note encodings unchanged and renumbers `ConfirmationStatus` +//! instead, bumping that record's own inner version byte from 0 to 1 so that +//! the status byte 4, `Failed`, becomes writable. +//! +//! Because the marks live in delegated records rather than in the top-level +//! layout, all three fixtures carry the same wallet contents: one unified key +//! store, one unified address, one transparent address, one outpoint-map entry +//! at this era's u16 output-index width, one written transaction record holding +//! one sapling note, and a `SyncState` holding one scan range. Keeping the +//! shape fixed is deliberate. It leaves the era's four moving bytes as the only +//! differences between neighbouring fixtures, which is exactly what a +//! Discriminator must key on. + +use super::util::{ + push_bytes, push_compact_size, push_optional_none, push_optional_some, push_u16_le, + push_u32_le, push_u64_le, push_u64_string, push_u8, +}; +use super::Fixture; + +/// The wallet-file version word every row in this era writes +/// (`LightWallet::serialized_version`). +const WALLET_VERSION: u64 = 39; + +/// The chain name, written by `utils::write_string` in the u64-length +/// discipline. `ChainType::Mainnet` displays as `main`. +const CHAIN_NAME: &str = "main"; + +/// The wallet's birthday height, chosen below the transaction record's height +/// so that the two agree. +const BIRTHDAY: u32 = 1_900_000; + +/// The height carried by the written transaction's `ConfirmationStatus`. It +/// sits inside mainnet's NU5 range, between that upgrade's activation at +/// 1_687_104 and NU6's at 2_726_400, so the consensus branch id embedded in the +/// transaction body below is unambiguous. +const TRANSACTION_HEIGHT: u32 = 2_000_000; + +/// The transaction the wallet holds a record of. +const TXID_TRANSACTION: [u8; 32] = [0xA1; 32]; + +/// The transaction the outpoint map's scan target points at. +const TXID_SCAN_TARGET: [u8; 32] = [0xB2; 32]; + +/// The four bytes that separate this era's three grammars, plus the trailing +/// vector one of them appends. Every other byte of the three fixtures is +/// shared. +struct Grammar { + /// `SyncState::serialized_version` at the Defining Commit. + sync_state_version: u8, + /// The `ScanPriority` discriminant written for the fixture's one scan + /// range. The value names a variant whose number the row moved. + scan_priority: u8, + /// `WalletNote::serialized_version` at the Defining Commit. + wallet_note_version: u8, + /// Whether the note writer appends the refetch-nullifier ranges vector, + /// which arrived with `WalletNote` version 1. + writes_refetch_nullifier_ranges: bool, + /// `ConfirmationStatus::serialized_version` at the Defining Commit. + confirmation_status_version: u8, + /// The `ConfirmationStatus` discriminant written for the fixture's one + /// transaction record. + confirmation_status: u8, +} + +/// Census row 66's grammar. +const GRAMMAR_66: Grammar = Grammar { + sync_state_version: 2, + scan_priority: 2, + wallet_note_version: 0, + writes_refetch_nullifier_ranges: false, + confirmation_status_version: 0, + confirmation_status: 3, +}; + +/// Census row 67's grammar. +const GRAMMAR_67: Grammar = Grammar { + sync_state_version: 3, + scan_priority: 0, + wallet_note_version: 1, + writes_refetch_nullifier_ranges: true, + confirmation_status_version: 0, + confirmation_status: 3, +}; + +/// Census row 68's grammar. +const GRAMMAR_68: Grammar = Grammar { + sync_state_version: 3, + scan_priority: 0, + wallet_note_version: 1, + writes_refetch_nullifier_ranges: true, + confirmation_status_version: 1, + confirmation_status: 4, +}; + +/// A built fixture together with the offsets of the bytes the tests inspect. +/// Recording the offsets while writing keeps the tests from restating the +/// layout arithmetic, which would only drift from the writer. The offsets +/// serve the tests alone, so a fixture build outside `cfg(test)` leaves them +/// unread. +#[cfg_attr(not(test), allow(dead_code))] +struct Built { + bytes: Vec, + /// Offset of the `ConfirmationStatus` record's inner version byte. The + /// status discriminant follows immediately. + confirmation_status_offset: usize, + /// Offset of the `SyncState` record's inner version byte. + sync_state_version_offset: usize, +} + +/// Census rows 66 through 68, in order. +pub fn fixtures() -> Vec { + vec![ + Fixture { + row: 66, + defining_commit: "b1c04e38c", + branch: "dev", + bytes: row_66(), + }, + Fixture { + row: 67, + defining_commit: "ff7ba3ec0", + branch: "dev", + bytes: row_67(), + }, + Fixture { + row: 68, + defining_commit: "f86717800", + branch: "dev", + bytes: row_68(), + }, + ] +} + +/// Row 66, Defining Commit `b1c04e38c` (the merge of PR #1837, +/// `low_memory_nonlinear_scanning`, whose grammar arrived through `ecbb35574` +/// and `1cd5c9221`). +/// +/// The top-level layout replicates `LightWallet::write` in +/// `zingolib/src/wallet/disk.rs` at that commit. The delegated records come +/// from `pepper-sync/src/wallet/serialization.rs` (`SyncState`, `ScanTarget`, +/// `NullifierMap`, `WalletTransaction`, `SaplingNote`, `ShardTrees`), +/// `pepper-sync/src/config.rs` (`SyncConfig`, `PerformanceLevel`), +/// `zingo-status/src/confirmation_status.rs` (`ConfirmationStatus`), +/// `zingolib/src/wallet/keys/unified.rs` (`UnifiedKeyStore`, +/// `UnifiedSpendingKey`, `ReceiverSelection`), and `zingo-price/src/lib.rs` +/// (`PriceList`). +/// +/// The row's mark is the `SyncState` inner version byte 2. Under that version +/// the reader maps the scan-priority byte 2 to `ScannedWithoutMapping`, a +/// variant this commit inserted, so the fixture's one scan range carries that +/// byte and the renumbering is visible rather than merely declared. Row 67 +/// writes the same version field as 3, so the two rows are always structurally +/// distinct. +/// +/// The wallet holds a mnemonic seed, one spending key store, one unified +/// address, one transparent address, one outpoint-map entry, and one +/// transaction record containing one sapling note. The note's `WalletNote` +/// inner version is 0 here and its record ends after the optional spending +/// transaction, with no refetch-nullifier vector. +fn row_66() -> Vec { + build(&GRAMMAR_66).bytes +} + +/// Row 67, Defining Commit `ff7ba3ec0` (the merge of PR #2156, +/// `backport_stable`, whose grammar arrived through `7dc66315c`). +/// +/// The sources replicated are the same files at this commit, and the top-level +/// `LightWallet::write` body is byte-identical to row 66's. Two delegated +/// records moved. `SyncState::serialized_version` became 3, and under that +/// version the scan-priority numbering shifts by one to make room for +/// `RefetchingNullifiers` at 0; the fixture's one scan range carries the byte 0 +/// so the new variant appears in the file. `WalletNote::serialized_version` +/// became 1, and both note writers now append a refetch-nullifier +/// `Vector>` after the optional spending transaction. The +/// writer emits that vector unconditionally, so the fixture gives it one entry, +/// which distinguishes a populated vector from an absent one as well as from an +/// empty one. +/// +/// This fixture is one half of a deliberate pair. Its written transaction +/// record carries a `ConfirmationStatus` whose status byte is 3, a value legal +/// under both this row's grammar and row 68's, and whose record version byte is +/// 0. Row 68 writes the same record with version byte 1 and status byte 4. The +/// pairing makes the census's design question concrete: under this row's +/// numbering the byte 3 means `Confirmed`, while under row 68's it means +/// `Calculated`, so a reader that ignores the record's inner version byte reads +/// this file with the wrong meaning. See `row_68` for the other half. +fn row_67() -> Vec { + build(&GRAMMAR_67).bytes +} + +/// Row 68, Defining Commit `f86717800` (the merge of PR #2181, +/// `rework_resend_and_remove_tx`; a dev-line commit the dev walk's file set +/// missed, inserted into the census from the stable-arm sweep). +/// +/// The top-level `LightWallet::write` body and every sync record are unchanged +/// from row 67. The single moving record is `ConfirmationStatus` in +/// `zingo-status/src/confirmation_status.rs`. Its writer gained a `Failed` +/// variant, renumbered the discriminants so that `Confirmed` is 0, `Mempool` is +/// 1, `Transmitted` is 2, `Calculated` is 3 and `Failed` is 4, and bumped the +/// record's own `serialized_version` from 0 to 1. +/// +/// This fixture is the other half of the pair described on `row_67`. Its +/// transaction record carries the status byte 4, `Failed`, which only this +/// row's grammar can write, so the file is unambiguously row 68. Row 67's +/// fixture, by contrast, carries the status byte 3, a value both grammars +/// accept, and therefore parses byte for byte under this row's grammar as well. +/// The record's inner version byte is what resolves the two: this row's reader +/// dispatches on it and so still reads row 67's file correctly, whereas row +/// 67's reader ignores it and would read this row's status bytes with row 67's +/// meanings. The census records the row as fully discriminable by that version +/// byte, and these two fixtures pin the asymmetry that makes it so. +fn row_68() -> Vec { + build(&GRAMMAR_68).bytes +} + +/// Write one complete wallet file in this era's shared layout, varying only the +/// bytes named by `grammar`. The order below follows `LightWallet::write` at +/// each Defining Commit statement for statement. +fn build(grammar: &Grammar) -> Built { + let mut out = Vec::new(); + + push_u64_le(&mut out, WALLET_VERSION); + push_u64_string(&mut out, CHAIN_NAME); + + // The mnemonic entropy, written as a `Vector` of bytes. Thirty-two bytes of + // entropy is a twenty-four-word mnemonic; the value is zeroed because the + // corpus never carries live key material. + push_compact_size(&mut out, 32); + push_bytes(&mut out, &[0u8; 32]); + + push_u32_le(&mut out, BIRTHDAY); + + // The key store map: `Vector<(u32 account id, UnifiedKeyStore)>`. + push_compact_size(&mut out, 1); + push_u32_le(&mut out, 0); + push_unified_key_store(&mut out); + + // The unified addresses: + // `Vector<(u32 account, u32 index, ReceiverSelection)>`. + push_compact_size(&mut out, 1); + push_u32_le(&mut out, 0); + push_u32_le(&mut out, 0); + push_receiver_selection(&mut out); + + // The transparent addresses: `Vector<(u32 account, u8 scope, u32 index)>`. + push_compact_size(&mut out, 1); + push_u32_le(&mut out, 0); + push_u8(&mut out, 0); + push_u32_le(&mut out, 0); + + // The wallet blocks. The era's marks need none, so the vector is empty. + push_compact_size(&mut out, 0); + + // The wallet transactions: one record, which carries the + // `ConfirmationStatus` byte this era's last row moved. + push_compact_size(&mut out, 1); + let confirmation_status_offset = push_wallet_transaction(&mut out, grammar); + + push_nullifier_map(&mut out); + + // The outpoint map: `Vector<(TxId, u16 output index, ScanTarget)>`. The u16 + // index width is this era's; census row 71 widens it to u32. + push_compact_size(&mut out, 1); + push_bytes(&mut out, &TXID_SCAN_TARGET); + push_u16_le(&mut out, 1); + push_scan_target(&mut out); + + push_shard_trees(&mut out); + + let sync_state_version_offset = push_sync_state(&mut out, grammar); + + push_sync_config(&mut out); + + // `min_confirmations`, appended by census row 65 (`ad6ded426`). + push_u32_le(&mut out, 3); + + push_price_list(&mut out); + + Built { + bytes: out, + confirmation_status_offset, + sync_state_version_offset, + } +} + +/// Write a `UnifiedKeyStore` holding a spending key, replicating the +/// `ReadableWriteable` impls in `zingolib/src/wallet/keys/unified.rs`: the +/// record's own version byte, then the key-type tag, then the key itself. +fn push_unified_key_store(out: &mut Vec) { + push_u8(out, 0); // UnifiedKeyStore::VERSION + push_u8(out, 2); // KEY_TYPE_SPEND + + // `UnifiedSpendingKey` writes a `CompactSize` length and then the opaque + // `to_bytes(Era::Orchard)` encoding. The corpus fills such blobs with dummy + // bytes, so the length here is representative rather than derived. + push_compact_size(out, 128); + push_bytes(out, &[0u8; 128]); +} + +/// Write a `ReceiverSelection` with both shielded receivers set. The record's +/// inner version has read 2 since census row 62 retired the transparent bit +/// from the bitmask. +fn push_receiver_selection(out: &mut Vec) { + push_u8(out, 2); // ReceiverSelection::VERSION + push_u8(out, 0b11); // orchard | sapling +} + +/// Write one `WalletTransaction` record and return the offset of the +/// `ConfirmationStatus` record's inner version byte, whose status discriminant +/// follows it. +fn push_wallet_transaction(out: &mut Vec, grammar: &Grammar) -> usize { + push_u8(out, 0); // WalletTransaction::serialized_version + push_bytes(out, &TXID_TRANSACTION); + + let confirmation_status_offset = out.len(); + push_u8(out, grammar.confirmation_status_version); + push_u8(out, grammar.confirmation_status); + push_u32_le(out, TRANSACTION_HEIGHT); + + push_transaction_body(out); + + push_u32_le(out, 1_700_000_000); // datetime + + push_compact_size(out, 0); // transparent coins + push_compact_size(out, 1); // sapling notes + push_sapling_note(out, grammar); + push_compact_size(out, 0); // orchard notes + push_compact_size(out, 0); // outgoing sapling notes + push_compact_size(out, 0); // outgoing orchard notes + + confirmation_status_offset +} + +/// Write the raw consensus encoding of the recorded transaction, which +/// `WalletTransaction::write` emits through `Transaction::write`. +/// +/// The body is a v5 transaction with no transparent inputs or outputs, no +/// sapling spends or outputs, and no orchard actions. Under ZIP 225 an empty +/// bundle collapses to its count alone, so the whole encoding is twenty-five +/// bytes, which keeps the fixture's opaque region as small as the format +/// permits. +fn push_transaction_body(out: &mut Vec) { + // ASSUMPTION: the ZIP 225 v5 transaction encoding, as `zcash_primitives` + // writes it at these commits. The header sets the overwintered bit above + // version 5; the version group id is v5's fixed 0x26A7270A; the consensus + // branch id is NU5's 0xC2D6D0B4, which matches TRANSACTION_HEIGHT on + // mainnet. `zcash_primitives` is not vendored locally at the pinned + // revisions, so these three constants come from the published format rather + // than from source read at the Defining Commits. + push_u32_le(out, 0x8000_0005); // header: overwintered | version 5 + push_u32_le(out, 0x26A7_270A); // nVersionGroupId + push_u32_le(out, 0xC2D6_D0B4); // nConsensusBranchId (NU5) + push_u32_le(out, 0); // lock_time + push_u32_le(out, 0); // nExpiryHeight + push_compact_size(out, 0); // tx_in count + push_compact_size(out, 0); // tx_out count + push_compact_size(out, 0); // nSpendsSapling + push_compact_size(out, 0); // nOutputsSapling + push_compact_size(out, 0); // nActionsOrchard +} + +/// Write one `SaplingNote`, the record whose inner version and trailing vector +/// census row 67 moved. +fn push_sapling_note(out: &mut Vec, grammar: &Grammar) { + push_u8(out, grammar.wallet_note_version); + + push_bytes(out, &TXID_TRANSACTION); + push_u16_le(out, 0); // output index + + push_u32_le(out, 0); // account id + push_u8(out, 0); // scope: External + + // ASSUMPTION: `sapling_crypto::PaymentAddress::to_bytes` is the + // forty-three-byte diversifier-and-public-key encoding. The corpus zeroes + // key material, so these bytes are a placeholder and would not decode to a + // point on the curve; the fixture pins the field's width and position, not + // its cryptographic validity. + push_bytes(out, &[0u8; 43]); + + push_u64_le(out, 100_000); // note value in zatoshis + + push_u8(out, 1); // rseed tag: AfterZip212 + push_bytes(out, &[0u8; 32]); + + push_optional_none(out); // nullifier + push_optional_some(out); // position + push_u64_le(out, 1_234); + + // ASSUMPTION: `Memo::Empty.encode()` is the five-hundred-and-twelve-byte + // array whose first byte is 0xF6 and whose remainder is zero, per ZIP 302. + let mut memo = [0u8; 512]; + memo[0] = 0xF6; + push_bytes(out, &memo); + + push_optional_none(out); // spending transaction + + if grammar.writes_refetch_nullifier_ranges { + // `Vector>`, appended by `WalletNote` version 1. + push_compact_size(out, 1); + push_u32_le(out, 1_990_000); + push_u32_le(out, 2_000_000); + } +} + +/// Write an empty `NullifierMap`. Its inner version has read 1 since the +/// outpoint values became `ScanTarget` records at census row 64. +fn push_nullifier_map(out: &mut Vec) { + push_u8(out, 1); // NullifierMap::serialized_version + push_compact_size(out, 0); // sapling + push_compact_size(out, 0); // orchard +} + +/// Write one `ScanTarget`, the outpoint map's value record since census row 64. +fn push_scan_target(out: &mut Vec) { + push_u8(out, 0); // ScanTarget::serialized_version + push_u32_le(out, 1_995_000); // block height + push_bytes(out, &TXID_SCAN_TARGET); + push_u8(out, 1); // narrow_scan_area +} + +/// Write an empty `ShardTrees` pair. Each tree writes an empty shard vector, an +/// empty checkpoint vector, and its cap. +fn push_shard_trees(out: &mut Vec) { + push_u8(out, 0); // ShardTrees::serialized_version + push_empty_shardtree(out); // sapling + push_empty_shardtree(out); // orchard +} + +/// Write one empty memory-backed shard tree. +fn push_empty_shardtree(out: &mut Vec) { + push_compact_size(out, 0); // located prunable trees + push_compact_size(out, 0); // checkpoints + + // ASSUMPTION: + // `zcash_client_backend::serialization::shardtree::write_shard` writes its + // version tag 1 and then the tree, and an empty tree is the single Nil tag + // 0. Verified against the locally available zcash_client_backend 0.23.0; + // the commits pin 0.18.0 and 0.21.0, where the same two constants hold. + push_bytes(out, &[1, 0]); +} + +/// Write the `SyncState` record and return the offset of its inner version +/// byte, the mark that separates census row 66 from row 67. +fn push_sync_state(out: &mut Vec, grammar: &Grammar) -> usize { + let version_offset = out.len(); + push_u8(out, grammar.sync_state_version); + + // The scan ranges: `Vector<(u32 start, u32 end, u8 priority)>`. + push_compact_size(out, 1); + push_u32_le(out, BIRTHDAY); + push_u32_le(out, TRANSACTION_HEIGHT); + push_u8(out, grammar.scan_priority); + + push_compact_size(out, 0); // sapling shard ranges + push_compact_size(out, 0); // orchard shard ranges + push_compact_size(out, 0); // scan targets + + version_offset +} + +/// Write the `SyncConfig` record, whose inner version has read 1 since census +/// row 65 (`ad6ded426`) appended the performance level. +fn push_sync_config(out: &mut Vec) { + push_u8(out, 1); // SyncConfig::serialized_version + push_u8(out, 20); // transparent address discovery gap limit + push_u8(out, 0b011); // scopes: external | internal + push_u8(out, 0); // PerformanceLevel::serialized_version + push_u8(out, 2); // PerformanceLevel::High +} + +/// Write an empty `PriceList` in the shape census row 60 settled, after that +/// row dropped the leading `Optional` census row 59 had appended with +/// the record itself. +fn push_price_list(out: &mut Vec) { + push_u8(out, 0); // PriceList::serialized_version + push_optional_none(out); // time historical prices last updated + push_optional_none(out); // current price + push_compact_size(out, 0); // daily prices +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The era holds the wallet-file version word still at 39: the three rows + /// are told apart by their sub-records, never by the header. + #[test] + fn every_row_writes_version_word_thirty_nine() { + for fixture in fixtures() { + let word = u64::from_le_bytes(fixture.bytes[..8].try_into().expect("eight bytes")); + assert_eq!(word, 39, "row {} header", fixture.row); + } + } + + /// The module contributes the rows it claims, in census order. + #[test] + fn fixtures_are_rows_sixty_six_through_sixty_eight() { + let rows: Vec = fixtures().iter().map(|f| f.row).collect(); + assert_eq!(rows, vec![66, 67, 68]); + } + + /// Row 66 writes the `SyncState` inner version byte 2 and rows 67 and 68 + /// write 3. This byte alone separates row 66 from the rest of the era. + #[test] + fn sync_state_version_separates_row_sixty_six() { + let built_66 = build(&GRAMMAR_66); + let built_67 = build(&GRAMMAR_67); + let built_68 = build(&GRAMMAR_68); + + assert_eq!(built_66.bytes[built_66.sync_state_version_offset], 2); + assert_eq!(built_67.bytes[built_67.sync_state_version_offset], 3); + assert_eq!(built_68.bytes[built_68.sync_state_version_offset], 3); + } + + /// Row 68 writes the status byte 4, `Failed`, which only its grammar can + /// produce, while rows 66 and 67 write a byte both numberings accept. The + /// record's inner version byte moves with the renumbering, from 0 to 1. + #[test] + fn confirmation_status_pairs_row_sixty_seven_against_row_sixty_eight() { + let built_66 = build(&GRAMMAR_66); + let built_67 = build(&GRAMMAR_67); + let built_68 = build(&GRAMMAR_68); + + let status = |b: &Built| b.bytes[b.confirmation_status_offset + 1]; + let version = |b: &Built| b.bytes[b.confirmation_status_offset]; + + assert!( + status(&built_66) <= 3, + "row 66 status must be legal under both numberings" + ); + assert!( + status(&built_67) <= 3, + "row 67 status must be legal under both numberings" + ); + assert_eq!( + status(&built_68), + 4, + "row 68 status must be the Failed byte" + ); + + assert_eq!(version(&built_66), 0); + assert_eq!(version(&built_67), 0); + assert_eq!(version(&built_68), 1); + } + + /// Rows 67 and 68 share every byte except the two the `ConfirmationStatus` + /// renumbering moved, and the record's own version byte is one of the two. + /// That is what makes the pair discriminable, as the census now records. + #[test] + fn rows_sixty_seven_and_sixty_eight_differ_only_in_the_status_record() { + let built_67 = build(&GRAMMAR_67); + let built_68 = build(&GRAMMAR_68); + assert_eq!(built_67.bytes.len(), built_68.bytes.len()); + + let differing: Vec = built_67 + .bytes + .iter() + .zip(built_68.bytes.iter()) + .enumerate() + .filter(|(_, (a, b))| a != b) + .map(|(index, _)| index) + .collect(); + assert_eq!( + differing, + vec![ + built_67.confirmation_status_offset, + built_67.confirmation_status_offset + 1 + ] + ); + } + + /// Adjacent census rows produce different files, which is the corpus's + /// reason to exist. + #[test] + fn adjacent_rows_are_byte_distinct() { + let fixtures = fixtures(); + for pair in fixtures.windows(2) { + assert_ne!( + pair[0].bytes, pair[1].bytes, + "rows {} and {} produced identical bytes", + pair[0].row, pair[1].row + ); + } + } +} diff --git a/tools/workbench/src/wallet_grammars/era_v32.rs b/tools/workbench/src/wallet_grammars/era_v32.rs new file mode 100644 index 0000000000..7dbcc09fdf --- /dev/null +++ b/tools/workbench/src/wallet_grammars/era_v32.rs @@ -0,0 +1,955 @@ +//! Format Census rows 56 through 65: the version-32 restructure and its +//! descendants. +//! +//! Row 56 rebuilds the Wallet File around the new sync engine. Everything +//! after the version word is new: the chain name, a seed `Vector`, a +//! four-byte birthday where eight bytes used to stand, the unified key +//! store, the unified- and transparent-address vectors, and the sync +//! engine's own structures. The nine rows that follow refine that layout +//! one delta at a time, and each delta is the row's grammar-unique mark. +//! +//! Every fixture here describes the same wallet: mainnet, one account, a +//! zeroed twenty-four-word seed, a spend-capable key store, one unified +//! address, one transparent address, no blocks and no transactions, one +//! outpoint-map entry, and one scan range. The outpoint entry is present in +//! every row because this era writes the output index as a `u16`; a later +//! era widens it to a `u32`, and the entry is what pins the width. + +use super::util::{ + push_bytes, push_compact_size, push_optional_none, push_optional_some, push_u16_le, + push_u32_le, push_u64_le, push_u64_string, push_u8, +}; +use super::Fixture; + +/// The chain name. This era writes it through `zingolib`'s historical +/// `utils::write_string`, so the framing is a u64 little-endian byte length +/// followed by the UTF-8 bytes, and `ChainType`'s `Display` renders mainnet +/// as `main`. +const CHAIN: &str = "main"; + +/// The wallet's seed entropy, zeroed. `bip0039::Mnemonic::into_entropy` +/// yields thirty-two bytes for a twenty-four-word mnemonic, which is what +/// this era's wallets carry. +const SEED_ENTROPY: [u8; 32] = [0u8; 32]; + +/// The single account these wallets hold. The zip32 account identifier is +/// written as a little-endian u32 wherever it appears. +const ACCOUNT_ID: u32 = 0; + +/// The wallet birthday, a plausible mainnet height. Row 56 narrowed this +/// field from u64 to u32. +const BIRTHDAY: u32 = 2_500_000; + +/// The address index of the one unified address and the one transparent +/// address. +const ADDRESS_INDEX: u32 = 0; + +/// `TransparentScope::External`, written as its enum discriminant. +const SCOPE_EXTERNAL: u8 = 0; + +/// The transaction identifier of the outpoint-map key. +const OUTPOINT_TXID: [u8; 32] = [0x11; 32]; + +/// The output index of the outpoint-map key. This era writes it as a u16. +const OUTPUT_INDEX: u16 = 1; + +/// The block height of the outpoint-map value and of the one sync-state +/// scan target. +const LOCATOR_HEIGHT: u32 = 2_500_100; + +/// The transaction identifier of the outpoint-map value and of the one +/// sync-state scan target. +const LOCATOR_TXID: [u8; 32] = [0x22; 32]; + +/// The one scan range's start height. +const SCAN_RANGE_START: u32 = BIRTHDAY; + +/// The one scan range's end height. +const SCAN_RANGE_END: u32 = BIRTHDAY + 1_000; + +/// `ScanPriority::Historic`, written as `priority as u8`. The discriminants +/// this era's reader accepts run `Ignored`, `Scanned`, `Historic`, +/// `OpenAdjacent`, `FoundNote`, `ChainTip`, `Verify`. +const SCAN_PRIORITY_HISTORIC: u8 = 2; + +/// The gap limit of `TransparentAddressDiscovery::minimal()`, which is what +/// `LightWallet` initialises its `SyncConfig` with throughout this era. +const GAP_LIMIT: u8 = 1; + +/// `TransparentAddressDiscoveryScopes::default()` packed into the writer's +/// bitmask: external set, internal clear, refund set. +const DISCOVERY_SCOPES: u8 = 0b101; + +/// `PerformanceLevel::High`, the default, written as the second byte of the +/// `PerformanceLevel` record. +const PERFORMANCE_LEVEL_HIGH: u8 = 2; + +/// The default `min_confirmations`, a `NonZeroU32` of one. +const MIN_CONFIRMATIONS: u32 = 1; + +/// Which of the two `PriceList` grammars a row writes. +/// +/// Both carry the same inner version byte of zero, which is what makes the +/// pair the census's third live misparse window: a file written in the +/// eight days between the two commits is read today under the later +/// grammar, and the missing `Optional` shifts every field after it. +#[derive(Clone, Copy, PartialEq)] +enum PriceListShape { + /// The record as row 59 minted it, opening with an `Optional` holding + /// the CoinCap API key. + WithApiKey, + /// The record from row 60 on, with that `Optional` gone. + WithoutApiKey, +} + +/// Which grammar each row's writer produces. The fields name the deltas the +/// census records for rows 56 through 65, so a row function reads as a +/// statement of what its Defining Commit changed. +struct Shape { + /// The version word at offset zero. + version: u64, + /// Whether the mnemonic's account index follows the seed vector. Row 61 + /// dropped it. + mnemonic_account_index: bool, + /// Whether the key store is a `Vector<(account, UnifiedKeyStore)>` + /// rather than a single bare record. Row 61 made the change. + account_keyed_key_store: bool, + /// The `ReceiverSelection` inner version byte. Row 62 moved it to two. + receiver_version: u8, + /// The `ReceiverSelection` bitmask. Row 62 retired the transparent bit. + receiver_mask: u8, + /// Whether the outpoint map's values and the sync state's scan targets + /// are `ScanTarget` records rather than bare `(height, txid)` locators. + /// Row 64 made the change. + scan_target_values: bool, + /// The `SyncState` inner version byte, which row 64 moved to one. + sync_state_version: u8, + /// The `SyncConfig` record's inner version, when the row writes the + /// record at all. Row 58 appended it; row 65 moved it to one. + sync_config_version: Option, + /// Whether `min_confirmations` follows the sync config. Row 65 appended + /// it. + min_confirmations: bool, + /// The `PriceList` record that closes the file, when the row writes one + /// at all. Row 59 appended it; row 60 dropped its leading `Optional`. + price_list: Option, + /// Whether the vestigial `WalletOptions` and `WalletZecPriceInfo` + /// records close the file. Only row 56 writes them; row 57 dropped them. + vestigial_tail: bool, +} + +/// The `UnifiedSpendingKey` blob, zeroed. +/// +/// The wallet writer treats this blob as opaque: it writes a CompactSize +/// length and then the bytes `UnifiedSpendingKey::to_bytes(Era::Orchard)` +/// returned. Only the dependency parses the interior, so the fixture +/// reproduces the container framing and zeroes the key material. +fn unified_spending_key_blob() -> Vec { + let mut out = Vec::new(); + // ASSUMPTION: `zcash_keys`' USK encoding writes the era identifier as a + // little-endian u32 holding the NU5 consensus branch id, then one + // (typecode, length, key) triple per pool with the typecode and the + // length each a CompactSize. The pools appear in the order orchard, + // sapling, transparent, and the typecodes are 3, 2 and 0. Read from the + // pinned `zingolabs/librustzcash` checkout of `zcash_keys/src/keys.rs`, + // whose `to_bytes` is unchanged across this era. + push_bytes(&mut out, &0xc2d6_d0b4u32.to_le_bytes()); + // ASSUMPTION: the orchard spending key is thirty-two bytes. + push_compact_size(&mut out, 3); + push_compact_size(&mut out, 32); + push_bytes(&mut out, &[0u8; 32]); + // ASSUMPTION: `sapling_crypto`'s `ExtendedSpendingKey::to_bytes` + // returns exactly 169 bytes (depth, parent tag, child index, chain + // code, expanded spending key, diversifier key). + push_compact_size(&mut out, 2); + push_compact_size(&mut out, 169); + push_bytes(&mut out, &[0u8; 169]); + // ASSUMPTION: `AccountPrivKey::to_bytes` returns the BIP 32 extended + // private key encoding without its four prefix bytes, so 74 of the 78 + // bytes. + push_compact_size(&mut out, 0); + push_compact_size(&mut out, 74); + push_bytes(&mut out, &[0u8; 74]); + out +} + +/// Append one `UnifiedKeyStore` record holding a spending key: the trait's +/// version byte, the `KEY_TYPE_SPEND` tag, and the CompactSize-framed +/// spending-key blob. +fn push_unified_key_store(out: &mut Vec) { + push_u8(out, 0); + push_u8(out, 2); + let blob = unified_spending_key_blob(); + push_compact_size(out, blob.len() as u64); + push_bytes(out, &blob); +} + +/// Append one `ScanTarget` record: its version byte, the block height, the +/// transaction identifier, and the narrow-scan-area flag row 64 introduced. +fn push_scan_target(out: &mut Vec) { + push_u8(out, 0); + push_u32_le(out, LOCATOR_HEIGHT); + push_bytes(out, &LOCATOR_TXID); + push_u8(out, 1); +} + +/// Append the empty `NullifierMap`: its version byte and two empty vectors, +/// sapling then orchard. +fn push_nullifier_map(out: &mut Vec) { + push_u8(out, 0); + push_compact_size(out, 0); + push_compact_size(out, 0); +} + +/// Append the empty `ShardTrees`: its version byte, then the sapling and +/// orchard trees, each an empty shard vector, an empty checkpoint vector +/// and a cap. +fn push_shard_trees(out: &mut Vec) { + push_u8(out, 0); + for _ in 0..2 { + push_compact_size(out, 0); + push_compact_size(out, 0); + // ASSUMPTION: `zcash_client_backend`'s `write_shard` writes a + // serialization-version byte of 1 followed by the tree, and an + // empty store's cap is a single `Nil` node whose tag is 0. + push_u8(out, 1); + push_u8(out, 0); + } +} + +/// Append the `SyncState`: its version byte, the scan ranges, the sapling +/// and orchard shard ranges, and finally the locators, which row 64 turned +/// into scan targets. +fn push_sync_state(out: &mut Vec, shape: &Shape) { + push_u8(out, shape.sync_state_version); + push_compact_size(out, 1); + push_u32_le(out, SCAN_RANGE_START); + push_u32_le(out, SCAN_RANGE_END); + push_u8(out, SCAN_PRIORITY_HISTORIC); + push_compact_size(out, 0); + push_compact_size(out, 0); + push_compact_size(out, 1); + if shape.scan_target_values { + push_scan_target(out); + } else { + push_u32_le(out, LOCATOR_HEIGHT); + push_bytes(out, &LOCATOR_TXID); + } +} + +/// Append the `SyncConfig` record: its version byte, the transparent +/// address-discovery gap limit and scope bitmask, and, from version one on, +/// a `PerformanceLevel` record. +fn push_sync_config(out: &mut Vec, version: u8) { + push_u8(out, version); + push_u8(out, GAP_LIMIT); + push_u8(out, DISCOVERY_SCOPES); + if version >= 1 { + push_u8(out, 0); + push_u8(out, PERFORMANCE_LEVEL_HIGH); + } +} + +/// Append the `PriceList` record for a wallet that has never fetched a +/// price: its version byte, the API key `Optional` while the record still +/// carried one, then no update time, no current price and no daily prices. +fn push_price_list(out: &mut Vec, shape: PriceListShape) { + push_u8(out, 0); + if shape == PriceListShape::WithApiKey { + push_optional_none(out); + } + push_optional_none(out); + push_optional_none(out); + push_compact_size(out, 0); +} + +/// Append the vestigial `WalletOptions` record at its defaults: version +/// two, `MemoDownloadOption::WalletMemos`, and a transaction size filter of +/// five hundred. +fn push_wallet_options(out: &mut Vec) { + push_u64_le(out, 2); + push_u8(out, 1); + push_optional_some(out); + push_u32_le(out, 500); +} + +/// Append the vestigial `WalletZecPriceInfo` record at its defaults: +/// version twenty, no fetch time, and a zero retry count. +fn push_zec_price_info(out: &mut Vec) { + push_u64_le(out, 20); + push_optional_none(out); + push_u64_le(out, 0); +} + +/// Build one Wallet File in the shape the given row's writer produces. The +/// field order follows `LightWallet::write` exactly: version word, chain +/// name, seed, birthday, key store, unified addresses, transparent +/// addresses, blocks, transactions, nullifier map, outpoint map, shard +/// trees, sync state, and then whichever tail records the row appends. +fn wallet(shape: &Shape) -> Vec { + let mut out = Vec::new(); + + push_u64_le(&mut out, shape.version); + push_u64_string(&mut out, CHAIN); + + push_compact_size(&mut out, SEED_ENTROPY.len() as u64); + push_bytes(&mut out, &SEED_ENTROPY); + if shape.mnemonic_account_index { + push_u32_le(&mut out, ACCOUNT_ID); + } + push_u32_le(&mut out, BIRTHDAY); + + if shape.account_keyed_key_store { + push_compact_size(&mut out, 1); + push_u32_le(&mut out, ACCOUNT_ID); + } + push_unified_key_store(&mut out); + + push_compact_size(&mut out, 1); + push_u32_le(&mut out, ACCOUNT_ID); + push_u32_le(&mut out, ADDRESS_INDEX); + push_u8(&mut out, shape.receiver_version); + push_u8(&mut out, shape.receiver_mask); + + push_compact_size(&mut out, 1); + push_u32_le(&mut out, ACCOUNT_ID); + push_u8(&mut out, SCOPE_EXTERNAL); + push_u32_le(&mut out, ADDRESS_INDEX); + + push_compact_size(&mut out, 0); + push_compact_size(&mut out, 0); + + push_nullifier_map(&mut out); + + push_compact_size(&mut out, 1); + push_bytes(&mut out, &OUTPOINT_TXID); + push_u16_le(&mut out, OUTPUT_INDEX); + if shape.scan_target_values { + push_scan_target(&mut out); + } else { + push_u32_le(&mut out, LOCATOR_HEIGHT); + push_bytes(&mut out, &LOCATOR_TXID); + } + + push_shard_trees(&mut out); + push_sync_state(&mut out, shape); + + if let Some(version) = shape.sync_config_version { + push_sync_config(&mut out, version); + } + if shape.min_confirmations { + push_u32_le(&mut out, MIN_CONFIRMATIONS); + } + if let Some(price_list) = shape.price_list { + push_price_list(&mut out, price_list); + } + if shape.vestigial_tail { + push_wallet_options(&mut out); + push_zec_price_info(&mut out); + } + + out +} + +/// This era's fixtures, census rows 56 through 65 in order. +pub fn fixtures() -> Vec { + vec![ + Fixture { + row: 56, + defining_commit: "44e6271cb", + branch: "dev", + bytes: row_56(), + }, + Fixture { + row: 57, + defining_commit: "8aaae992a", + branch: "dev", + bytes: row_57(), + }, + Fixture { + row: 58, + defining_commit: "82c61c0d3", + branch: "dev", + bytes: row_58(), + }, + Fixture { + row: 59, + defining_commit: "1ef03610b", + branch: "dev", + bytes: row_59(), + }, + Fixture { + row: 60, + defining_commit: "44baa11b4", + branch: "dev", + bytes: row_60(), + }, + Fixture { + row: 61, + defining_commit: "ccc1d681a", + branch: "dev", + bytes: row_61(), + }, + Fixture { + row: 62, + defining_commit: "e5e4a349f", + branch: "dev", + bytes: row_62(), + }, + Fixture { + row: 63, + defining_commit: "e6b02b0d8", + branch: "dev", + bytes: row_63(), + }, + Fixture { + row: 64, + defining_commit: "eae34880e", + branch: "dev", + bytes: row_64(), + }, + Fixture { + row: 65, + defining_commit: "ad6ded426", + branch: "dev", + bytes: row_65(), + }, + ] +} + +/// Row 56, Defining Commit `44e6271cb`, version 32. +/// +/// Replicates `LightWallet::write` in `zingolib/src/wallet/disk.rs`, +/// together with `utils::write_string`, `UnifiedKeyStore::write` and +/// `ReceiverSelection::write` in `zingolib/src/wallet/keys/unified.rs`, the +/// `NullifierMap`, `ShardTrees` and `SyncState` writers in +/// `pepper-sync/src/wallet/serialization.rs`, and the `WalletOptions` and +/// `WalletZecPriceInfo` writers in `zingolib/src/wallet.rs` and +/// `zingolib/src/wallet/data.rs`. +/// +/// The wallet is a mainnet wallet with a zeroed twenty-four-word seed whose +/// mnemonic account index follows the seed vector, a birthday of 2,500,000 +/// written in four bytes rather than the eight the previous grammar used, a +/// single spend-capable key store, one unified address, one transparent +/// address, no blocks, no transactions, an empty nullifier map, one +/// outpoint-map entry, empty shard trees, and a sync state holding one +/// historic scan range and one locator. The vestigial `WalletOptions` and +/// `WalletZecPriceInfo` records close the file; the version-32 reader never +/// consumes them, and row 57 removes them. +fn row_56() -> Vec { + wallet(&Shape { + version: 32, + mnemonic_account_index: true, + account_keyed_key_store: false, + receiver_version: 1, + receiver_mask: 0b111, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: None, + min_confirmations: false, + price_list: None, + vestigial_tail: true, + }) +} + +/// Row 57, Defining Commit `8aaae992a`, version 32 unbumped. +/// +/// Replicates the same writers as row 56. The wallet's contents are +/// identical; only the writer changed, dropping the trailing +/// `WalletOptions` and `WalletZecPriceInfo` records that the version-32 +/// reader never consumed. Two files therefore claim version 32 and differ +/// by a thirty-one-byte tail, which is the row's mark. +fn row_57() -> Vec { + wallet(&Shape { + version: 32, + mnemonic_account_index: true, + account_keyed_key_store: false, + receiver_version: 1, + receiver_mask: 0b111, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: None, + min_confirmations: false, + price_list: None, + vestigial_tail: false, + }) +} + +/// Row 58, Defining Commit `82c61c0d3`, version 33. +/// +/// Replicates row 57's writers plus `SyncConfig::write` in +/// `pepper-sync/src/sync.rs`. The wallet's contents are row 57's, and the +/// sync config is the one `LightWallet` initialises: version zero, the gap +/// limit of one that `TransparentAddressDiscovery::minimal` sets, and the +/// default scope bitmask with external and refund set. +fn row_58() -> Vec { + wallet(&Shape { + version: 33, + mnemonic_account_index: true, + account_keyed_key_store: false, + receiver_version: 1, + receiver_mask: 0b111, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: Some(0), + min_confirmations: false, + price_list: None, + vestigial_tail: false, + }) +} + +/// Row 59, Defining Commit `1ef03610b`, version 34. +/// +/// Replicates row 58's writers plus `PriceList::write` in +/// `zingo-price/src/lib.rs`. The wallet's contents are row 58's, and the +/// price list is the one `PriceList::new` builds: version zero, then four +/// fields, of which the first is the `Optional` holding the CoinCap API key +/// that row 60 removes. All four are empty here, since the wallet has never +/// fetched a price and no key has been set. +fn row_59() -> Vec { + wallet(&Shape { + version: 34, + mnemonic_account_index: true, + account_keyed_key_store: false, + receiver_version: 1, + receiver_mask: 0b111, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: Some(0), + min_confirmations: false, + price_list: Some(PriceListShape::WithApiKey), + vestigial_tail: false, + }) +} + +/// Row 60, Defining Commit `44baa11b4`, version 34 unbumped. +/// +/// Replicates row 59's writers with `PriceList::write` in +/// `zingo-price/src/lib.rs` rewritten for the Tor-fronted Gemini price +/// source, which needs no API key. The leading `Optional` disappears while +/// `PriceList::serialized_version` stands still at zero, so the record's +/// own version byte cannot tell the two grammars apart. Comparing +/// `zingolib/src/wallet/disk.rs` at `1ef03610b` and `44baa11b4` shows the +/// wallet writer itself untouched, so the fixture is row 59's bytes one +/// `Optional` shorter. +/// +/// This is the third of the census's live misparse windows: a file written +/// in the eight days between the two commits is read today under this +/// grammar, and the missing byte shifts the update time, the current price +/// and the daily-price vector. +fn row_60() -> Vec { + wallet(&Shape { + version: 34, + mnemonic_account_index: true, + account_keyed_key_store: false, + receiver_version: 1, + receiver_mask: 0b111, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: Some(0), + min_confirmations: false, + price_list: Some(PriceListShape::WithoutApiKey), + vestigial_tail: false, + }) +} + +/// Row 61, Defining Commit `ccc1d681a`, version 35. +/// +/// Replicates row 60's writers with `LightWallet::write` restructured for +/// multiple accounts. The single `UnifiedKeyStore` record becomes a +/// `Vector<(account, UnifiedKeyStore)>`, and the mnemonic account index +/// that used to follow the seed vector is gone. The wallet still holds one +/// account, so the vector carries one entry keyed by account zero, and the +/// birthday now follows the seed vector directly. +fn row_61() -> Vec { + wallet(&Shape { + version: 35, + mnemonic_account_index: false, + account_keyed_key_store: true, + receiver_version: 1, + receiver_mask: 0b111, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: Some(0), + min_confirmations: false, + price_list: Some(PriceListShape::WithoutApiKey), + vestigial_tail: false, + }) +} + +/// Row 62, Defining Commit `e5e4a349f`, version 35 unbumped. +/// +/// Replicates row 61's writers with `ReceiverSelection` in +/// `zingolib/src/wallet/keys/unified.rs` moved to inner version two, which +/// retires the transparent bit from the receiver bitmask. The wallet's +/// contents are row 61's, so its one unified address now writes the version +/// byte two and a bitmask of orchard and sapling alone. Two files therefore +/// claim version 35 and differ inside the unified-address vector. +fn row_62() -> Vec { + wallet(&Shape { + version: 35, + mnemonic_account_index: false, + account_keyed_key_store: true, + receiver_version: 2, + receiver_mask: 0b11, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: Some(0), + min_confirmations: false, + price_list: Some(PriceListShape::WithoutApiKey), + vestigial_tail: false, + }) +} + +/// Row 63, Defining Commit `e6b02b0d8`, version 36. +/// +/// Replicates row 62's writers unchanged. Comparing +/// `zingolib/src/wallet/disk.rs` at the two commits shows the writer +/// untouched: the commit moves the version word from 35 to 36 and teaches +/// the reader to regenerate addresses for anything older. The fixture is +/// therefore row 62's bytes with a different version word, and nothing +/// else distinguishes the two grammars. +fn row_63() -> Vec { + wallet(&Shape { + version: 36, + mnemonic_account_index: false, + account_keyed_key_store: true, + receiver_version: 2, + receiver_mask: 0b11, + scan_target_values: false, + sync_state_version: 0, + sync_config_version: Some(0), + min_confirmations: false, + price_list: Some(PriceListShape::WithoutApiKey), + vestigial_tail: false, + }) +} + +/// Row 64, Defining Commit `eae34880e`, version 37. +/// +/// Replicates row 63's writers with the locator re-encoded as a +/// `ScanTarget` record, whose writer joins `SyncState`'s in +/// `pepper-sync/src/wallet/serialization.rs`. The outpoint map's value and +/// the sync state's fourth vector both carried a bare block height and +/// transaction identifier; each now carries a versioned record that appends +/// a narrow-scan-area flag, and `SyncState`'s own inner version moves from +/// zero to one so its reader can tell the two encodings apart. The wallet's +/// contents are row 63's, and its one outpoint entry and one scan target +/// both show the new encoding with the flag set. +fn row_64() -> Vec { + wallet(&Shape { + version: 37, + mnemonic_account_index: false, + account_keyed_key_store: true, + receiver_version: 2, + receiver_mask: 0b11, + scan_target_values: true, + sync_state_version: 1, + sync_config_version: Some(0), + min_confirmations: false, + price_list: Some(PriceListShape::WithoutApiKey), + vestigial_tail: false, + }) +} + +/// Row 65, Defining Commit `ad6ded426`, version 38. +/// +/// Replicates row 64's writers with two additions. `LightWallet::write` +/// appends `min_confirmations` as a u32 between the sync config and the +/// price list, and `SyncConfig::write` in `pepper-sync/src/sync.rs` moves +/// to inner version one, which appends a `PerformanceLevel` record. The +/// wallet's contents are row 64's, with the default `min_confirmations` of +/// one and the default performance level of high. +fn row_65() -> Vec { + wallet(&Shape { + version: 38, + mnemonic_account_index: false, + account_keyed_key_store: true, + receiver_version: 2, + receiver_mask: 0b11, + scan_target_values: true, + sync_state_version: 1, + sync_config_version: Some(1), + min_confirmations: true, + price_list: Some(PriceListShape::WithoutApiKey), + vestigial_tail: false, + }) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The version word each row's writer emits, in row order. + const VERSION_WORDS: [u64; 10] = [32, 32, 33, 34, 34, 35, 35, 36, 37, 38]; + + fn version_word(bytes: &[u8]) -> u64 { + u64::from_le_bytes(bytes[0..8].try_into().expect("the header is eight bytes")) + } + + fn contains(haystack: &[u8], needle: &[u8]) -> bool { + haystack.windows(needle.len()).any(|w| w == needle) + } + + /// The offset of the byte that follows the seed vector: the version + /// word, the chain string, the seed's CompactSize count, and the seed. + const AFTER_SEED: usize = 8 + 8 + CHAIN.len() + 1 + SEED_ENTROPY.len(); + + fn read_u32(bytes: &[u8], offset: usize) -> u32 { + u32::from_le_bytes( + bytes[offset..offset + 4] + .try_into() + .expect("four bytes are in range"), + ) + } + + #[test] + fn every_row_writes_its_census_version_word_at_offset_zero() { + for (fixture, expected) in fixtures().iter().zip(VERSION_WORDS) { + assert_eq!( + version_word(&fixture.bytes), + expected, + "row {} wrote the wrong version word", + fixture.row + ); + } + } + + #[test] + fn every_row_is_a_mainnet_wallet_whose_chain_name_is_a_u64_framed_string() { + for fixture in fixtures() { + assert_eq!( + &fixture.bytes[8..20], + &[4, 0, 0, 0, 0, 0, 0, 0, b'm', b'a', b'i', b'n'], + "row {} framed the chain name differently", + fixture.row + ); + } + } + + /// Row 56's headline mark: the birthday is four bytes, not the eight + /// the preceding grammar wrote. In rows 56 through 60 the mnemonic + /// account index stands between the seed and the birthday. + #[test] + fn row_56_writes_the_birthday_in_four_bytes_after_the_account_index() { + let bytes = row_56(); + assert_eq!(read_u32(&bytes, AFTER_SEED), ACCOUNT_ID); + assert_eq!(read_u32(&bytes, AFTER_SEED + 4), BIRTHDAY); + } + + /// Row 57 drops the vestigial `WalletOptions` (fourteen bytes) and + /// `WalletZecPriceInfo` (seventeen bytes) records, and changes nothing + /// else. + #[test] + fn row_57_drops_the_vestigial_tail_and_leaves_the_rest_untouched() { + let (long, short) = (row_56(), row_57()); + assert_eq!(long.len() - short.len(), 31); + assert_eq!(&long[..short.len()], &short[..]); + } + + /// Row 58 appends the three-byte `SyncConfig` record. + #[test] + fn row_58_appends_the_sync_config_record() { + let (before, after) = (row_57(), row_58()); + assert_eq!(after.len() - before.len(), 3); + assert_eq!(&after[after.len() - 3..], &[0, GAP_LIMIT, DISCOVERY_SCOPES]); + } + + /// Row 59 appends the five-byte `PriceList` record of a wallet that has + /// never fetched a price: a version byte, the API key `Optional`, the + /// update-time and current-price `Optional`s, and an empty vector. + #[test] + fn row_59_appends_the_price_list_record_with_its_api_key_optional() { + let (before, after) = (row_58(), row_59()); + assert_eq!(after.len() - before.len(), 5); + assert_eq!(&after[after.len() - 5..], &[0, 0, 0, 0, 0]); + } + + /// Row 60's mark: the price list loses its leading `Optional` while its + /// inner version byte stands still at zero, so the two grammars differ + /// by exactly one byte at the end of the file and the record's own + /// version cannot separate them. + #[test] + fn row_60_drops_the_api_key_optional_without_bumping_the_inner_version() { + let (before, after) = (row_59(), row_60()); + assert_eq!(before.len() - after.len(), 1); + assert_eq!(version_word(&before), version_word(&after)); + // Everything up to the price list is common, and the price list's + // own version byte is zero in both, so the record cannot be told + // from its version. + let shared = after.len() - 4; + assert_eq!(&before[..shared], &after[..shared]); + assert_eq!(after[shared], 0); + assert_eq!(before[shared], 0); + } + + /// Only row 59 writes the API key `Optional`. Each price-list row's + /// file must end in exactly the records that follow the sync state, so + /// the price list's length is pinned by what precedes it rather than + /// counted in isolation: the sync config, then `min_confirmations` on + /// row 65, then the price list. + #[test] + fn only_row_59_writes_the_api_key_optional() { + const SYNC_CONFIG_V0: [u8; 3] = [0, GAP_LIMIT, DISCOVERY_SCOPES]; + const SYNC_CONFIG_V1: [u8; 5] = [1, GAP_LIMIT, DISCOVERY_SCOPES, 0, PERFORMANCE_LEVEL_HIGH]; + const MIN_CONFIRMATIONS_BYTES: [u8; 4] = [1, 0, 0, 0]; + const PRICE_LIST_WITH_API_KEY: [u8; 5] = [0, 0, 0, 0, 0]; + const PRICE_LIST_WITHOUT_API_KEY: [u8; 4] = [0, 0, 0, 0]; + + for fixture in fixtures() { + let mut expected_tail = Vec::new(); + match fixture.row { + 56..=58 => continue, + 59 => { + expected_tail.extend_from_slice(&SYNC_CONFIG_V0); + expected_tail.extend_from_slice(&PRICE_LIST_WITH_API_KEY); + } + 65 => { + expected_tail.extend_from_slice(&SYNC_CONFIG_V1); + expected_tail.extend_from_slice(&MIN_CONFIRMATIONS_BYTES); + expected_tail.extend_from_slice(&PRICE_LIST_WITHOUT_API_KEY); + } + _ => { + expected_tail.extend_from_slice(&SYNC_CONFIG_V0); + expected_tail.extend_from_slice(&PRICE_LIST_WITHOUT_API_KEY); + } + } + assert_eq!( + &fixture.bytes[fixture.bytes.len() - expected_tail.len()..], + &expected_tail[..], + "row {} wrote the wrong price list tail", + fixture.row + ); + } + } + + /// Row 61 drops the four-byte mnemonic account index and keys the key + /// store by account, so the birthday now follows the seed directly. + #[test] + fn row_61_keys_the_key_store_by_account_and_drops_the_mnemonic_index() { + let bytes = row_61(); + assert_eq!(read_u32(&bytes, AFTER_SEED), BIRTHDAY); + // The vector count of one, then the account identifier, then the + // key store's own version and spend-type bytes. + assert_eq!( + &bytes[AFTER_SEED + 4..AFTER_SEED + 11], + &[1, 0, 0, 0, 0, 0, 2] + ); + } + + /// Row 62 moves the `ReceiverSelection` inner version byte to two and + /// retires the transparent bit, while row 61 still writes version one + /// with all three bits set. + #[test] + fn row_62_writes_receiver_selection_version_two_without_the_transparent_bit() { + let unified_address = [0u8; 8]; + let mut row_61_record = unified_address.to_vec(); + row_61_record.extend_from_slice(&[1, 0b111]); + let mut row_62_record = unified_address.to_vec(); + row_62_record.extend_from_slice(&[2, 0b11]); + + assert!(contains(&row_61(), &row_61_record)); + assert!(!contains(&row_61(), &row_62_record)); + assert!(contains(&row_62(), &row_62_record)); + assert!(!contains(&row_62(), &row_61_record)); + } + + /// Rows 62 and 63 differ only in the version word: comparing + /// `zingolib/src/wallet/disk.rs` at `e5e4a349f` and `e6b02b0d8` shows + /// the writer untouched. + #[test] + fn rows_62_and_63_differ_only_in_the_version_word() { + let (older, newer) = (row_62(), row_63()); + assert_eq!(version_word(&older), 35); + assert_eq!(version_word(&newer), 36); + assert_eq!(&older[8..], &newer[8..]); + } + + /// Row 64 re-encodes the outpoint map's value and the sync state's + /// fourth vector as `ScanTarget` records, each of which adds a version + /// byte and a narrow-scan-area flag to the two bytes the bare locator + /// wrote, and moves `SyncState`'s inner version to one. + #[test] + fn row_64_re_encodes_locators_as_scan_targets() { + let (before, after) = (row_63(), row_64()); + assert_eq!(after.len() - before.len(), 4); + + let mut scan_target = vec![0u8]; + scan_target.extend_from_slice(&LOCATOR_HEIGHT.to_le_bytes()); + scan_target.extend_from_slice(&LOCATOR_TXID); + scan_target.push(1); + assert!(contains(&after, &scan_target)); + assert!(!contains(&before, &scan_target)); + } + + /// The whole era writes the outpoint map's output index as a u16, so + /// every fixture holds the outpoint key's transaction identifier + /// followed by exactly two index bytes. + #[test] + fn every_row_writes_the_outpoint_output_index_as_a_u16() { + let mut key = OUTPOINT_TXID.to_vec(); + key.extend_from_slice(&OUTPUT_INDEX.to_le_bytes()); + // The byte that follows the index starts the value: a locator's + // block height in rows 56 through 63, a `ScanTarget` version byte + // in rows 64 and 65. A u32 index would put a zero byte there in + // both cases, so pin the value's first byte too. + let mut locator_value = key.clone(); + locator_value.extend_from_slice(&LOCATOR_HEIGHT.to_le_bytes()[..1]); + let mut scan_target_value = key.clone(); + scan_target_value.push(0); + + for fixture in fixtures() { + assert!( + contains(&fixture.bytes, &locator_value) + || contains(&fixture.bytes, &scan_target_value), + "row {} did not write a u16 output index", + fixture.row + ); + } + } + + /// Row 65 appends the four-byte `min_confirmations` between the sync + /// config and the price list, and grows the sync config by the two-byte + /// `PerformanceLevel` record. + #[test] + fn row_65_appends_min_confirmations_and_a_performance_level() { + let (before, after) = (row_64(), row_65()); + assert_eq!(after.len() - before.len(), 6); + + let sync_config_and_confirmations = [ + 1, + GAP_LIMIT, + DISCOVERY_SCOPES, + 0, + PERFORMANCE_LEVEL_HIGH, + MIN_CONFIRMATIONS as u8, + 0, + 0, + 0, + ]; + assert!(contains(&after, &sync_config_and_confirmations)); + } + + /// Neighbouring rows are the pairs a recognizer is most likely to + /// confuse, so each adjacent pair must differ. + #[test] + fn adjacent_rows_are_byte_distinct() { + let fixtures = fixtures(); + for pair in fixtures.windows(2) { + assert_ne!( + pair[0].bytes, pair[1].bytes, + "rows {} and {} produced identical bytes", + pair[0].row, pair[1].row + ); + } + } + + /// The era contributes rows 56 through 65 in order, all minted on dev. + #[test] + fn the_era_covers_rows_56_through_65_on_dev() { + let fixtures = fixtures(); + let rows: Vec = fixtures.iter().map(|f| f.row).collect(); + assert_eq!(rows, (56..=65).collect::>()); + assert!(fixtures.iter().all(|f| f.branch == "dev")); + } +} diff --git a/tools/workbench/src/wallet_grammars/era_zkeys.rs b/tools/workbench/src/wallet_grammars/era_zkeys.rs new file mode 100644 index 0000000000..d1c668693f --- /dev/null +++ b/tools/workbench/src/wallet_grammars/era_zkeys.rs @@ -0,0 +1,1438 @@ +//! Census rows 22 through 39: the zecwallet-light-cli era, from the +//! collapse of the two sapling key vectors into one `Vector` +//! through the Blazesync restructure and on into the orchard additions of +//! 2022. +//! +//! Every fixture in this module is derived from the writer source at its +//! Defining Commit, read with `git show :` and the +//! sub-writers that file calls. Three file layouts appear here. +//! +//! Rows 22 through 29 use the flat layout, in which `LightWallet::write` in +//! `lib/src/lightwallet.rs` emits the key material inline. Rows 30 through +//! 35 use the layout Blazesync introduced at `87ad71c28`, in which a `Keys` +//! record holds the key material while the block and transaction sets +//! delegate to their own writers. Rows 36 through 39 continue that layout +//! with a `Keys` record that has absorbed the transparent addresses into +//! `WalletTKey` records and, from row 38, gained an orchard key vector; at +//! `a6f8a0bd6` the writer's home moves from `lib/src/lightwallet.rs` to +//! `lib/src/wallet.rs`. +//! +//! The wallet these fixtures describe is the same throughout: unencrypted, +//! born at height 1000000 on chain "main", holding one HD sapling key, one +//! transparent key with its address, one block, and one transaction that +//! carries one sapling note and one transparent output. From row 38, where +//! the grammar gains orchard vectors, it also holds one orchard key and one +//! orchard note. Key and seed material is zeroed; identifiers and hashes +//! carry repeated marker bytes so that a hexdump reads easily; opaque +//! length-prefixed blobs carry the encodings their own writers would have +//! produced. + +use super::util::{ + push_bytes, push_compact_size, push_i32_le, push_optional_none, push_optional_some, + push_u32_le, push_u64_le, push_u64_string, push_u8, +}; +use super::Fixture; + +/// The chain this wallet was configured for; `write_string` framed. From +/// `a6f8a0bd6` the writer takes the string from `Network`'s `Display` +/// implementation, whose mainnet arm still renders "main". +const CHAIN_NAME: &str = "main"; + +/// The wallet's birthday, written as a little-endian u64. +const BIRTHDAY: u64 = 1_000_000; + +/// The height of the single block and of the single transaction. +const BLOCK_HEIGHT: i32 = 1_000_000; + +/// The block's Unix mining time, and the transaction's datetime. +const BLOCK_TIME: u32 = 1_600_000_000; + +/// A serialized zip32 `ExtendedSpendingKey` or `ExtendedFullViewingKey`. +/// Both records occupy the same width: a depth byte, a four-byte parent +/// tag, a four-byte child index, a thirty-two-byte chain code, a +/// ninety-six-byte key body, and a thirty-two-byte diversifier key. +const EXTENDED_KEY_LEN: usize = 169; + +/// A serialized orchard `FullViewingKey`. +const ORCHARD_FVK_LEN: usize = 96; + +/// The width of the encrypted-seed field, which the writer emits raw and +/// unprefixed whether or not the wallet is encrypted. +const ENC_SEED_LEN: usize = 48; + +/// The width of the plaintext seed field, and of a secp256k1 or orchard +/// spending key. +const SEED_LEN: usize = 32; + +/// The single transaction's identifier. +const TXID: [u8; 32] = [0x11; 32]; + +/// The single sapling note's nullifier. +const NULLIFIER: [u8; 32] = [0x22; 32]; + +/// The single block's hash. +const BLOCK_HASH: [u8; 32] = [0x33; 32]; + +/// The single block's predecessor's hash. +const PREV_BLOCK_HASH: [u8; 32] = [0x44; 32]; + +/// The single note commitment carried by the block's compact transaction. +const CMU: [u8; 32] = [0x55; 32]; + +/// A note's diversifier. Sapling and orchard diversifiers are both eleven +/// bytes wide. +const DIVERSIFIER: [u8; 11] = [0x66; 11]; + +/// The sapling note's random seed material, written bare before row 25 and +/// behind a type tag from row 25 onward. +const RSEED: [u8; 32] = [0x77; 32]; + +/// The orchard note's rho, written inside the note record. +const ORCHARD_RHO: [u8; 32] = [0x88; 32]; + +/// The orchard note's random seed. +const ORCHARD_RSEED: [u8; 32] = [0x99; 32]; + +/// The orchard note's nullifier, written beside the note record. +const ORCHARD_NULLIFIER: [u8; 32] = [0xAA; 32]; + +/// The value of the single sapling note, in zatoshi. +const NOTE_VALUE: u64 = 100_000; + +/// The value of the single orchard note, in zatoshi. +const ORCHARD_NOTE_VALUE: u64 = 25_000; + +/// The value of the single transparent output, in zatoshi. +const UTXO_VALUE: u64 = 50_000; + +/// The wallet's single transparent address. The reader asserts that a +/// UTXO's address begins with `t`, so the fixture honors that. +const TADDR: &str = "t1FixtureTransparentAddress00000000"; + +/// The transparent output's script: the opening bytes of a +/// pay-to-pubkey-hash script, enough to exercise the length-prefixed +/// script vector. +const SCRIPT: [u8; 3] = [0x76, 0xa9, 0x14]; + +/// The `Rseed::AfterZip212` type tag, which `write_rseed` emits from +/// `28b795139` onward. +const RSEED_TAG_AFTER_ZIP212: u8 = 2; + +/// `MemoDownloadOption::WalletMemos`, the default the `WalletOptions` +/// record carries from `7f59c5320` onward. +const MEMO_DOWNLOAD_WALLET_MEMOS: u8 = 1; + +/// `MAX_TRANSACTION_SIZE_DEFAULT`, the transaction-size filter that +/// `2e8b86670` introduces and defaults to. +const TRANSACTION_SIZE_FILTER: u32 = 500; + +/// This era's fixtures, in census order. +pub fn fixtures() -> Vec { + vec![ + Fixture { + row: 22, + defining_commit: "fb1135328", + branch: "dev", + bytes: row_22(), + }, + Fixture { + row: 23, + defining_commit: "49ee4c406", + branch: "dev", + bytes: row_23(), + }, + Fixture { + row: 24, + defining_commit: "8e425fc6b", + branch: "dev", + bytes: row_24(), + }, + Fixture { + row: 25, + defining_commit: "28b795139", + branch: "dev", + bytes: row_25(), + }, + Fixture { + row: 26, + defining_commit: "b61175345", + branch: "dev", + bytes: row_26(), + }, + Fixture { + row: 27, + defining_commit: "bcf38a6fa", + branch: "dev", + bytes: row_27(), + }, + Fixture { + row: 28, + defining_commit: "7212e2bf1", + branch: "dev", + bytes: row_28(), + }, + Fixture { + row: 29, + defining_commit: "4a279179f", + branch: "dev", + bytes: row_29(), + }, + Fixture { + row: 30, + defining_commit: "87ad71c28", + branch: "dev", + bytes: row_30(), + }, + Fixture { + row: 31, + defining_commit: "ead95fe0a", + branch: "dev", + bytes: row_31(), + }, + Fixture { + row: 32, + defining_commit: "0cd53900b", + branch: "dev", + bytes: row_32(), + }, + Fixture { + row: 33, + defining_commit: "a1b9b0bbe", + branch: "dev", + bytes: row_33(), + }, + Fixture { + row: 34, + defining_commit: "ed3b21c09", + branch: "dev", + bytes: row_34(), + }, + Fixture { + row: 35, + defining_commit: "7f59c5320", + branch: "dev", + bytes: row_35(), + }, + Fixture { + row: 36, + defining_commit: "5e73adef4", + branch: "dev", + bytes: row_36(), + }, + Fixture { + row: 37, + defining_commit: "a6f8a0bd6", + branch: "dev", + bytes: row_37(), + }, + Fixture { + row: 38, + defining_commit: "6dd62d5e2", + branch: "dev", + bytes: row_38(), + }, + Fixture { + row: 39, + defining_commit: "2e8b86670", + branch: "dev", + bytes: row_39(), + }, + ] +} + +// --------------------------------------------------------------------------- +// Dependency-derived encodings +// --------------------------------------------------------------------------- + +/// Append a serialized zip32 extended key, spending or viewing. +// +// ASSUMPTION: librustzcash's `ExtendedSpendingKey::write` and +// `ExtendedFullViewingKey::write` are not vendored in this repository, so +// their widths come from the stable zip32 encoding: depth u8, parent +// fingerprint tag 4 bytes, child index u32, chain code 32 bytes, key body +// 96 bytes, diversifier key 32 bytes, for 169 bytes each. All-zero material +// is legal to write, since neither writer validates what it is given. +fn push_extended_key(out: &mut Vec) { + push_bytes(out, &[0u8; EXTENDED_KEY_LEN]); +} + +/// Append a serialized orchard full viewing key. +// +// ASSUMPTION: the `orchard` crate is not vendored here. Its +// `FullViewingKey::write` emits the raw ninety-six-byte encoding, the +// concatenation of `ak`, `nk`, and `rivk` at thirty-two bytes each, which +// the matching reader at `6dd62d5e2` confirms by reading `[0; 96]`. +fn push_orchard_full_viewing_key(out: &mut Vec) { + push_bytes(out, &[0u8; ORCHARD_FVK_LEN]); +} + +/// Append the serialization of an empty sapling `CommitmentTree`. +// +// ASSUMPTION: librustzcash's `CommitmentTree::write` emits +// `Optional`, `Optional`, and a `Vector>` of +// parents. An empty tree therefore serializes as three zero bytes: two +// `None` markers and a zero-length vector. +fn push_empty_commitment_tree(out: &mut Vec) { + push_optional_none(out); + push_optional_none(out); + push_compact_size(out, 0); +} + +/// Append a protobuf base-128 varint. +// +// ASSUMPTION: the block and tree-state blobs are produced by `prost`, whose +// encoding this helper and its callers reproduce: a key varint holding the +// field number and the wire type, then the payload, with proto3 default +// values omitted entirely. +fn push_proto_varint(out: &mut Vec, mut value: u64) { + loop { + if value < 0x80 { + push_u8(out, value as u8); + return; + } + push_u8(out, ((value & 0x7F) as u8) | 0x80); + value >>= 7; + } +} + +/// Append a protobuf field key: the field number and the wire type. +fn push_proto_key(out: &mut Vec, field: u64, wire_type: u64) { + push_proto_varint(out, (field << 3) | wire_type); +} + +/// Append a protobuf varint field, omitting it when the value is the +/// proto3 default. +fn push_proto_varint_field(out: &mut Vec, field: u64, value: u64) { + if value == 0 { + return; + } + push_proto_key(out, field, 0); + push_proto_varint(out, value); +} + +/// Append a protobuf length-delimited field, omitting it when the payload +/// is empty. +fn push_proto_bytes_field(out: &mut Vec, field: u64, payload: &[u8]) { + if payload.is_empty() { + return; + } + push_proto_key(out, field, 2); + push_proto_varint(out, payload.len() as u64); + push_bytes(out, payload); +} + +/// The protobuf encoding of the `CompactBlock` this wallet's single block +/// stores, as `lib/proto/compact_formats.proto` defines it at `87ad71c28`: +/// height in field 2, hash in field 3, prevHash in field 4, time in field +/// 5, and the compact transactions in field 7. `BlockData::new` clears the +/// header and each output's ciphertext and ephemeral key before encoding, +/// so the fixture omits them too. +fn compact_block_protobuf() -> Vec { + let mut spend = Vec::new(); + push_proto_bytes_field(&mut spend, 1, &NULLIFIER); + + let mut output = Vec::new(); + push_proto_bytes_field(&mut output, 1, &CMU); + + let mut transaction = Vec::new(); + push_proto_bytes_field(&mut transaction, 2, &TXID); + push_proto_bytes_field(&mut transaction, 4, &spend); + push_proto_bytes_field(&mut transaction, 5, &output); + + let mut block = Vec::new(); + push_proto_varint_field(&mut block, 2, BLOCK_HEIGHT as u64); + push_proto_bytes_field(&mut block, 3, &BLOCK_HASH); + push_proto_bytes_field(&mut block, 4, &PREV_BLOCK_HASH); + push_proto_varint_field(&mut block, 5, u64::from(BLOCK_TIME)); + push_proto_bytes_field(&mut block, 7, &transaction); + block +} + +/// The protobuf encoding of the `TreeState` that rows 32 onward store +/// behind an `Optional`, as `lib/proto/service.proto` defines it at +/// `0cd53900b`: network in field 1, height in field 2, hash in field 3, +/// time in field 4, and the hex-encoded sapling commitment tree in field 5. +fn tree_state_protobuf() -> Vec { + let mut empty_tree = Vec::new(); + push_empty_commitment_tree(&mut empty_tree); + let tree_hex: String = empty_tree.iter().map(|b| format!("{:02x}", b)).collect(); + let hash_hex: String = BLOCK_HASH.iter().map(|b| format!("{:02x}", b)).collect(); + + let mut state = Vec::new(); + push_proto_bytes_field(&mut state, 1, CHAIN_NAME.as_bytes()); + push_proto_varint_field(&mut state, 2, BIRTHDAY); + push_proto_bytes_field(&mut state, 3, hash_hex.as_bytes()); + push_proto_varint_field(&mut state, 4, u64::from(BLOCK_TIME)); + push_proto_bytes_field(&mut state, 5, tree_hex.as_bytes()); + state +} + +// --------------------------------------------------------------------------- +// Sub-records shared across the era +// --------------------------------------------------------------------------- + +/// Append one `WalletZKey`, replicating `WalletZKey::write` in +/// `lib/src/lightwallet/walletzkey.rs` and, from `a6f8a0bd6`, in +/// `lib/src/wallet/keys/sapling.rs`. That writer is byte-identical across +/// every row in this module. The fixture's key is an HD key, unlocked, +/// carrying both its spending key and its viewing key, at HD index zero. +fn push_wallet_zkey(out: &mut Vec) { + push_u8(out, 1); // The record's own version byte. + push_u32_le(out, 0); // WalletZKeyType::HdKey. + push_u8(out, 0); // Not locked. + push_optional_some(out); // The spending key is present. + push_extended_key(out); + push_extended_key(out); // The full viewing key, written unconditionally. + push_optional_some(out); // The HD key number is present. + push_u32_le(out, 0); + push_optional_none(out); // No encrypted key. + push_optional_none(out); // No nonce. +} + +/// Append one `WalletTKey`, replicating `WalletTKey::write` in +/// `lib/src/lightwallet/wallettkey.rs` at `5e73adef4` and, from +/// `a6f8a0bd6`, in `lib/src/wallet/keys/transparent.rs`. The record folds +/// the transparent address into the key, which is why the `Keys` record can +/// drop its separate address vector. The fixture's key is an HD key, +/// unlocked, at HD index zero. +fn push_wallet_tkey(out: &mut Vec) { + push_u8(out, 1); // The record's own version byte. + push_u32_le(out, 0); // WalletTKeyType::HdKey. + push_u8(out, 0); // Not locked. + push_optional_some(out); // The secret key is present. + push_bytes(out, &[0u8; SEED_LEN]); + push_u64_string(out, TADDR); + push_optional_some(out); // The HD key number is present. + push_u32_le(out, 0); + push_optional_none(out); // No encrypted key. + push_optional_none(out); // No nonce. +} + +/// Append one `WalletOKey`, replicating `OrchardKey::write` in +/// `lib/src/wallet/keys/orchard.rs` at `6dd62d5e2`. Note that this record's +/// own version byte is 0, not 1 as its sapling and transparent siblings +/// use, and that it writes a key-type discriminant byte rather than the u32 +/// the older records write. The unified address is derived on read and +/// never serialized. +fn push_wallet_okey(out: &mut Vec) { + push_u8(out, 0); // The record's own version byte. + push_u8(out, 0); // Not locked. + push_u8(out, 0); // WalletOKeyInner::HdKey. + push_bytes(out, &[0u8; SEED_LEN]); // The orchard spending key. + push_optional_some(out); // The HD key number is present. + push_u32_le(out, 0); + push_optional_none(out); // No encrypted key. + push_optional_none(out); // No nonce. +} + +/// Append the `WalletZecPriceInfo` record, replicating its writer in +/// `lib/src/lightwallet/data.rs` at `4a279179f`. That writer is unchanged +/// through `2e8b86670` apart from its version word, which moves from 1 to +/// 20 at the Blazesync restructure. The fixture has never fetched a +/// historical price. +fn push_price_info(out: &mut Vec, version: u64) { + push_u64_le(out, version); + push_optional_none(out); // last_historical_prices_fetched_at. + push_u64_le(out, 0); // historical_prices_retry_count. +} + +/// Append the `WalletOptions` record, replicating its writer in +/// `lib/src/lightwallet.rs` at `7f59c5320` and in `lib/src/wallet.rs` at +/// `2e8b86670`, which raises the record to version 2 and appends an +/// `Optional` transaction-size filter. The fixture downloads memos for +/// its own transactions and keeps the default filter. +fn push_wallet_options(out: &mut Vec, version: u64) { + push_u64_le(out, version); + push_u8(out, MEMO_DOWNLOAD_WALLET_MEMOS); + if version >= 2 { + push_optional_some(out); + push_u32_le(out, TRANSACTION_SIZE_FILTER); + } +} + +// --------------------------------------------------------------------------- +// The flat layout: rows 22 through 29 +// --------------------------------------------------------------------------- + +/// Append one `BlockData` in the flat era's encoding, replicating +/// `BlockData::write` in `lib/src/lightwallet/data.rs`, which is unchanged +/// from `fb1135328` through `4a279179f`: the height, the block hash, the +/// commitment tree, and a literal end tag of 11. +fn push_flat_block(out: &mut Vec) { + push_i32_le(out, BLOCK_HEIGHT); + push_bytes(out, &BLOCK_HASH); + push_empty_commitment_tree(out); + push_u64_le(out, 11); +} + +/// Append one `SaplingNoteData` as the writer at the given row's Defining +/// Commit emits it. The record's own version word tracks the row: 1 at +/// `fb1135328`, 2 once `49ee4c406` adds `spent_at_height`, 3 once +/// `8e425fc6b` adds the spendability flag, 4 once `28b795139` tags the note +/// randomness, and 5 once `7212e2bf1` adds the unconfirmed-spend option. +/// The note is unspent, has no memo, is not change, and is spendable. +fn push_flat_note(out: &mut Vec, row: u8) { + let version: u64 = match row { + 22 => 1, + 23 => 2, + 24 => 3, + 25..=27 => 4, + _ => 5, + }; + push_u64_le(out, version); + push_u64_le(out, 0); // The account index. + push_extended_key(out); // The note's extended full viewing key. + push_bytes(out, &DIVERSIFIER); + push_u64_le(out, NOTE_VALUE); + if row >= 25 { + // `28b795139` replaced the bare randomness with a tagged `Rseed`. + push_u8(out, RSEED_TAG_AFTER_ZIP212); + } + push_bytes(out, &RSEED); + push_compact_size(out, 0); // No witnesses are retained. + push_bytes(out, &NULLIFIER); + push_optional_none(out); // spent. + if row >= 23 { + push_optional_none(out); // spent_at_height, added at `49ee4c406`. + } + if row >= 28 { + push_optional_none(out); // unconfirmed_spent, added at `7212e2bf1`. + } + push_optional_none(out); // memo. + push_u8(out, 0); // is_change. + if row >= 24 { + // `8e425fc6b` wrote this as `is_spendable`; `b61175345` renamed it + // to `have_spending_key` without changing its width or position. + push_u8(out, 1); + } +} + +/// Append one `Utxo` as the writer at the given row's Defining Commit +/// emits it. The record's own version word moves from 1 to 2 at +/// `b61175345`, which adds `spent_at_height`, and to 3 at `7212e2bf1`, +/// which adds the unconfirmed-spend option. The address is framed by a u32 +/// length here, not by the u64 `write_string` framing that the top-level +/// address vector uses. +fn push_flat_utxo(out: &mut Vec, row: u8) { + let version: u64 = match row { + 22..=25 => 1, + 26 | 27 => 2, + _ => 3, + }; + push_u64_le(out, version); + push_u32_le(out, TADDR.len() as u32); + push_bytes(out, TADDR.as_bytes()); + push_bytes(out, &TXID); + push_u64_le(out, 0); // output_index. + push_u64_le(out, UTXO_VALUE); + push_i32_le(out, BLOCK_HEIGHT); + push_compact_size(out, SCRIPT.len() as u64); + push_bytes(out, &SCRIPT); + push_optional_none(out); // spent. + if row >= 26 { + push_optional_none(out); // spent_at_height, added at `b61175345`. + } + if row >= 28 { + push_optional_none(out); // unconfirmed_spent, added at `7212e2bf1`. + } +} + +/// Append one `WalletTx` in the flat era's encoding. The record's version +/// word is 4 through `7212e2bf1` and 5 from `4a279179f`, which appends the +/// per-transaction ZEC price. +fn push_flat_wallet_tx(out: &mut Vec, row: u8) { + let version: u64 = if row >= 29 { 5 } else { 4 }; + push_u64_le(out, version); + push_i32_le(out, BLOCK_HEIGHT); + push_u64_le(out, u64::from(BLOCK_TIME)); // datetime. + push_bytes(out, &TXID); + push_compact_size(out, 1); + push_flat_note(out, row); + push_compact_size(out, 1); + push_flat_utxo(out, row); + push_u64_le(out, 0); // total_shielded_value_spent. + push_u64_le(out, 0); // total_transparent_value_spent. + push_compact_size(out, 0); // No outgoing metadata. + push_u8(out, 1); // full_tx_scanned. + if row >= 29 { + push_optional_none(out); // zec_price, added at `4a279179f`. + } +} + +/// Build a flat-layout wallet file for one of rows 22 through 29, +/// replicating `LightWallet::write` in `lib/src/lightwallet.rs` at that +/// row's Defining Commit. +fn flat_layout(row: u8) -> Vec { + let version: u64 = match row { + 22 => 7, + 23 => 8, + 24 => 9, + 25 => 10, + 26 => 12, + 27 | 28 => 13, + _ => 14, + }; + let mut out = Vec::new(); + push_u64_le(&mut out, version); + push_u8(&mut out, 0); // Not encrypted. + push_bytes(&mut out, &[0u8; ENC_SEED_LEN]); // enc_seed, raw and unprefixed. + push_compact_size(&mut out, 0); // The nonce vector is empty. + push_bytes(&mut out, &[0u8; SEED_LEN]); // The seed, raw and unprefixed. + push_compact_size(&mut out, 1); // One `WalletZKey`. + push_wallet_zkey(&mut out); + push_compact_size(&mut out, 1); // One transparent key, 32 raw bytes. + push_bytes(&mut out, &[0u8; SEED_LEN]); + push_compact_size(&mut out, 1); // One transparent address. + push_u64_string(&mut out, TADDR); + push_compact_size(&mut out, 1); // One block. + push_flat_block(&mut out); + push_compact_size(&mut out, 1); // One transaction, keyed by its txid. + push_bytes(&mut out, &TXID); + push_flat_wallet_tx(&mut out, row); + push_u64_string(&mut out, CHAIN_NAME); + push_u64_le(&mut out, BIRTHDAY); + if row >= 27 { + push_u8(&mut out, 0); // sapling_tree_verified, added at `bcf38a6fa`. + } + if row >= 29 { + push_price_info(&mut out, 1); + } + out +} + +// --------------------------------------------------------------------------- +// The Blazesync layout: rows 30 through 39 +// --------------------------------------------------------------------------- + +/// The `Keys` record at the given row, replicating `Keys::write` in +/// `lib/src/lightwallet/keys.rs` and, from `a6f8a0bd6`, in +/// `lib/src/wallet/keys.rs`. Version 20 at `87ad71c28` writes the sapling +/// keys, the raw transparent secret keys, and the transparent addresses as +/// three separate vectors. Version 21 at `5e73adef4` collapses the last two +/// into one `Vector`. Version 22 at `6dd62d5e2` inserts a +/// `Vector` between the sapling and transparent vectors. +fn keys_record(row: u8) -> Vec { + let version: u64 = match row { + 30..=35 => 20, + 36 | 37 => 21, + _ => 22, + }; + let mut out = Vec::new(); + push_u64_le(&mut out, version); + push_u8(&mut out, 0); // Not encrypted. + push_bytes(&mut out, &[0u8; ENC_SEED_LEN]); + push_compact_size(&mut out, 0); // The nonce vector is empty. + push_bytes(&mut out, &[0u8; SEED_LEN]); + push_compact_size(&mut out, 1); // One sapling key. + push_wallet_zkey(&mut out); + if version >= 22 { + push_compact_size(&mut out, 1); // One orchard key. + push_wallet_okey(&mut out); + } + if version >= 21 { + push_compact_size(&mut out, 1); // One `WalletTKey`, address included. + push_wallet_tkey(&mut out); + } else { + push_compact_size(&mut out, 1); // One transparent key, 32 raw bytes. + push_bytes(&mut out, &[0u8; SEED_LEN]); + push_compact_size(&mut out, 1); // One transparent address. + push_u64_string(&mut out, TADDR); + } + out +} + +/// Append one `BlockData` in the opaque-blob encoding, replicating +/// `BlockData::write` in `lib/src/lightwallet/data.rs` at `87ad71c28`: the +/// height, the hash, an empty commitment tree, the record's version word of +/// 20, and the encoded compact block as a byte vector. The same writer +/// returns at `7f59c5320` after the one-commit detour through the compact +/// encoding, and it survives the move to `lib/src/wallet/data.rs`. +fn push_ecb_block(out: &mut Vec) { + let ecb = compact_block_protobuf(); + push_i32_le(out, BLOCK_HEIGHT); + push_bytes(out, &BLOCK_HASH); + push_empty_commitment_tree(out); + push_u64_le(out, 20); + push_compact_size(out, ecb.len() as u64); + push_bytes(out, &ecb); +} + +/// Append one `BlockData` in the compact encoding that `ed3b21c09` +/// introduced: the height, the hash, an empty commitment tree written only +/// so the reader can reach the version word, the record's version word of +/// 21, the predecessor's hash, the mining time, and a vector of +/// `CCompactTx`. Each `CCompactTx` carries its own version word of 21, its +/// hash, its nullifiers, and its note commitments. +fn push_compact_block(out: &mut Vec) { + push_i32_le(out, BLOCK_HEIGHT); + push_bytes(out, &BLOCK_HASH); + push_empty_commitment_tree(out); + push_u64_le(out, 21); + push_bytes(out, &PREV_BLOCK_HASH); + push_u32_le(out, BLOCK_TIME); + push_compact_size(out, 1); // One compact transaction. + push_u64_le(out, 21); // The `CCompactTx` version word. + push_bytes(out, &TXID); + push_compact_size(out, 1); // One nullifier. + push_bytes(out, &NULLIFIER); + push_compact_size(out, 1); // One note commitment. + push_bytes(out, &CMU); +} + +/// Append one sapling note record in the Blazesync encoding, replicating +/// the writer at `87ad71c28`. Its version word is 20 and its bytes stay +/// identical through `2e8b86670`, where the concrete writer has become the +/// blanket `ReadableWriteable` implementation for `NoteAndMetadata` in +/// `lib/src/wallet/traits.rs`. The account index and the separate +/// `spent_at_height` option are gone; the witness cache writes its top +/// height after the witness vector, and both spend records carry a height +/// beside the transaction identifier. +fn push_blaze_note(out: &mut Vec) { + push_u64_le(out, 20); + push_extended_key(out); + push_bytes(out, &DIVERSIFIER); + push_u64_le(out, NOTE_VALUE); + push_u8(out, RSEED_TAG_AFTER_ZIP212); + push_bytes(out, &RSEED); + push_compact_size(out, 0); // No witnesses are retained. + push_u64_le(out, BLOCK_HEIGHT as u64); // The witness cache's top height. + push_bytes(out, &NULLIFIER); + push_optional_none(out); // spent. + push_optional_none(out); // unconfirmed_spent. + push_optional_none(out); // memo. + push_u8(out, 0); // is_change. + push_u8(out, 1); // have_spending_key. +} + +/// Append one orchard note record, replicating the same blanket +/// `NoteAndMetadata` writer at `6dd62d5e2` with orchard's associated types. +/// The surrounding frame matches the sapling note exactly; only the key and +/// the note body differ, the latter being a value, a rho, and a random +/// seed where sapling writes a value and a tagged rseed. +fn push_orchard_note(out: &mut Vec) { + push_u64_le(out, 20); + push_orchard_full_viewing_key(out); + push_bytes(out, &DIVERSIFIER); + push_u64_le(out, ORCHARD_NOTE_VALUE); + push_bytes(out, &ORCHARD_RHO); + push_bytes(out, &ORCHARD_RSEED); + push_compact_size(out, 0); // No witnesses are retained. + push_u64_le(out, BLOCK_HEIGHT as u64); // The witness cache's top height. + push_bytes(out, &ORCHARD_NULLIFIER); + push_optional_none(out); // spent. + push_optional_none(out); // unconfirmed_spent. + push_optional_none(out); // memo. + push_u8(out, 0); // is_change. + push_u8(out, 1); // have_spending_key. +} + +/// Append one `Utxo` in the Blazesync encoding, replicating the writer at +/// `87ad71c28`, whose version word is 3 and which stays byte-identical +/// through `2e8b86670`. +fn push_blaze_utxo(out: &mut Vec) { + push_u64_le(out, 3); + push_u32_le(out, TADDR.len() as u32); + push_bytes(out, TADDR.as_bytes()); + push_bytes(out, &TXID); + push_u64_le(out, 0); // output_index. + push_u64_le(out, UTXO_VALUE); + push_i32_le(out, BLOCK_HEIGHT); + push_compact_size(out, SCRIPT.len() as u64); + push_bytes(out, &SCRIPT); + push_optional_none(out); // spent. + push_optional_none(out); // spent_at_height. + push_optional_none(out); // unconfirmed_spent. +} + +/// Append one transaction record in the Blazesync encoding, the type named +/// `WalletTx` until `6dd62d5e2` renames it `TransactionMetadata`. Its +/// version word is 20 at `87ad71c28`, 21 from `ead95fe0a`, which inserts +/// the `unconfirmed` flag after the block height, 22 from `a6f8a0bd6`, +/// which replaces the sapling-and-transparent value-spent pair with a +/// [transparent, sapling, orchard] triple and appends a second nullifier +/// vector, and 23 from `6dd62d5e2`, which inserts an orchard-note vector +/// after the sapling one. Note that the pair and the triple disagree on +/// order as well as on width: version 21 writes sapling before transparent, +/// while version 22 writes transparent first. +fn push_blaze_wallet_tx(out: &mut Vec, row: u8) { + let version: u64 = match row { + 30 => 20, + 31..=36 => 21, + 37 => 22, + _ => 23, + }; + push_u64_le(out, version); + push_i32_le(out, BLOCK_HEIGHT); + if version >= 21 { + push_u8(out, 0); // unconfirmed, added at `ead95fe0a`. + } + push_u64_le(out, u64::from(BLOCK_TIME)); // datetime. + push_bytes(out, &TXID); + push_compact_size(out, 1); // One sapling note. + push_blaze_note(out); + if version >= 23 { + push_compact_size(out, 1); // One orchard note, added at `6dd62d5e2`. + push_orchard_note(out); + } + push_compact_size(out, 1); // One transparent output. + push_blaze_utxo(out); + if version >= 22 { + push_u64_le(out, 0); // total_transparent_value_spent. + push_u64_le(out, 0); // total_sapling_value_spent. + push_u64_le(out, 0); // total_orchard_value_spent. + } else { + push_u64_le(out, 0); // total_sapling_value_spent. + push_u64_le(out, 0); // total_transparent_value_spent. + } + push_compact_size(out, 0); // No outgoing metadata. + push_u8(out, 1); // full_tx_scanned. + push_optional_none(out); // zec_price. + push_compact_size(out, 0); // No spent sapling nullifiers. + if version >= 22 { + push_compact_size(out, 0); // No spent orchard nullifiers. + } +} + +/// Append the transaction-set record, replicating `WalletTxns::write` in +/// `lib/src/lightwallet/wallet_txns.rs` and, from `a6f8a0bd6`, +/// `TransactionMetadataSet::write` in `lib/src/wallet/transactions.rs`. At +/// `87ad71c28` it writes its version word of 20, the confirmed +/// transactions, and a second vector of mempool transactions. At +/// `ead95fe0a` the version word becomes 21 and the mempool vector +/// disappears; it stays at 21 through `2e8b86670`. +fn push_wallet_txns(out: &mut Vec, row: u8) { + let version: u64 = if row >= 31 { 21 } else { 20 }; + push_u64_le(out, version); + push_compact_size(out, 1); // One confirmed transaction. + push_bytes(out, &TXID); + push_blaze_wallet_tx(out, row); + if row == 30 { + push_compact_size(out, 0); // The mempool vector, dropped at `ead95fe0a`. + } +} + +/// Append the `Optional` that `0cd53900b` introduced: the +/// `Optional` marker, then the protobuf-encoded tree state framed as a byte +/// vector. The fixture's wallet has verified its tree at its birthday +/// height. +fn push_optional_tree_state(out: &mut Vec) { + let state = tree_state_protobuf(); + push_optional_some(out); + push_compact_size(out, state.len() as u64); + push_bytes(out, &state); +} + +/// Build a Blazesync-layout wallet file for one of rows 30 through 39, +/// replicating `LightWallet::write` in `lib/src/lightwallet.rs` — and, from +/// `a6f8a0bd6`, in `lib/src/wallet.rs` — at that row's Defining Commit. +fn blaze_layout(row: u8) -> Vec { + let version: u64 = match row { + 30 => 20, + 31 => 21, + 32 => 22, + 33 => 23, + _ => 24, + }; + let mut out = Vec::new(); + push_u64_le(&mut out, version); + push_bytes(&mut out, &keys_record(row)); + push_compact_size(&mut out, 1); // One block. + if row == 34 { + push_compact_block(&mut out); + } else { + push_ecb_block(&mut out); + } + push_wallet_txns(&mut out, row); + push_u64_string(&mut out, CHAIN_NAME); + if row >= 35 { + // `7f59c5320` writes `WalletOptions` between the chain name and the + // birthday, not at the tail. `2e8b86670` raises it to version 2. + push_wallet_options(&mut out, if row >= 39 { 2 } else { 1 }); + } + push_u64_le(&mut out, BIRTHDAY); + if row <= 32 { + push_u8(&mut out, 0); // sapling_tree_verified, dropped at `a1b9b0bbe`. + } + if row >= 32 { + push_optional_tree_state(&mut out); + } + push_price_info(&mut out, 20); + out +} + +// --------------------------------------------------------------------------- +// The rows +// --------------------------------------------------------------------------- + +/// Row 22, Defining Commit `fb1135328` ("Viewing Keys (#32)", 2020-07-21), +/// version word 7. Replicates `LightWallet::write` in +/// `lib/src/lightwallet.rs` together with `WalletZKey::write` in +/// `lib/src/lightwallet/walletzkey.rs` and the `BlockData`, `WalletTx`, +/// `SaplingNoteData`, and `Utxo` writers in `lib/src/lightwallet/data.rs`. +/// The grammar's mark is that the separate spending-key and viewing-key +/// vectors have collapsed into one `Vector`, each element +/// carrying its own u8 version byte, so the fixture holds exactly one such +/// element. +fn row_22() -> Vec { + flat_layout(22) +} + +/// Row 23, Defining Commit `49ee4c406` ("Add spent_at_height for notes", +/// 2020-07-21), version word 8. Replicates the same writers as row 22 with +/// `SaplingNoteData::write` at version 2, which appends an +/// `Optional` after the spend option. The fixture's +/// single note is unspent, so the option is `None` and the record is +/// exactly one byte longer than row 22's. +fn row_23() -> Vec { + flat_layout(23) +} + +/// Row 24, Defining Commit `8e425fc6b` ("Don't update view key witnesses", +/// 2020-08-24), version word 9. `SaplingNoteData::write` reaches version 3 +/// and appends an `is_spendable` u8 after the change flag. The fixture's +/// note is spendable, so that byte is 1. +fn row_24() -> Vec { + flat_layout(24) +} + +/// Row 25, Defining Commit `28b795139` ("Update Librustzcash dependency +/// (#60)", 2020-10-15), version word 10. `SaplingNoteData::write` reaches +/// version 4 and replaces the bare thirty-two-byte randomness with the +/// tagged `Rseed` that `write_rseed` emits: a type byte, 1 for +/// `BeforeZip212` and 2 for `AfterZip212`, then the thirty-two bytes. The +/// fixture's note uses the post-ZIP-212 tag. +fn row_25() -> Vec { + flat_layout(25) +} + +/// Row 26, Defining Commit `b61175345` ("Speed up sync with multiple +/// parallel witness updates (#67)", 2020-12-01), version word 12; version +/// 11 was never minted. `Utxo::write` reaches version 2 and appends an +/// `Optional` after the spend option, so the fixture's +/// single unspent transparent output is one byte longer than row 25's. The +/// note's spendability flag is renamed `have_spending_key` at this commit +/// without changing its width or its position. +fn row_26() -> Vec { + flat_layout(26) +} + +/// Row 27, Defining Commit `bcf38a6fa` ("Fast Initial Sync (#69)", +/// 2021-04-22), version word 13. `LightWallet::write` appends a +/// `sapling_tree_verified` u8 after the birthday. The fixture has not +/// verified its tree, so the byte is 0 and it is the file's last byte. +fn row_27() -> Vec { + flat_layout(27) +} + +/// Row 28, Defining Commit `7212e2bf1` ("Add commands to track progress of +/// building and sending a transaction (#70)", 2021-05-05), version word 13 +/// reused. `SaplingNoteData::write` reaches version 5 and `Utxo::write` +/// reaches version 3, each appending an +/// `Optional<(txid[32], u32 height)>` unconfirmed-spend record: the note's +/// after `spent_at_height` and before the memo, the UTXO's after +/// `spent_at_height` and at the record's end. `WalletTx` stays at version 4 +/// and the file carries no price record, which is what separates this +/// grammar from row 29's. The fixture's note and output are both unspent, +/// so each new option is `None` and the file is two bytes longer than row +/// 27's under an unchanged version word. +fn row_28() -> Vec { + flat_layout(28) +} + +/// Row 29, Defining Commit `4a279179f` ("Prices (#71)", 2021-05-18), +/// version word 14. `LightWallet::write` appends the `WalletZecPriceInfo` +/// record — its own version word, an `Optional` fetch timestamp, and a +/// u64 retry count — after the `sapling_tree_verified` byte, and +/// `WalletTx::write` reaches version 5, gaining an `Optional` ZEC +/// price after the scan flag. The note and UTXO bumps that the 58-row +/// census table folded into this row belong to row 28, where `7212e2bf1` +/// minted them. +fn row_29() -> Vec { + flat_layout(29) +} + +/// Row 30, Defining Commit `87ad71c28` ("Blazesync (#74)", 2021-06-25), +/// version word 20; versions 15 through 19 were never minted. The file is +/// restructured: `LightWallet::write` in `lib/src/lightwallet.rs` now emits +/// its version word, a `Keys` record from `lib/src/lightwallet/keys.rs`, +/// the block vector, a `WalletTxns` record from +/// `lib/src/lightwallet/wallet_txns.rs`, the chain name, the birthday, the +/// `sapling_tree_verified` byte, and the price record. Every sub-record's +/// own version word moves to 20 as well, `BlockData` becomes an opaque +/// encoded compact block behind a byte vector, `SaplingNoteData` drops its +/// account index and its `spent_at_height` option while gaining a witness +/// top height, and `WalletTxns` writes a second, empty vector of mempool +/// transactions. +fn row_30() -> Vec { + blaze_layout(30) +} + +/// Row 31, Defining Commit `ead95fe0a` ("Mempool monitoring (#76)", +/// 2021-07-14), version word 21. `WalletTx::write` reaches version 21 and +/// inserts an `unconfirmed` u8 directly after the block height, while +/// `WalletTxns::write` reaches version 21 and stops writing the separate +/// mempool vector. The two changes cancel in length, so this fixture is +/// exactly as long as row 30's and differs from it only in content. +fn row_31() -> Vec { + blaze_layout(31) +} + +/// Row 32, Defining Commit `0cd53900b` ("Sapling tree verification", +/// 2021-07-27), version word 22. `LightWallet::write` appends an +/// `Optional` after the `sapling_tree_verified` byte, encoding +/// the protobuf message into a byte vector. The fixture's wallet has +/// verified a tree at its birthday height, so the option is `Some` and the +/// blob is a real `TreeState` encoding. +fn row_32() -> Vec { + blaze_layout(32) +} + +/// Row 33, Defining Commit `a1b9b0bbe` ("Clean up initial verification", +/// 2021-07-27), version word 23. `LightWallet::write` drops the +/// `sapling_tree_verified` byte that `bcf38a6fa` added, leaving the +/// `Optional` to stand alone between the birthday and the price +/// record. +/// +/// This grammar was minted twice. On 2021-08-05 the commit `c2c99265f` +/// ("Cleanup") reverted row 34's compact block encoding byte for byte and +/// restored version word 23, so a file written between 2021-08-05 and +/// `7f59c5320` on 2021-09-24 is indistinguishable from a file written in +/// this row's first window. The census therefore folds that second minting +/// into this row rather than giving it a row of its own, and this single +/// fixture stands for both windows. +fn row_33() -> Vec { + blaze_layout(33) +} + +/// Row 34, Defining Commit `ed3b21c09` ("use CCompactTx", 2021-07-29), +/// version word 24. `BlockData::write` in `lib/src/lightwallet/data.rs` +/// reaches version 21 and re-encodes each block structurally: after the +/// height, the hash, and an empty commitment tree written only so the +/// reader can reach the version word, it emits the predecessor's hash, the +/// mining time, and a `Vector`. The fixture's block carries one +/// `CCompactTx` holding one nullifier and one note commitment, the same +/// content that row 35 hides inside its opaque blob. +fn row_34() -> Vec { + blaze_layout(34) +} + +/// Row 35, Defining Commit `7f59c5320` ("Optionally download memos", +/// 2021-09-24), version word 24 reused. `BlockData::write` has returned to +/// the opaque encoded-compact-block encoding with its version word of 20, +/// and `LightWallet::write` writes a `WalletOptions` record — its own +/// version word and a `download_memos` u8 — between the chain name and the +/// birthday, not at the tail as the census summary suggests. The fixture's +/// wallet downloads memos for its own transactions, which is the default. +/// Rows 34 and 35 share the file's version word and are told apart only by +/// the block record's encoding. +fn row_35() -> Vec { + blaze_layout(35) +} + +/// Row 36, Defining Commit `5e73adef4` ("Taddress priv key import (#83)", +/// 2021-10-13), version word 24 reused. `Keys::write` in +/// `lib/src/lightwallet/keys.rs` reaches version 21: the raw transparent +/// secret keys and the transparent address strings, which were two +/// independent trailing vectors, collapse into one `Vector` +/// whose element writer lives in the new +/// `lib/src/lightwallet/wallettkey.rs`. Each `WalletTKey` carries its own +/// u8 version byte, a key-type u32, a locked flag, an optional secret key, +/// its address as a `write_string`, an optional HD index, and the encrypted +/// key and nonce options. Nothing outside the `Keys` record changes. +fn row_36() -> Vec { + blaze_layout(36) +} + +/// Row 37, Defining Commit `a6f8a0bd6` (merge of "keep_orcharding" via +/// `74bace493` and `9b1faacac`, 2022-07-23), version word 24 reused. The +/// writer's home has moved from `lib/src/lightwallet.rs` to +/// `lib/src/wallet.rs`, and `WalletTx::write` in `lib/src/wallet/data.rs` +/// reaches version 22. The sapling-and-transparent value-spent pair becomes +/// the `[transparent, sapling, orchard]` triple that `value_spent_by_pool` +/// returns, which both widens the field by eight bytes and reverses the +/// order of the two values that were already there, and a +/// `Vector` follows the sapling nullifier vector. The +/// `Keys` record stays at version 21, so the wallet still holds no orchard +/// key. +fn row_37() -> Vec { + blaze_layout(37) +} + +/// Row 38, Defining Commit `6dd62d5e2` (merge of "orchardize_more" via +/// `085cd0661` and `17f3f5b5b`, 2022-08-23), version word 24 reused. Two +/// marks land together. `Keys::write` in `lib/src/wallet/keys.rs` reaches +/// version 22 and inserts a `Vector` between the sapling and +/// transparent vectors, its element writer being `OrchardKey::write` in +/// `lib/src/wallet/keys/orchard.rs`. The transaction record, renamed +/// `TransactionMetadata`, reaches version 23 and inserts a +/// `Vector` between the sapling-note and UTXO vectors; both +/// note vectors are now written by the blanket `ReadableWriteable` +/// implementation for `NoteAndMetadata` in `lib/src/wallet/traits.rs`, +/// which frames orchard and sapling notes identically and differs only in +/// the key and note bodies. The fixture holds one orchard key and one +/// orchard note so that both marks appear in the bytes. +fn row_38() -> Vec { + blaze_layout(38) +} + +/// Row 39, Defining Commit `2e8b86670` (merge of +/// "transaction_filter_persists" via `649713ffb`, 2022-09-16), version word +/// 24 reused. `WalletOptions::write` in `lib/src/wallet.rs` reaches version +/// 2 and appends an `Optional` transaction-size filter after the +/// memo-download byte. Nothing else in the file changes: the accompanying +/// commit only reroutes the writer's field access through a +/// `TransactionContext`. The fixture keeps the record's own default of 500. +fn row_39() -> Vec { + blaze_layout(39) +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The version word a fixture opens with. + fn version_word(bytes: &[u8]) -> u64 { + u64::from_le_bytes(bytes[0..8].try_into().unwrap()) + } + + #[test] + fn every_row_opens_with_its_version_word() { + assert_eq!(version_word(&row_22()), 7); + assert_eq!(version_word(&row_23()), 8); + assert_eq!(version_word(&row_24()), 9); + assert_eq!(version_word(&row_25()), 10); + assert_eq!(version_word(&row_26()), 12); + assert_eq!(version_word(&row_27()), 13); + assert_eq!(version_word(&row_28()), 13); + assert_eq!(version_word(&row_29()), 14); + assert_eq!(version_word(&row_30()), 20); + assert_eq!(version_word(&row_31()), 21); + assert_eq!(version_word(&row_32()), 22); + assert_eq!(version_word(&row_33()), 23); + assert_eq!(version_word(&row_34()), 24); + assert_eq!(version_word(&row_35()), 24); + assert_eq!(version_word(&row_36()), 24); + assert_eq!(version_word(&row_37()), 24); + assert_eq!(version_word(&row_38()), 24); + assert_eq!(version_word(&row_39()), 24); + } + + /// Row 22's mark: one key vector, whose single element opens with the + /// `WalletZKey` version byte. The count sits at offset 90, after the + /// version word, the encrypted flag, the encrypted seed, the empty + /// nonce vector, and the seed. + #[test] + fn row_22_writes_one_wallet_zkey_vector() { + let bytes = row_22(); + let offset = 8 + 1 + ENC_SEED_LEN + 1 + SEED_LEN; + assert_eq!(offset, 90); + assert_eq!(bytes[offset], 1, "the key vector holds one element"); + assert_eq!(bytes[offset + 1], 1, "the element's own version byte"); + } + + /// Rows 23, 24, and 25 each widen the note record by exactly one byte: + /// the `spent_at_height` `None` marker, the spendability flag, and the + /// `Rseed` type tag in turn. + #[test] + fn rows_23_through_25_each_add_one_note_byte() { + assert_eq!(row_23().len(), row_22().len() + 1); + assert_eq!(row_24().len(), row_23().len() + 1); + assert_eq!(row_25().len(), row_24().len() + 1); + } + + /// Row 25's mark: the note's randomness now carries a type tag + /// immediately before its thirty-two bytes. + #[test] + fn row_25_tags_the_note_randomness() { + let mut untagged = Vec::new(); + push_flat_note(&mut untagged, 24); + let mut tagged = Vec::new(); + push_flat_note(&mut tagged, 25); + + let head = untagged + .windows(RSEED.len()) + .position(|w| w == RSEED) + .expect("the untagged note carries the randomness bare"); + assert_eq!(tagged[head], RSEED_TAG_AFTER_ZIP212); + assert_eq!(tagged[head + 1..head + 1 + RSEED.len()], RSEED); + assert_eq!(tagged.len(), untagged.len() + 1); + } + + /// Row 26's mark: the UTXO record gains one `Optional` marker. + #[test] + fn row_26_widens_the_utxo_record_by_one_byte() { + assert_eq!(row_26().len(), row_25().len() + 1); + let mut before = Vec::new(); + push_flat_utxo(&mut before, 25); + let mut after = Vec::new(); + push_flat_utxo(&mut after, 26); + assert_eq!(after.len(), before.len() + 1); + // Everything after the record's own version word is unchanged; only + // the trailing option is new. + assert_eq!(after[8..before.len()], before[8..]); + } + + /// Row 27's mark: a single `sapling_tree_verified` byte at the tail. + #[test] + fn row_27_appends_the_sapling_tree_verified_byte() { + let bytes = row_27(); + assert_eq!(bytes.len(), row_26().len() + 1); + assert_eq!(*bytes.last().unwrap(), 0); + } + + /// Row 28's mark: the note and the UTXO each gain one unconfirmed-spend + /// option under a version word that has not moved, so the file grows by + /// two bytes while offset 0 still reads 13. + #[test] + fn row_28_adds_two_unconfirmed_spend_options_without_a_version_bump() { + let previous = row_27(); + let bytes = row_28(); + assert_eq!(version_word(&bytes), version_word(&previous)); + assert_eq!(bytes.len(), previous.len() + 2); + + let mut note_before = Vec::new(); + push_flat_note(&mut note_before, 27); + let mut note_after = Vec::new(); + push_flat_note(&mut note_after, 28); + assert_eq!(note_after.len(), note_before.len() + 1); + assert_eq!(u64::from_le_bytes(note_after[0..8].try_into().unwrap()), 5); + + let mut utxo_before = Vec::new(); + push_flat_utxo(&mut utxo_before, 27); + let mut utxo_after = Vec::new(); + push_flat_utxo(&mut utxo_after, 28); + assert_eq!(utxo_after.len(), utxo_before.len() + 1); + assert_eq!(u64::from_le_bytes(utxo_after[0..8].try_into().unwrap()), 3); + } + + /// Row 29's mark: the `WalletZecPriceInfo` record closes the file. + #[test] + fn row_29_appends_the_price_record() { + let bytes = row_29(); + let mut price = Vec::new(); + push_price_info(&mut price, 1); + assert_eq!(price.len(), 17); + assert_eq!(&bytes[bytes.len() - price.len()..], &price[..]); + } + + /// Row 30's mark: a `Keys` record, carrying its own version word of 20, + /// begins immediately after the file's version word. + #[test] + fn row_30_delegates_the_key_material_to_a_keys_record() { + let bytes = row_30(); + let keys = keys_record(30); + assert_eq!(u64::from_le_bytes(keys[0..8].try_into().unwrap()), 20); + assert_eq!(&bytes[8..8 + keys.len()], &keys[..]); + } + + /// Row 31's mark: the transaction record gains an `unconfirmed` byte + /// while the mempool vector disappears, so the file's length is + /// unchanged and only its content moves. + #[test] + fn row_31_trades_the_mempool_vector_for_an_unconfirmed_byte() { + let previous = row_30(); + let bytes = row_31(); + assert_eq!(bytes.len(), previous.len()); + assert_ne!(bytes, previous); + } + + /// Row 32's mark: an `Optional` blob sits between the + /// `sapling_tree_verified` byte and the price record. + #[test] + fn row_32_appends_the_optional_tree_state() { + let bytes = row_32(); + let mut tree_state = Vec::new(); + push_optional_tree_state(&mut tree_state); + assert_eq!(bytes.len(), row_31().len() + tree_state.len()); + + let mut price = Vec::new(); + push_price_info(&mut price, 20); + let tail = bytes.len() - price.len(); + assert_eq!(&bytes[tail - tree_state.len()..tail], &tree_state[..]); + } + + /// Row 33's mark: the `sapling_tree_verified` byte is gone, so the file + /// is exactly one byte shorter than row 32's. + #[test] + fn row_33_drops_the_sapling_tree_verified_byte() { + assert_eq!(row_33().len(), row_32().len() - 1); + } + + /// Rows 34 and 35 share version word 24 and are told apart only by the + /// block record's own version word: 21 for the compact encoding and 20 + /// for the opaque encoded-compact-block encoding. That word sits after + /// the file version, the `Keys` record, the block vector's count, the + /// height, the hash, and the empty commitment tree. + #[test] + fn rows_34_and_35_differ_in_the_block_encoding_alone_at_version_24() { + let compact = row_34(); + let opaque = row_35(); + assert_eq!(version_word(&compact), version_word(&opaque)); + assert_ne!(compact, opaque); + + let offset = 8 + keys_record(34).len() + 1 + 4 + 32 + 3; + assert_eq!( + u64::from_le_bytes(compact[offset..offset + 8].try_into().unwrap()), + 21 + ); + assert_eq!( + u64::from_le_bytes(opaque[offset..offset + 8].try_into().unwrap()), + 20 + ); + } + + /// Row 35's mark: a `WalletOptions` record, its version word followed + /// by the memo-download option, sits between the chain name and the + /// birthday. + #[test] + fn row_35_writes_wallet_options_before_the_birthday() { + let bytes = row_35(); + let mut needle = Vec::new(); + push_u64_string(&mut needle, CHAIN_NAME); + push_wallet_options(&mut needle, 1); + push_u64_le(&mut needle, BIRTHDAY); + assert_eq!(needle.len(), 12 + 9 + 8); + assert!(bytes.windows(needle.len()).any(|w| w == needle)); + } + + /// Row 36's mark: the `Keys` record reaches version 21, and its two + /// trailing transparent vectors become one `Vector` whose + /// element opens with its own version byte and carries the address the + /// old vector held separately. + #[test] + fn row_36_collapses_the_transparent_vectors_into_wallet_tkeys() { + let previous = keys_record(35); + let keys = keys_record(36); + assert_eq!(u64::from_le_bytes(previous[0..8].try_into().unwrap()), 20); + assert_eq!(u64::from_le_bytes(keys[0..8].try_into().unwrap()), 21); + + // Both records agree from the encrypted flag through the sapling + // key vector; they diverge only in what follows it. + let mut zkey = Vec::new(); + push_wallet_zkey(&mut zkey); + let head = 8 + 1 + ENC_SEED_LEN + 1 + SEED_LEN + 1 + zkey.len(); + assert_eq!(keys[8..head], previous[8..head]); + + // Version 20 ended with a raw-key vector and an address vector. + assert_eq!( + previous.len() - head, + (1 + SEED_LEN) + (1 + 8 + TADDR.len()) + ); + + // Version 21 ends with one `Vector` instead. + let mut tail = Vec::new(); + push_compact_size(&mut tail, 1); + push_wallet_tkey(&mut tail); + assert_eq!(&keys[head..], &tail[..]); + assert_ne!(row_36(), row_35()); + } + + /// Row 37's mark: the transaction record reaches version 22, trading a + /// two-value spend field for a three-value one and appending a second + /// nullifier vector, which together add nine bytes. + #[test] + fn row_37_writes_the_pool_triple_and_a_second_nullifier_vector() { + let mut previous = Vec::new(); + push_blaze_wallet_tx(&mut previous, 36); + let mut bytes = Vec::new(); + push_blaze_wallet_tx(&mut bytes, 37); + assert_eq!(u64::from_le_bytes(previous[0..8].try_into().unwrap()), 21); + assert_eq!(u64::from_le_bytes(bytes[0..8].try_into().unwrap()), 22); + assert_eq!(bytes.len(), previous.len() + 8 + 1); + assert_eq!(row_37().len(), row_36().len() + 9); + } + + /// Row 38's two marks: the `Keys` record reaches version 22 and carries + /// a `Vector` between its sapling and transparent vectors, + /// and the transaction record reaches version 23 and carries a + /// `Vector` after its sapling-note vector. + #[test] + fn row_38_adds_the_orchard_key_and_orchard_note_vectors() { + let keys = keys_record(38); + assert_eq!(u64::from_le_bytes(keys[0..8].try_into().unwrap()), 22); + + let mut okey = Vec::new(); + push_wallet_okey(&mut okey); + let okey_at = keys + .windows(okey.len()) + .position(|w| w == okey) + .expect("the orchard key vector holds one element"); + let mut tkey = Vec::new(); + push_wallet_tkey(&mut tkey); + let tkey_at = keys + .windows(tkey.len()) + .position(|w| w == tkey) + .expect("the transparent key vector holds one element"); + assert!(okey_at < tkey_at, "orchard keys precede transparent keys"); + + let mut transaction = Vec::new(); + push_blaze_wallet_tx(&mut transaction, 38); + assert_eq!( + u64::from_le_bytes(transaction[0..8].try_into().unwrap()), + 23 + ); + let mut orchard_note = Vec::new(); + push_orchard_note(&mut orchard_note); + let mut sapling_note = Vec::new(); + push_blaze_note(&mut sapling_note); + let sapling_at = transaction + .windows(sapling_note.len()) + .position(|w| w == sapling_note) + .expect("the sapling note is present"); + let orchard_at = transaction + .windows(orchard_note.len()) + .position(|w| w == orchard_note) + .expect("the orchard note is present"); + assert!(sapling_at < orchard_at, "sapling notes come first"); + } + + /// Row 39's mark: the `WalletOptions` record reaches version 2 and + /// appends an `Optional` transaction-size filter, five bytes in + /// its `Some` form, and nothing else in the file moves. + #[test] + fn row_39_appends_the_transaction_size_filter() { + let mut previous = Vec::new(); + push_wallet_options(&mut previous, 1); + let mut options = Vec::new(); + push_wallet_options(&mut options, 2); + assert_eq!(options.len(), previous.len() + 5); + // Only the version word moves; the memo-download byte keeps its + // value and its position, and the filter follows it. + assert_eq!(options[8], previous[8]); + assert_eq!(options[9], 1, "the Optional Some marker"); + assert_eq!( + u32::from_le_bytes(options[10..14].try_into().unwrap()), + TRANSACTION_SIZE_FILTER + ); + + assert_eq!(row_39().len(), row_38().len() + 5); + let bytes = row_39(); + let mut needle = Vec::new(); + push_u64_string(&mut needle, CHAIN_NAME); + push_wallet_options(&mut needle, 2); + push_u64_le(&mut needle, BIRTHDAY); + assert!(bytes.windows(needle.len()).any(|w| w == needle)); + } + + /// Adjacent census rows are distinguishable, which is the claim the + /// census makes for every neighboring pair. + #[test] + fn adjacent_rows_are_byte_distinct() { + let all = fixtures(); + for pair in all.windows(2) { + assert_ne!( + pair[0].bytes, pair[1].bytes, + "rows {} and {} produced identical bytes", + pair[0].row, pair[1].row + ); + } + } + + /// The fixtures this module emits carry the row numbers the 77-row + /// census table assigns, in order and without gaps. + #[test] + fn the_era_covers_rows_22_through_39() { + let rows: Vec = fixtures().iter().map(|f| f.row).collect(); + assert_eq!(rows, (22..=39).collect::>()); + } +} diff --git a/tools/workbench/src/wallet_grammars/util.rs b/tools/workbench/src/wallet_grammars/util.rs new file mode 100644 index 0000000000..c239894914 --- /dev/null +++ b/tools/workbench/src/wallet_grammars/util.rs @@ -0,0 +1,119 @@ +//! Byte-level primitives shared by the era modules. +//! +//! The wallet writer's history uses three length disciplines, and telling +//! them apart is much of the census, so each has its own helper here: +//! little-endian scalars (byteorder style), Bitcoin CompactSize counts (as +//! `zcash_encoding::Vector` writes them), and u64-length strings (as the +//! historical `read_string`/`write_string` pair framed them). Every helper +//! appends to `out`. + +/// Append one byte. +pub fn push_u8(out: &mut Vec, v: u8) { + out.push(v); +} + +/// Append a little-endian u16. +pub fn push_u16_le(out: &mut Vec, v: u16) { + out.extend_from_slice(&v.to_le_bytes()); +} + +/// Append a little-endian u32. +pub fn push_u32_le(out: &mut Vec, v: u32) { + out.extend_from_slice(&v.to_le_bytes()); +} + +/// Append a little-endian u64. +pub fn push_u64_le(out: &mut Vec, v: u64) { + out.extend_from_slice(&v.to_le_bytes()); +} + +/// Append a little-endian i32. +pub fn push_i32_le(out: &mut Vec, v: i32) { + out.extend_from_slice(&v.to_le_bytes()); +} + +/// Append a little-endian i64. +pub fn push_i64_le(out: &mut Vec, v: i64) { + out.extend_from_slice(&v.to_le_bytes()); +} + +/// Append raw bytes. +pub fn push_bytes(out: &mut Vec, bytes: &[u8]) { + out.extend_from_slice(bytes); +} + +/// Append a Bitcoin CompactSize, the count encoding `zcash_encoding::Vector` +/// (and its librustzcash predecessors) writes before vector elements. +pub fn push_compact_size(out: &mut Vec, n: u64) { + match n { + 0..=0xFC => out.push(n as u8), + 0xFD..=0xFFFF => { + out.push(0xFD); + push_u16_le(out, n as u16); + } + 0x1_0000..=0xFFFF_FFFF => { + out.push(0xFE); + push_u32_le(out, n as u32); + } + _ => { + out.push(0xFF); + push_u64_le(out, n); + } + } +} + +/// Append a byte vector in `Vector` discipline: CompactSize count, then the +/// bytes as u8 elements. +pub fn push_compact_vec_u8(out: &mut Vec, bytes: &[u8]) { + push_compact_size(out, bytes.len() as u64); + push_bytes(out, bytes); +} + +/// Append a string in the historical `write_string` discipline: u64 +/// little-endian byte length, then the UTF-8 bytes. This is the framing whose +/// untrusted-length read (`read_string`) is the census's motivating defect. +pub fn push_u64_string(out: &mut Vec, s: &str) { + push_u64_le(out, s.len() as u64); + push_bytes(out, s.as_bytes()); +} + +/// Append an `Optional` None marker (`zcash_encoding::Optional` writes a 0u8). +pub fn push_optional_none(out: &mut Vec) { + out.push(0); +} + +/// Append an `Optional` Some marker (a 1u8); the caller appends the payload. +pub fn push_optional_some(out: &mut Vec) { + out.push(1); +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn compact_size_uses_the_bitcoin_thresholds() { + let mut out = Vec::new(); + push_compact_size(&mut out, 0xFC); + assert_eq!(out, [0xFC]); + + out.clear(); + push_compact_size(&mut out, 0xFD); + assert_eq!(out, [0xFD, 0xFD, 0x00]); + + out.clear(); + push_compact_size(&mut out, 0x1_0000); + assert_eq!(out, [0xFE, 0x00, 0x00, 0x01, 0x00]); + + out.clear(); + push_compact_size(&mut out, 0x1_0000_0000); + assert_eq!(out, [0xFF, 0, 0, 0, 0, 1, 0, 0, 0]); + } + + #[test] + fn u64_string_frames_length_then_utf8() { + let mut out = Vec::new(); + push_u64_string(&mut out, "main"); + assert_eq!(out, [4, 0, 0, 0, 0, 0, 0, 0, b'm', b'a', b'i', b'n']); + } +} diff --git a/zingolib/src/wallet/disk/testing/grammars/README.md b/zingolib/src/wallet/disk/testing/grammars/README.md new file mode 100644 index 0000000000..5c347cb1f6 --- /dev/null +++ b/zingolib/src/wallet/disk/testing/grammars/README.md @@ -0,0 +1,40 @@ +# Format Census example wallets + +One synthetic example Wallet File per row of the Format Census table in +[issue #2590](https://github.com/zingolabs/zingolib/issues/2590), which +enumerates the 77 distinguishable grammars the wallet writer has produced +across the linear (first-parent) histories of `dev` and `stable` (the +2026-07-29 revision; an item-level sweep of the full serializer closure +grew the table from its original 58 rows). + +Each file is named `NN_.dat`: `NN` is the census row number +(zero-padded so a directory listing sorts in census order), and the hash is +the row's Defining Commit — the format's identity, per the census's central +finding that the version word does not identify a format. Row numbers are +presentation order and have been renumbered once already; the hash is the +stable key, and the generator assigns numbers from a single manifest in +`tools/workbench/src/wallet_grammars.rs`. Row 70 (`70_5d8fda797.dat`) is +the one stable-only grammar; every other row was minted on dev's line. + +These files are synthetic, not archival: each was produced by replicating +the writer source at its Defining Commit (`git show :`) +in the generator at `tools/workbench/src/wallet_grammars/`, populated +minimally so that the row's grammar-unique mark is present in the bytes. +They contain zeroed dummy key material and hold no value on any network. +Provenance notes, including every assumption made where an encoding came +from a dependency crate, live as doc comments on the per-row functions in +the generator's era modules. + +Regenerate with: + +``` +cd tools/workbench +cargo run --bin wallet-grammar-fixtures +``` + +The corpus exists to pin Format Recognition: each row's Discriminator must +tell its file apart from the preceding and following rows' files. Two +archaeology results the corpus embodies ahead of the issue text: rows 1 +and 2 are byte-identical grammars (`7ebc8686e` already wrote the version +word; their fixtures differ only in contents, pending a table ruling), and +row 20's compressed body is a gzip frame (`libflate`), not zstd. From 3010e2bb7077a3060f6e094dc4d2057db93a235e Mon Sep 17 00:00:00 2001 From: zancas Date: Thu, 6 Aug 2026 09:16:06 -0700 Subject: [PATCH 2/2] chore(wallet): stage the Format Recognition module, unhooked The recognizer's spine: one WalletFormat arm per census row, each carrying its Defining Commit and its discriminator, plus the bounded Cursor whose length validation retires the read_string allocation class that aborts today's reader on a dev-v40 file. The corpus test and the dev-v40 refusal test ride along. The four era discriminator modules the spine declares (era_inception, era_keys, era_capability, era_modern) are not yet written. The module is therefore not yet declared in disk.rs and does not compile into zingolib; the hookup lands together with the discriminators. Co-Authored-By: Claude Fable 5 --- zingolib/src/wallet/disk/recognition.rs | 539 ++++++++++++++++++++++++ 1 file changed, 539 insertions(+) create mode 100644 zingolib/src/wallet/disk/recognition.rs diff --git a/zingolib/src/wallet/disk/recognition.rs b/zingolib/src/wallet/disk/recognition.rs new file mode 100644 index 0000000000..949bda37a1 --- /dev/null +++ b/zingolib/src/wallet/disk/recognition.rs @@ -0,0 +1,539 @@ +//! Format Recognition (see `zingolib/CONTEXT.md`, Persistence): the pure, +//! total judgment at the front of wallet ingestion that determines which +//! Shipped Format, if any, a candidate Wallet File's bytes conform to, +//! before any field of the file is interpreted. +//! +//! Issue zingolabs/zingolib#2590 is the census this module implements: one +//! enum arm per distinguishable writer grammar in the first-parent histories +//! of dev and stable, identified by its Defining Commit hash — never by the +//! serialized version number, which history reused, skipped, and decreased. +//! Each arm's discriminator structurally parses the entire buffer with +//! bounded reads and no allocation proportional to any claimed length; a +//! grammar conforms only when the parse consumes the buffer exactly. +//! +//! [`recognize`] runs every arm's discriminator and renders the complete +//! verdict: exactly one conformer is a recognition, several is an ambiguity +//! (the load refuses rather than guesses, per the version-42 precedent), and +//! none is a non-conformance carrying each arm's refusal evidence. + +pub(crate) mod era_capability; +pub(crate) mod era_inception; +pub(crate) mod era_keys; +pub(crate) mod era_modern; + +/// Why one grammar's discriminator refused the bytes: the byte offset the +/// parse died at and what the grammar demanded there. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub(crate) struct Refusal { + /// Byte offset at which the grammar's demand went unmet. + pub offset: usize, + /// The grammar's demand at that offset. + pub expected: &'static str, +} + +/// The Recognition Verdict: the complete outcome of Format Recognition. +#[derive(Debug, Clone, PartialEq, Eq)] +pub(crate) enum Recognition { + /// The bytes conform to exactly one Shipped Format. + Recognized(WalletFormat), + /// The bytes conform to more than one Shipped Format; the load refuses + /// rather than guesses, and Recovery Salvage remains available. + Ambiguous(Vec), + /// The bytes conform to no Shipped Format ever minted; the evidence is + /// each arm's refusal. + NotConforming(Vec<(WalletFormat, Refusal)>), +} + +/// A bounded reader over the candidate Wallet File's bytes. +/// +/// Every length claim is validated against the bytes remaining *before* any +/// use, so a corrupt or misgrammared length degrades to a [`Refusal`] instead +/// of an allocation — the `read_string` 854 PB abort class this module +/// retires. The cursor never allocates; it only hands out subslices. +pub(crate) struct Cursor<'a> { + bytes: &'a [u8], + pos: usize, +} + +impl<'a> Cursor<'a> { + pub(crate) fn new(bytes: &'a [u8]) -> Self { + Cursor { bytes, pos: 0 } + } + + pub(crate) fn offset(&self) -> usize { + self.pos + } + + pub(crate) fn remaining(&self) -> usize { + self.bytes.len() - self.pos + } + + /// Refuse at the current offset. + pub(crate) fn refuse(&self, expected: &'static str) -> Result { + Err(Refusal { + offset: self.pos, + expected, + }) + } + + /// A bounds-checked subslice of exactly `n` bytes. + pub(crate) fn bytes(&mut self, n: usize, expected: &'static str) -> Result<&'a [u8], Refusal> { + if self.remaining() < n { + return self.refuse(expected); + } + let s = &self.bytes[self.pos..self.pos + n]; + self.pos += n; + Ok(s) + } + + pub(crate) fn u8(&mut self, expected: &'static str) -> Result { + Ok(self.bytes(1, expected)?[0]) + } + + pub(crate) fn u16_le(&mut self, expected: &'static str) -> Result { + let b = self.bytes(2, expected)?; + Ok(u16::from_le_bytes([b[0], b[1]])) + } + + pub(crate) fn u32_le(&mut self, expected: &'static str) -> Result { + let b = self.bytes(4, expected)?; + Ok(u32::from_le_bytes([b[0], b[1], b[2], b[3]])) + } + + pub(crate) fn u64_le(&mut self, expected: &'static str) -> Result { + let b = self.bytes(8, expected)?; + Ok(u64::from_le_bytes([ + b[0], b[1], b[2], b[3], b[4], b[5], b[6], b[7], + ])) + } + + pub(crate) fn i32_le(&mut self, expected: &'static str) -> Result { + Ok(self.u32_le(expected)? as i32) + } + + pub(crate) fn i64_le(&mut self, expected: &'static str) -> Result { + Ok(self.u64_le(expected)? as i64) + } + + /// An exact little-endian u64 (the wallet-level version word). + pub(crate) fn exact_u64(&mut self, want: u64, expected: &'static str) -> Result<(), Refusal> { + let at = self.pos; + if self.u64_le(expected)? != want { + return Err(Refusal { + offset: at, + expected, + }); + } + Ok(()) + } + + /// An exact single byte (sub-record version bytes, discriminants). + pub(crate) fn exact_u8(&mut self, want: u8, expected: &'static str) -> Result<(), Refusal> { + let at = self.pos; + if self.u8(expected)? != want { + return Err(Refusal { + offset: at, + expected, + }); + } + Ok(()) + } + + /// A canonically-encoded Zcash CompactSize whose value fits the bytes + /// remaining. Non-minimal encodings refuse: no shipped writer emits them. + pub(crate) fn compact_size(&mut self, expected: &'static str) -> Result { + let at = self.pos; + let tag = self.u8(expected)?; + let n: u64 = match tag { + 0..=252 => u64::from(tag), + 253 => { + let v = u64::from(self.u16_le(expected)?); + if v < 253 { + return Err(Refusal { + offset: at, + expected, + }); + } + v + } + 254 => { + let v = u64::from(self.u32_le(expected)?); + if v <= u64::from(u16::MAX) { + return Err(Refusal { + offset: at, + expected, + }); + } + v + } + 255 => { + let v = self.u64_le(expected)?; + if v <= u64::from(u32::MAX) { + return Err(Refusal { + offset: at, + expected, + }); + } + v + } + }; + usize::try_from(n) + .ok() + .filter(|n| *n <= self.remaining()) + .ok_or(Refusal { + offset: at, + expected, + }) + } + + /// A u64 length claim, validated against the bytes remaining. + pub(crate) fn u64_len(&mut self, expected: &'static str) -> Result { + let at = self.pos; + let n = self.u64_le(expected)?; + usize::try_from(n) + .ok() + .filter(|n| *n <= self.remaining()) + .ok_or(Refusal { + offset: at, + expected, + }) + } + + /// The repo's `write_string` form: u64 length + that many bytes. + pub(crate) fn u64_string(&mut self, expected: &'static str) -> Result<&'a [u8], Refusal> { + let n = self.u64_len(expected)?; + self.bytes(n, expected) + } + + /// A `zcash_encoding::Vector`: CompactSize count, then `count` elements + /// parsed by `f`. + pub(crate) fn compact_vec( + &mut self, + expected: &'static str, + mut f: impl FnMut(&mut Cursor<'a>) -> Result<(), Refusal>, + ) -> Result<(), Refusal> { + let n = self.compact_size(expected)?; + for _ in 0..n { + f(self)?; + } + Ok(()) + } + + /// A `Vector` of raw bytes: CompactSize count + that many bytes. + pub(crate) fn compact_vec_u8(&mut self, expected: &'static str) -> Result<&'a [u8], Refusal> { + let n = self.compact_size(expected)?; + self.bytes(n, expected) + } + + /// A `zcash_encoding::Optional`: u8 0 (absent) or 1 (present, then `f`). + pub(crate) fn optional( + &mut self, + expected: &'static str, + f: impl FnOnce(&mut Cursor<'a>) -> Result<(), Refusal>, + ) -> Result<(), Refusal> { + let at = self.pos; + match self.u8(expected)? { + 0 => Ok(()), + 1 => f(self), + _ => Err(Refusal { + offset: at, + expected, + }), + } + } + + /// Conformance demands the grammar consume the buffer exactly. + pub(crate) fn finish(&self) -> Result<(), Refusal> { + if self.remaining() != 0 { + return self.refuse("end of file (trailing bytes refuse the grammar)"); + } + Ok(()) + } +} + +macro_rules! wallet_formats { + ($( $(#[doc = $doc:literal])+ $variant:ident = ($hash:literal, $parser:path), )+) => { + /// Every distinguishable wallet grammar ever minted (issue #2590), + /// one arm per Format Census row, identified by Defining Commit. + /// + /// Rows 1 and 2 of the census are byte-identical grammars + /// (`7ebc8686e` already wrote the u64 version word 1; `c2e26fbbc` + /// only refactored the literal), so they share the single arm + /// [`WalletFormat::F7ebc8686e`]: indistinguishable writer states + /// share an arm. + #[derive(Debug, Clone, Copy, PartialEq, Eq)] + #[allow(non_camel_case_types)] + pub(crate) enum WalletFormat { + $( $(#[doc = $doc])+ $variant, )+ + } + + impl WalletFormat { + /// All arms, census order. + pub(crate) const ALL: &'static [WalletFormat] = + &[ $( WalletFormat::$variant, )+ ]; + + /// The Defining Commit hash — the format's identity. + pub(crate) fn defining_commit(&self) -> &'static str { + match self { + $( WalletFormat::$variant => $hash, )+ + } + } + + /// The arm's discriminator: a full structural parse of the + /// entire buffer under this grammar. + pub(crate) fn discriminator(&self) -> fn(&[u8]) -> Result<(), Refusal> { + match self { + $( WalletFormat::$variant => $parser, )+ + } + } + } + }; +} + +wallet_formats! { + /// Rows 1–2 (2019-09-06): the writer's birth; u64 version word 1. Also + /// covers `c2e26fbbc` (byte-identical grammar). + F7ebc8686e = ("7ebc8686e", era_inception::parse_7ebc8686e), + /// Row 3 (2019-09-06): note `is_change`; txid and shielded-spent in tx. + F8ff6d15e3 = ("8ff6d15e3", era_inception::parse_8ff6d15e3), + /// Row 4 (2019-09-07): raw seed + ExtSK vector. + F5bd8b754d = ("5bd8b754d", era_inception::parse_5bd8b754d), + /// Row 5 (2019-09-13): tx `total_transparent_value_spent`. + Fdb549f5b6 = ("db549f5b6", era_inception::parse_db549f5b6), + /// Row 6 (2019-09-13): standalone Utxo vector. + Ff532b70ca = ("f532b70ca", era_inception::parse_f532b70ca), + /// Row 7 (2019-09-16): raw 32-byte tkey. + Fb24f174b5 = ("b24f174b5", era_inception::parse_b24f174b5), + /// Row 8 (2019-09-17): Utxos move into the tx record. + F0e8ab4d27 = ("0e8ab4d27", era_inception::parse_0e8ab4d27), + /// Row 9 (2019-09-17): Utxo drops unconfirmed-spent. + Ff93267507 = ("f93267507", era_inception::parse_f93267507), + /// Row 10 (2019-09-19): tx v2, outgoing metadata. + Fb0f7d8fcf = ("b0f7d8fcf", era_inception::parse_b0f7d8fcf), + /// Row 11 (2019-09-24): version word 2; chain-name string. + Fb3ca226ff = ("b3ca226ff", era_inception::parse_b3ca226ff), + /// Row 12 (2019-09-25): tx `full_tx_scanned`. + Fdf12ccf31 = ("df12ccf31", era_inception::parse_df12ccf31), + /// Row 13 (2019-09-27): birthday u64. + F88a80f574 = ("88a80f574", era_inception::parse_88a80f574), + /// Row 14 (2019-10-01): tkeys become a Vector. + Fba706ab7c = ("ba706ab7c", era_inception::parse_ba706ab7c), + /// Row 15 (2019-10-01): version word 3 (word-only; tx inner 2→3). + Fe3f972508 = ("e3f972508", era_inception::parse_e3f972508), + /// Row 16 (2019-10-18): tx v4, datetime. + Febf3c7133 = ("ebf3c7133", era_inception::parse_ebf3c7133), + /// Row 17 (2019-10-18): version word 4; locked byte + FVK vector. + Fe3a0fd2de = ("e3a0fd2de", era_inception::parse_e3a0fd2de), + /// Row 18 (2019-10-19): taddress vector. + Ffc15de568 = ("fc15de568", era_inception::parse_fc15de568), + /// Row 19 (2019-10-19): enc_seed + nonce vector. + F72548e077 = ("72548e077", era_inception::parse_72548e077), + /// Row 20 (2020-04-12): version word 5; gzip-compressed body. + F796663c97 = ("796663c97", era_inception::parse_796663c97), + /// Row 21 (2020-05-09): version word 6; plaintext restored. + Fcbffd69c6 = ("cbffd69c6", era_inception::parse_cbffd69c6), + /// Row 22 (2020-07-21): version word 7; WalletZKey vector. + Ffb1135328 = ("fb1135328", era_keys::parse_fb1135328), + /// Row 23 (2020-07-21): version word 8; note spent_at_height. + F49ee4c406 = ("49ee4c406", era_keys::parse_49ee4c406), + /// Row 24 (2020-08-24): version word 9; note is_spendable. + F8e425fc6b = ("8e425fc6b", era_keys::parse_8e425fc6b), + /// Row 25 (2020-10-15): version word 10; tagged rseed. + F28b795139 = ("28b795139", era_keys::parse_28b795139), + /// Row 26 (2020-12-01): version word 12; Utxo spent_at_height. + Fb61175345 = ("b61175345", era_keys::parse_b61175345), + /// Row 27 (2021-04-22): version word 13; tree_verified byte. + Fbcf38a6fa = ("bcf38a6fa", era_keys::parse_bcf38a6fa), + /// Row 28 (2021-05-05): note v5 / Utxo v3 pending-spent. + F7212e2bf1 = ("7212e2bf1", era_keys::parse_7212e2bf1), + /// Row 29 (2021-05-18): version word 14; price record; tx v5. + F4a279179f = ("4a279179f", era_keys::parse_4a279179f), + /// Row 30 (2021-06-25): version word 20; the Keys record (v20). + F87ad71c28 = ("87ad71c28", era_keys::parse_87ad71c28), + /// Row 31 (2021-07-14): version word 21; tx unconfirmed byte. + Fead95fe0a = ("ead95fe0a", era_keys::parse_ead95fe0a), + /// Row 32 (2021-07-27): version word 22; Optional TreeState. + F0cd53900b = ("0cd53900b", era_keys::parse_0cd53900b), + /// Row 33 (2021-07-27, reminted 2021-08-05): version word 23. + Fa1b9b0bbe = ("a1b9b0bbe", era_keys::parse_a1b9b0bbe), + /// Row 34 (2021-07-29): version word 24; compact-encoded blocks. + Fed3b21c09 = ("ed3b21c09", era_keys::parse_ed3b21c09), + /// Row 35 (2021-09-24): version word 24; WalletOptions record. + F7f59c5320 = ("7f59c5320", era_keys::parse_7f59c5320), + /// Row 36 (2021-10-13): Keys v21 (WalletTKey vector). + F5e73adef4 = ("5e73adef4", era_keys::parse_5e73adef4), + /// Row 37 (2022-07-23): tx v22 (pool triple + orchard nullifiers). + Fa6f8a0bd6 = ("a6f8a0bd6", era_keys::parse_a6f8a0bd6), + /// Row 38 (2022-08-23): Keys v22 (WalletOKey vector); tx v23. + F6dd62d5e2 = ("6dd62d5e2", era_keys::parse_6dd62d5e2), + /// Row 39 (2022-09-16): WalletOptions v2 (size filter). + F2e8b86670 = ("2e8b86670", era_keys::parse_2e8b86670), + /// Row 40 (2022-10-14): version word 25; orchard-anchor vector. + F6b6ed912e = ("6b6ed912e", era_capability::parse_6b6ed912e), + /// Row 41 (2022-10-26): WalletCapability v1 with trailing encrypted u8. + Fcc78c2358 = ("cc78c2358", era_capability::parse_cc78c2358), + /// Row 42 (2022-11-08): WalletCapability v1, encrypted byte trimmed. + Fb01873337 = ("b01873337", era_capability::parse_b01873337), + /// Row 43 (2022-11-16): version word 26; anchor vector removed. + F18014a7ee = ("18014a7ee", era_capability::parse_18014a7ee), + /// Row 44 (2023-02-28): version word 27; capability v2. + F939ef32b1 = ("939ef32b1", era_capability::parse_939ef32b1), + /// Row 45 (2023-04-19): note v2 (FVK removed). + F46eefb844 = ("46eefb844", era_capability::parse_46eefb844), + /// Row 46 (2023-06-13): note v3 (pending-spent dropped). + Fb9a984dc8 = ("b9a984dc8", era_capability::parse_b9a984dc8), + /// Row 47 (2023-08-19): note v4; WitnessTrees enter the file. + Fa3077c201 = ("a3077c201", era_capability::parse_a3077c201), + /// Row 48 (2023-09-28): version word 28; mnemonic account index. + F33daec1d1 = ("33daec1d1", era_capability::parse_33daec1d1), + /// Row 49 (2023-11-08): transparent output v4. + F9440d190d = ("9440d190d", era_capability::parse_9440d190d), + /// Row 50 (2024-10-07): version word 29; capability v3. + Ffd86965ea = ("fd86965ea", era_capability::parse_fd86965ea), + /// Row 51 (2024-10-15): version word 30; capability v4. + Feb2210e79 = ("eb2210e79", era_capability::parse_eb2210e79), + /// Row 52 (2024-10-24): note v5; ConfirmationStatus enters (v0). + F19f278670 = ("19f278670", era_capability::parse_19f278670), + /// Row 53 (2024-11-07): OutgoingTxData v0; tx record 24. + F03c191810 = ("03c191810", era_capability::parse_03c191810), + /// Row 54 (2025-02-02): version word 31; block vector dropped. + Fb82fbe17b = ("b82fbe17b", era_capability::parse_b82fbe17b), + /// Row 55 (2025-02-06): version word 31; key store dropped. + Fdb3f7f716 = ("db3f7f716", era_capability::parse_db3f7f716), + /// Row 56 (2025-03-17): version word 32; the v32 layout. + F44e6271cb = ("44e6271cb", era_modern::parse_44e6271cb), + /// Row 57 (2025-04-27): version word 32; vestigial tail dropped. + F8aaae992a = ("8aaae992a", era_modern::parse_8aaae992a), + /// Row 58 (2025-05-01): version word 33; SyncConfig. + F82c61c0d3 = ("82c61c0d3", era_modern::parse_82c61c0d3), + /// Row 59 (2025-05-15): version word 34; PriceList with api key. + F1ef03610b = ("1ef03610b", era_modern::parse_1ef03610b), + /// Row 60 (2025-05-23): version word 34; api key dropped, unbumped. + F44baa11b4 = ("44baa11b4", era_modern::parse_44baa11b4), + /// Row 61 (2025-05-27): version word 35; account-keyed key store. + Fccc1d681a = ("ccc1d681a", era_modern::parse_ccc1d681a), + /// Row 62 (2025-06-04): ReceiverSelection v2. + Fe5e4a349f = ("e5e4a349f", era_modern::parse_e5e4a349f), + /// Row 63 (2025-06-04): version word 36 (word-only). + Fe6b02b0d8 = ("e6b02b0d8", era_modern::parse_e6b02b0d8), + /// Row 64 (2025-06-13): version word 37; ScanTarget; SyncState v1. + Feae34880e = ("eae34880e", era_modern::parse_eae34880e), + /// Row 65 (2025-06-15): version word 38; min_confirmations. + Fad6ded426 = ("ad6ded426", era_modern::parse_ad6ded426), + /// Row 66 (2025-06-18): version word 39; SyncState v2. + Fb1c04e38c = ("b1c04e38c", era_modern::parse_b1c04e38c), + /// Row 67 (2026-02-12): version word 39; SyncState v3. + Fff7ba3ec0 = ("ff7ba3ec0", era_modern::parse_ff7ba3ec0), + /// Row 68 (2026-02-18): ConfirmationStatus v1. + Ff86717800 = ("f86717800", era_modern::parse_f86717800), + /// Row 69 (2026-03-25): version word 40, dev: chain-type u8. + Feda1dca85 = ("eda1dca85", era_modern::parse_eda1dca85), + /// Row 70 (2026-06-07, stable): version word 40: chain string, u32 + /// output indices. + F5d8fda797 = ("5d8fda797", era_modern::parse_5d8fda797), + /// Row 71 (2026-06-15): version word 41: chain u8 + u32 indices. + F6ae5c270d = ("6ae5c270d", era_modern::parse_6ae5c270d), + /// Row 72 (2026-07-13): version word 42, layout A (allow_v6 byte). + Ffffcc9e02 = ("fffcc9e02", era_modern::parse_fffcc9e02), + /// Row 73 (2026-07-14): version word 43. + F32261bb5f = ("32261bb5f", era_modern::parse_32261bb5f), + /// Row 74 (2026-07-14): version word 42, canonical layout. + F4158e20c2 = ("4158e20c2", era_modern::parse_4158e20c2), + /// Row 75 (2026-07-23): migration inner v2. + Fa6c1354ad = ("a6c1354ad", era_modern::parse_a6c1354ad), + /// Row 76 (2026-07-25): migration inner v3. + F894fe8e0a = ("894fe8e0a", era_modern::parse_894fe8e0a), + /// Row 77 (2026-07-25): migration inner v4. Today's writer. + Ff48b15c9e = ("f48b15c9e", era_modern::parse_f48b15c9e), +} + +/// Format Recognition: classify a complete Wallet File byte string to its +/// unique source grammar. +/// +/// Pure and total: every byte string maps to exactly one +/// [`Recognition`]. Runs every arm's discriminator over the whole buffer; +/// no prefix or single discriminator byte is ever sufficient evidence +/// (version 42's discriminating evidence sits at end of file). +pub(crate) fn recognize(bytes: &[u8]) -> Recognition { + let mut conformers = Vec::new(); + let mut refusals = Vec::new(); + for format in WalletFormat::ALL { + match (format.discriminator())(bytes) { + Ok(()) => conformers.push(*format), + Err(refusal) => refusals.push((*format, refusal)), + } + } + match conformers.len() { + 0 => Recognition::NotConforming(refusals), + 1 => Recognition::Recognized(conformers[0]), + _ => Recognition::Ambiguous(conformers), + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// The fixture corpus (written by `wallet-grammar-fixtures` into + /// `disk/testing/grammars/NN_.dat`) pins every + /// discriminator against every other arm: each fixture must recognize + /// as exactly its own row's format. Rows 1 and 2 share the merged + /// `F7ebc8686e` arm. Skips silently when the corpus has not been + /// generated yet. + #[test] + fn corpus_recognizes_uniquely() { + let dir = std::path::Path::new(env!("CARGO_MANIFEST_DIR")) + .join("src/wallet/disk/testing/grammars"); + let mut checked = 0usize; + let Ok(entries) = std::fs::read_dir(&dir) else { + return; + }; + for entry in entries.flatten() { + let name = entry.file_name(); + let name = name.to_string_lossy().into_owned(); + let Some(stem) = name.strip_suffix(".dat") else { + continue; + }; + let Some((_, hash)) = stem.split_once('_') else { + continue; + }; + // Rows 1 and 2 are one arm: c2e26fbbc's fixture recognizes as + // the merged 7ebc8686e arm. + let expected = if hash == "c2e26fbbc" { "7ebc8686e" } else { hash }; + let bytes = std::fs::read(entry.path()).unwrap(); + match recognize(&bytes) { + Recognition::Recognized(format) => { + assert_eq!( + format.defining_commit(), + expected, + "fixture {name} recognized as the wrong arm" + ); + } + other => panic!("fixture {name} did not recognize uniquely: {other:?}"), + } + checked += 1; + } + // When the corpus exists it must be complete. + if checked > 0 { + assert!(checked >= 77, "corpus present but only {checked} fixtures"); + } + } + + /// The motivating crash input: the first bytes of a dev-v40 wallet. + /// Under the census it must classify to dev's v40 arm — and whatever + /// the verdict, recognition must not allocate or abort on the length + /// field that killed the reader. + #[test] + fn dev_v40_prefix_refuses_stable_v40_without_allocating() { + // version word 40, chain byte 0 (Mainnet), CompactSize-32 seed, + // truncated: enough to prove the stable-v40 arm refuses early and + // cheaply rather than trusting a 854 PB length. + let mut bytes = 40u64.to_le_bytes().to_vec(); + bytes.push(0); + bytes.push(32); + bytes.extend_from_slice(&[0x55; 32]); + let refusal = (WalletFormat::F5d8fda797.discriminator())(&bytes) + .expect_err("a dev-v40 prefix must refuse the stable-v40 grammar"); + assert_eq!(refusal.offset, 8, "stable v40 demands a string length at offset 8"); + } +}