diff --git a/Cargo.lock b/Cargo.lock index 219ca5f6c70..026c040aa27 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,6 +108,25 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arbitrary-json" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08117a235f4bfba33f065e5f6941838fa9f77436c1823fca9c93c9b4a34a40e0" +dependencies = [ + "arbitrary", + "serde_json", +] + [[package]] name = "arc-swap" version = "1.9.2" @@ -582,6 +601,8 @@ dependencies = [ name = "azure_data_cosmos" version = "0.38.0" dependencies = [ + "arbitrary", + "arbitrary-json", "async-lock", "async-trait", "azure_core 1.1.0", @@ -591,6 +612,7 @@ dependencies = [ "base64", "clap", "futures", + "json-canon", "opentelemetry 0.32.0", "opentelemetry_sdk 0.32.1", "pin-project", @@ -598,6 +620,7 @@ dependencies = [ "serde", "serde_bytes", "serde_json", + "sha2", "time", "tokio", "tracing", @@ -1545,6 +1568,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -2405,6 +2439,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-canon" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447ae153a2bd47d61acc0d131295408e32ef87ed9785825a6f4ecef85afc0edb" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + [[package]] name = "json-patch" version = "4.2.0" @@ -3450,6 +3495,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9774ba4a74de5f7b1c1451ed6cd5285a32eddb5cccb8cc655a4e50009e06477f" +[[package]] +name = "ryu-js" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" + [[package]] name = "same-file" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index 6c4038d98f6..e84147dc1d4 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -88,6 +88,8 @@ version = "1.0.0" version = "1.0.0" [workspace.dependencies] +arbitrary = { version = "1.4", features = ["derive"] } +arbitrary-json = "0.1" async-lock = "3.4" async-stream = { version = "0.3.6" } async-trait = "0.1" @@ -117,6 +119,7 @@ hdrhistogram = "7.5" hostname = "0.4" hmac = { version = "0.12" } include-file = { version = "1.0.0", default-features = false } +json-canon = "0.1" openssl = { version = "0.10.79" } opentelemetry = { version = "0.32", features = ["trace"] } opentelemetry-appender-tracing = { version = "0.32" } diff --git a/eng/dict/crates.txt b/eng/dict/crates.txt index e40fc74f776..23767f27c79 100644 --- a/eng/dict/crates.txt +++ b/eng/dict/crates.txt @@ -1,3 +1,6 @@ +arbitrary +arbitrary_json +arbitrary-json arc_swap arc-swap async_lock @@ -70,8 +73,12 @@ hostname http include_file include-file +json_canon +json-canon json_patch json-patch +libfuzzer_sys +libfuzzer-sys litemap openssl opentelemetry diff --git a/sdk/cosmos/.cspell.json b/sdk/cosmos/.cspell.json index 302974ee913..14c9ba74969 100644 --- a/sdk/cosmos/.cspell.json +++ b/sdk/cosmos/.cspell.json @@ -33,6 +33,7 @@ "cabi", "canadacentral", "canadaeast", + "CBOR", "cbindgen", "CDLL", "cdriver", @@ -52,6 +53,7 @@ "chinaeast", "chinanorth", "chokepoint", + "canonicalizer", "cloneable", "codepoint", "codepoints", @@ -172,6 +174,7 @@ "libazurecosmosdriver", "libclang", "LIBCLANG", + "libfuzzer", "libqueryplaninterop", "QUERYPLANINTEROP", "linearizability", @@ -256,10 +259,14 @@ "RAII", "readfeed", "recompiles", + "redecoded", + "reencode", + "reencoded", "refetch", "refetched", "refetches", "refcounted", + "reparses", "reparsed", "Replicaset", "reqs", @@ -284,6 +291,7 @@ "serviceunavailable", "serviceversion", "sess", + "Signedness", "southafricanorth", "southafricawest", "southcentralus", @@ -313,11 +321,13 @@ "testdb", "thinclient", "threadsafe", + "tmin", "TOCTOU", "Tokio", "TOPCOUNT", "TPIO", "transcoders", + "trippable", "uaecentral", "uaenorth", "udfs", @@ -374,6 +384,35 @@ "writability", "xorshift", "xpart", + "artifactname", + "fsanitize", + "logissue", + "MSVC", + "AFORM", + "ANID", + "Chenault", + "devtestcoll", + "ENOENT", + "errno", + "feff", + "Flacco", + "logdata", + "lossily", + "Millett", + "modhash", + "MSNH", + "MUID", + "muid", + "nametype", + "Qsml", + "recclass", + "reclat", + "reclong", + "similars", + "Unschematized", + "vout", + "Xpert", + "xpert", "yxxx" ], "ignorePaths": [ diff --git a/sdk/cosmos/azure_data_cosmos/Cargo.toml b/sdk/cosmos/azure_data_cosmos/Cargo.toml index 95cd08e70b4..033658658de 100644 --- a/sdk/cosmos/azure_data_cosmos/Cargo.toml +++ b/sdk/cosmos/azure_data_cosmos/Cargo.toml @@ -52,6 +52,13 @@ azure_data_cosmos_driver = { path = "../azure_data_cosmos_driver", default-featu ] } azure_identity.workspace = true clap.workspace = true +# Test-only deps for the binary round-trip fuzzer +# (tests/binary_roundtrip_fuzzer.rs): it generates arbitrary JSON, canonicalizes +# it (RFC 8785), and hashes the canonical form for differential comparison. +arbitrary = { workspace = true } +arbitrary-json = { workspace = true } +json-canon = { workspace = true } +sha2 = { workspace = true } # Enabled here (dev-only) so the metrics and tracing handler tests can drive # in-memory OpenTelemetry exporters. `testing` provides `InMemoryMetricExporter` # and `InMemorySpanExporter`. @@ -144,6 +151,11 @@ name = "binary_encoding" path = "tests/binary_encoding.rs" required-features = ["key_auth", "control_plane", "fault_injection"] +[[test]] +name = "binary_roundtrip_fuzzer" +path = "tests/binary_roundtrip_fuzzer.rs" +required-features = ["key_auth", "control_plane", "fault_injection"] + [[test]] name = "gateway_v2" path = "tests/gateway_v2.rs" diff --git a/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs index d6dd4d95998..b11f7e9f77d 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs @@ -56,9 +56,13 @@ fn binary_encoding_options() -> TestOptions { } /// A document covering every JSON value shape the binary encoder emits: literal -/// and wide integers, an unsigned value beyond `i64::MAX`, a double, booleans, -/// `null`, unicode/empty strings, nested arrays and objects, and a vector of -/// objects. +/// and wide integers, a large unsigned value, a double, booleans, `null`, +/// unicode/empty strings, nested arrays and objects, and a vector of objects. +/// +/// Note: `huge` stays at or below `2^53` because the live Cosmos service +/// normalizes JSON numbers to IEEE-754 doubles. A value beyond `2^53` (e.g. +/// `u64::MAX`) is echoed back as a `Double` and can no longer be deserialized +/// into a `u64` field, so it does not round-trip against the real service. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] struct BinaryItem { id: String, @@ -95,11 +99,11 @@ fn sample_item(id: &str, partition_key: &str) -> BinaryItem { text: "hello binary".to_owned(), unicode: "cafΓ© β˜ƒ π„ž quotes:\" backslash:\\".to_owned(), empty: String::new(), - small_int: 7, // literal-int form (0..32) - big_int: 9_000_000_000, // Int64 form - negative: -1_234_567, // Int64 form - huge: u64::MAX, // UInt64 form (beyond i64::MAX) - ratio: 123.456_789, // Double form + small_int: 7, // literal-int form (0..32) + big_int: 9_000_000_000, // Int64 form + negative: -1_234_567, // Int64 form + huge: 9_007_199_254_740_992, // UInt64 form, exactly f64-representable (2^53) + ratio: 123.456_789, // Double form active: true, inactive: false, maybe: None, // null diff --git a/sdk/cosmos/azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs b/sdk/cosmos/azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs new file mode 100644 index 00000000000..8b18a76d662 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs @@ -0,0 +1,2834 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! End-to-end **round-trip fuzzer** for Cosmos binary JSON encoding. +//! +//! Generates random JSON documents with a seeded PRNG, stores and reads each one +//! back through a live Cosmos account under several binary-encoding +//! configurations, and asserts the value survives unchanged by comparing a +//! **Cosmos-compatible canonical form** of what was sent against what came back. +//! +//! See the design doc: +//! `azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md`. +//! +//! # Running +//! +//! ```bash +//! # Smoke run against a local emulator: +//! AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=...;AccountKey=...;' \ +//! AZURE_COSMOS_ALLOW_INVALID_CERT=true \ +//! RUSTFLAGS='--cfg test_category="binary_encoding"' \ +//! cargo test -p azure_data_cosmos --test binary_roundtrip_fuzzer --features key_auth,fault_injection,control_plane -- --nocapture +//! +//! # Multi-day soak (millions of docs), release build: +//! AZURE_COSMOS_CONNECTION_STRING='...' AZURE_COSMOS_FUZZ_ITERATIONS=5000000 \ +//! RUSTFLAGS='--cfg test_category="binary_encoding"' \ +//! cargo test -p azure_data_cosmos --test binary_roundtrip_fuzzer --features key_auth,fault_injection,control_plane --release -- --nocapture +//! +//! # Reproduce a failure exactly: +//! AZURE_COSMOS_FUZZ_SEED= ... cargo test ... +//! ``` +//! +//! Every run prints its seed; a failing document is reproduced deterministically +//! by re-running with `AZURE_COSMOS_FUZZ_SEED=`. + +#![allow(clippy::large_futures)] + +use std::error::Error; + +use arbitrary::{Arbitrary, Unstructured}; +use arbitrary_json::ArbitraryValue; +use azure_core::http::StatusCode; +use azure_data_cosmos::models::ContainerProperties; +use azure_data_cosmos::options::{ + BinaryEncodingOptions, ConnectionPoolOptions, ContentResponseOnWrite, ItemWriteOptions, + OperationOptions, Region, ServerCertificateValidation, +}; +use azure_data_cosmos::{ + AccountEndpoint, AccountReference, CosmosClient, CosmosRuntime, RoutingStrategy, SubStatusCode, +}; +use azure_data_cosmos_driver::models::ConnectionString; +use serde_json::{Map, Number, Value}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +const CONNECTION_STRING_ENV_VAR: &str = "AZURE_COSMOS_CONNECTION_STRING"; +const ALLOW_INVALID_CERT_ENV_VAR: &str = "AZURE_COSMOS_ALLOW_INVALID_CERT"; +const DATABASE_NAME_ENV_VAR: &str = "AZURE_COSMOS_BINARY_TEST_DATABASE"; +const CONTAINER_NAME_ENV_VAR: &str = "AZURE_COSMOS_BINARY_TEST_CONTAINER"; +const ITERATIONS_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_ITERATIONS"; +const SEED_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_SEED"; +const MAX_DEPTH_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_MAX_DEPTH"; +const WIDE_NUMBERS_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_WIDE_NUMBERS"; +const UNICODE_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_UNICODE"; +const BREADTH_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_BREADTH"; +const SHAPE_RATIO_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_SHAPE_RATIO"; +const SIZE_SCALE_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_SIZE_SCALE"; +const CALIBRATE_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_CALIBRATE"; +const PRINT_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_PRINT"; + +const DEFAULT_DATABASE_NAME: &str = "binary-fuzz-db"; +const DEFAULT_CONTAINER_NAME: &str = "binary-fuzz-ct"; +const PARTITION_KEY_PATH: &str = "/pk"; + +const DEFAULT_ITERATIONS: u64 = 200; +const DEFAULT_MAX_DEPTH: u32 = 6; +/// Default maximum number of child fields/elements generated at each container +/// level (the branching factor). Higher values produce larger, wider documents. +const DEFAULT_BREADTH: u32 = 6; +/// Default percent (0-100) of generated documents built in the shape of a real +/// corpus file (see `SHAPE_SAMPLERS`); the rest are free-form hybrid documents. +const DEFAULT_SHAPE_RATIO: u32 = 85; +/// Default multiplier applied to the internal array / collection sizes of the +/// corpus shape samplers. `1` keeps documents compact; larger values grow the +/// per-item payload (e.g. embedding-vector dimensions, nutrient / member / +/// keyword / similars arrays) toward corpus-scale sizes. +const DEFAULT_SIZE_SCALE: u32 = 1; + +// ───────────────────────────────────────────────────────────────────────────── +// Seeded PRNG (SplitMix64) β€” deterministic and dependency-feature-free, matching +// the codebase's in-tree fuzz convention so a failure reproduces from its seed. +// ───────────────────────────────────────────────────────────────────────────── + +struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next_u64(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Uniform integer in `[0, n)`. `n` must be non-zero. + fn below(&mut self, n: u64) -> u64 { + self.next_u64() % n + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Configuration +// ───────────────────────────────────────────────────────────────────────────── + +struct FuzzConfig { + iterations: u64, + seed: u64, + max_depth: u32, + wide_numbers: bool, + unicode: bool, + breadth: u32, + shape_ratio: u32, + size_scale: u32, + calibrate: bool, + print_docs: bool, +} + +impl FuzzConfig { + fn from_env() -> Self { + let seed = match std::env::var(SEED_ENV_VAR) { + // If set, it must parse β€” don't silently randomize (breaks reproduction). + Ok(v) => v.trim().parse::().unwrap_or_else(|_| { + panic!( + "{SEED_ENV_VAR} is set to {v:?} but is not a valid u64 seed; \ + provide a decimal u64 (e.g. 12345) or unset it to use a random seed" + ) + }), + // Unset: random seed from the wall clock. + Err(_) => std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0x1234_5678_9ABC_DEF0), + }; + Self { + iterations: env_u64(ITERATIONS_ENV_VAR, DEFAULT_ITERATIONS), + seed, + max_depth: env_u64(MAX_DEPTH_ENV_VAR, DEFAULT_MAX_DEPTH as u64) as u32, + wide_numbers: env_bool(WIDE_NUMBERS_ENV_VAR, false), + unicode: env_bool(UNICODE_ENV_VAR, true), + // Clamp into `[1, u32::MAX]` *before* the `as u32` cast: a raw value + // that is a multiple of 2^32 (e.g. 4294967296) would otherwise + // truncate to 0 and later panic in `rng.below(0)`. + breadth: env_u64(BREADTH_ENV_VAR, DEFAULT_BREADTH as u64).clamp(1, u32::MAX as u64) + as u32, + shape_ratio: env_u64(SHAPE_RATIO_ENV_VAR, DEFAULT_SHAPE_RATIO as u64).min(100) as u32, + size_scale: env_u64(SIZE_SCALE_ENV_VAR, DEFAULT_SIZE_SCALE as u64).max(1) as u32, + calibrate: env_bool(CALIBRATE_ENV_VAR, false), + print_docs: env_bool(PRINT_ENV_VAR, false), + } + } +} + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(default) +} + +fn env_bool(name: &str, default: bool) -> bool { + std::env::var(name) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(default) +} + +// ───────────────────────────────────────────────────────────────────────────── +// JSON generator (arbitrary-json, seeded from the PRNG) +// ───────────────────────────────────────────────────────────────────────────── + +/// Generates a random JSON **object** suitable as a Cosmos item body. +/// +/// Uses a **hybrid** strategy: a depth-controlled *skeleton* guarantees the +/// document actually reaches a target nesting depth (drawn from `[1, max_depth]`), +/// while every leaf and filler branch is irregular JSON β€” a mix of hand-rolled +/// typed scalars (integers, floats, alphabetic / alphanumeric / free-text / +/// non-ASCII strings, booleans, nulls, number arrays) and [`arbitrary_json`] +/// subtrees. This fixes the `arbitrary_iter` shallowness (it stops nesting +/// almost immediately regardless of byte budget), so `max_depth` now +/// meaningfully scales structure and `breadth` scales width. Everything is +/// driven by the [`SplitMix64`] seed stream, so the same `AZURE_COSMOS_FUZZ_SEED` +/// reproduces the same document. +/// +/// Every document also carries a **sampler** subtree ([`gen_sampler`]) that +/// guarantees at least one value of each category appears, so a single run +/// exercises numeric, alphabetic, alphanumeric, free-text, and non-ASCII data +/// under multi-level nesting. +/// +/// A [`bound_value`] pass applies the `wide_numbers` / `unicode` knobs: by +/// default numbers are clamped into the calibrated-safe envelope (design doc +/// Β§3.2) and strings to ASCII, unless explicitly widened. The hand-rolled +/// scalars already respect those knobs directly. +fn gen_object(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + // A configurable fraction of documents are built in the shape of a real + // corpus file (see SHAPE_SAMPLERS), so a run resembles the service corpus. + // The rest are the free-form hybrid documents below. The `_sampler` subtree + // is attached in both cases so every document still covers all categories. + let shaped = cfg.shape_ratio > 0 && rng.below(100) < cfg.shape_ratio as u64; + if shaped { + let mut map = gen_shaped_document(rng, cfg); + // Ensure the all-category sampler is present without clobbering a + // shape field of the same name (shapes never use `_sampler`). + map.insert("_sampler".to_string(), gen_sampler(rng, cfg)); + return map; + } + + let max_depth = cfg.max_depth.max(1); + // Target nesting depth for this document's spine, in [1, max_depth]. + let target_depth = 1 + rng.below(max_depth as u64) as u32; + + let mut map = Map::new(); + // A guaranteed sampler covering every value category (numeric, alphabetic, + // alphanumeric, free text, non-ASCII, boolean, null, number array, nested). + // Keyed distinctly from the caller-reserved `id`/`pk` so it is never + // overwritten. + map.insert("_sampler".to_string(), gen_sampler(rng, cfg)); + // A spread of irregular root fields (typed scalars + arbitrary-json subtrees). + for _ in 0..rng.below(cfg.breadth as u64 + 1) { + map.insert(gen_key(rng), gen_filler_value(rng, cfg)); + } + // The spine field guarantees the target depth is reached. Its key avoids the + // caller-reserved `id`/`pk`/`_sampler` (and empty) so nothing overwrites the + // deep subtree. + let mut spine_key = gen_key(rng); + while spine_key.is_empty() || spine_key == "id" || spine_key == "pk" || spine_key == "_sampler" + { + spine_key.push('_'); + } + map.insert(spine_key, gen_spine(rng, cfg, target_depth)); + + map +} + +/// Number of PRNG bytes fed to `arbitrary-json` for one filler subtree. A larger +/// budget lets `arbitrary-json` build bigger, deeper irregular subtrees. +const FILLER_BUDGET: usize = 256; + +/// Refills `n` bytes deterministically from the PRNG. +fn fill_bytes(rng: &mut SplitMix64, n: usize) -> Vec { + let mut bytes = Vec::with_capacity(n); + while bytes.len() < n { + bytes.extend_from_slice(&rng.next_u64().to_le_bytes()); + } + bytes +} + +/// A random object key from `arbitrary-json`'s string generator. +fn gen_key(rng: &mut SplitMix64) -> String { + let bytes = fill_bytes(rng, 16); + let mut u = Unstructured::new(&bytes); + String::arbitrary(&mut u).unwrap_or_default() +} + +// ───────────────────────────────────────────────────────────────────────────── +// Typed scalar generators (character classes + numbers), seeded from the PRNG. +// ───────────────────────────────────────────────────────────────────────────── + +/// ASCII letters, for the alphabetic string class. +const ALPHA_CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; +/// ASCII letters + digits, for the alphanumeric string class. +const ALPHANUMERIC_CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; +/// A spread of non-ASCII scalars across scripts, symbols, and astral-plane +/// emoji β€” exercises multi-byte UTF-8 and surrogate-pair paths in the codec. +const NON_ASCII_CHARS: &[char] = &[ + 'Γ©', 'Γ±', 'ΓΌ', 'ß', 'Γ§', 'Γ₯', 'ΓΈ', 'Ξ©', 'Ξ»', 'Ο€', 'Β΅', 'я', 'ΠΆ', 'Π΄', 'Ξ±', 'Ξ²', 'δΈ­', 'ζ–‡', + 'ζ—₯', '本', 'θͺž', 'ν•œ', 'κ΅­', 'ΨΉ', 'Ψ¨', '€', 'Β£', 'Β₯', 'Β©', 'β„’', 'β€”', '…', 'β†’', 'βˆ‘', 'β‰ˆ', 'β™ ', + 'β˜ƒ', 'πŸ˜€', 'πŸš€', '🌍', 'πŸŽ‰', 'π„ž', '𐍈', +]; + +/// A uniform integer in `[lo, hi]` (inclusive). `lo <= hi` required. +fn gen_int_in(rng: &mut SplitMix64, lo: i64, hi: i64) -> i64 { + let span = (hi - lo) as u64 + 1; + lo + rng.below(span) as i64 +} + +/// A random ASCII string drawn from `pool`, length in `[1, max_len]`. +fn gen_string_from(rng: &mut SplitMix64, pool: &[u8], max_len: usize) -> String { + let len = 1 + rng.below(max_len as u64) as usize; + (0..len) + .map(|_| pool[rng.below(pool.len() as u64) as usize] as char) + .collect() +} + +/// A string mixing alphanumeric and non-ASCII scalars, length in `[1, max_len]`. +/// Falls back to alphanumeric-only when `unicode` generation is disabled so the +/// ASCII envelope contract (design doc Β§3.2) still holds. +fn gen_unicode_string(rng: &mut SplitMix64, cfg: &FuzzConfig, max_len: usize) -> String { + if !cfg.unicode { + return gen_string_from(rng, ALPHANUMERIC_CHARS, max_len); + } + let len = 1 + rng.below(max_len as u64) as usize; + (0..len) + .map(|_| { + if rng.below(2) == 0 { + ALPHANUMERIC_CHARS[rng.below(ALPHANUMERIC_CHARS.len() as u64) as usize] as char + } else { + NON_ASCII_CHARS[rng.below(NON_ASCII_CHARS.len() as u64) as usize] + } + }) + .collect() +} + +/// An envelope-safe integer (`Β±1_000_000`). +fn gen_envelope_int(rng: &mut SplitMix64) -> Value { + Value::Number(Number::from(gen_int_in(rng, -1_000_000, 1_000_000))) +} + +/// An envelope-safe two-decimal float (`Β±100_000.00`). +fn gen_envelope_float(rng: &mut SplitMix64) -> Value { + let cents = gen_int_in(rng, -10_000_000, 10_000_000); + Number::from_f64(cents as f64 / 100.0) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(0))) +} + +/// A single number: envelope-safe by default; when `wide_numbers` is set, +/// occasionally a wide value beyond `2^53` that drives the calibrated +/// string-token comparison path (design doc Β§3.1). The wide branch spans signed +/// i64, true u64 above `i64::MAX`, and a wide non-integral float β€” all of which +/// the backend stores as lossy doubles, so [`normalize_number`] tokenizes each +/// via its rounded `f64`. +fn gen_number(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + if cfg.wide_numbers && rng.below(4) == 0 { + return match rng.below(3) { + // Signed wide integer (> 2^53) β†’ lossy double-token path. + 0 => Value::Number(Number::from(rng.next_u64() as i64)), + // True u64 above i64::MAX β†’ double-token path. + 1 => Value::Number(Number::from((i64::MAX as u64) + 1 + (rng.next_u64() >> 1))), + // Wide non-integral float β†’ double-token path. + _ => { + let scaled = (rng.next_u64() >> 8) as f64 * 1.000_000_1; + Number::from_f64(scaled) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(rng.next_u64() as i64))) + } + }; + } + if rng.below(2) == 0 { + gen_envelope_int(rng) + } else { + gen_envelope_float(rng) + } +} + +/// One rich scalar spanning the value taxonomy: integer, float, alphabetic, +/// alphanumeric, free text, non-ASCII, boolean, or null. +fn gen_scalar(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + match rng.below(8) { + 0 | 1 => gen_number(rng, cfg), + 2 => Value::String(gen_string_from(rng, ALPHA_CHARS, 24)), + 3 => Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 24)), + 4 => Value::String(gen_unicode_string(rng, cfg, 24)), + 5 => Value::String(gen_unicode_string(rng, cfg, 80)), // longer free text + 6 => Value::Bool(rng.below(2) == 0), + _ => Value::Null, + } +} + +/// A sampler object guaranteeing every value category appears in the document at +/// least once: integer, float, alphabetic, alphanumeric, free text, non-ASCII, +/// boolean, null, a homogeneous number array, and a small nested object. +fn gen_sampler(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + let mut map = Map::new(); + map.insert("int".into(), gen_envelope_int(rng)); + map.insert("float".into(), gen_envelope_float(rng)); + map.insert( + "alpha".into(), + Value::String(gen_string_from(rng, ALPHA_CHARS, 24)), + ); + map.insert( + "alphanumeric".into(), + Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 24)), + ); + map.insert( + "text".into(), + Value::String(gen_unicode_string(rng, cfg, 64)), + ); + map.insert( + "unicode".into(), + Value::String(gen_unicode_string(rng, cfg, 24)), + ); + map.insert("flag".into(), Value::Bool(rng.below(2) == 0)); + map.insert("empty".into(), Value::Null); + let count = 1 + rng.below(8); + let numbers = (0..count).map(|_| gen_envelope_int(rng)).collect(); + map.insert("numbers".into(), Value::Array(numbers)); + // A small nested object so the sampler itself has a second level. + let mut nested = Map::new(); + nested.insert("mixed".into(), gen_scalar(rng, cfg)); + nested.insert( + "list".into(), + Value::Array( + (0..1 + rng.below(4)) + .map(|_| gen_scalar(rng, cfg)) + .collect(), + ), + ); + map.insert("nested".into(), Value::Object(nested)); + Value::Object(map) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Domain-flavored scalar helpers (used by the corpus shape samplers below). +// ───────────────────────────────────────────────────────────────────────────── + +/// Lowercase hex string of `len` nibbles (e.g. hashes, GUIDs-as-hex). +fn gen_hex(rng: &mut SplitMix64, len: usize) -> String { + const HEX: &[u8] = b"0123456789abcdef"; + (0..len) + .map(|_| HEX[rng.below(16) as usize] as char) + .collect() +} + +/// Uppercase-hex string of `len` nibbles (e.g. GUID-like ids without dashes). +fn gen_hex_upper(rng: &mut SplitMix64, len: usize) -> String { + gen_hex(rng, len).to_ascii_uppercase() +} + +/// A canonical `8-4-4-4-12` UUID string (lowercase hex). +fn gen_uuid(rng: &mut SplitMix64) -> String { + format!( + "{}-{}-{}-{}-{}", + gen_hex(rng, 8), + gen_hex(rng, 4), + gen_hex(rng, 4), + gen_hex(rng, 4), + gen_hex(rng, 12) + ) +} + +/// An ISO-8601-ish timestamp string, optionally with fractional seconds and a +/// UTC `Z` suffix. Deterministic from the PRNG (not the wall clock). +fn gen_iso_datetime(rng: &mut SplitMix64) -> String { + let year = gen_int_in(rng, 1990, 2024); + let month = gen_int_in(rng, 1, 12); + let day = gen_int_in(rng, 1, 28); + let hour = gen_int_in(rng, 0, 23); + let min = gen_int_in(rng, 0, 59); + let sec = gen_int_in(rng, 0, 59); + match rng.below(3) { + 0 => format!("{year:04}-{month:02}-{day:02}T{hour:02}:{min:02}:{sec:02}"), + 1 => format!( + "{year:04}-{month:02}-{day:02}T{hour:02}:{min:02}:{sec:02}.{:07}Z", + gen_int_in(rng, 0, 9_999_999) + ), + _ => format!("{year:04}-{month:02}-{day:02} {hour:02}:{min:02}:{sec:02}"), + } +} + +/// A short uppercase alphabetic code (e.g. airport / entity codes). +fn gen_code(rng: &mut SplitMix64, len: usize) -> String { + const UPPER: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZ"; + (0..len) + .map(|_| UPPER[rng.below(26) as usize] as char) + .collect() +} + +/// A signed geographic coordinate (`Β±180`) with high precision, as an `f64`. +fn gen_coordinate(rng: &mut SplitMix64) -> Value { + let scaled = gen_int_in(rng, -180_000_000, 180_000_000); + Number::from_f64(scaled as f64 / 1_000_000.0) + .map(Value::Number) + .unwrap_or(Value::Null) +} + +/// A tiny normalized floating value in `[-1, 1]` (e.g. embedding components, +/// similarity scores) with many significant digits. +fn gen_unit_float(rng: &mut SplitMix64) -> Value { + let scaled = gen_int_in(rng, -100_000_000, 100_000_000); + Number::from_f64(scaled as f64 / 100_000_000.0) + .map(Value::Number) + .unwrap_or(Value::Null) +} + +/// A short human-ish name from a small fixed pool (keeps documents readable and +/// deterministic without a names dictionary). +fn gen_name(rng: &mut SplitMix64) -> String { + const NAMES: &[&str] = &[ + "Casual", + "Joe Flacco", + "Emmanuel", + "Aruba", + "Gary Stevens", + "Aachen", + "Atlanta", + "Chenault", + "Millett", + "Coffee", + "Volcano", + "Xpert", + "Reddit", + "Bitcoin", + ]; + NAMES[rng.below(NAMES.len() as u64) as usize].to_string() +} + +/// A GeoJSON linear-ring of `n` `[lon, lat]` coordinate pairs. +fn gen_coord_ring(rng: &mut SplitMix64, n: usize) -> Value { + Value::Array( + (0..n) + .map(|_| Value::Array(vec![gen_coordinate(rng), gen_coordinate(rng)])) + .collect(), + ) +} + +/// `n` items produced by `f`, as a JSON array. `n` is computed **before** the +/// call to avoid double-borrowing `rng`. +fn gen_array_of(rng: &mut SplitMix64, n: usize, mut f: F) -> Value +where + F: FnMut(&mut SplitMix64) -> Value, +{ + Value::Array((0..n).map(|_| f(rng)).collect()) +} + +/// A random, **size-scaled** count in `[min, min+span)` Γ— `cfg.size_scale` β€” a +/// small helper so array builders can pick their length in a `let` binding +/// (keeping the `gen_array_of` call from borrowing `rng` twice in one +/// expression). The `size_scale` knob multiplies every corpus-shape collection +/// length, so a run can grow per-item payloads toward corpus-scale sizes. +fn count(rng: &mut SplitMix64, cfg: &FuzzConfig, min: usize, span: u64) -> usize { + let scale = cfg.size_scale.max(1) as usize; + (min + rng.below(span) as usize) * scale +} + +// ───────────────────────────────────────────────────────────────────────────── +// Corpus shape samplers. +// +// Each `shape_*` produces a single document (a `Map`) matching the structural +// "shape" of one of the local `testdata/*.json` corpus files, populated with +// randomized alphanumeric / numeric / float / non-ASCII / boolean / null / +// datetime / GUID / coordinate values. The generator can emit any of these +// shapes at random (see [`gen_shaped_document`]), so a run resembles the real +// service corpus while staying fully synthetic and seed-reproducible. +// +// These reproduce the *shape*, not verbatim data β€” no corpus bytes are embedded. +// ───────────────────────────────────────────────────────────────────────────── + +/// `airline-delays-2003-2016.json`: nested Airport/Time/Statistics record. +fn shape_airline_delays(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let mut airport = Map::new(); + airport.insert("Code".into(), Value::String(gen_code(rng, 3))); + airport.insert( + "Name".into(), + Value::String(gen_unicode_string(rng, cfg, 40)), + ); + let mut time = Map::new(); + time.insert( + "Label".into(), + Value::String(format!( + "{:04}/{:02}", + gen_int_in(rng, 2003, 2016), + gen_int_in(rng, 1, 12) + )), + ); + time.insert( + "Month".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 12))), + ); + time.insert( + "Year".into(), + Value::Number(Number::from(gen_int_in(rng, 2003, 2016))), + ); + let mut stats = Map::new(); + stats.insert( + "Delayed".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 100_000))), + ); + stats.insert("OnTime".into(), gen_envelope_float(rng)); + let mut m = Map::new(); + m.insert("Airport".into(), Value::Object(airport)); + m.insert("Time".into(), Value::Object(time)); + m.insert("Statistics".into(), Value::Object(stats)); + m +} + +/// `bitcoin_transactions.json`: flat transaction with hash + numeric fields. +fn shape_bitcoin(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + let mut m = Map::new(); + m.insert("hash".into(), Value::String(gen_hex(rng, 64))); + m.insert( + "ver".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 2))), + ); + m.insert( + "vin_sz".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 10))), + ); + m.insert( + "vout_sz".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 10))), + ); + m.insert( + "size".into(), + Value::Number(Number::from(gen_int_in(rng, 100, 5000))), + ); + m.insert( + "fee".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 100_000))), + ); + m.insert("relayed_by".into(), Value::String("0.0.0.0".into())); + m +} + +/// `CombinedBingDocs.json`: blog post with a structured `postTime`. +fn shape_bing_docs(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let mut post_time = Map::new(); + post_time.insert( + "Month".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 12))), + ); + post_time.insert( + "Day".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 28))), + ); + post_time.insert( + "Year".into(), + Value::Number(Number::from(gen_int_in(rng, 1600, 2024))), + ); + post_time.insert( + "Hour".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 23))), + ); + post_time.insert( + "Minute".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 59))), + ); + let mut m = Map::new(); + m.insert("blogId".into(), Value::String(gen_code(rng, 9))); + m.insert( + "blogName".into(), + Value::String(gen_string_from(rng, ALPHA_CHARS, 12)), + ); + m.insert("postId".into(), Value::String(gen_code(rng, 9))); + m.insert("postTitle".into(), Value::String(gen_code(rng, 32))); + m.insert("authorName".into(), Value::String(gen_name(rng))); + m.insert("postTime".into(), Value::Object(post_time)); + m.insert( + "body".into(), + Value::String(gen_unicode_string(rng, cfg, 120)), + ); + m +} + +/// `CombinedScriptsData.Json`: entity with a `from` object + `actions` array. +fn shape_scripts_data(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let mut from = Map::new(); + from.insert("name".into(), Value::String(gen_name(rng))); + from.insert( + "id".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 1000))), + ); + let n = count(rng, cfg, 1, 4); + let actions = gen_array_of(rng, n, |_r| { + let mut a = Map::new(); + a.insert("name".into(), Value::String("Comment".into())); + a.insert( + "link".into(), + Value::String("http://www.facebook.com/X999/posts/Y999".into()), + ); + Value::Object(a) + }); + let mut m = Map::new(); + m.insert("entityId".into(), Value::String(gen_code(rng, 9))); + m.insert("from".into(), Value::Object(from)); + m.insert( + "message".into(), + Value::String(gen_unicode_string(rng, cfg, 60)), + ); + m.insert("actions".into(), actions); + m +} + +/// `countries.json` / `Volcanoes.json`: a GeoJSON `Feature` with geometry. +fn shape_geojson_feature(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let mut props = Map::new(); + props.insert("ADMIN".into(), Value::String(gen_name(rng))); + props.insert("ISO_A3".into(), Value::String(gen_code(rng, 3))); + props.insert( + "POP".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 1_000_000))), + ); + props.insert( + "NOTE".into(), + Value::String(gen_unicode_string(rng, cfg, 30)), + ); + let mut geometry = Map::new(); + if rng.below(2) == 0 { + geometry.insert("type".into(), Value::String("Point".into())); + geometry.insert( + "coordinates".into(), + Value::Array(vec![gen_coordinate(rng), gen_coordinate(rng)]), + ); + } else { + geometry.insert("type".into(), Value::String("Polygon".into())); + let ring_len = count(rng, cfg, 3, 5); + geometry.insert( + "coordinates".into(), + Value::Array(vec![gen_coord_ring(rng, ring_len)]), + ); + } + let mut m = Map::new(); + m.insert("type".into(), Value::String("Feature".into())); + m.insert("properties".into(), Value::Object(props)); + m.insert("geometry".into(), Value::Object(geometry)); + m +} + +/// `devtestcoll.json` / `runsCollection.json`: Cosmos-run metadata record with +/// `id`, state strings, and ISO timestamps. (The corpus files also carry a +/// `_self` resource link, but that is a Cosmos-reserved system property the +/// service owns; we model it as a non-reserved `resourceLink` string instead so +/// the document round-trips cleanly.) +fn shape_cosmos_run(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + const STATES: &[&str] = &["InProgress", "Completed", "Failed", "Queued"]; + let mut m = Map::new(); + m.insert("id".into(), Value::String(gen_hex(rng, 7))); + m.insert( + "resourceLink".into(), + Value::String(format!( + "dbs/{}==/colls/{}=/docs/{}==/", + gen_code(rng, 6), + gen_code(rng, 9), + gen_code(rng, 20) + )), + ); + m.insert( + "RunState".into(), + Value::String(STATES[rng.below(STATES.len() as u64) as usize].into()), + ); + m.insert("RunResult".into(), Value::String("Failed".into())); + m.insert( + "FederationName".into(), + Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 24)), + ); + m.insert("StartTime".into(), Value::String(gen_iso_datetime(rng))); + m.insert("CompletedTime".into(), Value::String(gen_iso_datetime(rng))); + m +} + +/// `earth-meteorite-landings.json`: flat record with **numbers-as-strings** and +/// a nested `geolocation`. +fn shape_meteorite(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + let lat = gen_int_in(rng, -90_000_000, 90_000_000) as f64 / 1_000_000.0; + let lon = gen_int_in(rng, -180_000_000, 180_000_000) as f64 / 1_000_000.0; + let mut geo = Map::new(); + geo.insert("type".into(), Value::String("Point".into())); + geo.insert( + "coordinates".into(), + Value::Array(vec![gen_coordinate(rng), gen_coordinate(rng)]), + ); + let mut m = Map::new(); + m.insert("name".into(), Value::String(gen_name(rng))); + m.insert( + "id".into(), + Value::String(gen_int_in(rng, 1, 100_000).to_string()), + ); + m.insert("nametype".into(), Value::String("Valid".into())); + m.insert("recclass".into(), Value::String(gen_code(rng, 2))); + m.insert( + "mass".into(), + Value::String(gen_int_in(rng, 1, 100_000).to_string()), + ); + m.insert("year".into(), Value::String(gen_iso_datetime(rng))); + m.insert("reclat".into(), Value::String(format!("{lat:.6}"))); + m.insert("reclong".into(), Value::String(format!("{lon:.6}"))); + m.insert("geolocation".into(), Value::Object(geo)); + m +} + +/// `Employee-Data-Skewed*.json`: user record with GUIDs, names, a cloud SID. +fn shape_employee(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let mut m = Map::new(); + m.insert("objectType".into(), Value::String("User".into())); + m.insert("id".into(), Value::String(gen_uuid(rng))); + m.insert("objectId".into(), Value::String(gen_uuid(rng))); + m.insert("tenantId".into(), Value::String(gen_uuid(rng))); + m.insert( + "firstName".into(), + Value::String(gen_string_from(rng, ALPHA_CHARS, 8)), + ); + m.insert( + "lastName".into(), + Value::String(gen_unicode_string(rng, cfg, 12)), + ); + m.insert( + "MailNickname".into(), + Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 10)), + ); + m.insert( + "cloudSid".into(), + Value::String(format!( + "S-1-12-1-{}-{}", + gen_int_in(rng, 1, 4_000_000_000i64), + gen_int_in(rng, 1, 4_000_000_000i64) + )), + ); + m.insert("isActive".into(), Value::Bool(rng.below(2) == 0)); + m +} + +/// `FuzzingStrings.json`: `{ "string": }`. Draws from a +/// pool of edge-case strings plus random unicode. +fn shape_fuzzing_string(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + const EDGE: &[&str] = &[ + "", + "undefined", + "null", + "NULL", + "#", + "\t", + "\n", + "true", + "false", + "0", + "-0", + "NaN", + "\\", + "\"", + "{}", + "[]", + "\u{0000}", + "πŸ˜€", + "δΈ­ζ–‡", + "\u{feff}", + ]; + let s = if rng.below(2) == 0 { + let edge = EDGE[rng.below(EDGE.len() as u64) as usize]; + if cfg.unicode { + edge.to_string() + } else { + // Honor AZURE_COSMOS_FUZZ_UNICODE=false: drop non-ASCII edge cases + // (πŸ˜€, δΈ­ζ–‡, BOM) so ASCII-only runs stay ASCII. + edge.chars().filter(char::is_ascii).collect() + } + } else { + gen_unicode_string(rng, cfg, 40) + }; + let mut m = Map::new(); + m.insert("string".into(), Value::String(s)); + m +} + +/// `lastfm.json` / `MillionSong1KDocuments.json`: artist track with a `similars` +/// array of `[trackId, score]` heterogeneous pairs. +fn shape_lastfm(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n = count(rng, cfg, 1, 6); + let similars = gen_array_of(rng, n, |r| { + Value::Array(vec![ + Value::String(format!("TR{}", gen_hex_upper(r, 16))), + gen_unit_float(r), + ]) + }); + let mut m = Map::new(); + m.insert( + "id".into(), + Value::String(format!("item {:03}", gen_int_in(rng, 0, 999))), + ); + m.insert("artist".into(), Value::String(gen_name(rng))); + m.insert("timestamp".into(), Value::String(gen_iso_datetime(rng))); + m.insert("similars".into(), similars); + m +} + +/// `MsnCollection.json`: food item with a deeply nested `Contents` of +/// unit-tagged numbers, some with scientific-notation-scale floats. +fn shape_msn_food(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + let mut calories = Map::new(); + calories.insert( + "InCalories".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 2000))), + ); + let mut carbs = Map::new(); + // Small scientific-scale magnitude, kept envelope-safe as a plain float. + carbs.insert("InKg".into(), gen_unit_float(rng)); + carbs.insert( + "PreferredUnit".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 5))), + ); + let mut contents = Map::new(); + contents.insert("TotalCalories".into(), Value::Object(calories)); + contents.insert("Carbohydrates".into(), Value::Object(carbs)); + let mut m = Map::new(); + m.insert( + "FoodId".into(), + Value::String(gen_int_in(rng, 1, 99999).to_string()), + ); + m.insert("FoodName".into(), Value::String(gen_name(rng))); + m.insert( + "ServingSize".into(), + Value::String("1 mug (8 fl oz)".into()), + ); + m.insert( + "NumberOfServings".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 10))), + ); + m.insert("Contents".into(), Value::Object(contents)); + m +} + +/// `NutritionData.json`: food doc with `tags` and `nutrients` object arrays. +fn shape_nutrition(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n_tags = count(rng, cfg, 1, 5); + let tags = gen_array_of(rng, n_tags, |r| { + let mut t = Map::new(); + t.insert( + "name".into(), + Value::String(gen_string_from(r, ALPHA_CHARS, 10)), + ); + Value::Object(t) + }); + let n_nutrients = count(rng, cfg, 1, 6); + let nutrients = gen_array_of(rng, n_nutrients, |r| { + let mut n = Map::new(); + n.insert( + "id".into(), + Value::String(gen_int_in(r, 1, 999).to_string()), + ); + n.insert( + "description".into(), + Value::String(gen_string_from(r, ALPHANUMERIC_CHARS, 8)), + ); + n.insert("nutritionValue".into(), gen_envelope_float(r)); + Value::Object(n) + }); + let mut m = Map::new(); + m.insert( + "id".into(), + Value::String(format!("{:05}", gen_int_in(rng, 0, 99999))), + ); + m.insert( + "description".into(), + Value::String(gen_unicode_string(rng, cfg, 50)), + ); + m.insert("tags".into(), tags); + m.insert( + "version".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 5))), + ); + m.insert("foodGroup".into(), Value::String(gen_name(rng))); + m.insert("nutrients".into(), nutrients); + m +} + +/// `OpenAI_3072dim.json`: `{ "vector": [ ] }`. +fn shape_embedding_vector(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + // Base 16-80 dims; `size_scale` grows it toward the corpus's 3072-dim + // vectors (e.g. scale=40 β†’ ~640-3200 dims). + let dims = count(rng, cfg, 16, 64); + let mut m = Map::new(); + m.insert("vector".into(), gen_array_of(rng, dims, gen_unit_float)); + m +} + +/// `store01C.json`: shop record with id arrays, a status, and **non-ASCII** +/// (CJK) name/description fields plus a `null`. +fn shape_store(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n = count(rng, cfg, 0, 4); + let cat_ids = gen_array_of(rng, n, |r| { + Value::String(gen_string_from(r, ALPHANUMERIC_CHARS, 22)) + }); + let mut m = Map::new(); + m.insert( + "ShopId".into(), + Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 22)), + ); + m.insert("CategoryIds".into(), cat_ids); + m.insert("CollectionIds".into(), Value::Array(Vec::new())); + m.insert("IsActive".into(), Value::Bool(rng.below(2) == 0)); + m.insert( + "Status".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 100))), + ); + m.insert( + "Name".into(), + Value::String(gen_unicode_string(rng, cfg, 20)), + ); + m.insert( + "Summary".into(), + Value::String(gen_unicode_string(rng, cfg, 20)), + ); + m.insert("Description".into(), Value::Null); + m +} + +/// `TicinoErrorBuckets.json`: error-bucket with a multiline stack-trace string +/// (embedded `\n`), a hash, and a hit count. +fn shape_error_bucket(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + let stack = format!( + "Error: spawn REG ENOENT\n at exports._errnoException (util.js:{}:11)\n at Process.ChildProcess._handle.onexit (child_process.js:{}:32)", + gen_int_in(rng, 100, 999), + gen_int_in(rng, 1000, 2000) + ); + let mut m = Map::new(); + m.insert("BucketId".into(), Value::String(stack)); + m.insert("BucketIdHash".into(), Value::String(gen_hex(rng, 32))); + m.insert( + "Hits".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 100_000))), + ); + m +} + +/// `XpertEvents.json`: telemetry event with nested `ingest`, ISO timestamps, a +/// GUID-bearing `userId`, and mixed numeric quality. +fn shape_xpert_event(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + let mut ingest = Map::new(); + ingest.insert("time".into(), Value::String(gen_iso_datetime(rng))); + ingest.insert("uploadTime".into(), Value::String(gen_iso_datetime(rng))); + ingest.insert( + "clientIp".into(), + Value::String(format!( + "{}.{}.{}.{}", + gen_int_in(rng, 0, 255), + gen_int_in(rng, 0, 255), + gen_int_in(rng, 0, 255), + gen_int_in(rng, 0, 255) + )), + ); + ingest.insert( + "quality".into(), + Value::Number(Number::from(gen_int_in(rng, 0, 5))), + ); + let mut m = Map::new(); + m.insert("ingest".into(), Value::Object(ingest)); + m.insert("time".into(), Value::String(gen_iso_datetime(rng))); + m.insert( + "userId".into(), + Value::String(format!("w:{{{}}}", gen_uuid(rng).to_ascii_uppercase())), + ); + m.insert("appId".into(), Value::String(gen_hex(rng, 40))); + m +} + +/// `ups1.json`: personalization payload with an opaque high-entropy `Vector` +/// string, an ANID GUID, and a small nested `payload`. +fn shape_ups(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + let mut payload = Map::new(); + payload.insert("ANID".into(), Value::String(gen_uuid(rng))); + payload.insert( + "MUID".into(), + Value::Array(vec![Value::String("muid".into())]), + ); + payload.insert("AppDomain".into(), Value::String("prime".into())); + payload.insert("Algo".into(), Value::String("lda".into())); + payload.insert("Culture".into(), Value::String("en-us".into())); + payload.insert("Version".into(), Value::Number(Number::from(1))); + payload.insert( + "Vector".into(), + Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 64)), + ); + let mut m = Map::new(); + m.insert("domain".into(), Value::String("Personalization".into())); + m.insert("lid".into(), Value::String("lda-prime-en-us-1".into())); + m.insert("payload".into(), Value::Object(payload)); + m +} + +/// `states_committees.json`: committee with a `members` array of role records. +fn shape_committee(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n = count(rng, cfg, 1, 6); + let members = gen_array_of(rng, n, |r| { + let mut mem = Map::new(); + mem.insert( + "leg_id".into(), + Value::String(format!("{}{:06}", gen_code(r, 3), gen_int_in(r, 0, 999999))), + ); + mem.insert("role".into(), Value::String("member".into())); + mem.insert( + "name".into(), + Value::String(format!("Representative {}", gen_name(r))), + ); + Value::Object(mem) + }); + let mut m = Map::new(); + m.insert("members".into(), members); + m +} + +/// `states_legislators.json`: legislator with `sources`, `old_roles` (object +/// keyed by term with role arrays containing `null`s), and name fields. +fn shape_legislator(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n = count(rng, cfg, 1, 2); + let sources = gen_array_of(rng, n, |r| { + let mut s = Map::new(); + s.insert( + "url".into(), + Value::String(format!( + "http://example.gov/legislator.php?id={}", + gen_string_from(r, ALPHA_CHARS, 4) + )), + ); + Value::Object(s) + }); + let n = count(rng, cfg, 1, 2); + let roles = gen_array_of(rng, n, |r| { + let mut role = Map::new(); + role.insert( + "term".into(), + Value::String(gen_int_in(r, 20, 30).to_string()), + ); + role.insert("end_date".into(), Value::Null); + role.insert("district".into(), Value::String(gen_code(r, 1))); + Value::Object(role) + }); + let mut old_roles = Map::new(); + old_roles.insert(gen_int_in(rng, 20, 30).to_string(), roles); + let mut m = Map::new(); + m.insert("last_name".into(), Value::String(gen_name(rng))); + m.insert("updated_at".into(), Value::String(gen_iso_datetime(rng))); + m.insert("sources".into(), sources); + m.insert("full_name".into(), Value::String(gen_name(rng))); + m.insert("old_roles".into(), Value::Object(old_roles)); + m +} + +/// `LogData.json`: impression log with GUID ids, a boolean, and an `Events` +/// array of nested event/page/requestInfo objects. +fn shape_logdata(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n = count(rng, cfg, 1, 3); + let events = gen_array_of(rng, n, |r| { + let mut page = Map::new(); + page.insert("Name".into(), Value::String("API.Qsml".into())); + let mut req = Map::new(); + req.insert("AFORM".into(), Value::String("MSNH2".into())); + req.insert( + "Bytes".into(), + Value::Number(Number::from(gen_int_in(r, 0, 10000))), + ); + let mut ev = Map::new(); + ev.insert("T".into(), Value::String("Event.Impression".into())); + ev.insert("EventId".into(), Value::String(gen_hex_upper(r, 32))); + ev.insert("Page".into(), Value::Object(page)); + ev.insert("RequestInfo".into(), Value::Object(req)); + Value::Object(ev) + }); + let mut m = Map::new(); + m.insert("AppNS".into(), Value::String("API".into())); + m.insert("ClientId".into(), Value::String(gen_hex_upper(rng, 32))); + m.insert( + "ImpressionGuid".into(), + Value::String(gen_hex_upper(rng, 32)), + ); + m.insert("ProvClientId".into(), Value::Bool(rng.below(2) == 0)); + m.insert("Events".into(), events); + m +} + +/// `sampleWorkload.json`: record with a `header` carrying nested `schema`. +fn shape_workload(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let mut schema = Map::new(); + schema.insert("name".into(), Value::String("_xdm.context.profile".into())); + schema.insert("version".into(), Value::String("1.0".into())); + let mut header = Map::new(); + header.insert("recordType".into(), Value::String("keyvalue".into())); + header.insert( + "tag".into(), + Value::String(format!("batchId-{}", gen_hex_upper(rng, 26))), + ); + header.insert("packetVersion".into(), Value::String("1.0".into())); + header.insert( + "component".into(), + Value::String(format!("{:03}", gen_int_in(rng, 0, 999))), + ); + header.insert("schema".into(), Value::Object(schema)); + let mut m = Map::new(); + m.insert("header".into(), Value::Object(header)); + m.insert( + "payload".into(), + Value::String(gen_unicode_string(rng, cfg, 40)), + ); + m +} + +/// `DefaultHybridRowSchema.json`: schema-policy document with booleans and a +/// nested `tableSchema.schemas` array (options flags). +fn shape_hybrid_schema(rng: &mut SplitMix64, _cfg: &FuzzConfig) -> Map { + let mut options = Map::new(); + options.insert( + "disallowUnschematized".into(), + Value::Bool(rng.below(2) == 0), + ); + options.insert( + "enablePropertyLevelTimestamp".into(), + Value::Bool(rng.below(2) == 0), + ); + options.insert("disableSystemPrefix".into(), Value::Bool(rng.below(2) == 0)); + let schema = { + let mut s = Map::new(); + s.insert("version".into(), Value::String("v1".into())); + s.insert("name".into(), Value::String("Row".into())); + s.insert("id".into(), Value::Number(Number::from(-1))); + s.insert("type".into(), Value::String("schema".into())); + s.insert("options".into(), Value::Object(options)); + Value::Object(s) + }; + let mut table_schema = Map::new(); + table_schema.insert("version".into(), Value::String("v1".into())); + table_schema.insert("name".into(), Value::String("tableSchema".into())); + table_schema.insert("schemas".into(), Value::Array(vec![schema])); + let mut policy = Map::new(); + policy.insert("tableSchema".into(), Value::Object(table_schema)); + let mut m = Map::new(); + m.insert("schemaPolicy".into(), Value::Object(policy)); + m +} + +/// `reddit_all.json` (single-record variant): a Listing with a nested `data` +/// carrying a `children` array of `t3` post objects and many `null`s. +fn shape_reddit(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n = count(rng, cfg, 1, 4); + let children = gen_array_of(rng, n, |r| { + let mut data = Map::new(); + data.insert("approved_at_utc".into(), Value::Null); + data.insert( + "title".into(), + Value::String(gen_unicode_string_for(r, cfg, 40)), + ); + data.insert( + "ups".into(), + Value::Number(Number::from(gen_int_in(r, 0, 100_000))), + ); + data.insert("over_18".into(), Value::Bool(r.below(2) == 0)); + data.insert("score".into(), gen_envelope_float(r)); + let mut child = Map::new(); + child.insert("kind".into(), Value::String("t3".into())); + child.insert("data".into(), Value::Object(data)); + Value::Object(child) + }); + let mut data = Map::new(); + data.insert( + "after".into(), + Value::String(format!( + "t3_{}", + gen_string_from(rng, ALPHANUMERIC_CHARS, 7) + )), + ); + data.insert( + "dist".into(), + Value::Number(Number::from(gen_int_in(rng, 1, 100))), + ); + data.insert("modhash".into(), Value::String(String::new())); + data.insert("geo_filter".into(), Value::Null); + data.insert("children".into(), children); + let mut m = Map::new(); + m.insert("kind".into(), Value::String("Listing".into())); + m.insert("data".into(), Value::Object(data)); + m +} + +/// `open-food-facts.json` (single-record variant): a product wrapper with a +/// `_keywords` string array and a nested `product`. +fn shape_open_food(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let n = count(rng, cfg, 3, 6); + let keywords = gen_array_of(rng, n, |r| { + Value::String(gen_string_from(r, ALPHA_CHARS, 10)) + }); + let mut product = Map::new(); + product.insert( + "_id".into(), + Value::String(gen_int_in(rng, 1, i64::MAX / 2).to_string()), + ); + product.insert("_keywords".into(), keywords); + product.insert("nutriments".into(), { + let mut n = Map::new(); + n.insert("energy".into(), gen_envelope_float(rng)); + n.insert("fat".into(), gen_envelope_float(rng)); + Value::Object(n) + }); + let mut m = Map::new(); + m.insert( + "code".into(), + Value::String(gen_int_in(rng, 1, i64::MAX / 2).to_string()), + ); + m.insert("product".into(), Value::Object(product)); + m +} + +/// Convenience: unicode string honoring the `unicode` knob (thin wrapper so the +/// closure-based array builders can call it with a `cfg`). +fn gen_unicode_string_for(rng: &mut SplitMix64, cfg: &FuzzConfig, max_len: usize) -> String { + gen_unicode_string(rng, cfg, max_len) +} + +/// The set of corpus-shape samplers. Each entry is `(name, fn)` where `name` +/// mirrors the originating `testdata/*.json` file and `fn` builds one document +/// in that shape. [`gen_shaped_document`] picks one uniformly at random. +type ShapeSampler = fn(&mut SplitMix64, &FuzzConfig) -> Map; + +const SHAPE_SAMPLERS: &[(&str, ShapeSampler)] = &[ + ("airline-delays", shape_airline_delays), + ("bitcoin", shape_bitcoin), + ("bing-docs", shape_bing_docs), + ("scripts-data", shape_scripts_data), + ("geojson-feature", shape_geojson_feature), + ("cosmos-run", shape_cosmos_run), + ("meteorite", shape_meteorite), + ("employee", shape_employee), + ("fuzzing-string", shape_fuzzing_string), + ("lastfm", shape_lastfm), + ("msn-food", shape_msn_food), + ("nutrition", shape_nutrition), + ("embedding-vector", shape_embedding_vector), + ("store", shape_store), + ("error-bucket", shape_error_bucket), + ("xpert-event", shape_xpert_event), + ("ups", shape_ups), + ("committee", shape_committee), + ("legislator", shape_legislator), + ("logdata", shape_logdata), + ("workload", shape_workload), + ("hybrid-schema", shape_hybrid_schema), + ("reddit", shape_reddit), + ("open-food", shape_open_food), +]; + +/// Builds one document matching a **randomly chosen** corpus shape (see +/// [`SHAPE_SAMPLERS`]). The returned map does not yet carry the caller-reserved +/// `id`/`pk` fields β€” the run loop inserts those. +fn gen_shaped_document(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let idx = rng.below(SHAPE_SAMPLERS.len() as u64) as usize; + SHAPE_SAMPLERS[idx].1(rng, cfg) +} + +/// A small, irregular filler value. Draws from typed scalars, mixed +/// arrays/objects of typed scalars, homogeneous number arrays (to exercise the +/// uniform-number wire forms), and `arbitrary-json` subtrees β€” so filler is both +/// varied and non-trivial in size. Already respects the `wide_numbers`/`unicode` +/// knobs. +fn gen_filler_value(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + match rng.below(10) { + // Homogeneous number array (uniform-number wire forms). + 0 => { + let len = rng.below(8); + let arr = (0..len).map(|_| gen_envelope_int(rng)).collect(); + Value::Array(arr) + } + // Typed scalars across the character/number classes. + 1..=3 => gen_scalar(rng, cfg), + // A short mixed-type array. + 4 => { + let len = 1 + rng.below(cfg.breadth as u64 + 1); + Value::Array((0..len).map(|_| gen_scalar(rng, cfg)).collect()) + } + // A small object of typed scalars. + 5 => { + let mut map = Map::new(); + for _ in 0..1 + rng.below(cfg.breadth as u64) { + map.insert(gen_key(rng), gen_scalar(rng, cfg)); + } + Value::Object(map) + } + // An `arbitrary-json` subtree (bigger byte budget), envelope-bounded. + _ => { + let bytes = fill_bytes(rng, FILLER_BUDGET); + let mut u = Unstructured::new(&bytes); + let mut v: Value = ArbitraryValue::arbitrary(&mut u) + .map(Into::into) + .unwrap_or(Value::Null); + bound_value(&mut v, cfg); + v + } + } +} + +/// Builds a nested container chain `depth` levels deep, with several irregular +/// filler siblings at each level, guaranteeing the document reaches `depth`. +/// Each level is randomly an object or an array; exactly one child continues the +/// spine deeper. The sibling count scales with `breadth`, so deeper documents +/// are also wider. +fn gen_spine(rng: &mut SplitMix64, cfg: &FuzzConfig, depth: u32) -> Value { + if depth == 0 { + return gen_filler_value(rng, cfg); + } + if rng.below(2) == 0 { + // Object: filler fields + one spine field going deeper. + let mut map = Map::new(); + for _ in 0..rng.below(cfg.breadth as u64 + 1) { + map.insert(gen_key(rng), gen_filler_value(rng, cfg)); + } + let mut key = gen_key(rng); + while key.is_empty() { + key.push('_'); + } + map.insert(key, gen_spine(rng, cfg, depth - 1)); + Value::Object(map) + } else { + // Array: filler elements + one spine element going deeper. + let mut arr = Vec::new(); + for _ in 0..rng.below(cfg.breadth as u64 + 1) { + arr.push(gen_filler_value(rng, cfg)); + } + arr.push(gen_spine(rng, cfg, depth - 1)); + Value::Array(arr) + } +} + +/// Recursively applies the generation bounds to a value: clamps numbers into the +/// calibrated-safe envelope unless `wide_numbers`, and drops non-ASCII from +/// strings unless `unicode`. Leaves structure otherwise untouched. +fn bound_value(value: &mut Value, cfg: &FuzzConfig) { + match value { + Value::Number(n) => { + if !cfg.wide_numbers { + *value = clamp_number_to_envelope(n); + } + } + Value::String(s) => { + if !cfg.unicode && !s.is_ascii() { + *s = s.chars().filter(char::is_ascii).collect(); + } + } + Value::Array(items) => { + for item in items.iter_mut() { + bound_value(item, cfg); + } + } + Value::Object(map) => { + // When Unicode is disabled, object *keys* must be ASCII-filtered + // too β€” property names go through the same binary string-encoding + // path as values, so leaving non-ASCII keys would not isolate the + // ASCII codec path. `arbitrary-json` can emit non-ASCII keys. + if !cfg.unicode && map.keys().any(|k| !k.is_ascii()) { + let rebuilt: Map = std::mem::take(map) + .into_iter() + .map(|(k, mut v)| { + bound_value(&mut v, cfg); + let key = if k.is_ascii() { + k + } else { + k.chars().filter(char::is_ascii).collect() + }; + (key, v) + }) + .collect(); + *map = rebuilt; + } else { + for v in map.values_mut() { + bound_value(v, cfg); + } + } + } + _ => {} + } +} + +/// Clamps a number into the **backend-safe** envelope (design doc Β§3.2): bounded +/// integers and two-decimal floats, matching what the calibrated +/// [`normalize_number`] models without `--wide-numbers`. +fn clamp_number_to_envelope(n: &Number) -> Value { + if let Some(i) = n.as_i64() { + Value::Number(Number::from(i.rem_euclid(2_000_001) - 1_000_000)) + } else if let Some(u) = n.as_u64() { + Value::Number(Number::from((u % 2_000_001) as i64 - 1_000_000)) + } else if let Some(f) = n.as_f64() { + // Two decimal places within Β±100_000 keeps it inside the calibrated + // envelope; a non-finite arbitrary float collapses to 0. + let bounded = if f.is_finite() { + ((f % 100_000.0) * 100.0).round() / 100.0 + } else { + 0.0 + }; + Number::from_f64(bounded) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(0))) + } else { + Value::Number(Number::from(0)) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Cosmos-compatible canonicalization (design doc Β§3) +// ───────────────────────────────────────────────────────────────────────────── + +/// Produces the canonical string for a JSON value: the calibrated Cosmos number +/// rewrite ([`normalize_numbers`]) followed by RFC 8785 (JCS) structural +/// canonicalization via [`json_canon`] β€” whitespace removed, object keys sorted, +/// strings minimally escaped. Two values with the same canonical string are +/// considered equal after a round-trip. +/// +/// Numbers are normalized **first** so the JCS serializer's own number +/// formatting no longer affects the comparison; the only Cosmos-specific step is +/// [`normalize_numbers`] (design doc Β§3.1). +fn canonicalize(value: &Value) -> String { + let normalized = normalize_numbers(value); + json_canon::to_string(&normalized).expect("normalized value always canonicalizes") +} + +/// Rewrites a single JSON number to its **Cosmos-calibrated** canonical +/// [`Value`]. **This is the tuning surface** β€” see the design doc Β§3.1. It is +/// the one number-specific step that must stay under our control (RFC 8785 / JCS +/// number formatting is *not* the backend's store-time rewrite); the structural +/// canonicalization around it can be delegated to a standard JCS serializer. +/// +/// Rules (calibrated against a live account, design doc Β§3.1): +/// - integers with magnitude `< 2^53` β†’ exact integer (JCS-safe); +/// - integers with magnitude `>= 2^53` (whether `i64` or `u64`) β†’ **tagged +/// wide-number value** ([`cosmos_wide_number_value`]) holding the `f64` token. +/// The backend stores *every* JSON number as an +/// IEEE-754 double, so an integer beyond `2^53` is not preserved exactly (e.g. +/// `28423844363879210` is echoed back as `28423844363879208`); tokenizing the +/// rounded double makes the sent and returned values compare equal; +/// - integral-valued floats below `2^53` (e.g. `1.0`) β†’ integer form (the +/// backend drops the trailing `.0`); +/// - integral-valued floats `>= 2^53` β†’ tagged wide-number value (matches the +/// lossy double case above); +/// - other finite floats β†’ kept as `f64` (JCS-safe); +/// - non-finite (`NaN` / `±∞`) β†’ `null`. +/// +/// The wide-number tokens are only ever compared for equality, so a tagged token +/// is sound: any two values Cosmos round-trips to each other produce the same +/// token, and the tag keeps them out of the plain-string domain (see +/// [`WIDE_NUMBER_TAG`]). +fn normalize_number(n: &Number) -> Value { + if let Some(i) = n.as_i64() { + if (i.unsigned_abs() as f64) < JCS_SAFE_INT_LIMIT { + Value::Number(Number::from(i)) + } else { + // i >= 2^53: the backend stores it as a lossy double, so tokenize the + // rounded f64 (not the exact decimal) β€” otherwise the returned, + // double-rounded value would mismatch. + cosmos_wide_number_value(i as f64) + } + } else if let Some(u) = n.as_u64() { + // u > i64::MAX: Cosmos stores it as a lossy double; token from the double. + cosmos_wide_number_value(u as f64) + } else if let Some(f) = n.as_f64() { + if !f.is_finite() { + Value::Null + } else if f.fract() == 0.0 && f.abs() < JCS_SAFE_INT_LIMIT { + Value::Number(Number::from(f as i64)) + } else if f.fract() == 0.0 { + // Integral but out of the JCS-safe range β†’ double token. + cosmos_wide_number_value(f) + } else { + // Non-integral finite float is JCS-safe as a number. + Number::from_f64(f) + .map(Value::Number) + .unwrap_or(Value::Null) + } + } else { + Value::Null + } +} + +/// `2^53`: the largest magnitude RFC 8785 (JCS) will emit as an integer. At or +/// beyond this, `json-canon` refuses integer output and Cosmos stores `u64` +/// above `i64::MAX` lossily as doubles, so such numbers are canonicalized as +/// string tokens (see [`normalize_number`]). +const JCS_SAFE_INT_LIMIT: f64 = 9_007_199_254_740_992.0; + +/// A stable decimal token for a Cosmos-stored double, from the `f64` value. +/// Both a sent `u64` and its returned scientific-notation double parse to the +/// same `f64`, so they produce the same token. +fn cosmos_double_token(f: f64) -> String { + format!("{f}") +} + +/// The object key that tags a normalized wide-number token. Wide numbers are +/// canonicalized to `{ WIDE_NUMBER_TAG: "" }` rather than a bare +/// [`Value::String`], keeping the token in a distinct type domain so a +/// number-to-string codec bug cannot canonicalize equal and pass silently. +const WIDE_NUMBER_TAG: &str = "$__cosmos_wide_number__"; + +/// Wraps a Cosmos-stored double's decimal token in the [`WIDE_NUMBER_TAG`] +/// envelope. The inner value stays a `String`, so [`normalize_numbers`] is +/// idempotent over the result. +fn cosmos_wide_number_value(f: f64) -> Value { + let mut map = serde_json::Map::new(); + map.insert( + WIDE_NUMBER_TAG.to_string(), + Value::String(cosmos_double_token(f)), + ); + Value::Object(map) +} + +/// Recursively rewrites every number in `value` to its Cosmos-calibrated form +/// (see [`normalize_number`]), leaving all other value kinds unchanged. The +/// result is a `Value` ready for a standard (JCS) structural canonicalization +/// pass β€” the number rewrite has already been applied, so the structural +/// serializer's own number formatting no longer changes the comparison. +fn normalize_numbers(value: &Value) -> Value { + match value { + Value::Number(n) => normalize_number(n), + Value::Array(items) => Value::Array(items.iter().map(normalize_numbers).collect()), + Value::Object(map) => Value::Object( + map.iter() + .map(|(k, v)| (k.clone(), normalize_numbers(v))) + .collect(), + ), + other => other.clone(), + } +} + +/// Canonicalizes and returns `(canonical_string, SHA-256 digest)`. +/// +/// The digest is a cryptographic hash of the canonical string, so it is stable +/// across runs and platforms β€” suitable for a durable corpus of expected `H0` +/// values ("store the hash once, compare later"). +fn canonical_hash(value: &Value) -> (String, [u8; 32]) { + let s = canonicalize(value); + let digest: [u8; 32] = Sha256::digest(s.as_bytes()).into(); + (s, digest) +} + +/// Formats a 32-byte digest as lowercase hex, for mismatch reporting. +fn hex(digest: &[u8; 32]) -> String { + let mut out = String::with_capacity(64); + for b in digest { + out.push_str(&format!("{b:02x}")); + } + out +} + +/// Normalizes a value by one JSON serializeβ†’parse pass. Any Cosmos round-trip +/// (through text or binary, plus the backend's own store rewrite) puts the value +/// through at least one serializeβ†’parse, which can shift a `from_f64` float to a +/// neighboring value with a shorter shortest-form. Computing the **sent** +/// canonical form from the normalized value puts sent and round-tripped +/// documents on equal footing. Normalization is idempotent (a parsed value's +/// shortest serialization round-trips back to itself). +fn normalize(value: &Value) -> Value { + let text = serde_json::to_string(value).expect("value always serializes"); + serde_json::from_str(&text).expect("serialized value always parses") +} + +/// Cosmos-reserved system properties: the service assigns and owns these, so a +/// document we author must never carry them (a random value would round-trip +/// back as the service's own value and cause a false mismatch). +const RESERVED_SYSTEM_KEYS: &[&str] = &["_rid", "_self", "_etag", "_ts", "_attachments"]; + +/// Removes any Cosmos-reserved system properties from `doc` in place. Applied to +/// every generated document before it is sent, so neither the corpus shapes nor +/// the free-form arbitrary-json generator can emit a reserved key the service +/// would overwrite. +fn strip_reserved_fields(doc: &mut Map) { + for key in RESERVED_SYSTEM_KEYS { + doc.remove(*key); + } +} + +/// Projects a returned document to only the keys present in `sent`, so +/// service-added system fields (`_rid`, `_etag`, `_ts`, ...) don't affect the +/// comparison. +fn project_to_sent_keys(sent: &Map, got: &Value) -> Value { + let got_obj = match got.as_object() { + Some(o) => o, + None => return got.clone(), + }; + let mut out = Map::new(); + for key in sent.keys() { + if let Some(v) = got_obj.get(key) { + out.insert(key.clone(), v.clone()); + } + } + Value::Object(out) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Client / account setup +// ───────────────────────────────────────────────────────────────────────────── + +/// One binary-encoding configuration exercised per generated document. +struct RunConfig { + label: &'static str, + binary: Option, +} + +fn run_configs() -> Vec { + vec![ + RunConfig { + label: "text-control", + binary: None, + }, + RunConfig { + label: "binary", + binary: Some(BinaryEncodingOptions::new().with_enabled(true)), + }, + RunConfig { + label: "binary+text-response", + binary: Some( + BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true), + ), + }, + ] +} + +/// Builds a Cosmos client for a given binary-encoding configuration. +async fn build_client( + binary: &Option, +) -> Result> { + let connection_string = std::env::var(CONNECTION_STRING_ENV_VAR).map_err(|_| { + format!("{CONNECTION_STRING_ENV_VAR} must be set to a Cosmos DB connection string") + })?; + let connection_string: ConnectionString = connection_string.parse()?; + + let endpoint: AccountEndpoint = connection_string.account_endpoint().parse()?; + let account = AccountReference::with_authentication_key( + endpoint, + connection_string.account_key().clone(), + ); + + let mut builder = CosmosClient::builder(); + if let Some(options) = binary { + builder = builder.with_binary_encoding_options(options.clone()); + } + + let allow_invalid_cert = env_bool(ALLOW_INVALID_CERT_ENV_VAR, false); + if allow_invalid_cert { + let runtime = CosmosRuntime::builder() + .with_connection_pool( + ConnectionPoolOptions::builder() + .with_server_certificate_validation( + ServerCertificateValidation::RequiredUnlessEmulator, + ) + .build()?, + ) + .build() + .await?; + builder = builder.with_runtime(runtime); + } + + let client = builder + .build(account, RoutingStrategy::ProximityTo(Region::EAST_US)) + .await?; + Ok(client) +} + +fn ignore_conflict(result: azure_data_cosmos::Result) -> Result<(), Box> { + match result { + Ok(_) => Ok(()), + Err(e) if e.status().status_code() == StatusCode::Conflict => Ok(()), + Err(e) => Err(e.into()), + } +} + +fn write_options_with_content() -> ItemWriteOptions { + let mut operation = OperationOptions::default(); + operation.content_response_on_write = Some(ContentResponseOnWrite::Enabled); + ItemWriteOptions::default().with_operation_options(operation) +} + +/// Maximum attempts for a single point operation before the run gives up. +const MAX_OP_ATTEMPTS: u32 = 6; + +/// Transient transport/service status codes worth retrying (429/408/503/...). +/// A response-body serialization failure (`500 / SERIALIZATION_RESPONSE_BODY_INVALID`) +/// is excluded: it is the decode corruption this fuzzer exists to catch, so it +/// must surface immediately rather than be retried into a masking 409. +fn is_transient(err: &azure_data_cosmos::CosmosError) -> bool { + let status = err.status(); + if status.sub_status() == Some(SubStatusCode::SERIALIZATION_RESPONSE_BODY_INVALID) { + return false; + } + matches!( + u16::from(status.status_code()), + 408 | 429 | 500 | 502 | 503 | 504 + ) +} + +/// Runs a point operation, retrying transient failures with exponential backoff +/// so a long soak survives network blips. Non-transient errors (notably a decode +/// failure β€” what this fuzzer exists to catch) return immediately. +async fn with_transient_retry( + op_name: &str, + context: &str, + mut op: F, +) -> Result> +where + F: FnMut() -> Fut, + Fut: std::future::Future>, +{ + let mut attempt = 0; + loop { + attempt += 1; + match op().await { + Ok(value) => return Ok(value), + Err(e) if is_transient(&e) && attempt < MAX_OP_ATTEMPTS => { + let backoff = + std::time::Duration::from_millis(200u64 * (1u64 << (attempt - 1)).min(16)); + eprintln!( + "{context}: {op_name} transient failure (attempt {attempt}/{MAX_OP_ATTEMPTS}), \ + retrying in {backoff:?}: {e}" + ); + tokio::time::sleep(backoff).await; + } + Err(e) => return Err(format!("{context}: {op_name} failed: {e}").into()), + } + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// The fuzzer test +// ───────────────────────────────────────────────────────────────────────────── + +#[tokio::test] +#[cfg_attr( + not(test_category = "binary_encoding"), + ignore = "requires test_category 'binary_encoding' and a live account connection string" +)] +async fn binary_encoding_roundtrip_fuzz() -> Result<(), Box> { + let cfg = FuzzConfig::from_env(); + + if cfg.calibrate { + return run_calibration().await; + } + + println!( + "binary_roundtrip_fuzzer: seed={} iterations={} max_depth={} breadth={} wide_numbers={} unicode={}", + cfg.seed, cfg.iterations, cfg.max_depth, cfg.breadth, cfg.wide_numbers, cfg.unicode + ); + println!("Reproduce this run with {SEED_ENV_VAR}={}", cfg.seed); + + let configs = run_configs(); + + // One client per config (binary encoding is resolved at build time). + let mut clients = Vec::new(); + for rc in &configs { + clients.push((rc.label, build_client(&rc.binary).await?)); + } + + // Ensure the target database + container exist on each client's account + // (they share the same account, so the first client suffices). + let database_name = + std::env::var(DATABASE_NAME_ENV_VAR).unwrap_or_else(|_| DEFAULT_DATABASE_NAME.to_string()); + let container_name = std::env::var(CONTAINER_NAME_ENV_VAR) + .unwrap_or_else(|_| DEFAULT_CONTAINER_NAME.to_string()); + + let setup_client = &clients[0].1; + ignore_conflict(setup_client.create_database(&database_name, None).await)?; + let setup_db = setup_client.database_client(&database_name); + ignore_conflict( + setup_db + .create_container( + ContainerProperties::new(container_name.clone(), PARTITION_KEY_PATH.into()), + None, + ) + .await, + )?; + + let mut rng = SplitMix64::new(cfg.seed); + let mut checked: u64 = 0; + + for iter in 0..cfg.iterations { + // Generate the document body once per iteration so all three configs + // test the *same value* three ways. Each config gets a distinct `id` + // below β€” the same document stored under multiple configs would + // otherwise collide on the `(pk, id)` key and fail with 409 Conflict. + let base_doc = gen_object(&mut rng, &cfg); + let pk = format!("pk-{}", rng.below(16)); + + // Optionally print the generated document (pretty JSON) so a run can be + // eyeballed. Enable with `AZURE_COSMOS_FUZZ_PRINT=true`. + if cfg.print_docs { + println!( + "--- iter {iter} (seed={}) ---\n{}", + cfg.seed, + serde_json::to_string_pretty(&Value::Object(base_doc.clone())) + .unwrap_or_else(|_| "".to_string()) + ); + } + + for (config_idx, (label, client)) in clients.iter().enumerate() { + let container = client + .database_client(&database_name) + .container_client(&container_name) + .await?; + + // Deterministic id per (seed, iteration, config) β€” derived without + // touching the document RNG stream so a rerun with the same + // AZURE_COSMOS_FUZZ_SEED reproduces the exact document *and* its + // canonical form (a random `Uuid` here would defeat that promise). + let id = format!("fuzz-{:016x}-{iter}-{config_idx}", cfg.seed); + let mut doc = base_doc.clone(); + doc.insert("id".to_string(), Value::String(id.clone())); + doc.insert("pk".to_string(), Value::String(pk.clone())); + // Never send Cosmos-reserved system properties (`_rid`, `_self`, + // `_etag`, `_ts`, `_attachments`): the service **owns** these and + // overwrites/assigns them, so a random value we send would come + // back different and cause a false round-trip mismatch. The corpus + // shapes (and free-form arbitrary-json) can incidentally emit them. + strip_reserved_fields(&mut doc); + + // Compute the sent canonical form from a normalized copy so it + // matches documents that have been through the backend's + // serializeβ†’parse. + let (sent_canon, sent_hash) = canonical_hash(&normalize(&Value::Object(doc.clone()))); + + let context = format!("iter={iter} config={label} id={id} seed={}", cfg.seed); + + // CREATE with content response (exercises the response decode + // path). Retry transient failures. A 409 Conflict is recovered on + // any attempt: the id is deterministic per (seed, iteration, + // config), so a replayed seed lands on an item an earlier run + // committed and the first create 409s. Reading it back keeps the + // assertion alive. This can't mask a broken create-response decode, + // which surfaces as `500 / SERIALIZATION_RESPONSE_BODY_INVALID` and + // is excluded from `is_transient`, so it returns immediately. + let created_doc: Value = { + let mut attempt = 0; + loop { + attempt += 1; + match container + .create_item(&pk, &id, &doc, Some(write_options_with_content())) + .await + { + Ok(resp) => { + break resp.into_model().map_err(|e| { + format!("{context}: create response decode failed: {e}") + })?; + } + Err(e) if e.status().status_code() == StatusCode::Conflict => { + let read = container.read_item(&pk, &id, None).await.map_err(|e| { + format!("{context}: create-conflict read failed: {e}") + })?; + break read.into_model().map_err(|e| { + format!("{context}: create-conflict decode failed: {e}") + })?; + } + Err(e) if is_transient(&e) && attempt < MAX_OP_ATTEMPTS => { + let backoff = std::time::Duration::from_millis( + 200u64 * (1u64 << (attempt - 1)).min(16), + ); + eprintln!( + "{context}: create transient failure (attempt {attempt}/{MAX_OP_ATTEMPTS}), retrying in {backoff:?}: {e}" + ); + tokio::time::sleep(backoff).await; + } + Err(e) => return Err(format!("{context}: create failed: {e}").into()), + } + } + }; + assert_roundtrip( + &doc, + &created_doc, + &sent_canon, + &sent_hash, + &context, + "create", + ); + + // READ back. + let read = + with_transient_retry("read", &context, || container.read_item(&pk, &id, None)) + .await?; + let read_doc: Value = read + .into_model() + .map_err(|e| format!("{context}: read response decode failed: {e}"))?; + assert_roundtrip(&doc, &read_doc, &sent_canon, &sent_hash, &context, "read"); + + // REPLACE the item with the same value (exercises the replace point + // op's request encode + response decode). Binary encoding is honored + // for replace, so this drives the encoder/decoder just like create. + let replaced = with_transient_retry("replace", &context, || { + container.replace_item(&pk, &id, &doc, Some(write_options_with_content())) + }) + .await?; + let replaced_doc: Value = replaced + .into_model() + .map_err(|e| format!("{context}: replace response decode failed: {e}"))?; + assert_roundtrip( + &doc, + &replaced_doc, + &sent_canon, + &sent_hash, + &context, + "replace", + ); + + // UPSERT the same value (upsert is a point op that also carries a + // body; here it updates the existing item). Covers the upsert + // request-encode + response-decode path. + let upserted = with_transient_retry("upsert", &context, || { + container.upsert_item(&pk, &id, &doc, Some(write_options_with_content())) + }) + .await?; + let upserted_doc: Value = upserted + .into_model() + .map_err(|e| format!("{context}: upsert response decode failed: {e}"))?; + assert_roundtrip( + &doc, + &upserted_doc, + &sent_canon, + &sent_hash, + &context, + "upsert", + ); + + // Four point-op round-trips this config: create, read, replace, upsert. + checked += 4; + } + + if (iter + 1) % 100 == 0 { + println!("... {} iterations, {checked} round-trips OK", iter + 1); + } + } + + println!( + "binary_roundtrip_fuzzer: DONE β€” {} documents Γ— {} configs Γ— 4 point ops = {checked} round-trips, all canonical-equal (seed={})", + cfg.iterations, + configs.len(), + cfg.seed + ); + Ok(()) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Calibration mode +// ───────────────────────────────────────────────────────────────────────────── + +/// A numeric edge case: a human label and the **exact JSON literal** to store. +/// The literal is parsed with `serde_json` so its precise form is preserved. +struct NumberProbe { + label: &'static str, + literal: &'static str, +} + +/// The spread of numeric forms whose backend rewrite we want to learn. These are +/// the cases the design doc (Β§3.1) flags as `[CALIBRATE]`. +const NUMBER_PROBES: &[NumberProbe] = &[ + NumberProbe { + label: "integer_zero", + literal: "0", + }, + NumberProbe { + label: "negative_zero", + literal: "-0", + }, + NumberProbe { + label: "integral_float_1.0", + literal: "1.0", + }, + NumberProbe { + label: "integral_float_20.0", + literal: "20.0", + }, + NumberProbe { + label: "integral_float_exp_2e1", + literal: "2e1", + }, + NumberProbe { + label: "small_fraction_0.5", + literal: "0.5", + }, + NumberProbe { + label: "repeating_0.1", + literal: "0.1", + }, + NumberProbe { + label: "sum_0.1_plus_0.2", + literal: "0.30000000000000004", + }, + NumberProbe { + label: "high_precision_pi", + literal: "3.141592653589793", + }, + NumberProbe { + label: "large_exponent", + literal: "1e20", + }, + NumberProbe { + label: "small_exponent", + literal: "1e-20", + }, + NumberProbe { + label: "negative_large_exp", + literal: "-1.5e18", + }, + NumberProbe { + label: "i64_max", + literal: "9223372036854775807", + }, + NumberProbe { + label: "i64_min", + literal: "-9223372036854775808", + }, + NumberProbe { + label: "u64_max_minus_1", + literal: "18446744073709551614", + }, + NumberProbe { + label: "just_above_i64", + literal: "9223372036854775808", + }, + NumberProbe { + label: "trailing_zeros_1.2300", + literal: "1.2300", + }, + NumberProbe { + label: "leading_int_0e0", + literal: "0e0", + }, +]; + +/// **Calibration mode** (design doc Β§3.1): stores each numeric probe through the +/// binary path, reads it back, and prints how the backend rewrote it alongside +/// how `canonicalize_number` currently renders it. Any `DIFF` row is a number +/// form the canonicalizer does not yet model β€” tune `canonicalize_number` (or +/// narrow the generator) until the calibration table is all `MATCH`. +/// +/// This is a **diagnostic** that prints a table; it does not assert (a `DIFF` is +/// expected the first time and is the signal to tune, not a test failure). Run +/// it with `AZURE_COSMOS_FUZZ_CALIBRATE=true` against a live account. +async fn run_calibration() -> Result<(), Box> { + println!("binary_roundtrip_fuzzer: CALIBRATION MODE β€” learning the backend's number rewrite"); + println!("(store each probe via binary encoding, read back, compare canonical forms)\n"); + + // Use the binary config so the full encodeβ†’storeβ†’decode path is exercised. + let client = build_client(&Some(BinaryEncodingOptions::new().with_enabled(true))).await?; + + let database_name = + std::env::var(DATABASE_NAME_ENV_VAR).unwrap_or_else(|_| DEFAULT_DATABASE_NAME.to_string()); + let container_name = std::env::var(CONTAINER_NAME_ENV_VAR) + .unwrap_or_else(|_| DEFAULT_CONTAINER_NAME.to_string()); + + ignore_conflict(client.create_database(&database_name, None).await)?; + let db = client.database_client(&database_name); + ignore_conflict( + db.create_container( + ContainerProperties::new(container_name.clone(), PARTITION_KEY_PATH.into()), + None, + ) + .await, + )?; + let container = db.container_client(&container_name).await?; + + println!( + "{:<26} {:<24} {:<24} {:<24} {}", + "probe", "sent-literal", "our-canonical", "backend-returned", "status" + ); + println!("{}", "-".repeat(120)); + + let mut diffs = 0u32; + for probe in NUMBER_PROBES { + // Parse the exact literal (skip probes serde_json cannot represent). + let Ok(number_value) = serde_json::from_str::(probe.literal) else { + println!( + "{:<26} {:<24} (serde_json cannot parse this literal)", + probe.label, probe.literal + ); + continue; + }; + + let id = Uuid::new_v4().to_string(); + let pk = "calibration".to_string(); + let doc = serde_json::json!({ "id": id, "pk": pk, "n": number_value }); + + container + .create_item(&pk, &id, &doc, Some(write_options_with_content())) + .await + .map_err(|e| format!("{}: create failed: {e}", probe.label))?; + let read = container + .read_item(&pk, &id, None) + .await + .map_err(|e| format!("{}: read failed: {e}", probe.label))?; + let read_doc: Value = read + .into_model() + .map_err(|e| format!("{}: read decode failed: {e}", probe.label))?; + + let returned_n = read_doc.get("n").cloned().unwrap_or(Value::Null); + // The backend's raw JSON text rendering of the number. + let backend_returned = serde_json::to_string(&returned_n).unwrap_or_default(); + // How our canonicalizer renders the sent value vs the returned value. + let (our_canonical, _) = canonical_hash(&number_value); + let (returned_canonical, _) = canonical_hash(&returned_n); + + let status = if our_canonical == returned_canonical { + "MATCH" + } else { + diffs += 1; + "DIFF <-- tune normalize_number" + }; + + println!( + "{:<26} {:<24} {:<24} {:<24} {}", + probe.label, probe.literal, our_canonical, backend_returned, status + ); + } + + println!("{}", "-".repeat(120)); + if diffs == 0 { + println!("CALIBRATION: all probes MATCH β€” normalize_number models the backend rewrite."); + } else { + println!( + "CALIBRATION: {diffs} probe(s) DIFF β€” update `normalize_number` to match the backend-returned column above." + ); + } + Ok(()) +} + +/// Asserts the returned document, projected to the sent keys, canonicalizes to +/// the same form (and hash) as what was sent. On mismatch, prints both canonical +/// forms and the reproduction seed. +fn assert_roundtrip( + sent: &Map, + got: &Value, + sent_canon: &str, + sent_hash: &[u8; 32], + context: &str, + phase: &str, +) { + let projected = project_to_sent_keys(sent, got); + let (got_canon, got_hash) = canonical_hash(&projected); + if &got_hash != sent_hash || got_canon != sent_canon { + panic!( + "{context}: {phase} round-trip MISMATCH\n sent (sha256 {}): {sent_canon}\n got (sha256 {}): {got_canon}\n reproduce with {SEED_ENV_VAR} from the context above", + hex(sent_hash), + hex(&got_hash), + ); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Offline unit tests β€” validate the canonicalizer and generator without a live +// account. These run under a normal `cargo test -p azure_data_cosmos`. +// ───────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn canon(value: &Value) -> String { + canonicalize(value) + } + + #[test] + fn canonicalize_sorts_object_keys() { + let a = serde_json::json!({ "b": 1, "a": 2, "c": 3 }); + let b = serde_json::json!({ "c": 3, "a": 2, "b": 1 }); + assert_eq!(canon(&a), canon(&b)); + assert_eq!(canon(&a), r#"{"a":2,"b":1,"c":3}"#); + } + + #[test] + fn canonicalize_drops_whitespace_and_preserves_array_order() { + let v: Value = serde_json::from_str(" [ 1 ,2, 3 ] ").unwrap(); + assert_eq!(canon(&v), "[1,2,3]"); + } + + #[test] + fn canonicalize_normalizes_integral_floats_to_integers() { + // 1.0 and 1 must canonicalize identically (mirrors the backend rewrite). + assert_eq!(canon(&serde_json::json!(1.0)), "1"); + assert_eq!(canon(&serde_json::json!(1)), "1"); + assert_eq!(canon(&serde_json::json!(20.0)), "20"); + assert_eq!(canon(&serde_json::json!(-0.0)), "0"); + } + + #[test] + fn canonicalize_keeps_non_integral_floats() { + assert_eq!(canon(&serde_json::json!(3.5)), "3.5"); + assert_eq!(canon(&serde_json::json!(-2.25)), "-2.25"); + } + + #[test] + fn canonicalize_large_unsigned_integer_matches_backend_double() { + // The backend stores integers above 2^53 as doubles, so they must + // canonicalize to the same string token as the rounded double form. + let sent_u64: Value = serde_json::from_str("18446744073709551614").unwrap(); + let backend_double: Value = serde_json::from_str("1.8446744073709552e+19").unwrap(); + assert_eq!(canon(&sent_u64), canon(&backend_double)); + + // 2^63 (just above i64::MAX). + let sent_2p63: Value = serde_json::from_str("9223372036854775808").unwrap(); + let backend_2p63: Value = serde_json::from_str("9.223372036854776e+18").unwrap(); + assert_eq!(canon(&sent_2p63), canon(&backend_2p63)); + + // Signed i64 above 2^53 is also stored lossily (regression: the i64 + // branch used to emit the exact decimal). Two real live-leg cases: + let sent_a: Value = serde_json::from_str("28423844363879210").unwrap(); + let backend_a: Value = serde_json::from_str("28423844363879208").unwrap(); + assert_eq!(canon(&sent_a), canon(&backend_a)); + let sent_b: Value = serde_json::from_str("39207287747660610").unwrap(); + let backend_b: Value = serde_json::from_str("39207287747660608").unwrap(); + assert_eq!(canon(&sent_b), canon(&backend_b)); + + // i64::MAX exceeds 2^53 β†’ lossy double token (2^63), not exact decimal. + let i64_max: Value = serde_json::from_str("9223372036854775807").unwrap(); + assert_eq!(canon(&i64_max), canon(&sent_2p63)); + + // A JCS-safe integer stays a bare number. + assert_eq!(canon(&serde_json::json!(1_000_000)), "1000000"); + } + + #[test] + fn wide_number_token_cannot_collide_with_a_plain_string() { + // Oracle type-safety: a wide number lives in its own tagged domain, so a + // codec bug that turns it into a JSON string with the same decimal text + // must NOT canonicalize equal β€” otherwise the corruption passes silently. + let wide_number: Value = serde_json::from_str("18446744073709551614").unwrap(); + let normalized = normalize_numbers(&wide_number); + let token = normalized[WIDE_NUMBER_TAG] + .as_str() + .expect("wide number normalizes to a tagged token object") + .to_string(); + + // The bare decimal string the backend would echo for a numberβ†’string bug. + let corrupted_string = Value::String(token); + assert_ne!(canon(&wide_number), canon(&corrupted_string)); + } + + #[test] + fn normalize_numbers_rewrites_every_number_in_the_tree() { + // The Cosmos-calibrated number rewrite applies recursively through + // arrays and nested objects, leaving non-number values untouched. + let input = serde_json::json!({ + "int": 5, + "integral_float": 1.0, + "fraction": 2.5, + "arr": [1.0, 2.0, 3.5], + "nested": { "big": 18446744073709551614u64, "s": "x", "b": true, "n": null } + }); + let out = normalize_numbers(&input); + + // Integral floats collapse to integers; fractions stay; big u64 β†’ f64. + assert_eq!(out["int"], serde_json::json!(5)); + assert_eq!(out["integral_float"], serde_json::json!(1)); + assert_eq!(out["fraction"], serde_json::json!(2.5)); + assert_eq!(out["arr"], serde_json::json!([1, 2, 3.5])); + assert_eq!( + out["nested"]["big"], + normalize_number(&serde_json::from_str::("18446744073709551614").unwrap()) + ); + // Non-number leaves pass through unchanged. + assert_eq!(out["nested"]["s"], serde_json::json!("x")); + assert_eq!(out["nested"]["b"], serde_json::json!(true)); + assert_eq!(out["nested"]["n"], Value::Null); + } + + #[test] + fn normalize_numbers_is_idempotent() { + // Applying the rewrite twice yields the same tree β€” a prerequisite for + // comparing a normalized sent doc against a normalized returned doc. + let input = serde_json::json!({ + "a": 1.0, "b": [2.0, 3.5, 18446744073709551614u64], "c": { "d": 9223372036854775807i64 } + }); + let once = normalize_numbers(&input); + let twice = normalize_numbers(&once); + assert_eq!(once, twice); + } + + #[test] + fn canonical_hash_is_stable_and_matches_json_canon() { + // The digest is a deterministic function of the canonical string, and + // the canonical string is the JCS form of the number-normalized value. + let v = serde_json::json!({ "b": 1.0, "a": [2.0, 3.5], "c": "x" }); + let (s1, h1) = canonical_hash(&v); + let (s2, h2) = canonical_hash(&v); + assert_eq!(s1, s2); + assert_eq!(h1, h2); + + // Structural equivalence (key order / whitespace / integral floats) maps + // to the same digest. + let equiv = serde_json::from_str::(r#" { "c":"x", "a":[2,3.5], "b":1 } "#).unwrap(); + let (_, h_equiv) = canonical_hash(&equiv); + assert_eq!(h1, h_equiv); + } + + #[test] + fn project_strips_service_fields() { + let sent: Map = serde_json::from_value(serde_json::json!({ + "id": "x", "pk": "p", "value": 1 + })) + .unwrap(); + let got = serde_json::json!({ + "id": "x", "pk": "p", "value": 1, + "_rid": "abc", "_etag": "\"y\"", "_ts": 123 + }); + let projected = project_to_sent_keys(&sent, &got); + assert_eq!(canon(&projected), canon(&Value::Object(sent))); + } + + #[test] + fn generator_is_deterministic_for_a_seed() { + let cfg = FuzzConfig { + iterations: 0, + seed: 42, + max_depth: 4, + wide_numbers: false, + unicode: true, + breadth: DEFAULT_BREADTH, + shape_ratio: DEFAULT_SHAPE_RATIO, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut a = SplitMix64::new(cfg.seed); + let mut b = SplitMix64::new(cfg.seed); + let doc_a = Value::Object(gen_object(&mut a, &cfg)); + let doc_b = Value::Object(gen_object(&mut b, &cfg)); + assert_eq!(canon(&doc_a), canon(&doc_b)); + } + + /// Nesting depth of a JSON value (scalars are depth 0). + fn depth_of(v: &Value) -> u32 { + match v { + Value::Array(items) => 1 + items.iter().map(depth_of).max().unwrap_or(0), + Value::Object(map) => 1 + map.values().map(depth_of).max().unwrap_or(0), + _ => 0, + } + } + + #[test] + fn generator_depth_scales_with_max_depth() { + // Guards the hybrid-skeleton generator: the average nesting depth must + // grow with `max_depth` (the old arbitrary-json-only generator was flat + // at ~1.3 regardless of the knob). We assert a conservative lower bound + // on the average and that the deepest doc reaches near the target. + fn avg_and_max_depth(max_depth: u32) -> (f64, u32) { + let cfg = FuzzConfig { + iterations: 0, + seed: 1784944014111583800, + max_depth, + wide_numbers: false, + unicode: true, + breadth: DEFAULT_BREADTH, + // Force the free-form hybrid-skeleton generator (shape_ratio=0): + // this test asserts that *its* depth scales with `max_depth`. + // Corpus shape samplers have fixed, sampler-defined depth + // independent of the knob, so they would dilute the signal at + // the production `DEFAULT_SHAPE_RATIO`. + shape_ratio: 0, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + let n = 1000u32; + let mut sum = 0u64; + let mut max_seen = 0u32; + for _ in 0..n { + let d = depth_of(&Value::Object(gen_object(&mut rng, &cfg))); + sum += d as u64; + max_seen = max_seen.max(d); + } + (sum as f64 / n as f64, max_seen) + } + + let (avg3, max3) = avg_and_max_depth(3); + let (avg8, max8) = avg_and_max_depth(8); + + // Depth clearly scales with the knob (not flat like the old generator). + assert!( + avg8 > avg3 + 1.0, + "avg depth should grow with max_depth: avg@3={avg3:.2}, avg@8={avg8:.2}" + ); + // The deepest documents actually approach the requested depth. + assert!( + max3 >= 3, + "max depth @3 should reach the target, got {max3}" + ); + assert!( + max8 >= 8, + "max depth @8 should reach the target, got {max8}" + ); + } + + #[test] + fn generated_documents_normalize_idempotently() { + // Sanity: after one serializeβ†’parse normalization, a generated doc is + // stable β€” a second round-trip does not change its canonical form. This + // is the invariant the fuzzer relies on to compare the (normalized) sent + // doc against a backend-round-tripped one. + let cfg = FuzzConfig { + iterations: 0, + seed: 7, + max_depth: 5, + wide_numbers: true, + unicode: true, + breadth: DEFAULT_BREADTH, + shape_ratio: DEFAULT_SHAPE_RATIO, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + for _ in 0..500 { + let doc = Value::Object(gen_object(&mut rng, &cfg)); + let once = normalize(&doc); + let twice = normalize(&once); + assert_eq!( + canon(&once), + canon(&twice), + "normalization not idempotent for doc: {}", + serde_json::to_string(&doc).unwrap() + ); + } + } + + #[test] + fn unicode_off_strips_non_ascii_from_object_keys_and_values() { + // AZURE_COSMOS_FUZZ_UNICODE=false must isolate the ASCII codec path: + // non-ASCII must be dropped from object *keys* as well as values + // (property names go through the same binary string encoding). + let cfg = FuzzConfig { + iterations: 0, + seed: 1, + max_depth: 3, + wide_numbers: false, + unicode: false, + breadth: DEFAULT_BREADTH, + shape_ratio: DEFAULT_SHAPE_RATIO, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut doc = serde_json::json!({ + "kΓ©y1": { "kΓ©y2": "vΓ l", "ok": 1 }, + "plain": ["a", "bΓ©"], + }); + bound_value(&mut doc, &cfg); + fn assert_ascii(v: &Value) { + match v { + Value::String(s) => assert!(s.is_ascii(), "non-ASCII value survived: {s:?}"), + Value::Array(items) => items.iter().for_each(assert_ascii), + Value::Object(map) => { + for (k, v) in map { + assert!(k.is_ascii(), "non-ASCII key survived: {k:?}"); + assert_ascii(v); + } + } + _ => {} + } + } + assert_ascii(&doc); + } + + #[test] + fn breadth_env_clamps_multiple_of_2_pow_32_to_a_nonzero_u32() { + // A raw breadth that is a multiple of 2^32 must NOT truncate to 0 (which + // would later panic in `rng.below(0)`). Mirror the `from_env` clamp so + // the test does not mutate the process environment (racy under the + // parallel harness). + let clamp = |raw: u64| raw.clamp(1, u32::MAX as u64) as u32; + assert_eq!(clamp(4_294_967_296), u32::MAX); // 2^32 β†’ would truncate to 0 + assert_eq!(clamp(0), 1); + assert_eq!(clamp(6), 6); + } + + #[test] + fn generated_documents_cover_all_value_categories() { + // The sampler subtree guarantees every value category appears in each + // document: integer, float, alphabetic, alphanumeric, non-ASCII string, + // boolean, null, a number array, and multi-level nesting. This asserts + // the "really complex JSON" contract holds for a spread of seeds. + fn walk(v: &Value, seen: &mut Categories, max_depth: &mut u32, depth: u32) { + *max_depth = (*max_depth).max(depth); + match v { + Value::Null => seen.null = true, + Value::Bool(_) => seen.boolean = true, + Value::Number(n) => { + if n.is_f64() { + seen.float = true; + } else { + seen.integer = true; + } + } + Value::String(s) => { + if !s.is_empty() && s.chars().all(|c| c.is_ascii_alphabetic()) { + seen.alphabetic = true; + } + if !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric()) { + seen.alphanumeric = true; + } + if !s.is_ascii() { + seen.non_ascii = true; + } + } + Value::Array(items) => { + seen.array = true; + for item in items { + walk(item, seen, max_depth, depth + 1); + } + } + Value::Object(map) => { + seen.object = true; + for child in map.values() { + walk(child, seen, max_depth, depth + 1); + } + } + } + } + + #[derive(Default)] + struct Categories { + integer: bool, + float: bool, + alphabetic: bool, + alphanumeric: bool, + non_ascii: bool, + boolean: bool, + null: bool, + array: bool, + object: bool, + } + + let cfg = FuzzConfig { + iterations: 0, + seed: 0xC0FFEE, + max_depth: 6, + wide_numbers: false, + unicode: true, + breadth: DEFAULT_BREADTH, + shape_ratio: DEFAULT_SHAPE_RATIO, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + + let mut all = Categories::default(); + let mut deepest = 0u32; + for _ in 0..50 { + let doc = Value::Object(gen_object(&mut rng, &cfg)); + walk(&doc, &mut all, &mut deepest, 0); + } + + assert!(all.integer, "no integer produced"); + assert!(all.float, "no float produced"); + assert!(all.alphabetic, "no alphabetic string produced"); + assert!(all.alphanumeric, "no alphanumeric string produced"); + assert!(all.non_ascii, "no non-ASCII string produced"); + assert!(all.boolean, "no boolean produced"); + assert!(all.null, "no null produced"); + assert!(all.array, "no array produced"); + assert!(all.object, "no nested object produced"); + // Multi-level nesting: the guaranteed sampler alone reaches depth β‰₯ 3, + // and the spine pushes documents deeper. + assert!( + deepest >= 4, + "documents should reach multi-level nesting, deepest={deepest}" + ); + } + + #[test] + fn every_corpus_shape_produces_a_valid_object() { + // Each corpus shape sampler must produce a non-empty JSON object that + // serializes and normalizes cleanly (the invariant the round-trip + // comparison relies on). Exercised across several seeds and both the + // wide-numbers / unicode knob settings. + for &wide in &[false, true] { + for &uni in &[false, true] { + let cfg = FuzzConfig { + iterations: 0, + seed: 0x5EED_1234, + max_depth: 6, + wide_numbers: wide, + unicode: uni, + breadth: DEFAULT_BREADTH, + shape_ratio: DEFAULT_SHAPE_RATIO, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + for (name, sampler) in SHAPE_SAMPLERS { + let doc = sampler(&mut rng, &cfg); + assert!(!doc.is_empty(), "shape {name} produced an empty object"); + let value = Value::Object(doc); + // Normalization is idempotent for a well-formed value. + let once = normalize(&value); + let twice = normalize(&once); + assert_eq!( + canon(&once), + canon(&twice), + "shape {name} not normalization-stable" + ); + } + } + } + } + + #[test] + fn no_shape_emits_reserved_system_fields() { + // Cosmos owns `_rid`/`_self`/`_etag`/`_ts`/`_attachments`; a generated + // document must never author them (the service overwrites them, causing + // a false round-trip mismatch). Assert every shape sampler is clean, and + // that the strip helper removes them if present. + let cfg = FuzzConfig { + iterations: 0, + seed: 0xBADF00D, + max_depth: 6, + wide_numbers: true, + unicode: true, + breadth: DEFAULT_BREADTH, + shape_ratio: DEFAULT_SHAPE_RATIO, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + for (name, sampler) in SHAPE_SAMPLERS { + let doc = sampler(&mut rng, &cfg); + for reserved in RESERVED_SYSTEM_KEYS { + assert!( + !doc.contains_key(*reserved), + "shape {name} emitted reserved system field {reserved}" + ); + } + } + // The strip helper removes reserved keys wherever they appear. + let mut with_reserved = Map::new(); + with_reserved.insert("_self".into(), Value::String("x".into())); + with_reserved.insert("_ts".into(), Value::Number(123.into())); + with_reserved.insert("keep".into(), Value::Bool(true)); + strip_reserved_fields(&mut with_reserved); + assert_eq!(with_reserved.len(), 1); + assert!(with_reserved.contains_key("keep")); + } + + #[test] + fn shaped_documents_are_emitted_when_ratio_is_full() { + // With shape_ratio = 100 every document is a corpus shape; assert that + // over many draws we see several distinct shapes (the id/pk-free base is + // shaped, then the run loop adds id/pk). Detect a shape by a signature + // key unique to a sampler. + let cfg = FuzzConfig { + iterations: 0, + seed: 0xA11CE, + max_depth: 6, + wide_numbers: false, + unicode: true, + breadth: DEFAULT_BREADTH, + shape_ratio: 100, + size_scale: DEFAULT_SIZE_SCALE, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + let mut seen_signature_keys = std::collections::BTreeSet::new(); + // Signature keys that appear only in specific shapes. + let signatures = [ + "hash", // bitcoin + "blogId", // bing-docs + "similars", // lastfm + "vector", // embedding + "BucketId", // error-bucket + "schemaPolicy", // hybrid-schema + "ShopId", // store + "cloudSid", // employee + ]; + for _ in 0..400 { + let doc = gen_object(&mut rng, &cfg); + // Every shaped document still carries the all-category sampler. + assert!(doc.contains_key("_sampler"), "shaped doc missing _sampler"); + for sig in &signatures { + if doc.contains_key(*sig) { + seen_signature_keys.insert(*sig); + } + } + } + assert!( + seen_signature_keys.len() >= 4, + "expected several distinct corpus shapes, saw signatures: {seen_signature_keys:?}" + ); + } + + /// Prints a few sample generated documents as pretty JSON so the generator + /// output can be eyeballed **offline** (no Cosmos account). Ignored by + /// default; run explicitly with `--ignored --nocapture`: + /// + /// ```bash + /// cargo test -p azure_data_cosmos --test binary_roundtrip_fuzzer --features key_auth,fault_injection,control_plane \ + /// print_sample_documents -- --ignored --nocapture + /// ``` + /// + /// Control shape/size via the same env vars as a live run, e.g. + /// `AZURE_COSMOS_FUZZ_SEED`, `AZURE_COSMOS_FUZZ_MAX_DEPTH`, + /// `AZURE_COSMOS_FUZZ_BREADTH`, `AZURE_COSMOS_FUZZ_WIDE_NUMBERS`; the count + /// defaults to 3 (override with `AZURE_COSMOS_FUZZ_PRINT_COUNT`). + #[test] + #[ignore = "prints sample JSON on demand; run with --ignored --nocapture"] + fn print_sample_documents() { + let cfg = FuzzConfig::from_env(); + let count = std::env::var("AZURE_COSMOS_FUZZ_PRINT_COUNT") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(3); + println!( + "print_sample_documents: seed={} max_depth={} breadth={} wide_numbers={} unicode={}", + cfg.seed, cfg.max_depth, cfg.breadth, cfg.wide_numbers, cfg.unicode + ); + let mut rng = SplitMix64::new(cfg.seed); + for i in 0..count { + let doc = Value::Object(gen_object(&mut rng, &cfg)); + println!( + "--- sample {i} ---\n{}", + serde_json::to_string_pretty(&doc).unwrap() + ); + } + } + + #[test] + fn calibration_probes_are_valid_and_unique() { + // Every calibration probe literal must parse as a JSON number, and the + // labels must be unique (they key the printed calibration table). + let mut labels: Vec<&str> = Vec::new(); + for probe in NUMBER_PROBES { + let value: Value = serde_json::from_str(probe.literal).unwrap_or_else(|e| { + panic!( + "probe {} literal {:?} invalid: {e}", + probe.label, probe.literal + ) + }); + assert!( + value.is_number(), + "probe {} literal {:?} is not a JSON number", + probe.label, + probe.literal + ); + labels.push(probe.label); + } + labels.sort_unstable(); + let count = labels.len(); + labels.dedup(); + assert_eq!(labels.len(), count, "duplicate calibration probe label"); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md new file mode 100644 index 00000000000..a65226b6ec5 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md @@ -0,0 +1,601 @@ +# Cosmos Binary JSON Encoding β€” Wire Format Specification + +**Status:** Draft Β· **Version:** 0.1 Β· **Audience:** SDK / codec implementers (Rust, .NET, C++, Java, Go, Python) + +> This document is a **normative, self-contained** description of the Cosmos +> Binary JSON wire format. A conforming encoder/decoder can be implemented from +> this document alone, without reference to any SDK source. It is modeled on the +> style of [RFC 8949 (CBOR)](https://datatracker.ietf.org/doc/html/rfc8949) and +> the [Protocol Buffers encoding guide](https://protobuf.dev/programming-guides/encoding/). +> +> **Provenance.** The wire constants are transcribed from the .NET reference +> implementation (`Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs`) +> and cross-checked against the Rust codec (`azure_data_cosmos_driver::binary_json`). +> Details that could not be confirmed from the Rust implementation alone are +> tagged **`[CROSS-VERIFY: .NET/C++]`** and MUST be validated against the .NET +> and C++ sources before this draft is promoted to a stable version. + +--- + +## 1. Introduction + +### 1.1 Purpose + +Cosmos Binary JSON is a compact, self-describing binary serialization of the +JSON data model used by Azure Cosmos DB for item request and response bodies. It +is **information-preserving with respect to the JSON value model** (null, +boolean, number, string, array, object) while being smaller and faster to +parse than UTF-8 JSON text. The service and every language SDK MUST agree on +this format byte-for-byte. + +### 1.2 Scope + +This specification defines: + +- the byte-level layout of every value kind (Β§3–§6), +- the **canonical** encoding a conforming encoder emits when multiple encodings + are valid (Β§7), +- decoder conformance requirements, including bounds and resource limits (Β§8), +- security considerations for decoding untrusted input (Β§9). + +It does **not** define: transport framing, HTTP/RNTBD negotiation headers, +per-account dictionary (user-string) construction policy, or the query wire +protocol. Those are layered above this format. + +### 1.3 Requirements language + +The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are +to be interpreted as described in [RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119). + +### 1.4 Relationship to the other binary-encoding artifacts + +This RFC is the **source of truth** for the wire format. Several other artifacts +either derive from it or validate against it β€” they do not redefine it: + +```mermaid +flowchart TD + RFC["BINARY_ENCODING_RFC.md
(normative spec: what correct means)"] + RFC -->|"Β§7 canonical encoding
Β§8 decoder conformance"| CONF["binary_json/conformance.rs
(encoder byte-exactness)"] + RFC -->|"Appendix A golden vectors"| CORPUS["testdata/binary_json_vectors.json
(fixed decode+encode oracle)"] + RFC -->|"value model + round-trip invariant"| FUZZ["BINARY_ENCODING_ROUNDTRIP_FUZZER
(random JSON, live service, at scale)"] + RFC -->|"Β§8 decoder + Β§9 security"| IN_FUZZ["binary_json/fuzz_tests.rs
(malformed/truncated buffers)"] + CORPUS --> CONF + CORPUS --> FUZZ + CORPUS --> IN_FUZZ +``` + +| Artifact | Role relative to this RFC | RFC sections it enforces | +| -------- | ------------------------- | ------------------------ | +| `testdata/binary_json_vectors.json` | Machine-readable golden corpus (this RFC's Appendix A) | Appendix A | +| `binary_json/conformance.rs` | Encoder byte-exactness + canonical-form snapshots | Β§3.1, Β§7 | +| `binary_json/reader.rs`, `de.rs` (tests) | Decoder conformance per form | Β§4–§6, Β§8 | +| `binary_json/fuzz_tests.rs` | Decoder never panics/hangs/over-allocates on malformed input | Β§8, Β§9 | +| [`BINARY_ENCODING_ROUNDTRIP_FUZZER.md`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md) + harness | End-to-end **round-trip invariant** on random JSON against the live service, at volume | Β§7 (round-trip), value model (Β§2) | + +Two connections are worth calling out explicitly: + +- **The round-trip fuzzer validates this RFC's round-trip invariant (Β§7), but + end-to-end rather than byte-level.** Where `conformance.rs` checks + `decode(encode(v)) == v` in-process on fixed vectors, the fuzzer checks the + *same* invariant across the full pipeline (Rust encode β†’ wire β†’ backend + store/rewrite β†’ wire β†’ Rust decode) on millions of random documents. +- **The fuzzer is the instrument that closes this RFC's open number-format + items.** The `[CROSS-VERIFY: .NET/C++]` tags and Β§7 canonical rules leave the + encoder-vs-backend number-normalization under-specified; the fuzzer's + canonicalization/calibration surface (its design doc Β§3.1) empirically + discovers the backend's actual number rewrite, and those findings feed back + into Β§7 here. + +--- + +## 2. Notation and terminology + +- **byte** β€” an 8-bit octet, written as two hexadecimal digits, e.g. `C8`. +- **marker** (or **type marker**) β€” the single leading byte that selects how the + following bytes are interpreted. Every encoded value begins with exactly one + marker (except literal small integers, where the marker byte *is* the value β€” + see Β§4.1). +- **spaced-hex** β€” the human-reviewable notation for a byte sequence used + throughout this document and in the shared test corpus, e.g. `80 CC 00 00 00 + 00 00 00 0C 40`. +- **preamble** β€” the single byte `0x80` that MUST prefix a complete buffer (Β§3.1). +- **little-endian (LE)** β€” multi-byte integers and floats are serialized + least-significant-byte first. This is **normative and independent of host + architecture**: an encoder on a big-endian host MUST still emit LE. +- **value model** β€” the abstract JSON value: `null | bool | number | string | + array | object`. Numbers are IEEE-754 doubles or integers in the ranges + encodable by the number markers in Β§4. + +--- + +## 3. Structure of an encoded value + +### 3.1 Buffer preamble and auto-detection + +A complete Cosmos Binary JSON buffer MUST begin with the **preamble byte** +`0x80`, followed by exactly one encoded value: + +``` +buffer = 0x80 value +``` + +Because `0x80` is also the marker for a zero-length encoded string (Β§4.4), the +preamble is **always consumed first**; a top-level empty string is therefore +`80 80` (preamble + zero-length-string marker). + +Consumers distinguish binary from UTF-8 JSON text by inspecting the **first +byte**: a payload whose first byte is `0x80` is Cosmos Binary JSON; any other +first byte (`{`, `[`, `"`, digit, `t`, `f`, `n`, whitespace, …) is UTF-8 text. +This is the `is_binary` predicate. + +> **Note.** UTF-8 text can never legitimately begin with `0x80` (a continuation +> byte), so the discriminator is unambiguous. + +### 3.2 Marker byte taxonomy + +The 256 marker values are partitioned into contiguous ranges. Ranges are written +`[MIN, MAX)` β€” MIN inclusive, MAX exclusive. + +| Range | Meaning | Section | +| ------------- | ------------------------------------------------------------- | ------- | +| `[0x00,0x20)` | Literal small integer (`value == marker`, range 0–31) | Β§4.1 | +| `[0x20,0x40)` | 1-byte **system** string (index into fixed dictionary) | Β§4.4.3 | +| `[0x40,0x60)` | 1-byte **user** string (index into per-buffer dictionary) | Β§4.4.4 | +| `[0x60,0x68)` | 2-byte user string | Β§4.4.4 | +| `[0x68,0x80)` | base64 / GUID-string / compressed-string forms | Β§4.4.5–6| +| `[0x80,0xC0)` | Encoded-length string (`len == marker & 0x7F`, 0–63) | Β§4.4.1 | +| `[0xC0,0xC8)` | Length-prefixed strings, reference strings, `NumberUInt64` | Β§4.4.1, Β§6, Β§4.2 | +| `[0xC8,0xD0)` | Fixed-width numbers (`UInt8`,`Int16/32/64`,`Double`,`Float*`) | Β§4.2–3 | +| `[0xD0,0xE0)` | null, bool, GUID value, extended sized ints, binary blobs | Β§4.1, Β§4.2, Β§4.7 | +| `[0xE0,0xE8)` | Arrays | Β§5.1 | +| `[0xE8,0xF0)` | Objects | Β§5.2 | +| `[0xF0,0xF8)` | Uniform (typed) number arrays | Β§5.3 | +| `[0xF8,0xFF]` | Reserved; `0xFF` == Invalid | Β§8 | + +The complete marker constant table is given in Appendix C. + +--- + +## 4. Scalars + +### 4.1 Null, boolean, and literal small integers + +| Value | Marker | Sample (with preamble) | +| --------------- | ------- | ---------------------- | +| `null` | `D0` | `80 D0` | +| `false` | `D1` | `80 D1` | +| `true` | `D2` | `80 D2` | +| integer `0`–`31`| `00`–`1F` | `0` β†’ `80 00`; `31` β†’ `80 1F` | + +For an integer `n` in `[0, 31]`, the marker byte itself is the value: the encoded +form is the single byte `n`. This is the most compact integer encoding and is the +canonical form for that range (Β§7). + +### 4.2 Integers + +Two families of integer markers exist, distinguished only by their historical +range. Both are LE. + +| Marker | Byte | Width | Signedness | Sample | +| ----------------- | ---- | ----- | ---------- | ----------------------------------- | +| `NumberUInt8` | `C8` | 1 | unsigned | `200` β†’ `80 C8 C8` | +| `NumberInt16` | `C9` | 2 | signed | `-1000` β†’ `80 C9 18 FC` | +| `NumberInt32` | `CA` | 4 | signed | `70000` β†’ `80 CA 70 11 01 00` | +| `NumberInt64` | `CB` | 8 | signed | `-5000000000` β†’ `80 CB 00 0E FA D5 FE FF FF FF` | +| `NumberUInt64` | `C7` | 8 | unsigned | `18446744073709551614` β†’ `80 C7 FE FF FF FF FF FF FF FF` | +| `UInt8` (ext.) | `D7` | 1 | unsigned | `[CROSS-VERIFY: .NET/C++]` | +| `Int8` (ext.) | `D8` | 1 | signed | `-5` β†’ `80 D8 FB` | +| `Int16` (ext.) | `D9` | 2 | signed | `-1000` β†’ `80 D9 18 FC` | +| `Int32` (ext.) | `DA` | 4 | signed | `-70000` β†’ `80 DA 90 EE FE FF` | +| `Int64` (ext.) | `DB` | 8 | signed | `-5000000000` β†’ `80 DB 00 0E FA D5 FE FF FF FF` | +| `UInt32` (ext.) | `DC` | 4 | unsigned | `4294967294` β†’ `80 DC FE FF FF FF` | + +Decoders MUST accept **both** families. Encoders emit the canonical family per +Β§7. The `NumberUInt64` marker (`C7`) is the only encoding able to carry +unsigned 64-bit values above `i64::MAX`. + +### 4.3 Floating-point numbers + +| Marker | Byte | Width | Encoding | Sample | +| -------------- | ---- | ----- | ---------------------------- | ------------------------------- | +| `NumberDouble` | `CC` | 8 | IEEE-754 binary64 (LE) | `3.5` β†’ `80 CC 00 00 00 00 00 00 0C 40` | +| `Float32` | `CD` | 4 | IEEE-754 binary32 (LE) | `1.5` β†’ `80 CD 00 00 C0 3F` | +| `Float64` | `CE` | 8 | IEEE-754 binary64 (LE) | `-2.25` β†’ `80 CE 00 00 00 00 00 00 02 C0` | +| `Float16` | `CF` | 2 | IEEE-754 binary16 (LE) | `[CROSS-VERIFY: .NET/C++]` | + +`NumberDouble` (`CC`) is the canonical JSON-number float form (Β§7). + +**Non-finite values.** JSON has no representation for `NaN` or `Β±Infinity`. +A conforming encoder MUST NOT emit a non-finite double; instead it MUST encode +`null` (`D0`), mirroring `serde_json` / JavaScript `JSON.stringify`. A conforming +decoder MUST reject a non-finite `NumberDouble`/`Float*` payload as an invalid +number. `[CROSS-VERIFY: .NET/C++]` β€” confirm the .NET encoder/decoder policy is +identical. + +### 4.4 Strings + +JSON strings are UTF-8. Several encodings exist; a decoder MUST accept all of +them, and an encoder selects the canonical one per Β§7. + +#### 4.4.1 Length-framed strings + +| Form | Marker | Length field | Sample | +| ------------------ | ------ | ------------------ | -------------------------- | +| Encoded-length | `80`–`BF` | in marker: `len = marker & 0x7F` (0–63) | `""` β†’ `80 80`; `"hi"` β†’ `80 82 68 69` | +| `StrL1` | `C0` | u8 | `"hello"` β†’ `80 C0 05 68 65 6C 6C 6F` | +| `StrL2` | `C1` | u16 LE | 300Γ—`"a"` β†’ `80 C1 2C 01 …` | +| `StrL4` | `C2` | u32 LE | (large strings) | + +The string's UTF-8 bytes follow the length field verbatim. + +#### 4.4.2 GUID strings + +| Marker | Byte | Meaning | Sample | +| ------ | ---- | ------------------------------------ | ------ | +| lower | `75` | 36-char lowercase GUID string | `80 75 00 01 … 0F` β†’ `"00010203-0405-0607-0809-0a0b0c0d0e0f"` | +| upper | `76` | 36-char uppercase GUID string | `80 76 …` β†’ uppercase | +| quoted | `77` | double-quoted lowercase GUID string | `80 77 …` β†’ `"\"…\""` | + +The 16 raw GUID bytes follow the marker; the decoder formats them as a hyphenated +GUID string (byte order per Appendix B). This is distinct from a **GUID value** +(Β§4.7). + +#### 4.4.3 System strings + +Markers `[0x20, 0x40)` encode a string by **index into a fixed, well-known +dictionary** of common Cosmos property names (`id`, `_rid`, `_etag`, …). The +encoded form is a single byte; `marker - 0x20` is the dictionary index. + +Example: `"id"` has system index `0x0C`, so `"id"` β†’ `80 2C` (`0x20 + 0x0C`). + +The full system-string table is normative and given in Appendix D. +`[CROSS-VERIFY: .NET/C++]` β€” the index assignments MUST match `JsonBinaryEncoding`. + +#### 4.4.4 User (per-buffer dictionary) strings + +Markers `[0x40, 0x60)` (1-byte) and `[0x60, 0x68)` (2-byte) encode a string by +index into a **per-buffer user-string dictionary**. The dictionary and its +construction policy are out of scope for this document (a decoder receives the +dictionary alongside the buffer, or the buffer contains no user-dictionary +references). `[CROSS-VERIFY: .NET/C++]` + +#### 4.4.5 base64 strings + +| Marker | Byte | Alphabet | Length field | Sample | +| ------ | ---- | ------------ | ------------ | ------ | +| `Base64Len1` | `71` | standard | u8 | `"Zm9v"` β†’ `80 71 01 00 66 6F 6F` | +| `Base64Len2` | `72` | standard | u16 | `"Zm9vYmFy"` β†’ `80 72 02 00 00 66 6F 6F 62 61 72` | +| `Base64UrlLen1` | `73` | URL-safe | u8 | `"-__-"` β†’ `80 73 01 00 FB FF FE` | +| `Base64UrlLen2` | `74` | URL-safe | u16 | (as above, 2-byte length) | + +The payload is the **decoded** bytes; the decoder re-encodes them to a base64 +string using the marker's alphabet. Padding handling (`=`) and the "omitted +padding" length-field convention are illustrated in Appendix B. +`[CROSS-VERIFY: .NET/C++]` for the exact padding/length-offset encoding. + +#### 4.4.6 Compressed strings + +Restricted-alphabet strings are bit-packed relative to a base character: + +| Marker | Byte | Packing | Sample | +| ------ | ---- | ------------------------------- | ------ | +| lower-hex | `78` | 4-bit hex digits (lowercase) | `"1a2b"` β†’ `80 78 04 A1 B2` | +| upper-hex | `79` | 4-bit hex digits (uppercase) | `"1A2B"` β†’ `80 79 04 A1 B2` | +| date-time | `7A` | 4-bit date-time charset | `"2024-01"` β†’ `80 7A 07 13 53 1C 02` | +| packed-4bit | `7B` | 4 bits/char + base | `"0123"` β†’ `80 7B 04 30 10 32` | +| packed-5bit | `7C` | 5 bits/char + base | `"abc"` β†’ `80 7C 03 61 20 08` | +| packed-6bit | `7D` | 6 bits/char + base | `"abcd"` β†’ `80 7D 04 61 40 20 0C` | +| packed-7bit-L1 | `7E` | 7 bits/char, u8 length | `"Hi"` β†’ `80 7E 02 C8 34` | +| packed-7bit-L2 | `7F` | 7 bits/char, u16 length | `"Hi"` β†’ `80 7F 02 00 C8 34` | + +The byte immediately after the marker is the **character count** (for L1/L2 forms +a 1-/2-byte length), followed by the packed bits. Appendix B gives a worked +unpacking example. `[CROSS-VERIFY: .NET/C++]` for the exact base/charset tables. + +### 4.7 GUID value and binary blobs + +| Marker | Byte | Meaning | Sample | +| ------ | ---- | -------------------------------- | ------ | +| `Guid` | `D3` | raw 16-byte GUID **value** | `80 D3 00 01 … 0F` β†’ `"03020100-0504-0706-0809-0a0b0c0d0e0f"` | +| `Binary1` | `DD` | blob, u8 length prefix | `0xDEADBEEF` β†’ `80 DD 04 DE AD BE EF` β†’ `"3q2+7w=="` | +| `Binary2` | `DE` | blob, u16 length prefix | `80 DE 03 00 01 02 03` β†’ `"AQID"` | +| `Binary4` | `DF` | blob, u32 length prefix | (large blobs) | + +A **binary blob** decodes to a **standard base64 string** in the JSON value +model. Note the byte order of a GUID *value* (`D3`) differs from a GUID *string* +(Β§4.4.2) β€” see Appendix B. `[CROSS-VERIFY: .NET/C++]` + +--- + +## 5. Containers + +### 5.1 Arrays + +| Marker | Byte | Framing | Sample | +| ------- | ---- | ---------------------------------------- | ------ | +| `Arr0` | `E0` | empty | `[]` β†’ `80 E0` | +| `Arr1` | `E1` | exactly one element, no length/count | `[true]` β†’ `80 E1 D2` | +| `ArrL1` | `E2` | u8 **byte-length** prefix | `[0,1,null]` β†’ `80 E2 03 00 01 D0` | +| `ArrL2` | `E3` | u16 byte-length | | +| `ArrL4` | `E4` | u32 byte-length | | +| `ArrLC1`| `E5` | u8 byte-length **+** u8 item-count | `[0,1,null]` β†’ `80 E5 03 03 00 01 D0` | +| `ArrLC2`| `E6` | u16 byte-length + u16 count | | +| `ArrLC4`| `E7` | u32 byte-length + u32 count | | + +For `L*` forms, the length is the **byte length of the element region** (not the +element count); the decoder reads elements until it has consumed exactly that +many bytes. For `LC*` forms, both the byte length **and** the element count are +given, and a conforming decoder MUST verify that reading `count` elements +consumes exactly the declared byte length (Β§8). + +### 5.2 Objects + +| Marker | Byte | Framing | Sample | +| ------- | ---- | ----------------------------------- | ------ | +| `Obj0` | `E8` | empty | `{}` β†’ `80 E8` | +| `Obj1` | `E9` | exactly one name/value pair | `{"id":true}` β†’ `80 E9 2C D2` | +| `ObjL1` | `EA` | u8 byte-length | `{"id":0,"type":1}` β†’ `80 EA 04 2C 00 3B 01` | +| `ObjL2` | `EB` | u16 byte-length | | +| `ObjL4` | `EC` | u32 byte-length | | +| `ObjLC1`| `ED` | u8 byte-length + u8 pair-count | `{"id":0,"type":1}` β†’ `80 ED 04 02 2C 00 3B 01` | +| `ObjLC2`| `EE` | u16 byte-length + u16 count | | +| `ObjLC4`| `EF` | u32 byte-length + u32 count | | + +Members are encoded as **name, value, name, value, …**. Each **name** is itself +an encoded string (any string form, including system strings β€” note `2C` = system +`"id"` in the samples). Member ordering is preserved as encoded. + +### 5.3 Uniform (typed) number arrays + +A homogeneous array of numbers of a single width is encoded compactly by writing +the item type marker once, then the raw item bytes with no per-item markers. + +| Marker | Byte | Layout | Sample | +| ----------- | ---- | -------------------------------------------------------- | ------ | +| `ArrNumC1` | `F0` | `itemMarker`, u8 count, then `count` bare numbers | `[1,2,3]` (Int32) β†’ `80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00` | +| `ArrNumC2` | `F1` | `itemMarker`, u16 count, then bare numbers | `[-1,0,1000]` (Int16) β†’ `80 F1 D9 03 00 FF FF 00 00 E8 03` | +| `ArrArrNumC1C1` | `F2` | innerMarker, itemMarker, u8 inner-count, u8 outer-count, then inner arrays | `[[1,2],[3,4]]` β†’ `80 F2 F0 DA 02 02 01 00 00 00 …` | +| `ArrArrNumC2C2` | `F3` | as above with u16 counts | | + +Further examples: `[10,20,30]` (UInt8) β†’ `80 F0 D7 03 0A 14 1E`; empty uniform +array β†’ `80 F0 DA 00`; `[1.5,-0.25]` (Float32) β†’ `80 F0 CD 02 00 00 C0 3F 00 00 +80 BE`. + +**Item markers.** The uniform-array item type MUST be one of the **extended** +number markers (`Int8`/`UInt8`/`Int16`/`Int32`/`Int64`/`UInt32`/`Float32`/ +`Float64`, i.e. `D7`–`DC`, `CD`, `CE`). The self-describing `Number*` markers +(`C7`–`CC`) MUST NOT appear as a uniform-array item type and a conforming decoder +MUST reject them there. `[CROSS-VERIFY: .NET/C++]` + +--- + +## 6. Reference strings + +A string that already appeared earlier in the same buffer MAY be encoded as a +**back-reference** to its byte offset, saving space for repeated keys/values. + +| Marker | Byte | Offset field | +| ------- | ---- | ------------ | +| `StrR1` | `C3` | u8 offset | +| `StrR2` | `C4` | u16 offset | +| `StrR3` | `C5` | u24 offset (3 bytes LE) | +| `StrR4` | `C6` | u32 offset | + +The offset is an **absolute byte offset into the buffer**, measured in the same +frame as the preamble (the preamble is offset `0`). The referenced offset MUST +hold a non-reference string; reference-to-reference chains are prohibited, which +makes cycles impossible and bounds resolution without recursion. + +**Decoder resource bound (normative).** Because many references can point at one +large string, a naΓ―ve decoder can be forced into O(SΒ²) output for a size-`S` +buffer. A conforming decoder MUST bound total materialized reference bytes by a +budget proportional to the input size (e.g. `max(16 Γ— buffer_len, 64 KiB)`) and +fail with an invalid-length error once exceeded (Β§9). + +--- + +## 7. Canonical encoding + +Multiple valid encodings exist for the same value (e.g. the integer `5` can be a +literal `05`, `NumberUInt8`, `Int8`, `Int16`, …; the string `"id"` can be a +system string, an encoded-length string, `StrL1`, …). A **decoder MUST accept +all valid encodings**. An **encoder MUST be deterministic**: for a given value it +MUST emit exactly one encoding. + +Determinism does **not** require the *narrowest* encoding. An encoder MAY emit +any valid form as long as it does so deterministically. The Rust reference +encoder (`ser.rs` / `writer.rs`, pinned by `conformance.rs`) deliberately emits a +**valid but non-minimal subset**: integers outside `[0,31]` use `Int64`/`UInt64` +(never the narrower `NumberUInt8`/`Int16`/`Int32`), strings use the +encoded-length or `StrL*` forms (never system/user/compressed/reference strings), +and containers always use `LC*` framing (never `Arr0`/`Arr1`/`Obj0`/`Obj1`). The +decoder still accepts every compact form the service may emit. + +The rules below describe the **widest canonical shape** each SDK is free to +narrow. Rules 1, 3 and 4 are **required** (they select the value's category); +rules 2, 5 and 6 describe **encoder-optional narrowing** β€” an encoder MAY emit a +wider valid form instead: + +1. **Integers in `[0,31]`** β†’ literal small integer (single byte). *(Required.)* +2. **Other integers** β†’ any fixed-width `Number*` marker that holds the value + (narrowest is `NumberUInt8` β†’ `NumberInt16` β†’ `NumberInt32` β†’ `NumberInt64`; + values above `i64::MAX` use `NumberUInt64`). *Encoder-optional:* the Rust + encoder always uses `Int64`/`UInt64`. `[CROSS-VERIFY: .NET/C++]` β€” confirm the + .NET encoder prefers `Number*` over the extended `D7`–`DC` markers. +3. **Floating-point (non-integer) numbers** β†’ `NumberDouble` (`CC`). *(Required.)* +4. **Non-finite floats** β†’ `null` (Β§4.3). *(Required.)* +5. **Strings** β†’ system string if the value is in the system dictionary; + otherwise encoded-length (`< 64` bytes), `StrL1`, `StrL2`, or `StrL4` by + length. System/compressed/base64/GUID/reference forms are **decoder-accepted + but encoder-optional** optimizations; a minimal conforming encoder need not + emit them (the Rust encoder never emits system or compressed strings). + `[CROSS-VERIFY: .NET/C++]` β€” enumerate which optimizations the .NET encoder + applies by default. +6. **Containers** β†’ `LC*` (length-and-count) framing. Narrowing to the compact + `Arr0`/`Arr1`/`Obj0`/`Obj1` forms is **encoder-optional**; the Rust encoder + always emits `LC*`. `[CROSS-VERIFY: .NET/C++]` + +> **Why deterministic encoding matters.** A snapshot / golden-vector test asserts +> `encode(value) == expected_bytes`, so each SDK's encoder must be deterministic +> for its own snapshots to be stable. Cross-SDK **byte-equality** tests are only +> valid between encoders that made the *same* narrowing choices; because the Rust +> encoder emits the verbose subset above, a cross-SDK byte-equality test must +> compare against that subset (or compare *decoded values* rather than bytes). + +--- + +## 8. Decoder conformance requirements + +A conforming decoder MUST: + +1. **Reject a missing preamble.** The first byte MUST be `0x80` (Β§3.1). +2. **Reject trailing bytes.** After decoding the single top-level value, no bytes + may remain. +3. **Bounds-check every read.** A length/offset field that would read past the + end of the buffer MUST fail with an unexpected-EOF / invalid-length error, not + panic or read out of bounds. +4. **Enforce a maximum nesting depth** to prevent stack exhaustion on deeply + nested containers. The reference limit is `256`. Both the reference (`Value`) + decoder and any streaming decoder MUST reject at the **same** depth. +5. **Validate `LC*` containers.** After reading `count` elements/members, the + cursor MUST be exactly at the declared byte-length boundary; a mismatch MUST + fail (this catches malformed length+count buffers that a count-only decoder + would silently under-read). +6. **Enforce the reference-string budget** (Β§6). +7. **Bound uniform-array output.** For `ArrArrNum*` with a zero inner count, the + outer count MUST NOT exceed the remaining buffer bytes (else a few bytes could + materialize `u16::MAX` empty arrays). +8. **Reject the `Invalid` marker (`0xFF`)** and any unassigned marker with an + invalid-marker error. +9. **Reject non-finite numbers** (Β§4.3). + +A conforming decoder MUST NOT panic, hang, or allocate unboundedly on any input, +because it parses **untrusted** service/network bytes (Β§9). + +--- + +## 9. Security considerations + +Decoders process bytes that may originate from a compromised or buggy service, a +MITM, or a corrupted cache. The threats and required mitigations: + +| Threat | Mitigation (normative) | +| ---------------------------------------- | ---------------------------------------------- | +| Out-of-bounds read via a large length | Bounds-check every read against buffer end (Β§8.3) | +| Stack exhaustion via deep nesting | Max depth limit, rejected identically by all decoders (Β§8.4) | +| O(SΒ²) memory via many back-references | Per-decode reference-expansion budget (Β§6) | +| Output amplification via empty uniform arrays | Bound outer count by remaining bytes (Β§8.7) | +| Malformed length+count under-read | Assert cursor == declared end (Β§8.5) | + +Encoders SHOULD reject values that cannot be represented (e.g. integer widths +beyond `u32::MAX` container framing) rather than silently truncating. + +--- + +## Appendix A β€” Golden test vectors (shared corpus) + +The normative, cross-SDK test corpus lives in machine-readable form at +`azure_data_cosmos_driver/testdata/binary_json_vectors.json`. Each entry pairs a +`name`, a spaced-hex `binary` buffer (including the `0x80` preamble), and the +`json` value it decodes to. A conforming decoder MUST reproduce every `json` from +its `binary`; a conforming encoder MUST reproduce the canonical `binary` for +every `json` that is in canonical form (Β§7). + +A representative subset (see the file for the full set): + +| name | binary | json | +| ---- | ------ | ---- | +| null | `80 D0` | `null` | +| true | `80 D2` | `true` | +| literal_int_max | `80 1F` | `31` | +| uint8 | `80 C8 C8` | `200` | +| double | `80 CC 00 00 00 00 00 00 0C 40` | `3.5` | +| system_string_id | `80 2C` | `"id"` | +| str_l1_hello | `80 C0 05 68 65 6C 6C 6F` | `"hello"` | +| binary_deadbeef | `80 DD 04 DE AD BE EF` | `"3q2+7w=="` | +| uniform_int32 | `80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00` | `[1,2,3]` | +| object_lc1 | `80 ED 04 02 2C 00 3B 01` | `{"id":0,"type":1}` | +| nested_containers | `80 E2 05 E1 00 E9 2C 01` | `[[0],{"id":1}]` | + +## Appendix B β€” Worked examples (byte-by-byte) + +**`{"id":0,"type":1}` as `ObjLC1` (`80 ED 04 02 2C 00 3B 01`):** + +``` +80 preamble +ED ObjLC1 marker (u8 byte-length + u8 count) +04 byte-length of member region = 4 +02 member (pair) count = 2 +2C name: system string 0x0C ("id") ← 0x20 + 0x0C +00 value: literal small int 0 +3B name: system string 0x1B ("type") ← 0x20 + 0x1B [CROSS-VERIFY] +01 value: literal small int 1 +``` + +**`[1,2,3]` as a uniform Int32 array (`80 F0 DA 03 01 00 00 00 …`):** + +``` +80 preamble +F0 ArrNumC1 (uniform number array, u8 count) +DA item type marker = Int32 +03 item count = 3 +01 00 00 00 1 (Int32 LE) +02 00 00 00 2 +03 00 00 00 3 +``` + +**base64 with omitted padding (`80 71 01 FD 41` β†’ `"QQ"`):** the length field +uses a signed offset convention to signal padding omission; see the codec's +base64 decoder for the exact rule. `[CROSS-VERIFY: .NET/C++]` + +## Appendix C β€” Complete marker constant table + +*(Authoritative values; transcribed from `markers.rs` / `JsonBinaryEncoding.TypeMarker.cs`.)* + +``` +Literal int 0x00–0x1F +System string 1B 0x20–0x3F +User string 1B 0x40–0x5F +User string 2B 0x60–0x67 +Base64Len1 0x71 Base64Len2 0x72 +Base64UrlLen1 0x73 Base64UrlLen2 0x74 +GuidLower 0x75 GuidUpper 0x76 GuidQuoted 0x77 +CompLowerHex 0x78 CompUpperHex 0x79 CompDateTime 0x7A +Packed4/5/6bit 0x7B/0x7C/0x7D +Packed7bitL1/L2 0x7E/0x7F +Encoded-len str 0x80–0xBF (len = marker & 0x7F) +StrL1/L2/L4 0xC0/0xC1/0xC2 +StrR1/R2/R3/R4 0xC3/0xC4/0xC5/0xC6 +NumberUInt64 0xC7 +NumberUInt8 0xC8 NumberInt16 0xC9 NumberInt32 0xCA NumberInt64 0xCB +NumberDouble 0xCC Float32 0xCD Float64 0xCE Float16 0xCF +Null 0xD0 False 0xD1 True 0xD2 Guid 0xD3 +UInt8 0xD7 Int8 0xD8 Int16 0xD9 Int32 0xDA Int64 0xDB UInt32 0xDC +Binary1/2/4 0xDD/0xDE/0xDF +Arr0..ArrLC4 0xE0–0xE7 +Obj0..ObjLC4 0xE8–0xEF +ArrNumC1/C2 0xF0/0xF1 +ArrArrNumC1C1/C2C2 0xF2/0xF3 +Invalid 0xFF +``` + +## Appendix D β€” System string dictionary + +The fixed system-string table (index β†’ string) is normative and MUST match +`JsonBinaryEncoding` across SDKs. It is defined in +`azure_data_cosmos_driver/src/binary_json/system_strings.rs`. +`[CROSS-VERIFY: .NET/C++]` β€” reproduce the full table here once confirmed against +the .NET source (only `id` = index `0x0C` and `type` = index `0x1B` are shown +inline in this draft's examples). + +--- + +## Open items before promotion to stable + +- Resolve every `[CROSS-VERIFY: .NET/C++]` tag against the .NET + (`Microsoft.Azure.Cosmos/src/Json/`) and C++ reference implementations. +- Complete Appendix D (full system-string table). +- Confirm the canonical-encoding rules in Β§7 match the .NET encoder's actual + output (needed for cross-SDK byte-equality snapshot tests). +- Specify the exact base64 padding/length-offset convention (Appendix B). +- Specify the compressed-string base/charset tables (Β§4.4.6). diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md new file mode 100644 index 00000000000..e2f78d684f9 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md @@ -0,0 +1,628 @@ +# Binary-Encoding Round-Trip Fuzzer β€” Design + +**Status:** Draft Β· **Companion harness:** `azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs` + +## 1. Goal + +Validate that **arbitrary JSON survives a full Cosmos round-trip unchanged**, +across binary-encoding configurations, at high volume. Where the deterministic +golden vectors ([`binary_json_vectors.json`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json)) +prove specific byte layouts and the in-tree fuzz suite hammers the *decoder* with +malformed input, this harness exercises the **end-to-end path**: + +``` +generate JSON β†’ Rust encode β†’ wire β†’ backend store+rewrite β†’ wire β†’ Rust decode β†’ compare +``` + +A single machine can validate **millions of distinct JSON structures over a few +days**, which no hand-written test set can approach. + +This directly implements the reviewer request (FabianMeiswinkel, PR #4671): + +> a tool that can produce random json structures and does e2e validation (via +> canonicalization + hash) β€” this would allow us to test millions of differently +> structured json objects and increase confidence level. + +## 2. Core idea: canonicalize + compare + +For each generated document `D`: + +1. `H0 = hash(canonicalize(D))` +2. For each config `C` (binary on/off, text-response on/off, …): + - drive every **body-carrying point op** β€” `create` β†’ `read` β†’ `replace` β†’ + `upsert` β€” each returning a document `R` (writes use content-response so the + response decode path is exercised too); + - `Hc = hash(canonicalize(project(R, keys(D))))` for each op's `R`; + - **assert `Hc == H0`** for every op β€” otherwise dump the seed + both + canonical forms. + +These are exactly the four point operations for which binary encoding is honored +(`create` / `read` / `replace` / `upsert`); `delete` carries no body, and +`patch` / transactional batch / bulk are deferred (see the SPEC/HLD), so they are +intentionally excluded. + +`project(R, keys(D))` strips the service-added system fields (`_rid`, `_etag`, +`_ts`, `_self`, `_attachments`) so only the fields we control are compared. + +> **Reserved fields are also stripped on the _send_ side.** Cosmos *owns* the +> `_rid`/`_self`/`_etag`/`_ts`/`_attachments` properties and overwrites any value +> a client authors. Because the corpus shapes are modeled on real exported +> service documents (which carry `_self`), and the free-form `arbitrary-json` +> generator can emit arbitrary keys, every generated document is passed through +> `strip_reserved_fields` **before** it is sent. Without this, a randomly +> generated `_self` would round-trip back as the service's own value and cause a +> **false** mismatch (this is exactly the failure the guard was added to fix). +> The `no_shape_emits_reserved_system_fields` offline test locks it in. + +The harness compares **canonical strings directly** (strongest signal β€” it can +print the exact diff) *and* logs a **SHA-256** digest so the "store `H0` once, +compare later" workflow is available for a persistent corpus. SHA-256 is stable +across runs and platforms; we are detecting *differences*, so the digest's role +is a compact, durable corpus key rather than collision defense. (The canonical +string is produced by RFC 8785 / `json_canon` over the number-normalized value β€” +see Β§9.) + +### 2.1 How this detects codec gaps + +The fuzzer needs no knowledge of *correct bytes* β€” it exploits one invariant: +**a value stored and read back must be identical.** It runs each generated +document through **three configs** (text control, binary, binary+text-response), +which is what lets a mismatch **localize the broken layer**: + +| Symptom in the mismatch dump | Where the bug is | +| ---------------------------- | ---------------- | +| Text config passes, **binary** config fails | The **encoder** (`ser.rs` / `writer.rs`) emitted wrong bytes for some value. | +| Binary write succeeds but the **read decodes wrong** | The **decoder** (`de.rs` / `reader.rs`) mishandles a wire form. | +| **binary+text-response** fails but plain binary passes | The **driver transcode** (`transcode_to_text`) loses something on binaryβ†’text. | +| **All configs fail identically** | Likely a backend rewrite the normalizer doesn't model yet β†’ a calibration gap (tune `normalize_number`, Β§3.1), or a genuine service behavior to escalate. | + +The specific classes of gap it is built to surface β€” the ones curated tests miss +because no human authored the triggering input: + +- **Encoder ↔ decoder disagreement on a wire form the *backend* emits.** The Rust + encoder emits only a *subset* of wire forms; the decoder accepts *all* of them + (system strings, compressed 4/5/6/7-bit strings, GUID/base64 forms, uniform + number arrays, reference strings). If the decoder mishandles a compact form the + **backend** produces but the Rust encoder never does, no unit test exercises + it β€” only a live round-trip does. +- **Number precision/representation edges** β€” exactly what calibration surfaces: + non-finite handling, `-0`, integers beyond `2^53` (backend stores as + doubles), high-precision floats. +- **Unicode / string escaping** β€” astral code points, control characters, + characters needing JSON escaping; a mismatch here is an encoder/decoder + UTF-8/length bug. +- **Container framing** β€” deep nesting, mixed vs. uniform arrays, empty + containers, arrays-of-objects; catches off-by-one length/count bugs. +- **The transcode path specifically** β€” the `binary+text-response` config is the + *only* end-to-end exercise of `transcode_to_text` against real binary the + backend produced. + +**The debugging loop:** a failure prints the exact document, the config, and the +seed. Re-run with `AZURE_COSMOS_FUZZ_SEED=` to reproduce deterministically, +reduce to the minimal triggering value, add it as a golden vector, and fix the +codec β€” then the new vector guards against regression. + +**Limitations to keep in mind:** the fuzzer is only as good as its calibration +and its generator's range. Under-calibrated `normalize_number` β†’ false +positives (noise); a form the generator never emits β†’ false negatives (blind +spots). Calibrate first (Β§3.1), then widen coverage progressively with +`--wide-numbers` / `max_depth` / `unicode`. + +### 2.2 How it works, visualized + +**The per-document loop.** Every generated document is canonicalized once to get +the expected hash `H0`, then stored + read back under each config and compared: + +```mermaid +flowchart TD + SEED["Seed (SplitMix64)\nAZURE_COSMOS_FUZZ_SEED"] --> GEN + GEN["arbitrary-json\ngenerate random Value D"] --> BOUND["bound_value\nclamp numbers/strings\nto calibrated envelope"] + BOUND --> NORM0["normalize_numbers(D)\nCosmos number rewrite"] + NORM0 --> CANON0["json-canon (RFC 8785)\ncanonical string"] + CANON0 --> HASH0["SHA-256 -> H0\n(expected)"] + + BOUND --> STORE["for each config:\ncreate -> read -> replace -> upsert\n(each returns R)"] + STORE --> PROJ["project(R, keys(D))\nstrip _rid/_etag/_ts/..."] + PROJ --> NORM1["normalize_numbers"] + NORM1 --> CANON1["json-canon"] + CANON1 --> HASH1["SHA-256 -> Hc\n(actual)"] + + HASH0 --> CMP{"Hc == H0 ?"} + HASH1 --> CMP + CMP -->|yes| OK["round-trip OK\nnext doc"] + CMP -->|no| FAIL["MISMATCH\ndump seed + both canonical forms"] +``` + +**Three configs localize the broken layer.** The same document `D` runs through +three client configurations; because only one variable differs, *which* config +fails points at *which* layer is broken: + +```mermaid +flowchart LR + D["Document D"] --> C1 & C2 & C3 + + subgraph C1["Config A - text control"] + A1["binary = off\nwire = text JSON"] + end + subgraph C2["Config B - binary"] + B1["binary = on\nwire = binary both ways"] + end + subgraph C3["Config C - binary + text-response"] + G1["binary = on\nrequest_text_response = on\ndriver transcodes response to text"] + end + + C1 --> R{"compare canonical(sent)\nvs canonical(returned)"} + C2 --> R + C3 --> R +``` + +| What fails | Where the bug is | +| ---------- | ---------------- | +| **Text (A) fails** | Not the codec β€” a canonicalization gap or a real backend rewrite; escalate. | +| **Binary (B) fails, text (A) passes** | The **encoder** (`ser.rs`/`writer.rs`) emitted wrong bytes. | +| Binary write OK but **read decodes wrong** | The **decoder** (`de.rs`/`reader.rs`) mishandles a wire form. | +| **C fails, B passes** | The **driver transcode** (`transcode_to_text`) loses something binary to text. | +| **All three fail identically** | A backend rewrite the canonicalizer doesn't model yet (tune `normalize_number`), or a genuine service behavior to escalate. | + +**The debugging loop.** Every failure is deterministically reproducible and +reduces to a permanent regression guard: + +```mermaid +flowchart LR + F["MISMATCH\nprints seed + config\n+ both canonical forms"] --> REPRO["Reproduce:\nAZURE_COSMOS_FUZZ_SEED=\ndeterministic replay"] + REPRO --> REDUCE["Reduce to the minimal\ntriggering value"] + REDUCE --> CLASS{"Classify"} + CLASS -->|codec bug| FIX["Fix ser/de + add a\ngolden vector (regression)"] + CLASS -->|canonicalization gap| TUNE["Tune normalize_number\n(re-run calibration -> MATCH)"] + CLASS -->|backend rewrite| ESC["Escalate - genuine\nservice behavior difference"] +``` + +**Calibration is the safety valve.** Before a soak is trustworthy, calibration +proves the number model matches *this* account, so a mismatch is a real bug and +not modeling noise (it prints a table; it does not assert β€” see Β§3.1): + +```mermaid +flowchart LR + P["NUMBER_PROBES\n(1e20, i64::MAX, u64-1, -0, 1.0, ...)"] --> ST["store via binary\n-> read back"] + ST --> T{"our-canonical ==\nbackend-returned ?"} + T -->|MATCH| GOOD["number model correct\n-> soak results are trustworthy"] + T -->|DIFF| BAD["tune normalize_number\nbefore soaking"] +``` + +## 3. Canonicalization (the hard part) + +Two JSON texts are "the same value" if they canonicalize identically. Rules: + +| Aspect | Rule | +| ----------- | ---- | +| Whitespace | removed entirely | +| Object keys | sorted lexicographically (by UTF-16 code unit, per RFC 8785) | +| Strings | minimally JSON-escaped (via `serde_json`) | +| Arrays | order preserved | +| **Numbers** | **Cosmos-compatible normalization β€” see Β§3.1** | + +### 3.1 Number normalization (the tuning surface) + +The subtlety the reviewer flagged: **the backend rewrites numbers on store**, and +its rewrite is *not* identical to a strict canonicalizer like +[JCS / RFC 8785](https://datatracker.ietf.org/doc/html/rfc8785). If we +canonicalized with JCS but the backend renders `1.0` as `1`, a faithful +round-trip would *falsely* report a mismatch. + +The harness therefore uses a **Cosmos-compatible** number canonicalizer, not JCS: + +- **Integers with magnitude `< 2^53`**: emit plain decimal, no decimal point, no + exponent, no leading zeros. `-0` β†’ `0`. +- **Integers with magnitude `>= 2^53`** (whether `i64` or `u64`): routed through + `f64` (see the calibration finding below). The backend stores *every* JSON + number as an IEEE-754 double, so integers beyond `2^53` are not preserved + exactly; a sent integer and its returned (rounded) double must canonicalize + identically. +- **Integral-valued floats** (e.g. `1.0`, `2.0e1`): normalized to their integer + form (`1`, `20`) β€” this mirrors the observed backend rewrite where a trailing + `.0` is dropped. +- **Non-integral floats**: shortest round-trippable decimal (Rust's `ryu`, via + `serde_json`'s `f64` formatting). + +> **Calibrated against a live account (Β§6).** The first calibration run +> (18 probes) confirmed **16/18 forms already match**, including the tricky ones: +> integral floats and integral exponents collapse to integers (`1.0`, `2e1` β†’ +> `1`, `20`), `-0` β†’ `0`, trailing zeros are dropped (`1.2300` β†’ `1.23`), +> repeating/high-precision floats and `0.1 + 0.2` round-trip exactly, and the +> backend renders large/small exponents in scientific notation (`1e20`, +> `1e-20`) which reparses to the same `f64`. The **two DIFFs** were integers +> beyond `2^53`: the backend stores them as IEEE-754 doubles (lossy) and +> returns scientific notation β€” `18446744073709551614` β†’ `1.8446744073709552e+19` +> and `2^63` β†’ `9.223372036854776e+18`. `normalize_number` now models this by +> routing every integer beyond `2^53` (both `i64` and `u64`) through `f64`, so +> both sides canonicalize to the same double form. Re-running calibration after +> this change yields all `MATCH`. +> +> To re-calibrate after any change (or against a different account/config), run +> calibration mode (`AZURE_COSMOS_FUZZ_CALIBRATE=true`, see Β§6): it stores each +> probe in `NUMBER_PROBES` through the binary path, reads it back, and prints a +> table comparing `normalize_number`'s rendering against the backend's +> returned form. Every `DIFF` is a form to model; calibration is a **diagnostic** +> (prints the table, does not assert), since a `DIFF` is the signal to tune, not +> a failure. + +### 3.2 Generator stays inside the calibrated envelope + +To avoid false positives from *un-calibrated* number forms, the generator emits +numbers in **backend-safe ranges by default** (bounded integers, bounded-precision +floats). A `--wide-numbers` flag widens the range once the canonicalizer is +calibrated for those forms β€” this is how you progressively expand coverage. + +## 4. Generator + +A seeded PRNG (`SplitMix64`, seed logged for exact reproduction) produces a +random JSON **object** (Cosmos items are objects) using a **hybrid** strategy: + +- a **depth-controlled skeleton** builds a nested container *spine* to a target + depth drawn from `[1, max_depth]`, guaranteeing the document actually reaches + that depth (each level is randomly an object or an array, with a few irregular + filler siblings); +- every **leaf and filler branch** is irregular JSON from + [`arbitrary-json`](https://docs.rs/arbitrary-json) β€” random keys (incl. empty, + control-char, and Unicode), mixed-type arrays, nested sub-objects and + arrays-of-objects, occasional homogeneous number arrays, and the scalars + `null` / `true` / `false`; +- numbers are clamped to the calibrated envelope (Β§3.2) unless `--wide-numbers`; + strings to ASCII unless `unicode`. + +> **Why hybrid?** `arbitrary-json`'s `arbitrary_iter` decides whether to recurse +> from *remaining bytes* and stops almost immediately, so an `arbitrary-json`-only +> generator produced near-flat documents (avg depth β‰ˆ 1.3, unchanged by +> `max_depth`). The explicit skeleton restores real depth: measured average depth +> now scales with the knob (β‰ˆ 3.9 at `max_depth=3`, β‰ˆ 8.5 at `max_depth=12`, with +> deepest docs reaching 11–17 levels), which is what exercises the codec's +> container framing (length/count prefixes, nested arrays-of-objects) and the +> decoder's `MAX_DEPTH` guard. A `generator_depth_scales_with_max_depth` offline +> test locks this in. + +### 4.1 Corpus-shaped documents + +Beyond the free-form hybrid documents, a configurable fraction of each run +(`AZURE_COSMOS_FUZZ_SHAPE_RATIO`, default 85%) generates documents in the +**shape of the real service testdata corpus**. A set of ~24 *shape samplers* +(`SHAPE_SAMPLERS`) each reproduce the structure of one `testdata/*.json` family +β€” GeoJSON features, embedding vectors, blog/telemetry/log records, Cosmos-run +metadata, user records with GUIDs, nutrition/food docs, error buckets with +multiline stack traces, legislator/committee records, and more β€” populated with +randomized, seed-reproducible data spanning every datatype: integers, floats, +high-precision unit/coordinate doubles, alphabetic / alphanumeric / free-text / +**non-ASCII + emoji** strings, booleans, `null`s, ISO-8601 timestamps, hex +hashes, and UUIDs. These reproduce the *shape*, not verbatim corpus bytes (no +data is embedded). Every shaped document still carries the all-category +`_sampler` subtree, so category coverage holds regardless of shape. Set +`AZURE_COSMOS_FUZZ_SHAPE_RATIO=0` for purely free-form docs, or `=100` for an +all-corpus run. Offline tests (`every_corpus_shape_produces_a_valid_object`, +`shaped_documents_are_emitted_when_ratio_is_full`) lock this in. + +`AZURE_COSMOS_FUZZ_SIZE_SCALE` (default `1`) multiplies every corpus-shape +collection length β€” embedding-vector dimensions, nutrient / member / keyword / +`similars` arrays, etc. β€” so a run can grow per-item payloads toward corpus-scale +sizes (e.g. `SIZE_SCALE=40` yields ~18-40 KB items with hundreds-of-dimension +vectors, versus ~1 KB at the default). The corpus's overall *bulk* is a function +of item **count**, so scale volume with `AZURE_COSMOS_FUZZ_ITERATIONS` (live) or +`AZURE_COSMOS_FUZZ_PRINT_COUNT` (offline printer). + +Every run prints its seed; a failing run is reproduced exactly with +`AZURE_COSMOS_FUZZ_SEED=`. + +## 5. Configurations exercised + +Each config is a separate `CosmosClient` (binary encoding is resolved once at +build time): + +| Config | `enabled` | `request_text_response` | +| ------ | --------- | ----------------------- | +| control (text) | false | β€” | +| binary | true | false | +| binary + text response | true | true | + +> **Not yet implemented:** running each document through a *second* account (to +> cover dictionary encoding on/off) is a planned extension. The harness reads a +> single `AZURE_COSMOS_CONNECTION_STRING`; there is no second-account lookup. + +## 6. Running the harness + +In CI, the harness runs automatically on the **`binary_encoding` live leg** +(`live-platform-matrix.json` β†’ `Session SingleWrite BinaryEncodingRoundtripFuzz`, which sets +`testCategory = 'binary_encoding'`). That leg's bicep emits +`--cfg=test_category="binary_encoding"` into `RUSTFLAGS` and provides the live +`AZURE_COSMOS_CONNECTION_STRING`, so `binary_encoding_roundtrip_fuzz` (and the +sibling `binary_encoding` item tests) stop being ignored. The per-run iteration +budget is set by `AZURE_COSMOS_FUZZ_ITERATIONS` in `sdk/cosmos/ci.yml` +(default 200 there). Live tests only run on the weekly schedule or when a build +is queued with **Run live tests** enabled. + +```bash +# One-shot smoke run (a few hundred docs), local emulator: +AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=...;AccountKey=...;' \ +AZURE_COSMOS_ALLOW_INVALID_CERT=true \ +RUSTFLAGS='--cfg test_category="binary_encoding"' \ + cargo test -p azure_data_cosmos --test binary_roundtrip_fuzzer --features key_auth,fault_injection,control_plane -- --nocapture + +# Multi-day soak (millions of docs): +AZURE_COSMOS_CONNECTION_STRING='...' \ +AZURE_COSMOS_FUZZ_ITERATIONS=5000000 \ +AZURE_COSMOS_FUZZ_MAX_DEPTH=6 \ +RUSTFLAGS='--cfg test_category="binary_encoding"' \ + cargo test -p azure_data_cosmos --test binary_roundtrip_fuzzer --features key_auth,fault_injection,control_plane --release -- --nocapture + +# Reproduce a failure: +AZURE_COSMOS_FUZZ_SEED=12345678901234567890 ... cargo test ... + +# Calibrate number canonicalization against the account (prints a table, no assert): +AZURE_COSMOS_CONNECTION_STRING='...' AZURE_COSMOS_FUZZ_CALIBRATE=true \ +RUSTFLAGS='--cfg test_category="binary_encoding"' \ + cargo test -p azure_data_cosmos --test binary_roundtrip_fuzzer --features key_auth,fault_injection,control_plane -- --nocapture +``` + +### Environment knobs + +| Variable | Default | Meaning | +| -------- | ------- | ------- | +| `AZURE_COSMOS_CONNECTION_STRING` | β€” (required) | live account (endpoint + key) | +| `AZURE_COSMOS_ALLOW_INVALID_CERT` | false | accept emulator cert | +| `AZURE_COSMOS_FUZZ_ITERATIONS` | 200 | number of generated docs | +| `AZURE_COSMOS_FUZZ_SEED` | random | PRNG seed (for reproduction) | +| `AZURE_COSMOS_FUZZ_MAX_DEPTH` | 6 | max JSON nesting depth | +| `AZURE_COSMOS_FUZZ_WIDE_NUMBERS` | false | widen numeric range (post-calibration) | +| `AZURE_COSMOS_FUZZ_UNICODE` | true | include Unicode strings | +| `AZURE_COSMOS_FUZZ_CALIBRATE` | false | number-calibration mode (Β§3.1) | +| `AZURE_COSMOS_BINARY_TEST_DATABASE` / `_CONTAINER` | `binary-fuzz-*` | target names | + +## 7. What a failure tells you + +A mismatch is one of: + +1. **A real codec bug** β€” Rust encoded or decoded a value wrong. (The golden + vectors + in-tree fuzz should also then be extended with the reduced case.) +2. **A canonicalization gap** β€” the backend rewrote a number/string in a form the + canonicalizer doesn't yet model. Fix `normalize_number` (Β§3.1) and, if the + form is legitimately out of scope, narrow the generator. +3. **A backend rewrite difference** β€” genuinely different value after store; this + is the highest-value finding and should be escalated. + +The harness prints the seed, the config, and both canonical forms so the case is +immediately reproducible and reducible. + +## 8. Relationship to the other test layers + +The normative wire format is defined by +[`BINARY_ENCODING_RFC.md`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md); this harness is one of the +mechanisms that **validates Rust against that spec** β€” specifically the RFC's Β§7 +round-trip invariant, exercised end-to-end at scale (see the RFC's Β§1.4 diagram +for how all the artifacts relate). + +| Layer | Input | Checks | Location | +| ----- | ----- | ------ | -------- | +| RFC | β€” | normative wire spec (source of truth) | `docs/BINARY_ENCODING_RFC.md` | +| Golden vectors | fixed corpus | exact byte layout, decode parity | `binary_json/vectors.rs` | +| Encoder conformance | fixed corpus | encode byte-exactness, canonical form | `binary_json/conformance.rs` | +| In-tree fuzz | random/truncated buffers | decoder never panics; `decode` ≑ `from_slice` | `binary_json/fuzz_tests.rs` | +| Coverage-guided codec fuzz | mutated **bytes** | decoder/serde/transcode no-crash + `decode∘encode` idempotence | `fuzz/` (cargo-fuzz) | +| **Round-trip fuzzer** | **random JSON** | **end-to-end value fidelity across configs** | **this harness** | + +These are complementary: golden vectors + conformance pin *the format*, in-tree +fuzz and the coverage-guided `cargo-fuzz` targets harden *the decoder* against +mis-encoded bytes (the latter mutating outward from valid frames with libFuzzer, +offline), and this harness validates *the whole pipeline against the live +service* β€” and its number-canonicalization calibration (Β§3.1) feeds the +backend's observed rewrite rules back into the RFC's Β§7. + +### 8.1 This round-trip fuzzer vs. the `cargo-fuzz` codec crate + +The two most easily-confused layers are the **two fuzzers**. They sit at +*opposite ends of the same pipeline* and answer different questions β€” neither +replaces the other. + +``` + round-trip fuzzer starts here (random JSON value) + β”‚ + JSON value ──encode──► binary bytes ──wire──► service ──► bytes ──decode──► JSON value + β–² β”‚ + └──────────── compare (round-trip) β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”€β”˜ + + cargo-fuzz starts HERE (random/mutated bytes) ──► decode ──► Ok/Err (must never crash) +``` + +| | **cargo-fuzz crate** (protocol/byte fuzzer) | **round-trip fuzzer** (value fuzzer) | +| --- | --- | --- | +| Location | `azure_data_cosmos_driver/fuzz/` | `azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs` | +| Fuzzes | the **byte/wire format** (the codec) | **value fidelity** (end-to-end pipeline) | +| Input space | random / mutated **raw bytes** | random **JSON values** (arbitrary-json + corpus shapes) | +| Decoder sees | **mis-encoded** frames the encoder never emits | only **well-formed** encoder output | +| Question | "does the decoder ever crash / hang / OOM on garbage?" | "does a value survive store β†’ read back unchanged?" | +| Oracle | robustness (never panic) + `decode∘encode` idempotence | `canonicalize(sent) == canonicalize(returned)` | +| Cosmos account | ❌ offline, in-process | βœ… live service | +| Engine | coverage-guided (libFuzzer) | seeded PRNG (SplitMix64), deterministic per seed | +| Toolchain / OS | **nightly + Linux** (libFuzzer) | **stable, any OS** | +| Command | `cargo +nightly fuzz run ` | `cargo test … binary_roundtrip_fuzzer` | + +**Why both are needed:** the round-trip fuzzer only ever feeds the decoder bytes +our *own encoder* produced, so it validates **semantic correctness** on the happy +path against the real service but **cannot** reach the decoder's error-handling +code (a correct encoder never emits truncated buffers, bad length prefixes, or +unknown markers). The `cargo-fuzz` crate feeds the decoder **arbitrary garbage**, +so it validates **robustness/hardening** on the malformed path, offline β€” but +says nothing about round-trip value fidelity. This is exactly the reviewer's +point that arbitrary-json alone does not fuzz the *protocol*. + +**Which to use when:** + +- Changed the **decoder / reader / codec** (`reader.rs`, `de.rs`, `markers.rs`, + `transcode_to_text`) or want offline hardening against malformed input, or are + running the weekly deep soak β†’ **cargo-fuzz crate** (`cargo +nightly fuzz run …`, + Linux/nightly). +- Changed the **encoder** or want to prove data survives the **real service** + unchanged, or to exercise realistic corpus shapes/sizes and the three binary + configs, or to reproduce a specific document by seed β†’ **round-trip fuzzer** + (`cargo test … binary_roundtrip_fuzzer`, stable + live account). +- Changed **codec internals** β†’ run **both**. +- Quick, always-on, Windows-friendly decoder no-crash coverage without nightly β†’ + the in-tree `binary_json/fuzz_tests.rs` (`cargo test -p azure_data_cosmos_driver + --lib fuzz`). + + + +This section captures an agreed enhancement plan for the harness. The current harness uses a hand-rolled seeded generator (Β§4) and a hand-rolled canonicalizer (Β§3). Three well-maintained crates can replace the parts of that machinery that are pure boilerplate, while we **keep** the one part that is genuinely Cosmos-specific. + +### 9.1 The crates and what each replaces + +| Crate | Role | Replaces | +| ----- | ---- | -------- | +| [`arbitrary-json`](https://docs.rs/arbitrary-json) | Turns raw fuzzer/PRNG bytes into a random, structurally-valid `serde_json::Value` (via the `arbitrary` crate). | Our hand-rolled `gen_object` / `gen_value` / `gen_array` generator (Β§4). | +| [`json-canon`](https://docs.rs/json-canon) | RFC 8785 (JCS) canonical serialization β€” object-key sort, whitespace removal, string escaping. | The **structural** part of our `canonicalize` (Β§3): keys, whitespace, strings, array order. | +| [`sha2`](https://docs.rs/sha2) | SHA-256 over the canonical string, enabling a durable cross-run corpus of `H0` hashes. | Our `DefaultHasher` (SipHash) 64-bit hash. | + +### 9.2 The critical constraint β€” keep Cosmos number canonicalization + +**JCS number formatting is *not* Cosmos number formatting.** This is the whole reason the harness exists (Β§3.1). RFC 8785 uses ES6 `Number.prototype.toString` (shortest round-trippable), which differs from the backend's observed store-time rewrite: + +- the backend stores integers beyond `2^53` as IEEE-754 **doubles** and returns scientific notation (`18446744073709551614` β†’ `1.8446744073709552e+19`); +- integral floats/exponents collapse to integers (`2e1` β†’ `20`). + +If we canonicalized numbers with raw JCS, a *faithful* round-trip would report **false-positive** mismatches on exactly the number edges we most want to test. So the plan is a **hybrid**, not a wholesale swap: + +> **Normalize numbers with our calibrated `normalize_number` first (produce a number-normalized `Value`), then run that `Value` through `json_canon` for the structural pass, then `sha2` the result.** + +`json-canon`'s own docs also note it emits `null` for `NaN`/`Inf` β€” incidentally aligned with Cosmos, but we do not want to rely on that incidentally, so number handling stays under our control. + +### 9.3 Target pipeline + +``` +generate: bytes ──arbitrary-json──▢ Value +normalize: Value ──our normalize_numbers (calibrated Β§3.1)──▢ Valueβ€² +canonicalize: Valueβ€² ──json_canon (RFC 8785 structural)──▢ canonical String +hash: String ──sha2 (SHA-256)──▢ H +compare: H(sent) == H(project(returned, keys(sent))) +``` + +Only **step 2** is Cosmos-specific and stays in our code; steps 1, 3, 4 become library calls. + +### 9.4 Two harness shapes (we will land both, in order) + +1. **Live-service round-trip (this harness, evolved).** Keep the `#[tokio::test]` soak driven by a seeded PRNG, but feed the PRNG bytes into `arbitrary-json` for generation and swap the structural canonicalizer to `json_canon` + `sha2`. This is the primary deliverable β€” it validates the whole pipeline against a real account, which per-doc network I/O makes unsuitable for a coverage-guided engine. +2. **Offline codec fuzzer (new, no account).** A `cargo-fuzz` crate that feeds **raw/mutated bytes** straight into the decoder with **no network**, so libfuzzer's coverage guidance and speed apply. As landed it goes *beyond* the originally-planned round-trip check: it fuzzes the **binary protocol itself** (mis-encoded frames the encoder never produces), not just encoder-produced buffers. See Β§9.8 for the target set. This complements the decoder-only `fuzz_tests.rs` with coverage-guided, byte-level hardening. + +### 9.5 Work plan + +1. Add `arbitrary`, `arbitrary-json`, `json-canon`, and `sha2` as **dev-dependencies** of the harness crate (test-only; not shipped in the SDK). *(Originally landed in `azure_data_cosmos_perf`; later moved with the harness to `azure_data_cosmos` β€” see Β§9.7.)* +2. Extract the current number logic into a standalone `normalize_numbers(&Value) -> Value` that applies the calibrated `normalize_number` rules and leave calibration mode (Β§6) pointing at it. +3. Replace `canonicalize` internals with: `normalize_numbers` β†’ `json_canon::to_string` β†’ `sha2` digest. Keep the `project_to_sent_keys` step (Β§2) unchanged. +4. Replace `gen_object`/`gen_value` with an `arbitrary-json`-backed generator seeded from the existing `SplitMix64` byte stream (so runs stay reproducible via `AZURE_COSMOS_FUZZ_SEED`). +5. Re-run **calibration** (Β§6) against a live account to confirm `normalize_numbers` still yields all `MATCH` after the refactor; fold any new `DIFF` back in. +6. (Separate change) Add the `cargo-fuzz` offline codec target from Β§9.4(2). *(Landed β€” see Β§9.8; scope broadened to byte-level protocol fuzzing.)* +7. Update Β§3, Β§4, and the layer table (Β§8) to reference the crates once landed. + +### 9.6 Acceptance + +- [x] Live harness produces identical pass/fail decisions to the pre-refactor version on a fixed seed set (no behavior regression), with cleaner internals. +- [x] Number edges (`> i64::MAX`, integral floats, `-0`, high-precision) still round-trip without false positives β€” verified by calibration `MATCH`. +- [x] `sha2` hashes are stable across runs for the same canonical input (enables a persistent corpus). +- [x] Offline `cargo-fuzz` codec crate lands (byte-level decoder/serde/transcode no-crash + `decode∘encode` idempotence). See Β§9.8. + +### 9.7 Implementation status (landed) + +Phases 1–4 of Β§9.5 are implemented in `binary_roundtrip_fuzzer.rs` across four commits: + +| Phase | Change | Status | +| ----- | ------ | ------ | +| 1 | Dev-deps `arbitrary`, `arbitrary-json`, `json-canon`, `sha2` wired into the harness crate (initially `azure_data_cosmos_perf`; later relocated with the harness to `azure_data_cosmos`, so the live leg actually runs it). | βœ… landed | +| 2 | `normalize_number` / `normalize_numbers` extracted as the sole Cosmos-specific number transform (behavior-preserving). | βœ… landed | +| 3 | `canonicalize` now = `normalize_numbers` β†’ `json_canon::to_string` (RFC 8785); differential hash switched to SHA-256. | βœ… landed | +| 4 | Generator replaced with `arbitrary-json`, seeded from `SplitMix64` (deterministic per `AZURE_COSMOS_FUZZ_SEED`); a `bound_value` pass keeps the `wide_numbers`/`unicode` envelope contract. | βœ… landed | +| 5 | Live re-calibration + soak against a real account. | βœ… landed (see below) | +| 6 | Offline `cargo-fuzz` codec crate (byte-level protocol fuzzing). | βœ… landed (see Β§9.8) | + +#### Key finding β€” `json-canon` rejects integers β‰₯ 2⁡³ + +RFC 8785 / `json-canon` refuses to serialize any integer at or beyond the JSON +"max safe integer" (`2^53`), returning `Error("u64 must be less than JSON max +safe integer")`. Cosmos stores *every* JSON number as an IEEE-754 double, so no +integer beyond `2^53` is preserved exactly. To bridge this, `normalize_number` +maps JCS-unsafe numbers to **stable string tokens** (the `f64` form of the +rounded double, for both large `i64` and `u64`), which are +only ever compared for equality β€” never parsed back. This keeps the sent and +round-tripped values comparable without tripping the JCS safe-integer guard, and +is the concrete realization of the Β§9.2 "keep Cosmos number canonicalization" +constraint. JCS-safe numbers still serialize as bare JSON numbers. + +#### Remaining manual step (Phase 5) + +Run calibration and a short soak against a live account to confirm the refactor +did not regress the number model (all `MATCH`) β€” see Β§6 for the commands. This is +the one step that cannot run in CI or offline because it requires a real Cosmos +endpoint. + +#### Live validation (Phase 5, recorded) + +Run against a real Cosmos account after the crate refactor: + +- **Calibration:** all **18/18** number probes `MATCH` β€” every JCS-unsafe edge + (`1e20`, `-1.5e18`, `i64::MAX`/`MIN`, `u64::MAX-1`, `2^63`) canonicalizes to the + same stable string token on both the sent and backend-returned sides, and the + integral-float/`-0`/trailing-zero rewrites all match. +- **Soak (initial, create + read):** 500 documents Γ— 3 configs = 1500 + round-trips, all canonical-equal (seed `1784934026943565900`). +- **Soak (all four point ops):** **1000 documents Γ— 3 configs Γ— 4 point ops + (create/read/replace/upsert) = 12,000 round-trips, all canonical-equal** + (seed `1784944014111583800`), plus the offline unit tests. No mismatches. + +This confirms no behavior regression from the `arbitrary-json` + `json-canon` + +SHA-256 refactor, and that the request-encode + response-decode paths for +`replace` and `upsert` round-trip identically to `create`/`read`. Closes Β§9.6's +first three acceptance items. + +## 9.8 Offline codec fuzzer (`cargo-fuzz`) β€” landed + +Phase 6 lands as a self-contained `cargo-fuzz` crate at +[`azure_data_cosmos_driver/fuzz/`](https://github.com/Azure/azure-sdk-for-rust/tree/main/sdk/cosmos/azure_data_cosmos_driver/fuzz) (see its +[`README.md`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md)). It closes a gap this live harness structurally +cannot: because the round-trip fuzzer generates random *JSON values* and only +ever feeds the decoder **encoder-produced** (well-formed) bytes, it exercises +happy-path encode/decode symmetry but never the decoder's defensive paths. The +`cargo-fuzz` crate feeds **arbitrary and mutated bytes** straight into the codec, +so it fuzzes the **binary protocol itself** β€” truncated buffers, bad length +prefixes, unknown/misused markers, reference and depth bombs, non-UTF-8 string +payloads, and trailing bytes. + +### Why a separate crate + +`cargo-fuzz` builds targets on **nightly** with **libFuzzer**, whereas the repo +workspace builds on stable. The crate therefore carries its own empty +`[workspace]` table so it is *excluded* from the parent workspace; it is not a +workspace member and does not affect stable builds. It depends on the driver by +path and is **offline** (no Cosmos account), so unlike this live harness it is +cheap enough for a CI nightly or a time-boxed PR smoke run. + +### Targets + +| Target | Entry point | Oracle | +| ------ | ----------- | ------ | +| `decode` | `binary_json::decode` | no panic/hang/over-alloc on any bytes | +| `from_slice` | `binary_json::from_slice::` | same, for the native serde streaming path | +| `transcode_to_text` | `binary_json::transcode_to_text` | same, for the driver-side binaryβ†’text response path | +| `decode_reencode_roundtrip` | `decode` + `encode` | **differential**: `decode(encode(decode(x))) == decode(x)` on decoder-accepted input | + +The first three assert the robustness oracle (terminate with `Ok`/`Err`, never +crash); the fourth adds a semantic oracle catching reader/writer disagreements +that fixed golden vectors don't enumerate. + +### Corpus seeding + +libFuzzer starts from a corpus of **valid** frames so it mutates outward from +real wire shapes. The [golden vectors](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json) +(every marker family, as hex) seed it directly β€” the crate README carries the +one-liner (PowerShell / jq+xxd) that materializes them into `fuzz/corpus/decode`. + +### Relationship to the other layers + +This `cargo-fuzz` crate and `fuzz_tests.rs` both harden the *decoder* against +mis-encoded bytes; the difference is coverage-guided mutation and scale +(libFuzzer) versus a fixed in-tree sweep. Neither replaces this live harness, +which is the only layer that validates the **whole pipeline against the real +service**. See the layer table in Β§8. diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUND_TRIP_FINDINGS.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUND_TRIP_FINDINGS.md new file mode 100644 index 00000000000..c2ff1dfabd2 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUND_TRIP_FINDINGS.md @@ -0,0 +1,84 @@ +# Binary Encoding Round-Trip Findings + +A running log of findings related to Cosmos binary JSON encoding round-trips. +Add new findings as additional numbered sections below. + +--- + +## Finding 1: Integers beyond `2^53` do not round-trip through a live account + +### Summary + +Large unsigned integers beyond `2^53` do **not** round-trip losslessly through a +live Azure Cosmos DB account, regardless of whether the item body is encoded as +**Cosmos binary JSON** or as **text JSON**. This is a property of the service's +number model, not a defect in the SDK's binary codec. + +### Background + +The live Cosmos DB service normalizes every JSON number to an IEEE-754 +double-precision float (`f64`) on ingestion. Doubles can represent integers +exactly only up to `2^53` (`9_007_199_254_740_992`). Any integer larger than +that is snapped to the nearest representable double when the service stores it. + +Concretely, `u64::MAX` (`18446744073709551615`, i.e. `2^64 βˆ’ 1`) is not exactly +representable as an `f64`. The nearest double is `2^64` +(`18446744073709551616`), which the service stores and echoes back. That value +no longer fits in a `u64` field (`u64` tops out at `2^64 βˆ’ 1`). + +### Observed failure + +The live integration test +`binary_encoding_tests::cosmos_binary_encoding::binary_encoding_item_crud_round_trips` +failed in the pipeline (macOS `SessionSingleWriteBinaryEncodingRoundtripFuzz` +leg) with: + +```text +CosmosError: 500/20020 (SerializationResponseBodyInvalid): failed to deserialize response body +Caused by: + 0: Custom("invalid type: floating point `18446744073709552000.0`, expected u64") +``` + +The test document set `huge: u64::MAX`. The service echoed the value back as the +double `18446744073709552000.0`, and deserialization into the `u64` field failed. + +### Why this is not a binary-encoding regression + +The failure happens **after** the service normalizes the number, so both wire +formats are affected identically: + +- **Binary path:** the echoed number carries the `NUMBER_DOUBLE` marker. The + decoder correctly maps it via `visit_f64`, and serde rejects the float for a + `u64` field: *"invalid type: floating point … expected u64"*. +- **Text path:** the service echoes the value as a JSON number. `serde_json` + attempts to fit `2^64` into the `u64` field and fails with an + out-of-range/overflow error (`2^64` exceeds `u64::MAX`). + +Either way the round-trip fails. The binary codec behaves correctly and +consistently with the text path β€” it faithfully decodes exactly what the service +returned. + +### Why the in-memory emulator test passed + +The analogous in-memory-emulator round-trip test uses `u64::MAX` and passes, +because the emulator preserves the exact `UInt64` wire form rather than +normalizing numbers to doubles. It therefore does not exercise the live +service's double-normalization behavior. + +### Resolution + +The defect was in the **test data**, not the codec. A value at or below `2^53` +round-trips losslessly. The live test now uses `2^53` +(`9_007_199_254_740_992`) β€” the largest integer exactly representable as an +`f64`, while still being a `u64` β€” to exercise the wide-unsigned encoder form +without hitting the service's precision ceiling. + +### Guidance + +- Do not assert exact round-trip equality for integers greater than `2^53` when + testing against a live Cosmos DB account. +- If an application must preserve integers beyond `2^53`, store them as strings + (or a struct with explicit high/low words); numeric fields will be subject to + double precision on the service. +- This constraint is independent of the binary-encoding feature and applies to + text JSON as well. diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore b/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore new file mode 100644 index 00000000000..1a45eee7760 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml b/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml new file mode 100644 index 00000000000..f54748f8658 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml @@ -0,0 +1,60 @@ +# cargo-fuzz crate for the Cosmos binary JSON codec. +# +# This is a SEPARATE crate with its own `[workspace]` (see the empty table at +# the bottom) so it is NOT pulled into the parent stable workspace: cargo-fuzz +# builds these targets on nightly with libFuzzer, while the repo workspace +# builds on stable. +# +# Usage (from this directory or the driver crate root): +# rustup toolchain install nightly +# cargo install cargo-fuzz +# cargo +nightly fuzz run decode +# See README.md for target descriptions and corpus seeding. + +[package] +name = "azure_data_cosmos_driver-fuzz" +version = "0.0.0" +publish = false +edition = "2021" +license = "MIT" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +serde_json = "1" + +[dependencies.azure_data_cosmos_driver] +path = ".." + +[[bin]] +name = "decode" +path = "fuzz_targets/decode.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "from_slice" +path = "fuzz_targets/from_slice.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "transcode_to_text" +path = "fuzz_targets/transcode_to_text.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "decode_reencode_roundtrip" +path = "fuzz_targets/decode_reencode_roundtrip.rs" +test = false +doc = false +bench = false + +# Isolate this crate from the parent workspace (nightly + libFuzzer only). +[workspace] diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md new file mode 100644 index 00000000000..e7a3cdf8f5d --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md @@ -0,0 +1,174 @@ +# Binary JSON codec fuzzing (`cargo-fuzz`) + +Coverage-guided, **byte-level** fuzzing for the Cosmos binary JSON codec +(`azure_data_cosmos_driver::binary_json`). Where the live +[round-trip fuzzer](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs) +generates random JSON *values* and only ever feeds the decoder **encoder-produced** +(well-formed) bytes, these targets feed **arbitrary and mutated bytes** straight +into the decoder β€” so they exercise the *format*/protocol itself: truncated +buffers, bad length prefixes, unknown or misused markers, reference/depth bombs, +non-UTF-8 string payloads, and trailing bytes. + +This is a **separate crate** with its own empty `[workspace]` in `Cargo.toml`, so +it stays isolated from the stable repo workspace: cargo-fuzz builds it on nightly +with libFuzzer. + +## Prerequisites + +```bash +rustup toolchain install nightly +cargo install cargo-fuzz +``` + +## Targets + +| Target | Entry point | What it checks | +| --- | --- | --- | +| `decode` | `binary_json::decode` | `Value` decode never panics/hangs/over-allocates on any bytes. | +| `from_slice` | `binary_json::from_slice::` | Native serde streaming decode honors the same no-crash contract. | +| `transcode_to_text` | `binary_json::transcode_to_text` | Driver-side binaryβ†’text response transcode never panics on a malformed body. | +| `decode_reencode_roundtrip` | `decode` + `encode` | **Differential**: any buffer the decoder accepts must satisfy `decode(encode(decode(x))) == decode(x)` β€” catches reader/writer disagreements. | + +All four assert the **robustness oracle**: for *any* input the codec terminates +and returns `Ok`/`Err` β€” never panics, hangs, or allocates beyond the buffer. +The last one adds a **semantic** oracle on decoder-accepted inputs. + +## Running + +From this `fuzz/` directory (or the driver crate root): + +```bash +# Explore one target (Ctrl-C to stop): +cargo +nightly fuzz run decode + +# Time-boxed CI-style smoke run (60s), 4 workers: +cargo +nightly fuzz run decode -- -max_total_time=60 -workers=4 + +# Reproduce a crash from a saved artifact: +cargo +nightly fuzz run decode fuzz/artifacts/decode/crash- + +# Minimize a crashing input: +cargo +nightly fuzz tmin decode fuzz/artifacts/decode/crash- +``` + +## Thorough manual run on a Linux VM + +Weekly CI only replays the committed corpus once (`-runs=0`, no mutation). To +perform coverage-guided mutation and deeper fuzzing, run it by hand on any Linux +box (or WSL2), without a wall-clock cap: + +```bash +# 1. Toolchain (one-time) +rustup toolchain install nightly --component rust-src +cargo install cargo-fuzz --locked + +# 2. Get the code and seed the corpus from the golden vectors (recommended β€” +# lets libFuzzer mutate outward from real wire frames). +cd sdk/cosmos/azure_data_cosmos_driver +mkdir -p fuzz/corpus/decode +jq -r '.[] | "\(.name) \(.binary)"' testdata/binary_json_vectors.json | +while read -r name hex; do + echo "$hex" | tr -d ' ' | xxd -r -p > "fuzz/corpus/decode/$name" +done + +# 3a. Run one target for a fixed budget (e.g. 1 hour), 8 parallel workers: +cargo +nightly fuzz run decode -- -max_total_time=3600 -workers=8 -jobs=8 -print_final_stats=1 + +# 3b. Or run it open-ended until you Ctrl-C (a true soak): +cargo +nightly fuzz run decode -- -workers=8 -jobs=8 + +# 4. Repeat for the other targets (they share the same corpus format): +cargo +nightly fuzz run from_slice -- -max_total_time=3600 -workers=8 +cargo +nightly fuzz run transcode_to_text -- -max_total_time=3600 -workers=8 +cargo +nightly fuzz run decode_reencode_roundtrip -- -max_total_time=3600 -workers=8 + +# 5. Or drive all four with the CI helper (installs deps, seeds corpus, runs each): +pwsh ../eng/scripts/Run-BinaryJsonFuzz.ps1 -MaxTotalTimeSeconds 3600 -Workers 8 +``` + +**If a crash is found**, libFuzzer writes the triggering input to +`fuzz/artifacts//crash-`. Reproduce and minimize it: + +```bash +cargo +nightly fuzz run decode fuzz/artifacts/decode/crash- # reproduce +cargo +nightly fuzz tmin decode fuzz/artifacts/decode/crash- # minimize +``` + +Then add the minimized input as a golden vector / unit test in +`src/binary_json/` and fix the codec. The **corpus in `fuzz/corpus//` +persists across runs** β€” keep it (or copy it between machines) to accelerate +subsequent sessions. + +Sizing guidance: `job time β‰ˆ 1 min (compile) + N_targets Γ— per-target budget`. +On an 8-vCPU VM, `-workers=8` roughly 2Γ— the throughput seen in CI (~3.4K +exec/s/worker in the first run), so a 1-hour/target soak explores tens of +millions of inputs per target. + +## Seeding the corpus from the golden vectors + +Seeding libFuzzer with **valid** frames lets it mutate outward from real wire +shapes and reach the interesting error paths far faster than blind byte flips. +The [golden vectors](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json) already contain every +marker family as space-separated hex. Materialize them into the `decode` corpus: + +PowerShell: + +```powershell +$dir = "fuzz/corpus/decode"; New-Item -ItemType Directory -Force $dir | Out-Null +(Get-Content ../testdata/binary_json_vectors.json | ConvertFrom-Json) | ForEach-Object { + $bytes = $_.binary -split '\s+' | ForEach-Object { [Convert]::ToByte($_, 16) } + [IO.File]::WriteAllBytes("$dir/$($_.name)", [byte[]]$bytes) +} +``` + +bash + jq + xxd: + +```bash +mkdir -p fuzz/corpus/decode +jq -r '.[] | "\(.name) \(.binary)"' ../testdata/binary_json_vectors.json | +while read -r name hex; do + echo "$hex" | tr -d ' ' | xxd -r -p > "fuzz/corpus/decode/$name" +done +``` + +The same corpus works for `from_slice`, `transcode_to_text`, and +`decode_reencode_roundtrip` (all consume raw binary buffers); copy or point +`--corpus` at `fuzz/corpus/decode`. + +## Notes + +- `corpus/`, `artifacts/`, and `target/` are git-ignored (regenerated locally / in CI). +- These targets are **offline** (no live account), so they are cheap enough to + run in CI as a nightly job or a time-boxed smoke check on PRs touching + `binary_json`. +- A reproducible crash should be reduced with `cargo fuzz tmin`, added as a + golden vector / unit test in `src/binary_json/`, and fixed there. + +## Windows + +`cargo-fuzz` builds on **libFuzzer** (`-fsanitize=fuzzer`), which the Windows +MSVC target does not support β€” `cargo fuzz run` fails to link on Windows. Use +**WSL2** or a **Linux** box. On Windows, the always-on decoder robustness +coverage lives in `src/binary_json/fuzz_tests.rs` (random / truncated / corrupted +buffers into `decode`) and runs on stable via `cargo test -p +azure_data_cosmos_driver --lib fuzz`. + +## CI + +Fuzzing runs as a **non-blocking leg of the existing `sdk/cosmos/ci.yml`** β€” a +Build-stage `MatrixConfigs` entry ([`sdk/cosmos/fuzz-matrix.json`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/fuzz-matrix.json)) +that adds one **Linux + nightly** job (cargo-fuzz/libFuzzer is Linux-only), gated +to the **weekly / scheduled** build only (not per-PR). It carries +`ContinueOnError: "true"`, so a discovered crash reports "succeeded with issues" +instead of blocking merge. The job's test-setup hook +([`Invoke-CosmosTestSetup.ps1`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1), +gated on `AZURE_COSMOS_FUZZ=1`) calls +[`Run-BinaryJsonFuzz.ps1`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1) +**with `-ValidateOnly`**, which installs cargo-fuzz, seeds each corpus from the +golden vectors, and **replays the committed vectors once** (libFuzzer `-runs=0`, +no mutation, no time budget) to prove they still decode without panicking. + +Coverage-guided mutation soaks (`-max_total_time`) are **manual / local only** β€” +CI never runs an unattended time-boxed soak (see the manual-run section above). +Crash inputs are published as the `fuzz-crashes` build artifact so a failure can +be reproduced and minimized (`cargo fuzz tmin`). diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs new file mode 100644 index 00000000000..d5d158df996 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the binary-JSON **decoder**. +//! +//! libFuzzer feeds arbitrary (and, once seeded, mutated-from-valid) byte +//! buffers straight into [`decode`]. This is the format fuzzer the live +//! round-trip test can't be: it explores mis-encoded frames β€” truncated +//! buffers, bad length prefixes, unknown/misused markers, reference and +//! depth bombs, non-UTF-8 string payloads, trailing bytes β€” that the encoder +//! never produces. +//! +//! Oracle: for **any** input the decoder must terminate and return either +//! `Ok(Value)` or `Err(BinaryError)` β€” never panic, hang, or allocate beyond +//! what the buffer can back. A crash or hang here is a decoder-hardening bug. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::decode; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = decode(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs new file mode 100644 index 00000000000..661bc1a9349 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Differential fuzz target: decode β†’ encode β†’ decode idempotence. +//! +//! Unlike the plain `decode` no-crash target, this asserts a **semantic** +//! invariant on every buffer the decoder *accepts*: re-encoding the decoded +//! value and decoding it again must reproduce the exact same value. It catches +//! the class of bug the live round-trip fuzzer cannot β€” a buffer the decoder +//! accepts but the encoder would round-trip to a *different* value (marker or +//! number-form disagreements between the reader and writer). libFuzzer's +//! mutation reaches decoder-accepted-but-unusual frames that hand-written +//! golden vectors don't enumerate. +//! +//! Oracle: `decode(data) = Ok(v)` β‡’ `decode(encode(v)) = Ok(v)`. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::{decode, encode}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + if let Ok(value) = decode(data) { + let reencoded = encode(&value); + let redecoded = + decode(&reencoded).expect("re-encoding a decoded value must itself decode"); + assert_eq!( + value, redecoded, + "decode∘encode∘decode is not idempotent for a decoder-accepted buffer" + ); + } +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs new file mode 100644 index 00000000000..6c0046c654c --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the native serde **deserializer**. +//! +//! [`from_slice`] is the zero-`Value` streaming decode path used by the SDK's +//! typed reads; it drives a different code path from [`decode`] (it streams +//! tokens into a serde visitor instead of materializing a +//! [`serde_json::Value`]). Fuzzing it independently ensures the streaming +//! deserializer honors the same no-crash contract on malformed input. +//! +//! Oracle: for any input, deserialization must terminate with `Ok`/`Err` β€” +//! never panic, hang, or over-allocate. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::from_slice; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = from_slice::(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs new file mode 100644 index 00000000000..199c0371196 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the driver-side response transcode. +//! +//! [`transcode_to_text`] is what the driver runs on a binary response body when +//! a text-only host asked for text back: it decodes the binary buffer and +//! re-serializes it as UTF-8 text JSON (or passes text/empty input through +//! unchanged). It sits on the FFI/text-host response path, so a panic here on a +//! malformed service body would take down the host. +//! +//! Oracle: for any input, transcoding must terminate with `Ok`/`Err` β€” never +//! panic, hang, or over-allocate. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::transcode_to_text; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = transcode_to_text(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs new file mode 100644 index 00000000000..2f70f89b185 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Encode-direction conformance tests for the Cosmos binary JSON codec. +//! +//! These tests implement the encoder conformance requirements of the +//! wire-format spec (`docs/BINARY_ENCODING_RFC.md` Β§7 "Canonical encoding" and +//! Appendix A "Golden test vectors"). Decoder conformance (Β§8) is covered by the +//! per-form tests in [`reader`](super::reader) and [`de`](super::de); this +//! module fills the previously-missing encode side. +//! +//! Two guarantees are asserted: +//! +//! 1. **Round-trip validity** β€” for every golden-corpus value, `decode(encode(v)) +//! == v`. The encoder MUST emit a valid buffer that decodes back to the input +//! (RFC Β§7: the encoder emits a conformant *subset* of the wire forms; the +//! decoder accepts all of them). +//! 2. **Canonical output snapshots** β€” the encoder is deterministic, so its exact +//! bytes for representative values are pinned as regression snapshots. This +//! documents the Rust encoder's actual canonical form, which is a valid +//! subset that does **not** use the most compact forms (system strings, +//! `Arr0`/`Arr1`, narrowest `Number*`, etc.) β€” see the notes below. +//! +//! The module is compiled only under `cfg(test)`. + +use super::{decode, encode, is_binary, PREAMBLE}; +use serde_json::{json, Value}; + +/// Parses a spaced-hex string (e.g. `"80 D0"`) into bytes, matching the corpus +/// notation used throughout the RFC. +fn hex(s: &str) -> Vec { + s.split_whitespace() + .map(|b| u8::from_str_radix(b, 16).expect("valid hex byte")) + .collect() +} + +/// RFC Β§7 (round-trip validity): every value in the shared golden corpus MUST +/// re-encode to a buffer that decodes back to the identical value. This is the +/// encode-direction counterpart to `reader::decodes_golden_corpus`. +#[test] +fn encode_round_trips_golden_corpus() { + for vector in super::vectors::golden_vectors() { + let expected: Value = + serde_json::from_str(&vector.json).expect("corpus json is valid JSON"); + let encoded = encode(&expected); + assert!( + is_binary(&encoded), + "{}: encoder output missing preamble", + vector.name + ); + let decoded = decode(&encoded) + .unwrap_or_else(|e| panic!("{}: re-encoded buffer failed to decode: {e}", vector.name)); + assert_eq!( + decoded, expected, + "{}: encodeβ†’decode did not round-trip", + vector.name + ); + } +} + +/// RFC Β§7 (deterministic canonical output): the encoder emits exactly these +/// bytes for representative values. These snapshots are the regression bar for +/// the Rust encoder's canonical form. +/// +/// Note the encoder deliberately emits a **valid but non-minimal subset** of the +/// wire forms (RFC Β§7): integers outside `[0,31]` use `Int64`/`UInt64` (never the +/// narrower `NumberUInt8`/`Int16`/`Int32`), strings use the encoded-length or +/// `StrL*` forms (never system/user/compressed strings), and containers always +/// use the `LC*` length+count framing (never `Arr0`/`Arr1`/`Obj0`/`Obj1`). The +/// decoder accepts the compact forms the service may emit; the encoder need not +/// produce them. +#[test] +fn encode_produces_expected_canonical_bytes() { + let cases: &[(Value, &str)] = &[ + // Singletons. + (json!(null), "80 D0"), + (json!(false), "80 D1"), + (json!(true), "80 D2"), + // Literal small integers (value == marker), 0..=31. + (json!(0), "80 00"), + (json!(31), "80 1F"), + // Integers outside [0,31] use Int64 (not the narrower Number* forms). + (json!(32), "80 CB 20 00 00 00 00 00 00 00"), + (json!(200), "80 CB C8 00 00 00 00 00 00 00"), + (json!(-5), "80 CB FB FF FF FF FF FF FF FF"), + // Values above i64::MAX use UInt64. + ( + json!(18446744073709551614u64), + "80 C7 FE FF FF FF FF FF FF FF", + ), + // Non-integral numbers use NumberDouble. + (json!(3.5), "80 CC 00 00 00 00 00 00 0C 40"), + // Strings ≀ 63 bytes use the encoded-length form (length baked into the + // marker), including "hello" β€” the corpus stores it as StrL1, but the + // encoder's canonical form is encoded-length. + (json!(""), "80 80"), + (json!("hi"), "80 82 68 69"), + (json!("hello"), "80 85 68 65 6C 6C 6F"), + // Containers always use LC* framing (byte-length + count). + (json!([]), "80 E5 00 00"), + (json!([true]), "80 E5 01 01 D2"), + (json!({}), "80 ED 00 00"), + (json!({"id": 0}), "80 ED 04 01 82 69 64 00"), + ]; + + for (value, expected_hex) in cases { + let encoded = encode(value); + let expected = hex(expected_hex); + assert_eq!( + encoded, expected, + "encoder output for {value} did not match the canonical snapshot\n expected: {expected_hex}\n actual: {}", + spaced_hex(&encoded), + ); + } +} + +/// RFC Β§7 (valid subset): where the golden corpus stores a **compact** wire form +/// the encoder does not emit (system strings, `Arr0`, `NumberUInt8`, …), the +/// encoder's own output differs byte-wise but still decodes to the same value. +/// This pins the intentional asymmetry so a future "make the encoder compact" +/// change is a conscious decision rather than a silent regression. +#[test] +fn encoder_emits_valid_subset_for_compact_corpus_forms() { + // (value, the corpus's compact encoding) β€” the encoder produces a *different* + // buffer, but both decode to `value`. + let compact_cases: &[(Value, &str)] = &[ + (json!(200), "80 C8 C8"), // corpus: NumberUInt8 + (json!("id"), "80 2C"), // corpus: system string + (json!([]), "80 E0"), // corpus: Arr0 + (json!({}), "80 E8"), // corpus: Obj0 + (json!([true]), "80 E1 D2"), // corpus: Arr1 + ( + json!([1, 2, 3]), + "80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00", + ), // uniform array + ]; + + for (value, compact_hex) in compact_cases { + let compact = hex(compact_hex); + // The compact form is valid and decodes to `value` ... + assert_eq!( + decode(&compact).unwrap(), + *value, + "compact corpus form {compact_hex} did not decode to {value}", + ); + // ... but the encoder emits a different (verbose) buffer. + let encoded = encode(value); + assert_ne!( + encoded, compact, + "encoder unexpectedly produced the compact form for {value}; update this test if the encoder was made compact", + ); + // ... which still decodes to the same value. + assert_eq!( + decode(&encoded).unwrap(), + *value, + "encoder's verbose form for {value} did not round-trip", + ); + } +} + +/// RFC Β§3.1: a complete buffer begins with the preamble and the encoder always +/// emits it. +#[test] +fn encoder_always_emits_preamble() { + for value in [ + json!(null), + json!(1), + json!("x"), + json!([1]), + json!({"a": 1}), + ] { + let encoded = encode(&value); + assert_eq!( + encoded.first(), + Some(&PREAMBLE), + "missing preamble for {value}" + ); + } +} + +/// Formats bytes as spaced uppercase hex for assertion messages. +fn spaced_hex(bytes: &[u8]) -> String { + bytes + .iter() + .map(|b| format!("{b:02X}")) + .collect::>() + .join(" ") +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs index b3607cd6a7b..7e2ffb3a72d 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs @@ -36,6 +36,8 @@ pub mod ser; pub mod system_strings; pub mod writer; +#[cfg(test)] +mod conformance; #[cfg(test)] mod fuzz_tests; #[cfg(test)] diff --git a/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json b/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json index 84515b918d5..d971b646866 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json +++ b/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json @@ -19,6 +19,10 @@ { "name": "float32", "binary": "80 CD 00 00 C0 3F", "json": 1.5 }, { "name": "float64", "binary": "80 CE 00 00 00 00 00 00 02 C0", "json": -2.25 }, { "name": "system_string_id", "binary": "80 2C", "json": "id" }, + { "name": "system_string_first", "binary": "80 20", "json": "$s" }, + { "name": "system_string_rid", "binary": "80 25", "json": "_rid" }, + { "name": "system_string_geometry_collection", "binary": "80 2B", "json": "GeometryCollection" }, + { "name": "system_string_last", "binary": "80 3F", "json": "_id" }, { "name": "encoded_length_string_empty", "binary": "80 80", "json": "" }, { "name": "encoded_length_string_hi", "binary": "80 82 68 69", "json": "hi" }, { "name": "str_l1_hello", "binary": "80 C0 05 68 65 6C 6C 6F", "json": "hello" }, @@ -51,6 +55,10 @@ { "name": "uniform_empty", "binary": "80 F0 DA 00", "json": [] }, { "name": "uniform_int16_c2", "binary": "80 F1 D9 03 00 FF FF 00 00 E8 03", "json": [-1,0,1000] }, { "name": "uniform_float32", "binary": "80 F0 CD 02 00 00 C0 3F 00 00 80 BE", "json": [1.5,-0.25] }, + { "name": "uniform_int8", "binary": "80 F0 D8 03 FB 00 05", "json": [-5,0,5] }, + { "name": "uniform_int64", "binary": "80 F0 DB 02 01 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00", "json": [1,2] }, + { "name": "uniform_uint32", "binary": "80 F0 DC 02 FE FF FF FF 00 00 00 00", "json": [4294967294,0] }, + { "name": "uniform_float64", "binary": "80 F0 CE 02 00 00 00 00 00 00 F8 3F 00 00 00 00 00 00 02 C0", "json": [1.5,-2.25] }, { "name": "uniform_arr_of_arr", "binary": "80 F2 F0 DA 02 02 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00", "json": [[1,2],[3,4]] }, { "name": "empty_array", "binary": "80 E0", "json": [] }, { "name": "empty_object", "binary": "80 E8", "json": {} }, @@ -60,5 +68,21 @@ { "name": "array_lc1", "binary": "80 E5 03 03 00 01 D0", "json": [0,1,null] }, { "name": "object_l1", "binary": "80 EA 04 2C 00 3B 01", "json": {"id":0,"type":1} }, { "name": "object_lc1", "binary": "80 ED 04 02 2C 00 3B 01", "json": {"id":0,"type":1} }, - { "name": "nested_containers", "binary": "80 E2 05 E1 00 E9 2C 01", "json": [[0],{"id":1}] } + { "name": "nested_containers", "binary": "80 E2 05 E1 00 E9 2C 01", "json": [[0],{"id":1}] }, + { "name": "str_l4_hello", "binary": "80 C2 05 00 00 00 68 65 6C 6C 6F", "json": "hello" }, + { "name": "base64_url_len2", "binary": "80 74 01 00 00 FB FF FE", "json": "-__-" }, + { "name": "binary_4byte", "binary": "80 DF 04 00 00 00 DE AD BE EF", "json": "3q2+7w==" }, + { "name": "array_l2", "binary": "80 E3 03 00 00 01 D0", "json": [0,1,null] }, + { "name": "array_l4", "binary": "80 E4 03 00 00 00 00 01 D0", "json": [0,1,null] }, + { "name": "array_lc2", "binary": "80 E6 03 00 03 00 00 01 D0", "json": [0,1,null] }, + { "name": "array_lc4", "binary": "80 E7 03 00 00 00 03 00 00 00 00 01 D0", "json": [0,1,null] }, + { "name": "object_l2", "binary": "80 EB 04 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "object_l4", "binary": "80 EC 04 00 00 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "object_lc2", "binary": "80 EE 04 00 02 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "object_lc4", "binary": "80 EF 04 00 00 00 02 00 00 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "uniform_arr_of_arr_c2c2", "binary": "80 F3 F1 DA 02 00 02 00 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00", "json": [[1,2],[3,4]] }, + { "name": "reference_string_r1", "binary": "80 E2 05 82 68 69 C3 03", "json": ["hi","hi"] }, + { "name": "reference_string_r2", "binary": "80 E2 06 82 68 69 C4 03 00", "json": ["hi","hi"] }, + { "name": "reference_string_r3", "binary": "80 E2 07 82 68 69 C5 03 00 00", "json": ["hi","hi"] }, + { "name": "reference_string_r4", "binary": "80 E2 08 82 68 69 C6 03 00 00 00", "json": ["hi","hi"] } ] \ No newline at end of file diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_account_metadata_failover.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_account_metadata_failover.rs index 1d74f6030db..5b679a6bd33 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_account_metadata_failover.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/emulator_tests/driver_account_metadata_failover.rs @@ -14,6 +14,10 @@ use std::sync::Arc; /// Persistent 503 on GET / must surface as upstream HTTP status β€” never relabeled as /// `SERIALIZATION_RESPONSE_BODY_INVALID` ("missing field `_self`"). #[tokio::test] +#[cfg_attr( + not(any(test_category = "emulator", test_category = "emulator_vnext")), + ignore = "requires test_category 'emulator' or 'emulator_vnext'" +)] pub async fn account_metadata_503_surfaces_as_status_error() -> Result<(), Box> { // Persistent 503 on every MetadataReadDatabaseAccount so the first lazy fetch // (via create_driver) hits the fault. diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index ca60f0f2c0c..b08e8c43aa6 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -49,6 +49,13 @@ extends: EnvVars: RUST_TEST_THREADS: '1' RUST_BACKTRACE: '1' + # Live budget for the binary-encoding round-trip fuzzer + # (azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs). Only consumed on + # the `binary_encoding` live leg (live-platform-matrix.json); a no-op + # elsewhere. Each iteration exercises 3 encoding configs x 4 point ops, so + # 200 => ~2400 round-trips β€” bounded to fit the live-test time cap. Bump it + # for a deeper soak. + AZURE_COSMOS_FUZZ_ITERATIONS: '200' AZURE_COSMOS_GW_V2_ENDPOINT: $(thinclient-test-endpoint) AZURE_COSMOS_GW_V2_KEY: $(thinclient-test-key) AZURE_COSMOS_GW_V2_MULTI_REGION_ENDPOINT: $(thin-client-canary-multi-region-session-endpoint) @@ -61,6 +68,31 @@ extends: Selection: sparse NonSparseParameters: RustToolchainName GenerateVMJobs: true + # Non-blocking, byte-level codec fuzzing (cargo-fuzz). Adds a single + # Linux + nightly job to the **Build stage** (MatrixConfigs is consumed + # only by the Build stage, unlike AdditionalMatrixConfigs which the + # archetype also appends to the Live Test stage). Its test-setup hook + # (Invoke-CosmosTestSetup.ps1, gated on AZURE_COSMOS_FUZZ=1) seeds each + # fuzz target's corpus from the golden vectors and replays it once + # (libFuzzer -runs=0) to prove the committed vectors still decode without + # panicking. cargo-fuzz/libFuzzer only works on Linux + nightly, so it + # cannot ride the cross-platform matrix. The entry carries + # `ContinueOnError: "true"`, so a discovered crash surfaces as "succeeded + # with issues" rather than blocking merge; crash inputs are published as + # build artifacts. Runner: eng/scripts/Run-BinaryJsonFuzz.ps1. + # + # WEEKLY ONLY: gated to the scheduled/weekly run so it does not run on the + # default PR gate (the golden vectors are already covered on every PR by + # the offline `decodes_golden_corpus` / `encode_round_trips_golden_corpus` + # unit tests in the normal cross-platform matrix). This mirrors the + # `RunLiveTests` condition above. Net: the byte-level codec validation + # runs in the weekly **Build** stage, while the value-space round-trip + # fuzzer runs in the weekly **Live Test** stage (live-platform-matrix.json). + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), endsWith(variables['Build.DefinitionName'], '- weekly')) }}: + - Name: Cosmos_binary_codec_golden_vector_validation + Path: sdk/cosmos/fuzz-matrix.json + Selection: all + GenerateVMJobs: true # Emulator jobs run only in Build. `MatrixConfigs` (unlike # `AdditionalMatrixConfigs`) is never consumed by the live-test stage in # archetype-sdk-client.yml, so these entries don't enter live stages, diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 index 296276565fb..acb035b638c 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 @@ -14,6 +14,40 @@ if ($env:COSMOS_RUSTFLAGS) { Write-Host "RUSTFLAGS appended with COSMOS_RUSTFLAGS: $env:RUSTFLAGS" } +# Byte-level binary-JSON codec fuzzing (cargo-fuzz). Triggered by +# AZURE_COSMOS_FUZZ=1 (set as a matrix variable on the fuzz leg in +# sdk/cosmos/fuzz-matrix.json β€” a Linux + nightly job). On the Build stage this +# runs as a FAST golden-vector regression: it replays only the committed golden +# vectors through every codec fuzz target (libFuzzer -runs=0, no mutation, no +# time budget) to prove they still decode without panicking. The long, +# coverage-guided value-space soak lives in the live round-trip fuzzer +# (azure_data_cosmos_perf) instead β€” a time-boxed byte soak here previously +# overran the archetype's 90-minute TestTimeoutInMinutes. cargo-fuzz/libFuzzer is +# Linux-only, and the leg carries ContinueOnError=true so a crash reports +# "succeeded with issues" rather than blocking merge. Guarded so it runs once +# even though Test-Setup.ps1 fires per crate. +if ($env:AZURE_COSMOS_FUZZ -eq '1' -and -not $env:AZURE_COSMOS_FUZZ_RAN) { + $env:AZURE_COSMOS_FUZZ_RAN = '1' + if (-not $IsLinux) { + Write-Host "AZURE_COSMOS_FUZZ=1 but not on Linux; cargo-fuzz is Linux-only. Skipping." + } + else { + Write-Host "==> Cosmos binary-JSON fuzz: golden-vector corpus validation (-runs=0)" + & "$PSScriptRoot\Run-BinaryJsonFuzz.ps1" -ValidateOnly + } + # A fuzz leg does no live testing. Strip any test_category cfg that + # COSMOS_RUSTFLAGS may have injected (e.g. emulator) so the archetype's + # subsequent cargo build/test on this leg compiles and runs only the + # always-on offline unit tests β€” never emulator/live-gated tests, which + # would panic here with no connection string. Then skip the rest of the + # setup (no emulator, no connection string). + if ($env:RUSTFLAGS -match 'test_category') { + $env:RUSTFLAGS = ($env:RUSTFLAGS -replace '--cfg[= ]test_category="[^"]*"', '' -replace '\s+', ' ').Trim() + Write-Host "Stripped test_category from RUSTFLAGS on fuzz leg: '$env:RUSTFLAGS'" + } + return +} + # Hosted in-memory emulator path. The additional CI matrix sets one of the two # flavors below so the existing emulator suites run against both Gateway V1 # and Gateway 2.0 over cleartext HTTP/2. diff --git a/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 b/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 new file mode 100644 index 00000000000..542682fd26d --- /dev/null +++ b/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 @@ -0,0 +1,132 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Runs the coverage-guided cargo-fuzz targets for the Cosmos binary-JSON codec. + +.DESCRIPTION + Installs (idempotently) the nightly toolchain + cargo-fuzz, seeds each + target's corpus from the committed golden vectors, then runs every fuzz + target under `azure_data_cosmos_driver/fuzz` for a bounded wall-clock budget. + + This is a byte-level *protocol* fuzzer: it feeds arbitrary/mutated bytes + straight into the decoder (decode / from_slice / transcode_to_text) plus a + differential decode->encode->decode idempotence check. It is the complement + to the value-space live round-trip fuzzer in + azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs. + + LINUX ONLY. libFuzzer (the -fsanitize=fuzzer backend) is not supported on the + Windows MSVC target, so this script no-ops with a warning off Linux. + +.PARAMETER MaxTotalTimeSeconds + Wall-clock budget PER TARGET passed to libFuzzer as `-max_total_time`. + PR smoke runs use ~90s; weekly deep runs use ~1800s (30 min). Ignored when + -ValidateOnly is set. + +.PARAMETER ValidateOnly + Regression mode: replay ONLY the seeded golden-vector corpus through each + target once (libFuzzer `-runs=0`, no mutation, no time budget) and assert no + crash. This is a fast, deterministic gate suitable for the Build stage β€” it + proves the committed golden vectors still decode without panicking, without + the multi-minute coverage-guided soak (which belongs on a live/weekly leg). + +.PARAMETER Toolchain + Nightly toolchain to use. Defaults to the repo's pinned nightly if the + RUST_NIGHTLY_TOOLCHAIN env var is set, else plain `nightly`. + +.PARAMETER Targets + Which fuzz targets to run. Defaults to all four. + +.EXAMPLE + ./Run-BinaryJsonFuzz.ps1 -MaxTotalTimeSeconds 90 # PR smoke + ./Run-BinaryJsonFuzz.ps1 -MaxTotalTimeSeconds 1800 # weekly deep run +#> +[CmdletBinding()] +param( + [int] $MaxTotalTimeSeconds = 90, + [switch] $ValidateOnly, + [string] $Toolchain = $(if ($env:RUST_NIGHTLY_TOOLCHAIN) { $env:RUST_NIGHTLY_TOOLCHAIN } else { 'nightly' }), + [string[]] $Targets = @('decode', 'from_slice', 'transcode_to_text', 'decode_reencode_roundtrip'), + [int] $Workers = 0 # 0 => libFuzzer default (single); CI can raise it +) + +$ErrorActionPreference = 'Stop' + +if (-not $IsLinux) { + Write-Warning "cargo-fuzz / libFuzzer is only supported on Linux; skipping on this OS. Use WSL or a Linux CI leg." + exit 0 +} + +$driverDir = Resolve-Path "$PSScriptRoot/../../azure_data_cosmos_driver" +$fuzzDir = Join-Path $driverDir 'fuzz' +$vectorsPath = Join-Path $driverDir 'testdata/binary_json_vectors.json' + +Write-Host "==> Ensuring nightly toolchain '$Toolchain' + cargo-fuzz" +rustup toolchain install $Toolchain --profile minimal --component rust-src +# Detect cargo-fuzz via the installed-binaries list (always exits 0). Probing +# with `cargo fuzz --help` would exit non-zero when missing and, under +# $ErrorActionPreference='Stop' on PowerShell 7.4+, abort before we can install. +$fuzzInstalled = (cargo "+$Toolchain" install --list 2>$null) -match 'cargo-fuzz' +if (-not $fuzzInstalled) { + Write-Host "==> cargo-fuzz not found; installing" + cargo "+$Toolchain" install cargo-fuzz --locked +} + +# Seed each target's corpus from the committed golden vectors so libFuzzer +# mutates outward from real wire frames instead of blind byte flips. +if (Test-Path $vectorsPath) { + Write-Host "==> Seeding corpora from golden vectors" + $vectors = Get-Content $vectorsPath -Raw | ConvertFrom-Json + foreach ($t in $Targets) { + $corpus = Join-Path $fuzzDir "corpus/$t" + New-Item -ItemType Directory -Force $corpus | Out-Null + foreach ($v in $vectors) { + $bytes = ($v.binary -split '\s+' | ForEach-Object { [Convert]::ToByte($_, 16) }) + [IO.File]::WriteAllBytes((Join-Path $corpus $v.name), [byte[]]$bytes) + } + } +} + +Push-Location $driverDir +try { + $failed = @() + foreach ($t in $Targets) { + if ($ValidateOnly) { + # `-runs=0` replays the (golden-vector-seeded) corpus once with no + # mutation and exits: a fast, deterministic "do the golden vectors + # still decode without panicking?" regression gate. + Write-Host "==> Validating '$t' against the golden-vector corpus (-runs=0)" + $runArgs = @("+$Toolchain", 'fuzz', 'run', $t, '--', '-runs=0', '-print_final_stats=1') + } + else { + Write-Host "==> Fuzzing '$t' for ${MaxTotalTimeSeconds}s" + $runArgs = @("+$Toolchain", 'fuzz', 'run', $t, '--', "-max_total_time=$MaxTotalTimeSeconds", '-print_final_stats=1') + if ($Workers -gt 0) { $runArgs += "-workers=$Workers"; $runArgs += "-jobs=$Workers" } + } + & cargo @runArgs + if ($LASTEXITCODE -ne 0) { + $failed += $t + Write-Host "##vso[task.logissue type=error]Fuzz target '$t' found a crash. Minimize with: cargo +$Toolchain fuzz tmin $t " + } + } + + # Publish any crash inputs so a failure can be reproduced off-agent. + $artifactsDir = Join-Path $fuzzDir 'artifacts' + if (Test-Path $artifactsDir) { + $crashes = Get-ChildItem $artifactsDir -Recurse -File -ErrorAction SilentlyContinue + if ($crashes -and $env:SYSTEM_TEAMPROJECTID) { + Write-Host "##vso[task.logissue type=warning]Fuzz crash artifacts found; published as 'fuzz-crashes'." + Write-Host "##vso[artifact.upload artifactname=fuzz-crashes]$((Resolve-Path $artifactsDir).Path)" + } + } + + if ($failed.Count -gt 0) { + throw "Fuzz targets reported crashes: $($failed -join ', ')" + } + Write-Host "==> All fuzz targets clean." +} +finally { + Pop-Location +} diff --git a/sdk/cosmos/fuzz-matrix.json b/sdk/cosmos/fuzz-matrix.json new file mode 100644 index 00000000000..3f9b1025d9b --- /dev/null +++ b/sdk/cosmos/fuzz-matrix.json @@ -0,0 +1,18 @@ +{ + "displayNames": { + "golden": "binary_codec_golden_vector_validation", + "1": "", + "true": "" + }, + "matrix": { + "Agent": { + "golden": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "RustToolchainName": ["nightly"], + "AZURE_COSMOS_FUZZ": ["1"], + "ContinueOnError": ["true"] + } +} diff --git a/sdk/cosmos/live-platform-matrix.json b/sdk/cosmos/live-platform-matrix.json index 9b337474dc2..9c547a385eb 100644 --- a/sdk/cosmos/live-platform-matrix.json +++ b/sdk/cosmos/live-platform-matrix.json @@ -54,6 +54,20 @@ "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableMultipleRegions = $true; testCategory = 'gateway_v2_multi_region' }" } } + }, + { + "Agent": { + "ubuntu": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "RustToolchainName": ["stable"], + "Account Settings": { + "Session SingleWrite BinaryEncodingRoundtripFuzz": { + "ArmTemplateParameters": "@{ defaultConsistencyLevel = 'Session'; enableAutomaticFailover = $false; testCategory = 'binary_encoding' }" + } + } } ] }