From 10613dbb7bffd0adc8e28eb0b55fd62fecbfcbe5 Mon Sep 17 00:00:00 2001 From: kundadebdatta Date: Mon, 3 Aug 2026 12:16:06 -0700 Subject: [PATCH 1/2] test(cosmos): add byte-level binary JSON codec cargo-fuzz crate + spec Byte/wire-format validation for the Cosmos binary JSON codec, independent of any live account: - azure_data_cosmos_driver/fuzz: isolated cargo-fuzz crate (nightly/libFuzzer) with decode, from_slice, transcode_to_text, decode_reencode_roundtrip targets - binary_json/conformance.rs: offline golden-vector conformance unit tests - testdata/binary_json_vectors.json: golden byte vectors - fuzz-matrix.json + ci.yml MatrixConfigs: weekly Build-stage -runs=0 replay - Run-BinaryJsonFuzz.ps1 + Invoke-CosmosTestSetup.ps1 fuzz hook - BINARY_ENCODING_RFC.md: wire-format spec (pending backend-team review) Split out of the original combined PR (#4898); the live e2e round-trip fuzzer is delivered separately. --- eng/dict/crates.txt | 2 + sdk/cosmos/.cspell.json | 39 ++ .../cosmos_binary_encoding.rs | 20 +- .../docs/BINARY_ENCODING_RFC.md | 601 ++++++++++++++++++ .../azure_data_cosmos_driver/fuzz/.gitignore | 4 + .../azure_data_cosmos_driver/fuzz/Cargo.toml | 59 ++ .../azure_data_cosmos_driver/fuzz/README.md | 174 +++++ .../fuzz/fuzz_targets/decode.rs | 24 + .../fuzz_targets/decode_reencode_roundtrip.rs | 32 + .../fuzz/fuzz_targets/from_slice.rs | 22 + .../fuzz/fuzz_targets/transcode_to_text.rs | 22 + .../src/binary_json/conformance.rs | 188 ++++++ .../src/binary_json/mod.rs | 2 + .../testdata/binary_json_vectors.json | 26 +- sdk/cosmos/ci.yml | 13 + .../eng/scripts/Invoke-CosmosTestSetup.ps1 | 31 + sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 | 131 ++++ sdk/cosmos/fuzz-matrix.json | 18 + 18 files changed, 1399 insertions(+), 9 deletions(-) create mode 100644 sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md create mode 100644 sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore create mode 100644 sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml create mode 100644 sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md create mode 100644 sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs create mode 100644 sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs create mode 100644 sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs create mode 100644 sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs create mode 100644 sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs create mode 100644 sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 create mode 100644 sdk/cosmos/fuzz-matrix.json diff --git a/eng/dict/crates.txt b/eng/dict/crates.txt index e40fc74f776..b8fc2109c80 100644 --- a/eng/dict/crates.txt +++ b/eng/dict/crates.txt @@ -72,6 +72,8 @@ include_file include-file json_patch json-patch +libfuzzer_sys +libfuzzer-sys litemap openssl opentelemetry diff --git a/sdk/cosmos/.cspell.json b/sdk/cosmos/.cspell.json index 302974ee913..14c9ba74969 100644 --- a/sdk/cosmos/.cspell.json +++ b/sdk/cosmos/.cspell.json @@ -33,6 +33,7 @@ "cabi", "canadacentral", "canadaeast", + "CBOR", "cbindgen", "CDLL", "cdriver", @@ -52,6 +53,7 @@ "chinaeast", "chinanorth", "chokepoint", + "canonicalizer", "cloneable", "codepoint", "codepoints", @@ -172,6 +174,7 @@ "libazurecosmosdriver", "libclang", "LIBCLANG", + "libfuzzer", "libqueryplaninterop", "QUERYPLANINTEROP", "linearizability", @@ -256,10 +259,14 @@ "RAII", "readfeed", "recompiles", + "redecoded", + "reencode", + "reencoded", "refetch", "refetched", "refetches", "refcounted", + "reparses", "reparsed", "Replicaset", "reqs", @@ -284,6 +291,7 @@ "serviceunavailable", "serviceversion", "sess", + "Signedness", "southafricanorth", "southafricawest", "southcentralus", @@ -313,11 +321,13 @@ "testdb", "thinclient", "threadsafe", + "tmin", "TOCTOU", "Tokio", "TOPCOUNT", "TPIO", "transcoders", + "trippable", "uaecentral", "uaenorth", "udfs", @@ -374,6 +384,35 @@ "writability", "xorshift", "xpart", + "artifactname", + "fsanitize", + "logissue", + "MSVC", + "AFORM", + "ANID", + "Chenault", + "devtestcoll", + "ENOENT", + "errno", + "feff", + "Flacco", + "logdata", + "lossily", + "Millett", + "modhash", + "MSNH", + "MUID", + "muid", + "nametype", + "Qsml", + "recclass", + "reclat", + "reclong", + "similars", + "Unschematized", + "vout", + "Xpert", + "xpert", "yxxx" ], "ignorePaths": [ diff --git a/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs index d6dd4d95998..b11f7e9f77d 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs @@ -56,9 +56,13 @@ fn binary_encoding_options() -> TestOptions { } /// A document covering every JSON value shape the binary encoder emits: literal -/// and wide integers, an unsigned value beyond `i64::MAX`, a double, booleans, -/// `null`, unicode/empty strings, nested arrays and objects, and a vector of -/// objects. +/// and wide integers, a large unsigned value, a double, booleans, `null`, +/// unicode/empty strings, nested arrays and objects, and a vector of objects. +/// +/// Note: `huge` stays at or below `2^53` because the live Cosmos service +/// normalizes JSON numbers to IEEE-754 doubles. A value beyond `2^53` (e.g. +/// `u64::MAX`) is echoed back as a `Double` and can no longer be deserialized +/// into a `u64` field, so it does not round-trip against the real service. #[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] struct BinaryItem { id: String, @@ -95,11 +99,11 @@ fn sample_item(id: &str, partition_key: &str) -> BinaryItem { text: "hello binary".to_owned(), unicode: "café ☃ 𝄞 quotes:\" backslash:\\".to_owned(), empty: String::new(), - small_int: 7, // literal-int form (0..32) - big_int: 9_000_000_000, // Int64 form - negative: -1_234_567, // Int64 form - huge: u64::MAX, // UInt64 form (beyond i64::MAX) - ratio: 123.456_789, // Double form + small_int: 7, // literal-int form (0..32) + big_int: 9_000_000_000, // Int64 form + negative: -1_234_567, // Int64 form + huge: 9_007_199_254_740_992, // UInt64 form, exactly f64-representable (2^53) + ratio: 123.456_789, // Double form active: true, inactive: false, maybe: None, // null diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md new file mode 100644 index 00000000000..a65226b6ec5 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md @@ -0,0 +1,601 @@ +# Cosmos Binary JSON Encoding — Wire Format Specification + +**Status:** Draft · **Version:** 0.1 · **Audience:** SDK / codec implementers (Rust, .NET, C++, Java, Go, Python) + +> This document is a **normative, self-contained** description of the Cosmos +> Binary JSON wire format. A conforming encoder/decoder can be implemented from +> this document alone, without reference to any SDK source. It is modeled on the +> style of [RFC 8949 (CBOR)](https://datatracker.ietf.org/doc/html/rfc8949) and +> the [Protocol Buffers encoding guide](https://protobuf.dev/programming-guides/encoding/). +> +> **Provenance.** The wire constants are transcribed from the .NET reference +> implementation (`Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs`) +> and cross-checked against the Rust codec (`azure_data_cosmos_driver::binary_json`). +> Details that could not be confirmed from the Rust implementation alone are +> tagged **`[CROSS-VERIFY: .NET/C++]`** and MUST be validated against the .NET +> and C++ sources before this draft is promoted to a stable version. + +--- + +## 1. Introduction + +### 1.1 Purpose + +Cosmos Binary JSON is a compact, self-describing binary serialization of the +JSON data model used by Azure Cosmos DB for item request and response bodies. It +is **information-preserving with respect to the JSON value model** (null, +boolean, number, string, array, object) while being smaller and faster to +parse than UTF-8 JSON text. The service and every language SDK MUST agree on +this format byte-for-byte. + +### 1.2 Scope + +This specification defines: + +- the byte-level layout of every value kind (§3–§6), +- the **canonical** encoding a conforming encoder emits when multiple encodings + are valid (§7), +- decoder conformance requirements, including bounds and resource limits (§8), +- security considerations for decoding untrusted input (§9). + +It does **not** define: transport framing, HTTP/RNTBD negotiation headers, +per-account dictionary (user-string) construction policy, or the query wire +protocol. Those are layered above this format. + +### 1.3 Requirements language + +The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are +to be interpreted as described in [RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119). + +### 1.4 Relationship to the other binary-encoding artifacts + +This RFC is the **source of truth** for the wire format. Several other artifacts +either derive from it or validate against it — they do not redefine it: + +```mermaid +flowchart TD + RFC["BINARY_ENCODING_RFC.md
(normative spec: what correct means)"] + RFC -->|"§7 canonical encoding
§8 decoder conformance"| CONF["binary_json/conformance.rs
(encoder byte-exactness)"] + RFC -->|"Appendix A golden vectors"| CORPUS["testdata/binary_json_vectors.json
(fixed decode+encode oracle)"] + RFC -->|"value model + round-trip invariant"| FUZZ["BINARY_ENCODING_ROUNDTRIP_FUZZER
(random JSON, live service, at scale)"] + RFC -->|"§8 decoder + §9 security"| IN_FUZZ["binary_json/fuzz_tests.rs
(malformed/truncated buffers)"] + CORPUS --> CONF + CORPUS --> FUZZ + CORPUS --> IN_FUZZ +``` + +| Artifact | Role relative to this RFC | RFC sections it enforces | +| -------- | ------------------------- | ------------------------ | +| `testdata/binary_json_vectors.json` | Machine-readable golden corpus (this RFC's Appendix A) | Appendix A | +| `binary_json/conformance.rs` | Encoder byte-exactness + canonical-form snapshots | §3.1, §7 | +| `binary_json/reader.rs`, `de.rs` (tests) | Decoder conformance per form | §4–§6, §8 | +| `binary_json/fuzz_tests.rs` | Decoder never panics/hangs/over-allocates on malformed input | §8, §9 | +| [`BINARY_ENCODING_ROUNDTRIP_FUZZER.md`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md) + harness | End-to-end **round-trip invariant** on random JSON against the live service, at volume | §7 (round-trip), value model (§2) | + +Two connections are worth calling out explicitly: + +- **The round-trip fuzzer validates this RFC's round-trip invariant (§7), but + end-to-end rather than byte-level.** Where `conformance.rs` checks + `decode(encode(v)) == v` in-process on fixed vectors, the fuzzer checks the + *same* invariant across the full pipeline (Rust encode → wire → backend + store/rewrite → wire → Rust decode) on millions of random documents. +- **The fuzzer is the instrument that closes this RFC's open number-format + items.** The `[CROSS-VERIFY: .NET/C++]` tags and §7 canonical rules leave the + encoder-vs-backend number-normalization under-specified; the fuzzer's + canonicalization/calibration surface (its design doc §3.1) empirically + discovers the backend's actual number rewrite, and those findings feed back + into §7 here. + +--- + +## 2. Notation and terminology + +- **byte** — an 8-bit octet, written as two hexadecimal digits, e.g. `C8`. +- **marker** (or **type marker**) — the single leading byte that selects how the + following bytes are interpreted. Every encoded value begins with exactly one + marker (except literal small integers, where the marker byte *is* the value — + see §4.1). +- **spaced-hex** — the human-reviewable notation for a byte sequence used + throughout this document and in the shared test corpus, e.g. `80 CC 00 00 00 + 00 00 00 0C 40`. +- **preamble** — the single byte `0x80` that MUST prefix a complete buffer (§3.1). +- **little-endian (LE)** — multi-byte integers and floats are serialized + least-significant-byte first. This is **normative and independent of host + architecture**: an encoder on a big-endian host MUST still emit LE. +- **value model** — the abstract JSON value: `null | bool | number | string | + array | object`. Numbers are IEEE-754 doubles or integers in the ranges + encodable by the number markers in §4. + +--- + +## 3. Structure of an encoded value + +### 3.1 Buffer preamble and auto-detection + +A complete Cosmos Binary JSON buffer MUST begin with the **preamble byte** +`0x80`, followed by exactly one encoded value: + +``` +buffer = 0x80 value +``` + +Because `0x80` is also the marker for a zero-length encoded string (§4.4), the +preamble is **always consumed first**; a top-level empty string is therefore +`80 80` (preamble + zero-length-string marker). + +Consumers distinguish binary from UTF-8 JSON text by inspecting the **first +byte**: a payload whose first byte is `0x80` is Cosmos Binary JSON; any other +first byte (`{`, `[`, `"`, digit, `t`, `f`, `n`, whitespace, …) is UTF-8 text. +This is the `is_binary` predicate. + +> **Note.** UTF-8 text can never legitimately begin with `0x80` (a continuation +> byte), so the discriminator is unambiguous. + +### 3.2 Marker byte taxonomy + +The 256 marker values are partitioned into contiguous ranges. Ranges are written +`[MIN, MAX)` — MIN inclusive, MAX exclusive. + +| Range | Meaning | Section | +| ------------- | ------------------------------------------------------------- | ------- | +| `[0x00,0x20)` | Literal small integer (`value == marker`, range 0–31) | §4.1 | +| `[0x20,0x40)` | 1-byte **system** string (index into fixed dictionary) | §4.4.3 | +| `[0x40,0x60)` | 1-byte **user** string (index into per-buffer dictionary) | §4.4.4 | +| `[0x60,0x68)` | 2-byte user string | §4.4.4 | +| `[0x68,0x80)` | base64 / GUID-string / compressed-string forms | §4.4.5–6| +| `[0x80,0xC0)` | Encoded-length string (`len == marker & 0x7F`, 0–63) | §4.4.1 | +| `[0xC0,0xC8)` | Length-prefixed strings, reference strings, `NumberUInt64` | §4.4.1, §6, §4.2 | +| `[0xC8,0xD0)` | Fixed-width numbers (`UInt8`,`Int16/32/64`,`Double`,`Float*`) | §4.2–3 | +| `[0xD0,0xE0)` | null, bool, GUID value, extended sized ints, binary blobs | §4.1, §4.2, §4.7 | +| `[0xE0,0xE8)` | Arrays | §5.1 | +| `[0xE8,0xF0)` | Objects | §5.2 | +| `[0xF0,0xF8)` | Uniform (typed) number arrays | §5.3 | +| `[0xF8,0xFF]` | Reserved; `0xFF` == Invalid | §8 | + +The complete marker constant table is given in Appendix C. + +--- + +## 4. Scalars + +### 4.1 Null, boolean, and literal small integers + +| Value | Marker | Sample (with preamble) | +| --------------- | ------- | ---------------------- | +| `null` | `D0` | `80 D0` | +| `false` | `D1` | `80 D1` | +| `true` | `D2` | `80 D2` | +| integer `0`–`31`| `00`–`1F` | `0` → `80 00`; `31` → `80 1F` | + +For an integer `n` in `[0, 31]`, the marker byte itself is the value: the encoded +form is the single byte `n`. This is the most compact integer encoding and is the +canonical form for that range (§7). + +### 4.2 Integers + +Two families of integer markers exist, distinguished only by their historical +range. Both are LE. + +| Marker | Byte | Width | Signedness | Sample | +| ----------------- | ---- | ----- | ---------- | ----------------------------------- | +| `NumberUInt8` | `C8` | 1 | unsigned | `200` → `80 C8 C8` | +| `NumberInt16` | `C9` | 2 | signed | `-1000` → `80 C9 18 FC` | +| `NumberInt32` | `CA` | 4 | signed | `70000` → `80 CA 70 11 01 00` | +| `NumberInt64` | `CB` | 8 | signed | `-5000000000` → `80 CB 00 0E FA D5 FE FF FF FF` | +| `NumberUInt64` | `C7` | 8 | unsigned | `18446744073709551614` → `80 C7 FE FF FF FF FF FF FF FF` | +| `UInt8` (ext.) | `D7` | 1 | unsigned | `[CROSS-VERIFY: .NET/C++]` | +| `Int8` (ext.) | `D8` | 1 | signed | `-5` → `80 D8 FB` | +| `Int16` (ext.) | `D9` | 2 | signed | `-1000` → `80 D9 18 FC` | +| `Int32` (ext.) | `DA` | 4 | signed | `-70000` → `80 DA 90 EE FE FF` | +| `Int64` (ext.) | `DB` | 8 | signed | `-5000000000` → `80 DB 00 0E FA D5 FE FF FF FF` | +| `UInt32` (ext.) | `DC` | 4 | unsigned | `4294967294` → `80 DC FE FF FF FF` | + +Decoders MUST accept **both** families. Encoders emit the canonical family per +§7. The `NumberUInt64` marker (`C7`) is the only encoding able to carry +unsigned 64-bit values above `i64::MAX`. + +### 4.3 Floating-point numbers + +| Marker | Byte | Width | Encoding | Sample | +| -------------- | ---- | ----- | ---------------------------- | ------------------------------- | +| `NumberDouble` | `CC` | 8 | IEEE-754 binary64 (LE) | `3.5` → `80 CC 00 00 00 00 00 00 0C 40` | +| `Float32` | `CD` | 4 | IEEE-754 binary32 (LE) | `1.5` → `80 CD 00 00 C0 3F` | +| `Float64` | `CE` | 8 | IEEE-754 binary64 (LE) | `-2.25` → `80 CE 00 00 00 00 00 00 02 C0` | +| `Float16` | `CF` | 2 | IEEE-754 binary16 (LE) | `[CROSS-VERIFY: .NET/C++]` | + +`NumberDouble` (`CC`) is the canonical JSON-number float form (§7). + +**Non-finite values.** JSON has no representation for `NaN` or `±Infinity`. +A conforming encoder MUST NOT emit a non-finite double; instead it MUST encode +`null` (`D0`), mirroring `serde_json` / JavaScript `JSON.stringify`. A conforming +decoder MUST reject a non-finite `NumberDouble`/`Float*` payload as an invalid +number. `[CROSS-VERIFY: .NET/C++]` — confirm the .NET encoder/decoder policy is +identical. + +### 4.4 Strings + +JSON strings are UTF-8. Several encodings exist; a decoder MUST accept all of +them, and an encoder selects the canonical one per §7. + +#### 4.4.1 Length-framed strings + +| Form | Marker | Length field | Sample | +| ------------------ | ------ | ------------------ | -------------------------- | +| Encoded-length | `80`–`BF` | in marker: `len = marker & 0x7F` (0–63) | `""` → `80 80`; `"hi"` → `80 82 68 69` | +| `StrL1` | `C0` | u8 | `"hello"` → `80 C0 05 68 65 6C 6C 6F` | +| `StrL2` | `C1` | u16 LE | 300×`"a"` → `80 C1 2C 01 …` | +| `StrL4` | `C2` | u32 LE | (large strings) | + +The string's UTF-8 bytes follow the length field verbatim. + +#### 4.4.2 GUID strings + +| Marker | Byte | Meaning | Sample | +| ------ | ---- | ------------------------------------ | ------ | +| lower | `75` | 36-char lowercase GUID string | `80 75 00 01 … 0F` → `"00010203-0405-0607-0809-0a0b0c0d0e0f"` | +| upper | `76` | 36-char uppercase GUID string | `80 76 …` → uppercase | +| quoted | `77` | double-quoted lowercase GUID string | `80 77 …` → `"\"…\""` | + +The 16 raw GUID bytes follow the marker; the decoder formats them as a hyphenated +GUID string (byte order per Appendix B). This is distinct from a **GUID value** +(§4.7). + +#### 4.4.3 System strings + +Markers `[0x20, 0x40)` encode a string by **index into a fixed, well-known +dictionary** of common Cosmos property names (`id`, `_rid`, `_etag`, …). The +encoded form is a single byte; `marker - 0x20` is the dictionary index. + +Example: `"id"` has system index `0x0C`, so `"id"` → `80 2C` (`0x20 + 0x0C`). + +The full system-string table is normative and given in Appendix D. +`[CROSS-VERIFY: .NET/C++]` — the index assignments MUST match `JsonBinaryEncoding`. + +#### 4.4.4 User (per-buffer dictionary) strings + +Markers `[0x40, 0x60)` (1-byte) and `[0x60, 0x68)` (2-byte) encode a string by +index into a **per-buffer user-string dictionary**. The dictionary and its +construction policy are out of scope for this document (a decoder receives the +dictionary alongside the buffer, or the buffer contains no user-dictionary +references). `[CROSS-VERIFY: .NET/C++]` + +#### 4.4.5 base64 strings + +| Marker | Byte | Alphabet | Length field | Sample | +| ------ | ---- | ------------ | ------------ | ------ | +| `Base64Len1` | `71` | standard | u8 | `"Zm9v"` → `80 71 01 00 66 6F 6F` | +| `Base64Len2` | `72` | standard | u16 | `"Zm9vYmFy"` → `80 72 02 00 00 66 6F 6F 62 61 72` | +| `Base64UrlLen1` | `73` | URL-safe | u8 | `"-__-"` → `80 73 01 00 FB FF FE` | +| `Base64UrlLen2` | `74` | URL-safe | u16 | (as above, 2-byte length) | + +The payload is the **decoded** bytes; the decoder re-encodes them to a base64 +string using the marker's alphabet. Padding handling (`=`) and the "omitted +padding" length-field convention are illustrated in Appendix B. +`[CROSS-VERIFY: .NET/C++]` for the exact padding/length-offset encoding. + +#### 4.4.6 Compressed strings + +Restricted-alphabet strings are bit-packed relative to a base character: + +| Marker | Byte | Packing | Sample | +| ------ | ---- | ------------------------------- | ------ | +| lower-hex | `78` | 4-bit hex digits (lowercase) | `"1a2b"` → `80 78 04 A1 B2` | +| upper-hex | `79` | 4-bit hex digits (uppercase) | `"1A2B"` → `80 79 04 A1 B2` | +| date-time | `7A` | 4-bit date-time charset | `"2024-01"` → `80 7A 07 13 53 1C 02` | +| packed-4bit | `7B` | 4 bits/char + base | `"0123"` → `80 7B 04 30 10 32` | +| packed-5bit | `7C` | 5 bits/char + base | `"abc"` → `80 7C 03 61 20 08` | +| packed-6bit | `7D` | 6 bits/char + base | `"abcd"` → `80 7D 04 61 40 20 0C` | +| packed-7bit-L1 | `7E` | 7 bits/char, u8 length | `"Hi"` → `80 7E 02 C8 34` | +| packed-7bit-L2 | `7F` | 7 bits/char, u16 length | `"Hi"` → `80 7F 02 00 C8 34` | + +The byte immediately after the marker is the **character count** (for L1/L2 forms +a 1-/2-byte length), followed by the packed bits. Appendix B gives a worked +unpacking example. `[CROSS-VERIFY: .NET/C++]` for the exact base/charset tables. + +### 4.7 GUID value and binary blobs + +| Marker | Byte | Meaning | Sample | +| ------ | ---- | -------------------------------- | ------ | +| `Guid` | `D3` | raw 16-byte GUID **value** | `80 D3 00 01 … 0F` → `"03020100-0504-0706-0809-0a0b0c0d0e0f"` | +| `Binary1` | `DD` | blob, u8 length prefix | `0xDEADBEEF` → `80 DD 04 DE AD BE EF` → `"3q2+7w=="` | +| `Binary2` | `DE` | blob, u16 length prefix | `80 DE 03 00 01 02 03` → `"AQID"` | +| `Binary4` | `DF` | blob, u32 length prefix | (large blobs) | + +A **binary blob** decodes to a **standard base64 string** in the JSON value +model. Note the byte order of a GUID *value* (`D3`) differs from a GUID *string* +(§4.4.2) — see Appendix B. `[CROSS-VERIFY: .NET/C++]` + +--- + +## 5. Containers + +### 5.1 Arrays + +| Marker | Byte | Framing | Sample | +| ------- | ---- | ---------------------------------------- | ------ | +| `Arr0` | `E0` | empty | `[]` → `80 E0` | +| `Arr1` | `E1` | exactly one element, no length/count | `[true]` → `80 E1 D2` | +| `ArrL1` | `E2` | u8 **byte-length** prefix | `[0,1,null]` → `80 E2 03 00 01 D0` | +| `ArrL2` | `E3` | u16 byte-length | | +| `ArrL4` | `E4` | u32 byte-length | | +| `ArrLC1`| `E5` | u8 byte-length **+** u8 item-count | `[0,1,null]` → `80 E5 03 03 00 01 D0` | +| `ArrLC2`| `E6` | u16 byte-length + u16 count | | +| `ArrLC4`| `E7` | u32 byte-length + u32 count | | + +For `L*` forms, the length is the **byte length of the element region** (not the +element count); the decoder reads elements until it has consumed exactly that +many bytes. For `LC*` forms, both the byte length **and** the element count are +given, and a conforming decoder MUST verify that reading `count` elements +consumes exactly the declared byte length (§8). + +### 5.2 Objects + +| Marker | Byte | Framing | Sample | +| ------- | ---- | ----------------------------------- | ------ | +| `Obj0` | `E8` | empty | `{}` → `80 E8` | +| `Obj1` | `E9` | exactly one name/value pair | `{"id":true}` → `80 E9 2C D2` | +| `ObjL1` | `EA` | u8 byte-length | `{"id":0,"type":1}` → `80 EA 04 2C 00 3B 01` | +| `ObjL2` | `EB` | u16 byte-length | | +| `ObjL4` | `EC` | u32 byte-length | | +| `ObjLC1`| `ED` | u8 byte-length + u8 pair-count | `{"id":0,"type":1}` → `80 ED 04 02 2C 00 3B 01` | +| `ObjLC2`| `EE` | u16 byte-length + u16 count | | +| `ObjLC4`| `EF` | u32 byte-length + u32 count | | + +Members are encoded as **name, value, name, value, …**. Each **name** is itself +an encoded string (any string form, including system strings — note `2C` = system +`"id"` in the samples). Member ordering is preserved as encoded. + +### 5.3 Uniform (typed) number arrays + +A homogeneous array of numbers of a single width is encoded compactly by writing +the item type marker once, then the raw item bytes with no per-item markers. + +| Marker | Byte | Layout | Sample | +| ----------- | ---- | -------------------------------------------------------- | ------ | +| `ArrNumC1` | `F0` | `itemMarker`, u8 count, then `count` bare numbers | `[1,2,3]` (Int32) → `80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00` | +| `ArrNumC2` | `F1` | `itemMarker`, u16 count, then bare numbers | `[-1,0,1000]` (Int16) → `80 F1 D9 03 00 FF FF 00 00 E8 03` | +| `ArrArrNumC1C1` | `F2` | innerMarker, itemMarker, u8 inner-count, u8 outer-count, then inner arrays | `[[1,2],[3,4]]` → `80 F2 F0 DA 02 02 01 00 00 00 …` | +| `ArrArrNumC2C2` | `F3` | as above with u16 counts | | + +Further examples: `[10,20,30]` (UInt8) → `80 F0 D7 03 0A 14 1E`; empty uniform +array → `80 F0 DA 00`; `[1.5,-0.25]` (Float32) → `80 F0 CD 02 00 00 C0 3F 00 00 +80 BE`. + +**Item markers.** The uniform-array item type MUST be one of the **extended** +number markers (`Int8`/`UInt8`/`Int16`/`Int32`/`Int64`/`UInt32`/`Float32`/ +`Float64`, i.e. `D7`–`DC`, `CD`, `CE`). The self-describing `Number*` markers +(`C7`–`CC`) MUST NOT appear as a uniform-array item type and a conforming decoder +MUST reject them there. `[CROSS-VERIFY: .NET/C++]` + +--- + +## 6. Reference strings + +A string that already appeared earlier in the same buffer MAY be encoded as a +**back-reference** to its byte offset, saving space for repeated keys/values. + +| Marker | Byte | Offset field | +| ------- | ---- | ------------ | +| `StrR1` | `C3` | u8 offset | +| `StrR2` | `C4` | u16 offset | +| `StrR3` | `C5` | u24 offset (3 bytes LE) | +| `StrR4` | `C6` | u32 offset | + +The offset is an **absolute byte offset into the buffer**, measured in the same +frame as the preamble (the preamble is offset `0`). The referenced offset MUST +hold a non-reference string; reference-to-reference chains are prohibited, which +makes cycles impossible and bounds resolution without recursion. + +**Decoder resource bound (normative).** Because many references can point at one +large string, a naïve decoder can be forced into O(S²) output for a size-`S` +buffer. A conforming decoder MUST bound total materialized reference bytes by a +budget proportional to the input size (e.g. `max(16 × buffer_len, 64 KiB)`) and +fail with an invalid-length error once exceeded (§9). + +--- + +## 7. Canonical encoding + +Multiple valid encodings exist for the same value (e.g. the integer `5` can be a +literal `05`, `NumberUInt8`, `Int8`, `Int16`, …; the string `"id"` can be a +system string, an encoded-length string, `StrL1`, …). A **decoder MUST accept +all valid encodings**. An **encoder MUST be deterministic**: for a given value it +MUST emit exactly one encoding. + +Determinism does **not** require the *narrowest* encoding. An encoder MAY emit +any valid form as long as it does so deterministically. The Rust reference +encoder (`ser.rs` / `writer.rs`, pinned by `conformance.rs`) deliberately emits a +**valid but non-minimal subset**: integers outside `[0,31]` use `Int64`/`UInt64` +(never the narrower `NumberUInt8`/`Int16`/`Int32`), strings use the +encoded-length or `StrL*` forms (never system/user/compressed/reference strings), +and containers always use `LC*` framing (never `Arr0`/`Arr1`/`Obj0`/`Obj1`). The +decoder still accepts every compact form the service may emit. + +The rules below describe the **widest canonical shape** each SDK is free to +narrow. Rules 1, 3 and 4 are **required** (they select the value's category); +rules 2, 5 and 6 describe **encoder-optional narrowing** — an encoder MAY emit a +wider valid form instead: + +1. **Integers in `[0,31]`** → literal small integer (single byte). *(Required.)* +2. **Other integers** → any fixed-width `Number*` marker that holds the value + (narrowest is `NumberUInt8` → `NumberInt16` → `NumberInt32` → `NumberInt64`; + values above `i64::MAX` use `NumberUInt64`). *Encoder-optional:* the Rust + encoder always uses `Int64`/`UInt64`. `[CROSS-VERIFY: .NET/C++]` — confirm the + .NET encoder prefers `Number*` over the extended `D7`–`DC` markers. +3. **Floating-point (non-integer) numbers** → `NumberDouble` (`CC`). *(Required.)* +4. **Non-finite floats** → `null` (§4.3). *(Required.)* +5. **Strings** → system string if the value is in the system dictionary; + otherwise encoded-length (`< 64` bytes), `StrL1`, `StrL2`, or `StrL4` by + length. System/compressed/base64/GUID/reference forms are **decoder-accepted + but encoder-optional** optimizations; a minimal conforming encoder need not + emit them (the Rust encoder never emits system or compressed strings). + `[CROSS-VERIFY: .NET/C++]` — enumerate which optimizations the .NET encoder + applies by default. +6. **Containers** → `LC*` (length-and-count) framing. Narrowing to the compact + `Arr0`/`Arr1`/`Obj0`/`Obj1` forms is **encoder-optional**; the Rust encoder + always emits `LC*`. `[CROSS-VERIFY: .NET/C++]` + +> **Why deterministic encoding matters.** A snapshot / golden-vector test asserts +> `encode(value) == expected_bytes`, so each SDK's encoder must be deterministic +> for its own snapshots to be stable. Cross-SDK **byte-equality** tests are only +> valid between encoders that made the *same* narrowing choices; because the Rust +> encoder emits the verbose subset above, a cross-SDK byte-equality test must +> compare against that subset (or compare *decoded values* rather than bytes). + +--- + +## 8. Decoder conformance requirements + +A conforming decoder MUST: + +1. **Reject a missing preamble.** The first byte MUST be `0x80` (§3.1). +2. **Reject trailing bytes.** After decoding the single top-level value, no bytes + may remain. +3. **Bounds-check every read.** A length/offset field that would read past the + end of the buffer MUST fail with an unexpected-EOF / invalid-length error, not + panic or read out of bounds. +4. **Enforce a maximum nesting depth** to prevent stack exhaustion on deeply + nested containers. The reference limit is `256`. Both the reference (`Value`) + decoder and any streaming decoder MUST reject at the **same** depth. +5. **Validate `LC*` containers.** After reading `count` elements/members, the + cursor MUST be exactly at the declared byte-length boundary; a mismatch MUST + fail (this catches malformed length+count buffers that a count-only decoder + would silently under-read). +6. **Enforce the reference-string budget** (§6). +7. **Bound uniform-array output.** For `ArrArrNum*` with a zero inner count, the + outer count MUST NOT exceed the remaining buffer bytes (else a few bytes could + materialize `u16::MAX` empty arrays). +8. **Reject the `Invalid` marker (`0xFF`)** and any unassigned marker with an + invalid-marker error. +9. **Reject non-finite numbers** (§4.3). + +A conforming decoder MUST NOT panic, hang, or allocate unboundedly on any input, +because it parses **untrusted** service/network bytes (§9). + +--- + +## 9. Security considerations + +Decoders process bytes that may originate from a compromised or buggy service, a +MITM, or a corrupted cache. The threats and required mitigations: + +| Threat | Mitigation (normative) | +| ---------------------------------------- | ---------------------------------------------- | +| Out-of-bounds read via a large length | Bounds-check every read against buffer end (§8.3) | +| Stack exhaustion via deep nesting | Max depth limit, rejected identically by all decoders (§8.4) | +| O(S²) memory via many back-references | Per-decode reference-expansion budget (§6) | +| Output amplification via empty uniform arrays | Bound outer count by remaining bytes (§8.7) | +| Malformed length+count under-read | Assert cursor == declared end (§8.5) | + +Encoders SHOULD reject values that cannot be represented (e.g. integer widths +beyond `u32::MAX` container framing) rather than silently truncating. + +--- + +## Appendix A — Golden test vectors (shared corpus) + +The normative, cross-SDK test corpus lives in machine-readable form at +`azure_data_cosmos_driver/testdata/binary_json_vectors.json`. Each entry pairs a +`name`, a spaced-hex `binary` buffer (including the `0x80` preamble), and the +`json` value it decodes to. A conforming decoder MUST reproduce every `json` from +its `binary`; a conforming encoder MUST reproduce the canonical `binary` for +every `json` that is in canonical form (§7). + +A representative subset (see the file for the full set): + +| name | binary | json | +| ---- | ------ | ---- | +| null | `80 D0` | `null` | +| true | `80 D2` | `true` | +| literal_int_max | `80 1F` | `31` | +| uint8 | `80 C8 C8` | `200` | +| double | `80 CC 00 00 00 00 00 00 0C 40` | `3.5` | +| system_string_id | `80 2C` | `"id"` | +| str_l1_hello | `80 C0 05 68 65 6C 6C 6F` | `"hello"` | +| binary_deadbeef | `80 DD 04 DE AD BE EF` | `"3q2+7w=="` | +| uniform_int32 | `80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00` | `[1,2,3]` | +| object_lc1 | `80 ED 04 02 2C 00 3B 01` | `{"id":0,"type":1}` | +| nested_containers | `80 E2 05 E1 00 E9 2C 01` | `[[0],{"id":1}]` | + +## Appendix B — Worked examples (byte-by-byte) + +**`{"id":0,"type":1}` as `ObjLC1` (`80 ED 04 02 2C 00 3B 01`):** + +``` +80 preamble +ED ObjLC1 marker (u8 byte-length + u8 count) +04 byte-length of member region = 4 +02 member (pair) count = 2 +2C name: system string 0x0C ("id") ← 0x20 + 0x0C +00 value: literal small int 0 +3B name: system string 0x1B ("type") ← 0x20 + 0x1B [CROSS-VERIFY] +01 value: literal small int 1 +``` + +**`[1,2,3]` as a uniform Int32 array (`80 F0 DA 03 01 00 00 00 …`):** + +``` +80 preamble +F0 ArrNumC1 (uniform number array, u8 count) +DA item type marker = Int32 +03 item count = 3 +01 00 00 00 1 (Int32 LE) +02 00 00 00 2 +03 00 00 00 3 +``` + +**base64 with omitted padding (`80 71 01 FD 41` → `"QQ"`):** the length field +uses a signed offset convention to signal padding omission; see the codec's +base64 decoder for the exact rule. `[CROSS-VERIFY: .NET/C++]` + +## Appendix C — Complete marker constant table + +*(Authoritative values; transcribed from `markers.rs` / `JsonBinaryEncoding.TypeMarker.cs`.)* + +``` +Literal int 0x00–0x1F +System string 1B 0x20–0x3F +User string 1B 0x40–0x5F +User string 2B 0x60–0x67 +Base64Len1 0x71 Base64Len2 0x72 +Base64UrlLen1 0x73 Base64UrlLen2 0x74 +GuidLower 0x75 GuidUpper 0x76 GuidQuoted 0x77 +CompLowerHex 0x78 CompUpperHex 0x79 CompDateTime 0x7A +Packed4/5/6bit 0x7B/0x7C/0x7D +Packed7bitL1/L2 0x7E/0x7F +Encoded-len str 0x80–0xBF (len = marker & 0x7F) +StrL1/L2/L4 0xC0/0xC1/0xC2 +StrR1/R2/R3/R4 0xC3/0xC4/0xC5/0xC6 +NumberUInt64 0xC7 +NumberUInt8 0xC8 NumberInt16 0xC9 NumberInt32 0xCA NumberInt64 0xCB +NumberDouble 0xCC Float32 0xCD Float64 0xCE Float16 0xCF +Null 0xD0 False 0xD1 True 0xD2 Guid 0xD3 +UInt8 0xD7 Int8 0xD8 Int16 0xD9 Int32 0xDA Int64 0xDB UInt32 0xDC +Binary1/2/4 0xDD/0xDE/0xDF +Arr0..ArrLC4 0xE0–0xE7 +Obj0..ObjLC4 0xE8–0xEF +ArrNumC1/C2 0xF0/0xF1 +ArrArrNumC1C1/C2C2 0xF2/0xF3 +Invalid 0xFF +``` + +## Appendix D — System string dictionary + +The fixed system-string table (index → string) is normative and MUST match +`JsonBinaryEncoding` across SDKs. It is defined in +`azure_data_cosmos_driver/src/binary_json/system_strings.rs`. +`[CROSS-VERIFY: .NET/C++]` — reproduce the full table here once confirmed against +the .NET source (only `id` = index `0x0C` and `type` = index `0x1B` are shown +inline in this draft's examples). + +--- + +## Open items before promotion to stable + +- Resolve every `[CROSS-VERIFY: .NET/C++]` tag against the .NET + (`Microsoft.Azure.Cosmos/src/Json/`) and C++ reference implementations. +- Complete Appendix D (full system-string table). +- Confirm the canonical-encoding rules in §7 match the .NET encoder's actual + output (needed for cross-SDK byte-equality snapshot tests). +- Specify the exact base64 padding/length-offset convention (Appendix B). +- Specify the compressed-string base/charset tables (§4.4.6). diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore b/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore new file mode 100644 index 00000000000..1a45eee7760 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml b/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml new file mode 100644 index 00000000000..6b4f445cb6d --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml @@ -0,0 +1,59 @@ +# cargo-fuzz crate for the Cosmos binary JSON codec. +# +# A SEPARATE crate with its own `[workspace]` (empty table at the bottom) so it +# is not pulled into the parent stable workspace: cargo-fuzz builds these +# targets on nightly with libFuzzer. +# +# Usage: +# rustup toolchain install nightly +# cargo install cargo-fuzz +# cargo +nightly fuzz run decode +# See README.md for target descriptions and corpus seeding. + +[package] +name = "azure_data_cosmos_driver-fuzz" +version = "0.0.0" +publish = false +edition = "2021" +license = "MIT" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +serde_json = "1" + +[dependencies.azure_data_cosmos_driver] +path = ".." + +[[bin]] +name = "decode" +path = "fuzz_targets/decode.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "from_slice" +path = "fuzz_targets/from_slice.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "transcode_to_text" +path = "fuzz_targets/transcode_to_text.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "decode_reencode_roundtrip" +path = "fuzz_targets/decode_reencode_roundtrip.rs" +test = false +doc = false +bench = false + +# Isolate this crate from the parent workspace (nightly + libFuzzer only). +[workspace] diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md new file mode 100644 index 00000000000..e7a3cdf8f5d --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md @@ -0,0 +1,174 @@ +# Binary JSON codec fuzzing (`cargo-fuzz`) + +Coverage-guided, **byte-level** fuzzing for the Cosmos binary JSON codec +(`azure_data_cosmos_driver::binary_json`). Where the live +[round-trip fuzzer](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs) +generates random JSON *values* and only ever feeds the decoder **encoder-produced** +(well-formed) bytes, these targets feed **arbitrary and mutated bytes** straight +into the decoder — so they exercise the *format*/protocol itself: truncated +buffers, bad length prefixes, unknown or misused markers, reference/depth bombs, +non-UTF-8 string payloads, and trailing bytes. + +This is a **separate crate** with its own empty `[workspace]` in `Cargo.toml`, so +it stays isolated from the stable repo workspace: cargo-fuzz builds it on nightly +with libFuzzer. + +## Prerequisites + +```bash +rustup toolchain install nightly +cargo install cargo-fuzz +``` + +## Targets + +| Target | Entry point | What it checks | +| --- | --- | --- | +| `decode` | `binary_json::decode` | `Value` decode never panics/hangs/over-allocates on any bytes. | +| `from_slice` | `binary_json::from_slice::` | Native serde streaming decode honors the same no-crash contract. | +| `transcode_to_text` | `binary_json::transcode_to_text` | Driver-side binary→text response transcode never panics on a malformed body. | +| `decode_reencode_roundtrip` | `decode` + `encode` | **Differential**: any buffer the decoder accepts must satisfy `decode(encode(decode(x))) == decode(x)` — catches reader/writer disagreements. | + +All four assert the **robustness oracle**: for *any* input the codec terminates +and returns `Ok`/`Err` — never panics, hangs, or allocates beyond the buffer. +The last one adds a **semantic** oracle on decoder-accepted inputs. + +## Running + +From this `fuzz/` directory (or the driver crate root): + +```bash +# Explore one target (Ctrl-C to stop): +cargo +nightly fuzz run decode + +# Time-boxed CI-style smoke run (60s), 4 workers: +cargo +nightly fuzz run decode -- -max_total_time=60 -workers=4 + +# Reproduce a crash from a saved artifact: +cargo +nightly fuzz run decode fuzz/artifacts/decode/crash- + +# Minimize a crashing input: +cargo +nightly fuzz tmin decode fuzz/artifacts/decode/crash- +``` + +## Thorough manual run on a Linux VM + +Weekly CI only replays the committed corpus once (`-runs=0`, no mutation). To +perform coverage-guided mutation and deeper fuzzing, run it by hand on any Linux +box (or WSL2), without a wall-clock cap: + +```bash +# 1. Toolchain (one-time) +rustup toolchain install nightly --component rust-src +cargo install cargo-fuzz --locked + +# 2. Get the code and seed the corpus from the golden vectors (recommended — +# lets libFuzzer mutate outward from real wire frames). +cd sdk/cosmos/azure_data_cosmos_driver +mkdir -p fuzz/corpus/decode +jq -r '.[] | "\(.name) \(.binary)"' testdata/binary_json_vectors.json | +while read -r name hex; do + echo "$hex" | tr -d ' ' | xxd -r -p > "fuzz/corpus/decode/$name" +done + +# 3a. Run one target for a fixed budget (e.g. 1 hour), 8 parallel workers: +cargo +nightly fuzz run decode -- -max_total_time=3600 -workers=8 -jobs=8 -print_final_stats=1 + +# 3b. Or run it open-ended until you Ctrl-C (a true soak): +cargo +nightly fuzz run decode -- -workers=8 -jobs=8 + +# 4. Repeat for the other targets (they share the same corpus format): +cargo +nightly fuzz run from_slice -- -max_total_time=3600 -workers=8 +cargo +nightly fuzz run transcode_to_text -- -max_total_time=3600 -workers=8 +cargo +nightly fuzz run decode_reencode_roundtrip -- -max_total_time=3600 -workers=8 + +# 5. Or drive all four with the CI helper (installs deps, seeds corpus, runs each): +pwsh ../eng/scripts/Run-BinaryJsonFuzz.ps1 -MaxTotalTimeSeconds 3600 -Workers 8 +``` + +**If a crash is found**, libFuzzer writes the triggering input to +`fuzz/artifacts//crash-`. Reproduce and minimize it: + +```bash +cargo +nightly fuzz run decode fuzz/artifacts/decode/crash- # reproduce +cargo +nightly fuzz tmin decode fuzz/artifacts/decode/crash- # minimize +``` + +Then add the minimized input as a golden vector / unit test in +`src/binary_json/` and fix the codec. The **corpus in `fuzz/corpus//` +persists across runs** — keep it (or copy it between machines) to accelerate +subsequent sessions. + +Sizing guidance: `job time ≈ 1 min (compile) + N_targets × per-target budget`. +On an 8-vCPU VM, `-workers=8` roughly 2× the throughput seen in CI (~3.4K +exec/s/worker in the first run), so a 1-hour/target soak explores tens of +millions of inputs per target. + +## Seeding the corpus from the golden vectors + +Seeding libFuzzer with **valid** frames lets it mutate outward from real wire +shapes and reach the interesting error paths far faster than blind byte flips. +The [golden vectors](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json) already contain every +marker family as space-separated hex. Materialize them into the `decode` corpus: + +PowerShell: + +```powershell +$dir = "fuzz/corpus/decode"; New-Item -ItemType Directory -Force $dir | Out-Null +(Get-Content ../testdata/binary_json_vectors.json | ConvertFrom-Json) | ForEach-Object { + $bytes = $_.binary -split '\s+' | ForEach-Object { [Convert]::ToByte($_, 16) } + [IO.File]::WriteAllBytes("$dir/$($_.name)", [byte[]]$bytes) +} +``` + +bash + jq + xxd: + +```bash +mkdir -p fuzz/corpus/decode +jq -r '.[] | "\(.name) \(.binary)"' ../testdata/binary_json_vectors.json | +while read -r name hex; do + echo "$hex" | tr -d ' ' | xxd -r -p > "fuzz/corpus/decode/$name" +done +``` + +The same corpus works for `from_slice`, `transcode_to_text`, and +`decode_reencode_roundtrip` (all consume raw binary buffers); copy or point +`--corpus` at `fuzz/corpus/decode`. + +## Notes + +- `corpus/`, `artifacts/`, and `target/` are git-ignored (regenerated locally / in CI). +- These targets are **offline** (no live account), so they are cheap enough to + run in CI as a nightly job or a time-boxed smoke check on PRs touching + `binary_json`. +- A reproducible crash should be reduced with `cargo fuzz tmin`, added as a + golden vector / unit test in `src/binary_json/`, and fixed there. + +## Windows + +`cargo-fuzz` builds on **libFuzzer** (`-fsanitize=fuzzer`), which the Windows +MSVC target does not support — `cargo fuzz run` fails to link on Windows. Use +**WSL2** or a **Linux** box. On Windows, the always-on decoder robustness +coverage lives in `src/binary_json/fuzz_tests.rs` (random / truncated / corrupted +buffers into `decode`) and runs on stable via `cargo test -p +azure_data_cosmos_driver --lib fuzz`. + +## CI + +Fuzzing runs as a **non-blocking leg of the existing `sdk/cosmos/ci.yml`** — a +Build-stage `MatrixConfigs` entry ([`sdk/cosmos/fuzz-matrix.json`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/fuzz-matrix.json)) +that adds one **Linux + nightly** job (cargo-fuzz/libFuzzer is Linux-only), gated +to the **weekly / scheduled** build only (not per-PR). It carries +`ContinueOnError: "true"`, so a discovered crash reports "succeeded with issues" +instead of blocking merge. The job's test-setup hook +([`Invoke-CosmosTestSetup.ps1`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1), +gated on `AZURE_COSMOS_FUZZ=1`) calls +[`Run-BinaryJsonFuzz.ps1`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1) +**with `-ValidateOnly`**, which installs cargo-fuzz, seeds each corpus from the +golden vectors, and **replays the committed vectors once** (libFuzzer `-runs=0`, +no mutation, no time budget) to prove they still decode without panicking. + +Coverage-guided mutation soaks (`-max_total_time`) are **manual / local only** — +CI never runs an unattended time-boxed soak (see the manual-run section above). +Crash inputs are published as the `fuzz-crashes` build artifact so a failure can +be reproduced and minimized (`cargo fuzz tmin`). diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs new file mode 100644 index 00000000000..d5d158df996 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the binary-JSON **decoder**. +//! +//! libFuzzer feeds arbitrary (and, once seeded, mutated-from-valid) byte +//! buffers straight into [`decode`]. This is the format fuzzer the live +//! round-trip test can't be: it explores mis-encoded frames — truncated +//! buffers, bad length prefixes, unknown/misused markers, reference and +//! depth bombs, non-UTF-8 string payloads, trailing bytes — that the encoder +//! never produces. +//! +//! Oracle: for **any** input the decoder must terminate and return either +//! `Ok(Value)` or `Err(BinaryError)` — never panic, hang, or allocate beyond +//! what the buffer can back. A crash or hang here is a decoder-hardening bug. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::decode; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = decode(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs new file mode 100644 index 00000000000..661bc1a9349 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Differential fuzz target: decode → encode → decode idempotence. +//! +//! Unlike the plain `decode` no-crash target, this asserts a **semantic** +//! invariant on every buffer the decoder *accepts*: re-encoding the decoded +//! value and decoding it again must reproduce the exact same value. It catches +//! the class of bug the live round-trip fuzzer cannot — a buffer the decoder +//! accepts but the encoder would round-trip to a *different* value (marker or +//! number-form disagreements between the reader and writer). libFuzzer's +//! mutation reaches decoder-accepted-but-unusual frames that hand-written +//! golden vectors don't enumerate. +//! +//! Oracle: `decode(data) = Ok(v)` ⇒ `decode(encode(v)) = Ok(v)`. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::{decode, encode}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + if let Ok(value) = decode(data) { + let reencoded = encode(&value); + let redecoded = + decode(&reencoded).expect("re-encoding a decoded value must itself decode"); + assert_eq!( + value, redecoded, + "decode∘encode∘decode is not idempotent for a decoder-accepted buffer" + ); + } +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs new file mode 100644 index 00000000000..6c0046c654c --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the native serde **deserializer**. +//! +//! [`from_slice`] is the zero-`Value` streaming decode path used by the SDK's +//! typed reads; it drives a different code path from [`decode`] (it streams +//! tokens into a serde visitor instead of materializing a +//! [`serde_json::Value`]). Fuzzing it independently ensures the streaming +//! deserializer honors the same no-crash contract on malformed input. +//! +//! Oracle: for any input, deserialization must terminate with `Ok`/`Err` — +//! never panic, hang, or over-allocate. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::from_slice; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = from_slice::(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs new file mode 100644 index 00000000000..199c0371196 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the driver-side response transcode. +//! +//! [`transcode_to_text`] is what the driver runs on a binary response body when +//! a text-only host asked for text back: it decodes the binary buffer and +//! re-serializes it as UTF-8 text JSON (or passes text/empty input through +//! unchanged). It sits on the FFI/text-host response path, so a panic here on a +//! malformed service body would take down the host. +//! +//! Oracle: for any input, transcoding must terminate with `Ok`/`Err` — never +//! panic, hang, or over-allocate. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::transcode_to_text; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = transcode_to_text(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs new file mode 100644 index 00000000000..2f70f89b185 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Encode-direction conformance tests for the Cosmos binary JSON codec. +//! +//! These tests implement the encoder conformance requirements of the +//! wire-format spec (`docs/BINARY_ENCODING_RFC.md` §7 "Canonical encoding" and +//! Appendix A "Golden test vectors"). Decoder conformance (§8) is covered by the +//! per-form tests in [`reader`](super::reader) and [`de`](super::de); this +//! module fills the previously-missing encode side. +//! +//! Two guarantees are asserted: +//! +//! 1. **Round-trip validity** — for every golden-corpus value, `decode(encode(v)) +//! == v`. The encoder MUST emit a valid buffer that decodes back to the input +//! (RFC §7: the encoder emits a conformant *subset* of the wire forms; the +//! decoder accepts all of them). +//! 2. **Canonical output snapshots** — the encoder is deterministic, so its exact +//! bytes for representative values are pinned as regression snapshots. This +//! documents the Rust encoder's actual canonical form, which is a valid +//! subset that does **not** use the most compact forms (system strings, +//! `Arr0`/`Arr1`, narrowest `Number*`, etc.) — see the notes below. +//! +//! The module is compiled only under `cfg(test)`. + +use super::{decode, encode, is_binary, PREAMBLE}; +use serde_json::{json, Value}; + +/// Parses a spaced-hex string (e.g. `"80 D0"`) into bytes, matching the corpus +/// notation used throughout the RFC. +fn hex(s: &str) -> Vec { + s.split_whitespace() + .map(|b| u8::from_str_radix(b, 16).expect("valid hex byte")) + .collect() +} + +/// RFC §7 (round-trip validity): every value in the shared golden corpus MUST +/// re-encode to a buffer that decodes back to the identical value. This is the +/// encode-direction counterpart to `reader::decodes_golden_corpus`. +#[test] +fn encode_round_trips_golden_corpus() { + for vector in super::vectors::golden_vectors() { + let expected: Value = + serde_json::from_str(&vector.json).expect("corpus json is valid JSON"); + let encoded = encode(&expected); + assert!( + is_binary(&encoded), + "{}: encoder output missing preamble", + vector.name + ); + let decoded = decode(&encoded) + .unwrap_or_else(|e| panic!("{}: re-encoded buffer failed to decode: {e}", vector.name)); + assert_eq!( + decoded, expected, + "{}: encode→decode did not round-trip", + vector.name + ); + } +} + +/// RFC §7 (deterministic canonical output): the encoder emits exactly these +/// bytes for representative values. These snapshots are the regression bar for +/// the Rust encoder's canonical form. +/// +/// Note the encoder deliberately emits a **valid but non-minimal subset** of the +/// wire forms (RFC §7): integers outside `[0,31]` use `Int64`/`UInt64` (never the +/// narrower `NumberUInt8`/`Int16`/`Int32`), strings use the encoded-length or +/// `StrL*` forms (never system/user/compressed strings), and containers always +/// use the `LC*` length+count framing (never `Arr0`/`Arr1`/`Obj0`/`Obj1`). The +/// decoder accepts the compact forms the service may emit; the encoder need not +/// produce them. +#[test] +fn encode_produces_expected_canonical_bytes() { + let cases: &[(Value, &str)] = &[ + // Singletons. + (json!(null), "80 D0"), + (json!(false), "80 D1"), + (json!(true), "80 D2"), + // Literal small integers (value == marker), 0..=31. + (json!(0), "80 00"), + (json!(31), "80 1F"), + // Integers outside [0,31] use Int64 (not the narrower Number* forms). + (json!(32), "80 CB 20 00 00 00 00 00 00 00"), + (json!(200), "80 CB C8 00 00 00 00 00 00 00"), + (json!(-5), "80 CB FB FF FF FF FF FF FF FF"), + // Values above i64::MAX use UInt64. + ( + json!(18446744073709551614u64), + "80 C7 FE FF FF FF FF FF FF FF", + ), + // Non-integral numbers use NumberDouble. + (json!(3.5), "80 CC 00 00 00 00 00 00 0C 40"), + // Strings ≤ 63 bytes use the encoded-length form (length baked into the + // marker), including "hello" — the corpus stores it as StrL1, but the + // encoder's canonical form is encoded-length. + (json!(""), "80 80"), + (json!("hi"), "80 82 68 69"), + (json!("hello"), "80 85 68 65 6C 6C 6F"), + // Containers always use LC* framing (byte-length + count). + (json!([]), "80 E5 00 00"), + (json!([true]), "80 E5 01 01 D2"), + (json!({}), "80 ED 00 00"), + (json!({"id": 0}), "80 ED 04 01 82 69 64 00"), + ]; + + for (value, expected_hex) in cases { + let encoded = encode(value); + let expected = hex(expected_hex); + assert_eq!( + encoded, expected, + "encoder output for {value} did not match the canonical snapshot\n expected: {expected_hex}\n actual: {}", + spaced_hex(&encoded), + ); + } +} + +/// RFC §7 (valid subset): where the golden corpus stores a **compact** wire form +/// the encoder does not emit (system strings, `Arr0`, `NumberUInt8`, …), the +/// encoder's own output differs byte-wise but still decodes to the same value. +/// This pins the intentional asymmetry so a future "make the encoder compact" +/// change is a conscious decision rather than a silent regression. +#[test] +fn encoder_emits_valid_subset_for_compact_corpus_forms() { + // (value, the corpus's compact encoding) — the encoder produces a *different* + // buffer, but both decode to `value`. + let compact_cases: &[(Value, &str)] = &[ + (json!(200), "80 C8 C8"), // corpus: NumberUInt8 + (json!("id"), "80 2C"), // corpus: system string + (json!([]), "80 E0"), // corpus: Arr0 + (json!({}), "80 E8"), // corpus: Obj0 + (json!([true]), "80 E1 D2"), // corpus: Arr1 + ( + json!([1, 2, 3]), + "80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00", + ), // uniform array + ]; + + for (value, compact_hex) in compact_cases { + let compact = hex(compact_hex); + // The compact form is valid and decodes to `value` ... + assert_eq!( + decode(&compact).unwrap(), + *value, + "compact corpus form {compact_hex} did not decode to {value}", + ); + // ... but the encoder emits a different (verbose) buffer. + let encoded = encode(value); + assert_ne!( + encoded, compact, + "encoder unexpectedly produced the compact form for {value}; update this test if the encoder was made compact", + ); + // ... which still decodes to the same value. + assert_eq!( + decode(&encoded).unwrap(), + *value, + "encoder's verbose form for {value} did not round-trip", + ); + } +} + +/// RFC §3.1: a complete buffer begins with the preamble and the encoder always +/// emits it. +#[test] +fn encoder_always_emits_preamble() { + for value in [ + json!(null), + json!(1), + json!("x"), + json!([1]), + json!({"a": 1}), + ] { + let encoded = encode(&value); + assert_eq!( + encoded.first(), + Some(&PREAMBLE), + "missing preamble for {value}" + ); + } +} + +/// Formats bytes as spaced uppercase hex for assertion messages. +fn spaced_hex(bytes: &[u8]) -> String { + bytes + .iter() + .map(|b| format!("{b:02X}")) + .collect::>() + .join(" ") +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs index b3607cd6a7b..7e2ffb3a72d 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs @@ -36,6 +36,8 @@ pub mod ser; pub mod system_strings; pub mod writer; +#[cfg(test)] +mod conformance; #[cfg(test)] mod fuzz_tests; #[cfg(test)] diff --git a/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json b/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json index 84515b918d5..d971b646866 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json +++ b/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json @@ -19,6 +19,10 @@ { "name": "float32", "binary": "80 CD 00 00 C0 3F", "json": 1.5 }, { "name": "float64", "binary": "80 CE 00 00 00 00 00 00 02 C0", "json": -2.25 }, { "name": "system_string_id", "binary": "80 2C", "json": "id" }, + { "name": "system_string_first", "binary": "80 20", "json": "$s" }, + { "name": "system_string_rid", "binary": "80 25", "json": "_rid" }, + { "name": "system_string_geometry_collection", "binary": "80 2B", "json": "GeometryCollection" }, + { "name": "system_string_last", "binary": "80 3F", "json": "_id" }, { "name": "encoded_length_string_empty", "binary": "80 80", "json": "" }, { "name": "encoded_length_string_hi", "binary": "80 82 68 69", "json": "hi" }, { "name": "str_l1_hello", "binary": "80 C0 05 68 65 6C 6C 6F", "json": "hello" }, @@ -51,6 +55,10 @@ { "name": "uniform_empty", "binary": "80 F0 DA 00", "json": [] }, { "name": "uniform_int16_c2", "binary": "80 F1 D9 03 00 FF FF 00 00 E8 03", "json": [-1,0,1000] }, { "name": "uniform_float32", "binary": "80 F0 CD 02 00 00 C0 3F 00 00 80 BE", "json": [1.5,-0.25] }, + { "name": "uniform_int8", "binary": "80 F0 D8 03 FB 00 05", "json": [-5,0,5] }, + { "name": "uniform_int64", "binary": "80 F0 DB 02 01 00 00 00 00 00 00 00 02 00 00 00 00 00 00 00", "json": [1,2] }, + { "name": "uniform_uint32", "binary": "80 F0 DC 02 FE FF FF FF 00 00 00 00", "json": [4294967294,0] }, + { "name": "uniform_float64", "binary": "80 F0 CE 02 00 00 00 00 00 00 F8 3F 00 00 00 00 00 00 02 C0", "json": [1.5,-2.25] }, { "name": "uniform_arr_of_arr", "binary": "80 F2 F0 DA 02 02 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00", "json": [[1,2],[3,4]] }, { "name": "empty_array", "binary": "80 E0", "json": [] }, { "name": "empty_object", "binary": "80 E8", "json": {} }, @@ -60,5 +68,21 @@ { "name": "array_lc1", "binary": "80 E5 03 03 00 01 D0", "json": [0,1,null] }, { "name": "object_l1", "binary": "80 EA 04 2C 00 3B 01", "json": {"id":0,"type":1} }, { "name": "object_lc1", "binary": "80 ED 04 02 2C 00 3B 01", "json": {"id":0,"type":1} }, - { "name": "nested_containers", "binary": "80 E2 05 E1 00 E9 2C 01", "json": [[0],{"id":1}] } + { "name": "nested_containers", "binary": "80 E2 05 E1 00 E9 2C 01", "json": [[0],{"id":1}] }, + { "name": "str_l4_hello", "binary": "80 C2 05 00 00 00 68 65 6C 6C 6F", "json": "hello" }, + { "name": "base64_url_len2", "binary": "80 74 01 00 00 FB FF FE", "json": "-__-" }, + { "name": "binary_4byte", "binary": "80 DF 04 00 00 00 DE AD BE EF", "json": "3q2+7w==" }, + { "name": "array_l2", "binary": "80 E3 03 00 00 01 D0", "json": [0,1,null] }, + { "name": "array_l4", "binary": "80 E4 03 00 00 00 00 01 D0", "json": [0,1,null] }, + { "name": "array_lc2", "binary": "80 E6 03 00 03 00 00 01 D0", "json": [0,1,null] }, + { "name": "array_lc4", "binary": "80 E7 03 00 00 00 03 00 00 00 00 01 D0", "json": [0,1,null] }, + { "name": "object_l2", "binary": "80 EB 04 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "object_l4", "binary": "80 EC 04 00 00 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "object_lc2", "binary": "80 EE 04 00 02 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "object_lc4", "binary": "80 EF 04 00 00 00 02 00 00 00 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "uniform_arr_of_arr_c2c2", "binary": "80 F3 F1 DA 02 00 02 00 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00", "json": [[1,2],[3,4]] }, + { "name": "reference_string_r1", "binary": "80 E2 05 82 68 69 C3 03", "json": ["hi","hi"] }, + { "name": "reference_string_r2", "binary": "80 E2 06 82 68 69 C4 03 00", "json": ["hi","hi"] }, + { "name": "reference_string_r3", "binary": "80 E2 07 82 68 69 C5 03 00 00", "json": ["hi","hi"] }, + { "name": "reference_string_r4", "binary": "80 E2 08 82 68 69 C6 03 00 00 00", "json": ["hi","hi"] } ] \ No newline at end of file diff --git a/sdk/cosmos/ci.yml b/sdk/cosmos/ci.yml index ca60f0f2c0c..e101b60dae7 100644 --- a/sdk/cosmos/ci.yml +++ b/sdk/cosmos/ci.yml @@ -61,6 +61,19 @@ extends: Selection: sparse NonSparseParameters: RustToolchainName GenerateVMJobs: true + # Non-blocking, byte-level codec fuzzing (cargo-fuzz). Adds a single + # Linux + nightly job to the Build stage (MatrixConfigs is Build-only). + # Its test-setup hook (Invoke-CosmosTestSetup.ps1, gated on + # AZURE_COSMOS_FUZZ=1) replays the golden vectors once (libFuzzer -runs=0) + # to prove they still decode without panicking. ContinueOnError=true, so a + # crash reports "succeeded with issues"; crash inputs are published as + # artifacts. Weekly-only: the golden vectors are already covered per-PR by + # the offline conformance unit tests. Runner: eng/scripts/Run-BinaryJsonFuzz.ps1. + - ${{ if or(eq(variables['Build.Reason'], 'Schedule'), endsWith(variables['Build.DefinitionName'], '- weekly')) }}: + - Name: Cosmos_binary_codec_golden_vector_validation + Path: sdk/cosmos/fuzz-matrix.json + Selection: all + GenerateVMJobs: true # Emulator jobs run only in Build. `MatrixConfigs` (unlike # `AdditionalMatrixConfigs`) is never consumed by the live-test stage in # archetype-sdk-client.yml, so these entries don't enter live stages, diff --git a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 index 296276565fb..30c18535996 100644 --- a/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 +++ b/sdk/cosmos/eng/scripts/Invoke-CosmosTestSetup.ps1 @@ -14,6 +14,37 @@ if ($env:COSMOS_RUSTFLAGS) { Write-Host "RUSTFLAGS appended with COSMOS_RUSTFLAGS: $env:RUSTFLAGS" } +# Byte-level binary-JSON codec fuzzing (cargo-fuzz). Triggered by +# AZURE_COSMOS_FUZZ=1 (matrix variable on the Linux + nightly fuzz leg in +# sdk/cosmos/fuzz-matrix.json). Replays the committed golden vectors through +# every codec fuzz target (libFuzzer -runs=0, no mutation) to prove they still +# decode without panicking. Linux-only; the leg carries ContinueOnError=true. +# Guarded so it runs once even though Test-Setup.ps1 fires per crate. +if ($env:AZURE_COSMOS_FUZZ -eq '1' -and -not $env:AZURE_COSMOS_FUZZ_RAN) { + $env:AZURE_COSMOS_FUZZ_RAN = '1' + if (-not $IsLinux) { + Write-Host "AZURE_COSMOS_FUZZ=1 but not on Linux; cargo-fuzz is Linux-only. Skipping." + } + else { + Write-Host "==> Cosmos binary-JSON fuzz: golden-vector corpus validation (-runs=0)" + & "$PSScriptRoot\Run-BinaryJsonFuzz.ps1" -ValidateOnly + } + # Strip any test_category cfg COSMOS_RUSTFLAGS injected so the subsequent + # cargo build/test runs only the always-on offline unit tests (no + # emulator/live-gated tests, which would panic with no connection string). + if ($env:RUSTFLAGS -match 'test_category') { + $env:RUSTFLAGS = ($env:RUSTFLAGS -replace '--cfg[= ]test_category="[^"]*"', '' -replace '\s+', ' ').Trim() + Write-Host "Stripped test_category from RUSTFLAGS on fuzz leg: '$env:RUSTFLAGS'" + } + # No live account is provisioned on the fuzz leg. Mark the test mode as + # skipped so `resolve_test_env` treats the unset connection string as a skip + # (not a fatal "required" panic) when the archetype's subsequent + # `cargo test` builds the account-backed driver tests on this Azure + # Pipelines job (SYSTEM_TEAMPROJECTID is set). + $env:AZURE_COSMOS_TEST_MODE = 'skipped' + return +} + # Hosted in-memory emulator path. The additional CI matrix sets one of the two # flavors below so the existing emulator suites run against both Gateway V1 # and Gateway 2.0 over cleartext HTTP/2. diff --git a/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 b/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 new file mode 100644 index 00000000000..dd0fdd344a1 --- /dev/null +++ b/sdk/cosmos/eng/scripts/Run-BinaryJsonFuzz.ps1 @@ -0,0 +1,131 @@ +#!/usr/bin/env pwsh +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +<# +.SYNOPSIS + Runs the coverage-guided cargo-fuzz targets for the Cosmos binary-JSON codec. + +.DESCRIPTION + Installs (idempotently) the nightly toolchain + cargo-fuzz, seeds each + target's corpus from the committed golden vectors, then runs every fuzz + target under `azure_data_cosmos_driver/fuzz` for a bounded wall-clock budget. + + This is a byte-level *protocol* fuzzer: it feeds arbitrary/mutated bytes + straight into the decoder (decode / from_slice / transcode_to_text) plus a + differential decode->encode->decode idempotence check. It is the complement + to the value-space live round-trip fuzzer in + azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs. + + LINUX ONLY. libFuzzer (the -fsanitize=fuzzer backend) is not supported on the + Windows MSVC target, so this script no-ops with a warning off Linux. + +.PARAMETER MaxTotalTimeSeconds + Wall-clock budget PER TARGET passed to libFuzzer as `-max_total_time`. + PR smoke runs use ~90s; weekly deep runs use ~1800s (30 min). Ignored when + -ValidateOnly is set. + +.PARAMETER ValidateOnly + Regression mode: replay ONLY the seeded golden-vector corpus through each + target once (libFuzzer `-runs=0`, no mutation, no time budget) and assert no + crash. This is a fast, deterministic gate suitable for the Build stage — it + proves the committed golden vectors still decode without panicking, without + the multi-minute coverage-guided soak (which belongs on a live/weekly leg). + +.PARAMETER Toolchain + Nightly toolchain to use. Defaults to the repo's pinned nightly if the + RUST_NIGHTLY_TOOLCHAIN env var is set, else plain `nightly`. + +.PARAMETER Targets + Which fuzz targets to run. Defaults to all four. + +.EXAMPLE + ./Run-BinaryJsonFuzz.ps1 -MaxTotalTimeSeconds 90 # PR smoke + ./Run-BinaryJsonFuzz.ps1 -MaxTotalTimeSeconds 1800 # weekly deep run +#> +[CmdletBinding()] +param( + [int] $MaxTotalTimeSeconds = 90, + [switch] $ValidateOnly, + [string] $Toolchain = $(if ($env:RUST_NIGHTLY_TOOLCHAIN) { $env:RUST_NIGHTLY_TOOLCHAIN } else { 'nightly' }), + [string[]] $Targets = @('decode', 'from_slice', 'transcode_to_text', 'decode_reencode_roundtrip'), + [int] $Workers = 0 # 0 => libFuzzer default (single); CI can raise it +) + +$ErrorActionPreference = 'Stop' + +if (-not $IsLinux) { + Write-Warning "cargo-fuzz / libFuzzer is only supported on Linux; skipping on this OS. Use WSL or a Linux CI leg." + exit 0 +} + +$driverDir = Resolve-Path "$PSScriptRoot/../../azure_data_cosmos_driver" +$fuzzDir = Join-Path $driverDir 'fuzz' +$vectorsPath = Join-Path $driverDir 'testdata/binary_json_vectors.json' + +Write-Host "==> Ensuring nightly toolchain '$Toolchain' + cargo-fuzz" +rustup toolchain install $Toolchain --profile minimal --component rust-src +# Detect cargo-fuzz via the installed-binaries list (always exits 0), since +# `cargo fuzz --help` would exit non-zero when missing and abort under +# $ErrorActionPreference='Stop'. +$fuzzInstalled = (cargo "+$Toolchain" install --list 2>$null) -match 'cargo-fuzz' +if (-not $fuzzInstalled) { + Write-Host "==> cargo-fuzz not found; installing" + cargo "+$Toolchain" install cargo-fuzz --locked +} + +# Seed each target's corpus from the committed golden vectors so libFuzzer +# mutates outward from real wire frames instead of blind byte flips. +if (Test-Path $vectorsPath) { + Write-Host "==> Seeding corpora from golden vectors" + $vectors = Get-Content $vectorsPath -Raw | ConvertFrom-Json + foreach ($t in $Targets) { + $corpus = Join-Path $fuzzDir "corpus/$t" + New-Item -ItemType Directory -Force $corpus | Out-Null + foreach ($v in $vectors) { + $bytes = ($v.binary -split '\s+' | ForEach-Object { [Convert]::ToByte($_, 16) }) + [IO.File]::WriteAllBytes((Join-Path $corpus $v.name), [byte[]]$bytes) + } + } +} + +Push-Location $driverDir +try { + $failed = @() + foreach ($t in $Targets) { + if ($ValidateOnly) { + # `-runs=0` replays the golden-vector-seeded corpus once with no + # mutation: a fast, deterministic regression gate. + Write-Host "==> Validating '$t' against the golden-vector corpus (-runs=0)" + $runArgs = @("+$Toolchain", 'fuzz', 'run', $t, '--', '-runs=0', '-print_final_stats=1') + } + else { + Write-Host "==> Fuzzing '$t' for ${MaxTotalTimeSeconds}s" + $runArgs = @("+$Toolchain", 'fuzz', 'run', $t, '--', "-max_total_time=$MaxTotalTimeSeconds", '-print_final_stats=1') + if ($Workers -gt 0) { $runArgs += "-workers=$Workers"; $runArgs += "-jobs=$Workers" } + } + & cargo @runArgs + if ($LASTEXITCODE -ne 0) { + $failed += $t + Write-Host "##vso[task.logissue type=error]Fuzz target '$t' found a crash. Minimize with: cargo +$Toolchain fuzz tmin $t " + } + } + + # Publish any crash inputs so a failure can be reproduced off-agent. + $artifactsDir = Join-Path $fuzzDir 'artifacts' + if (Test-Path $artifactsDir) { + $crashes = Get-ChildItem $artifactsDir -Recurse -File -ErrorAction SilentlyContinue + if ($crashes -and $env:SYSTEM_TEAMPROJECTID) { + Write-Host "##vso[task.logissue type=warning]Fuzz crash artifacts found; published as 'fuzz-crashes'." + Write-Host "##vso[artifact.upload artifactname=fuzz-crashes]$((Resolve-Path $artifactsDir).Path)" + } + } + + if ($failed.Count -gt 0) { + throw "Fuzz targets reported crashes: $($failed -join ', ')" + } + Write-Host "==> All fuzz targets clean." +} +finally { + Pop-Location +} diff --git a/sdk/cosmos/fuzz-matrix.json b/sdk/cosmos/fuzz-matrix.json new file mode 100644 index 00000000000..3f9b1025d9b --- /dev/null +++ b/sdk/cosmos/fuzz-matrix.json @@ -0,0 +1,18 @@ +{ + "displayNames": { + "golden": "binary_codec_golden_vector_validation", + "1": "", + "true": "" + }, + "matrix": { + "Agent": { + "golden": { + "OSVmImage": "env:LINUXVMIMAGE", + "Pool": "env:LINUXPOOL" + } + }, + "RustToolchainName": ["nightly"], + "AZURE_COSMOS_FUZZ": ["1"], + "ContinueOnError": ["true"] + } +} From 0151dce942ec831777c4680b29e0095e634f810e Mon Sep 17 00:00:00 2001 From: kundadebdatta Date: Mon, 3 Aug 2026 17:03:36 -0700 Subject: [PATCH 2/2] docs(cosmos): delink Part A files from fuzz README and RFC (link check) --- sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md | 2 +- sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md index a65226b6ec5..5443e304c18 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md @@ -70,7 +70,7 @@ flowchart TD | `binary_json/conformance.rs` | Encoder byte-exactness + canonical-form snapshots | §3.1, §7 | | `binary_json/reader.rs`, `de.rs` (tests) | Decoder conformance per form | §4–§6, §8 | | `binary_json/fuzz_tests.rs` | Decoder never panics/hangs/over-allocates on malformed input | §8, §9 | -| [`BINARY_ENCODING_ROUNDTRIP_FUZZER.md`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md) + harness | End-to-end **round-trip invariant** on random JSON against the live service, at volume | §7 (round-trip), value model (§2) | +| `BINARY_ENCODING_ROUNDTRIP_FUZZER.md` + harness | End-to-end **round-trip invariant** on random JSON against the live service, at volume | §7 (round-trip), value model (§2) | Two connections are worth calling out explicitly: diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md index e7a3cdf8f5d..c89f4d0ed3c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md @@ -2,7 +2,7 @@ Coverage-guided, **byte-level** fuzzing for the Cosmos binary JSON codec (`azure_data_cosmos_driver::binary_json`). Where the live -[round-trip fuzzer](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs) +round-trip fuzzer (`azure_data_cosmos/tests/binary_roundtrip_fuzzer.rs`) generates random JSON *values* and only ever feeds the decoder **encoder-produced** (well-formed) bytes, these targets feed **arbitrary and mutated bytes** straight into the decoder — so they exercise the *format*/protocol itself: truncated