diff --git a/Cargo.lock b/Cargo.lock index 5a906b143e7..489d90556f5 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,6 +108,25 @@ version = "1.0.104" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "330a5ed07fa54e4702c9d6c4174f74427fc0ef6e214bbd677ae50a5099946470" +[[package]] +name = "arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3d036a3c4ab069c7b410a2ce876bd74808d2d0888a82667669f8e783a898bf1" +dependencies = [ + "derive_arbitrary", +] + +[[package]] +name = "arbitrary-json" +version = "0.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "08117a235f4bfba33f065e5f6941838fa9f77436c1823fca9c93c9b4a34a40e0" +dependencies = [ + "arbitrary", + "serde_json", +] + [[package]] name = "arc-swap" version = "1.9.2" @@ -548,6 +567,8 @@ dependencies = [ "azure_core 1.1.0", "azure_data_cosmos_driver", "criterion", + "serde", + "serde_json", "tokio", "url", ] @@ -621,6 +642,8 @@ dependencies = [ name = "azure_data_cosmos_perf" version = "0.1.0" dependencies = [ + "arbitrary", + "arbitrary-json", "async-trait", "azure_core 1.1.0", "azure_data_cosmos", @@ -631,9 +654,11 @@ dependencies = [ "futures", "hdrhistogram", "hostname", + "json-canon", "rand 0.10.2", "serde", "serde_json", + "sha2", "sysinfo", "time", "tokio", @@ -1458,6 +1483,17 @@ dependencies = [ "serde_core", ] +[[package]] +name = "derive_arbitrary" +version = "1.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e567bd82dcff979e4b03460c307b3cdc9e96fde3d73bed1496d2bc75d9dd62a" +dependencies = [ + "proc-macro2", + "quote", + "syn 2.0.119", +] + [[package]] name = "digest" version = "0.10.7" @@ -2318,6 +2354,17 @@ dependencies = [ "wasm-bindgen", ] +[[package]] +name = "json-canon" +version = "0.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "447ae153a2bd47d61acc0d131295408e32ef87ed9785825a6f4ecef85afc0edb" +dependencies = [ + "ryu-js", + "serde", + "serde_json", +] + [[package]] name = "json-patch" version = "4.2.0" @@ -3351,6 +3398,12 @@ version = "1.0.23" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "cf54715a573b99ac80df0bc206da022bcd442c974952c7b9720069370852e21f" +[[package]] +name = "ryu-js" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6518fc26bced4d53678a22d6e423e9d8716377def84545fe328236e3af070e7f" + [[package]] name = "same-file" version = "1.0.6" diff --git a/Cargo.toml b/Cargo.toml index 4efe6ae96ad..082d0ed0ece 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -87,6 +87,8 @@ version = "1.0.0" version = "1.0.0" [workspace.dependencies] +arbitrary = { version = "1.4", features = ["derive"] } +arbitrary-json = "0.1" async-lock = "3.4" async-stream = { version = "0.3.6" } async-trait = "0.1" @@ -115,6 +117,7 @@ hdrhistogram = "7.5" hostname = "0.4" hmac = { version = "0.12" } include-file = { version = "1.0.0", default-features = false } +json-canon = "0.1" openssl = { version = "0.10.79" } opentelemetry = { version = "0.32", features = ["trace"] } opentelemetry-appender-tracing = { version = "0.32" } diff --git a/sdk/cosmos/.cspell.json b/sdk/cosmos/.cspell.json index ab818543976..d514287e77f 100644 --- a/sdk/cosmos/.cspell.json +++ b/sdk/cosmos/.cspell.json @@ -8,6 +8,7 @@ "ALPN", "apacsoutheast", "Appleby's", + "AQID", "argtypes", "asyncio", "australiacentral", @@ -19,6 +20,7 @@ "azurecosmos", "azurecosmosdriver", "backoff", + "backoffs", "backpressure", "BADFUNC", "bindgen", @@ -30,6 +32,7 @@ "cabi", "canadacentral", "canadaeast", + "CBOR", "cbindgen", "CDLL", "cdriver", @@ -49,6 +52,7 @@ "chinaeast", "chinanorth", "chokepoint", + "canonicalizer", "cloneable", "codepoint", "codepoints", @@ -61,6 +65,7 @@ "consumingly", "Conv", "cooldown", + "cosmosbinary", "cosmosclient", "cosmosdriver", "correlator", @@ -78,6 +83,7 @@ "dcount", "dedicatedgateway", "deprioritized", + "dkunda", "deprioritizes", "derefs", "dhat", @@ -125,6 +131,7 @@ "Geospatial", "fshort", "funct", + "fuzzable", "gatewayversion", "germanycentral", "germanynorth", @@ -148,10 +155,12 @@ "inclusivity", "inlines", "inmemory", + "ints", "INVALIDARG", "isquery", "japaneast", "japanwest", + "jsontext", "keepalive", "keyspace", "koreacentral", @@ -163,6 +172,7 @@ "libazurecosmosdriver", "libclang", "LIBCLANG", + "libfuzzer", "libqueryplaninterop", "QUERYPLANINTEROP", "linearizability", @@ -198,6 +208,7 @@ "nocoll", "noretry", "normalises", + "notneg", "northcentralus", "northeurope", "norwayeast", @@ -233,6 +244,7 @@ "precomputation", "preimage", "Prereqs", + "PRNG", "ptrs", "pushback", "pushdown", @@ -242,10 +254,15 @@ "RAII", "readfeed", "recompiles", + "redecoded", + "reencode", + "reencoded", "refetch", "refetched", "refetches", "refcounted", + "reparses", + "reparsed", "Replicaset", "reqs", "restype", @@ -262,9 +279,11 @@ "RUPM", "rwcache", "sbyte", + "seqs", "serviceunavailable", "serviceversion", "sess", + "Signedness", "southafricanorth", "southafricawest", "southcentralus", @@ -275,6 +294,7 @@ "sprocs", "staticlib", "stdlib", + "streamable", "subrange", "submitret", "subsec", @@ -293,10 +313,13 @@ "testdb", "thinclient", "threadsafe", + "tmin", "TOCTOU", "Tokio", "TOPCOUNT", "TPIO", + "transcoders", + "trippable", "uaecentral", "uaenorth", "udfs", @@ -307,6 +330,7 @@ "unavail", "uncollapsed", "uncontended", + "undecoded", "undrained", "unfaulted", "ungoverned", @@ -356,7 +380,8 @@ "**/query_plan_native/QueryPlanInterop.h", "**/query_plan_native/bindgen_wrapper.h", "**/query_plan_native/generated/native_bindings.rs", - "vnext-emulator-matrix.json" + "vnext-emulator-matrix.json", + "**/azure_data_cosmos_perf/testdata/**" ], "overrides": [ { diff --git a/sdk/cosmos/azure_data_cosmos/CHANGELOG.md b/sdk/cosmos/azure_data_cosmos/CHANGELOG.md index b9abdbbab1e..93bf9d0eb14 100644 --- a/sdk/cosmos/azure_data_cosmos/CHANGELOG.md +++ b/sdk/cosmos/azure_data_cosmos/CHANGELOG.md @@ -4,11 +4,7 @@ ### Features Added -### Breaking Changes - -### Bugs Fixed - -### Other Changes +- Added opt-in Cosmos binary JSON encoding for item operations (`create`/`read`/`replace`/`upsert`). Enable it via `CosmosClientBuilder::with_binary_encoding_options` (or the `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment-variable fallback). Off by default; when disabled, requests and responses are byte-for-byte unchanged. ([#4671](https://github.com/Azure/azure-sdk-for-rust/pull/4671)) ## 0.37.1 (2026-07-23) diff --git a/sdk/cosmos/azure_data_cosmos/Cargo.toml b/sdk/cosmos/azure_data_cosmos/Cargo.toml index ae71ef1e03c..1b08ee88111 100644 --- a/sdk/cosmos/azure_data_cosmos/Cargo.toml +++ b/sdk/cosmos/azure_data_cosmos/Cargo.toml @@ -122,6 +122,11 @@ name = "split" path = "tests/split.rs" required-features = ["key_auth", "fault_injection"] +[[test]] +name = "binary_encoding" +path = "tests/binary_encoding.rs" +required-features = ["key_auth", "fault_injection"] + [[test]] name = "gateway_v2" path = "tests/gateway_v2.rs" diff --git a/sdk/cosmos/azure_data_cosmos/build.rs b/sdk/cosmos/azure_data_cosmos/build.rs index a75eef56071..4a1c47219ae 100644 --- a/sdk/cosmos/azure_data_cosmos/build.rs +++ b/sdk/cosmos/azure_data_cosmos/build.rs @@ -7,6 +7,6 @@ fn main() { // Allow `#[cfg_attr(not(test_category = "..."), ignore)]` in `tests/*.rs`. println!( - "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"multi_write\", \"split\", \"gateway_v2\", \"gateway_v2_multi_region\"))" + "cargo:rustc-check-cfg=cfg(test_category, values(\"emulator\", \"emulator_vnext\", \"multi_write\", \"split\", \"binary_encoding\", \"gateway_v2\", \"gateway_v2_multi_region\"))" ); } diff --git a/sdk/cosmos/azure_data_cosmos/src/clients/container_client.rs b/sdk/cosmos/azure_data_cosmos/src/clients/container_client.rs index ed0778ecad7..519fde31682 100644 --- a/sdk/cosmos/azure_data_cosmos/src/clients/container_client.rs +++ b/sdk/cosmos/azure_data_cosmos/src/clients/container_client.rs @@ -8,10 +8,10 @@ use crate::{ models::{BatchResponse, ChangeFeedItem, ItemResponse, ResourceResponse}, models::{ContainerProperties, PatchInstructions, ThroughputProperties}, options::{ - BatchOptions, ChangeFeedOptions, ChangeFeedStartFrom, DeleteContainerOptions, - ItemReadOptions, ItemWriteOptions, PatchItemOptions, Precondition, QueryOptions, - ReadContainerOptions, ReadFeedRangesOptions, ReplaceContainerOptions, SessionToken, - ThroughputOptions, + BatchOptions, BinaryEncodingOptions, ChangeFeedOptions, ChangeFeedStartFrom, + DeleteContainerOptions, ItemReadOptions, ItemWriteOptions, OperationOptions, + PatchItemOptions, Precondition, QueryOptions, ReadContainerOptions, ReadFeedRangesOptions, + ReplaceContainerOptions, SessionToken, ThroughputOptions, }, PartitionKey, Query, }; @@ -308,7 +308,9 @@ impl ContainerClient { options: Option, ) -> crate::Result { let options = options.unwrap_or_default(); - let body = serde_json::to_vec(&item)?; + let (operation_options, binary) = + resolve_binary_encoding(options.operation, &self.context.binary_encoding); + let body = serialize_item_body(&item, binary.enabled)?; // Build the driver's item reference from our stored container metadata. let item_ref = ItemReference::from_name( @@ -321,11 +323,12 @@ impl ContainerClient { let operation = CosmosOperation::create_item(item_ref).with_body(body); let operation = apply_item_options(operation, options.session_token, options.precondition); - // Execute through the driver. + // Execute through the driver, with binary encoding on the operation + // options so the driver negotiates the wire format and transcoding. let driver_response = self .context .driver - .execute_singleton_operation(operation, options.operation) + .execute_singleton_operation(operation, operation_options) .await?; // Bridge the driver response to the SDK response type. @@ -406,7 +409,9 @@ impl ContainerClient { options: Option, ) -> crate::Result { let options = options.unwrap_or_default(); - let body = serde_json::to_vec(&item)?; + let (operation_options, binary) = + resolve_binary_encoding(options.operation, &self.context.binary_encoding); + let body = serialize_item_body(&item, binary.enabled)?; // Build the driver's item reference from our stored container metadata. let item_ref = ItemReference::from_name( @@ -419,11 +424,12 @@ impl ContainerClient { let operation = CosmosOperation::replace_item(item_ref).with_body(body); let operation = apply_item_options(operation, options.session_token, options.precondition); - // Execute through the driver. + // Execute through the driver, with binary encoding on the operation + // options so the driver negotiates the wire format and transcoding. let driver_response = self .context .driver - .execute_singleton_operation(operation, options.operation) + .execute_singleton_operation(operation, operation_options) .await?; // Bridge the driver response to the SDK response type. @@ -614,7 +620,9 @@ impl ContainerClient { options: Option, ) -> crate::Result { let options = options.unwrap_or_default(); - let body = serde_json::to_vec(&item)?; + let (operation_options, binary) = + resolve_binary_encoding(options.operation, &self.context.binary_encoding); + let body = serialize_item_body(&item, binary.enabled)?; // Build the driver's item reference from our stored container metadata. let item_ref = ItemReference::from_name( @@ -627,11 +635,12 @@ impl ContainerClient { let operation = CosmosOperation::upsert_item(item_ref).with_body(body); let operation = apply_item_options(operation, options.session_token, options.precondition); - // Execute through the driver. + // Execute through the driver, with binary encoding on the operation + // options so the driver negotiates the wire format and transcoding. let driver_response = self .context .driver - .execute_singleton_operation(operation, options.operation) + .execute_singleton_operation(operation, operation_options) .await?; // Bridge the driver response to the SDK response type. @@ -675,6 +684,8 @@ impl ContainerClient { options: Option, ) -> crate::Result { let options = options.unwrap_or_default(); + let (operation_options, _binary) = + resolve_binary_encoding(options.operation, &self.context.binary_encoding); // Build the driver's item reference from our stored container metadata. let item_ref = ItemReference::from_name( @@ -687,11 +698,12 @@ impl ContainerClient { let operation = CosmosOperation::read_item(item_ref); let operation = apply_item_options(operation, options.session_token, options.precondition); - // Execute through the driver. + // Execute through the driver, with binary encoding on the operation + // options so the driver negotiates the wire format and transcoding. let driver_response = self .context .driver - .execute_singleton_operation(operation, options.operation) + .execute_singleton_operation(operation, operation_options) .await?; // Bridge the driver response to the SDK response type. @@ -1228,6 +1240,55 @@ fn apply_item_options( operation } +/// Serializes an item write body as either Cosmos binary JSON (`binary`) or +/// UTF-8 text JSON. +/// +/// The binary path uses the driver's native serde serializer +/// [`binary_json::to_vec`](azure_data_cosmos_driver::binary_json::to_vec), +/// encoding `T` straight to Cosmos binary JSON without an intermediate +/// [`serde_json::Value`]; the text path is the original [`serde_json::to_vec`]. +/// Both produce a body the service accepts — the binary form begins with the +/// `0x80` preamble, which the service detects from the first byte, so the +/// request `Content-Type` stays `application/json`. +fn serialize_item_body(item: &T, binary: bool) -> crate::Result> { + if binary { + let body = azure_data_cosmos_driver::binary_json::to_vec(item) + .map_err(crate::error::convert_binary_encode_error)?; + tracing::debug!( + binary_encoding = true, + "binary encoding applied to item write body" + ); + Ok(body) + } else { + tracing::debug!( + binary_encoding = false, + "item write body serialized as text JSON" + ); + Ok(serde_json::to_vec(item)?) + } +} + +/// Resolves the effective binary encoding for an item operation, preferring a +/// caller-set per-operation value over the client-level default. +/// +/// Returns the resolved options alongside the updated [`OperationOptions`] so +/// the caller drives body serialization from the same decision. The operation +/// field is normalized to `Some(effective)` when enabled (the driver negotiates +/// the binary wire) and `None` when disabled (byte-for-byte unchanged). +fn resolve_binary_encoding( + mut options: OperationOptions, + client_default: &BinaryEncodingOptions, +) -> (OperationOptions, BinaryEncodingOptions) { + let effective = options + .binary_encoding + .take() + .unwrap_or_else(|| client_default.clone()); + // Write `Some` (never `None`, which means "inherit") so a resolved disable + // overrides driver-layer defaults; the wire is unchanged when disabled. + options.binary_encoding = Some(effective.clone()); + (options, effective) +} + /// Applies [`BatchOptions`] fields to a [`CosmosOperation`]. /// /// [`BatchOptions`] carries a session token but no precondition (ETag-based @@ -1263,7 +1324,112 @@ fn _assert_futures_are_send() { #[cfg(test)] mod tests { - use super::should_force_refresh_feed_ranges; + //! These are sanity checks that [`serialize_item_body`] picks the right + //! path (text vs binary) and that binary encoding is actually applied — + //! not full serialize/deserialize coverage. Byte-level codec correctness + //! lives in the driver's `binary_json` snapshot, golden-vector, and parity + //! tests. + use super::*; + use serde_json::json; + + #[test] + fn serialize_item_body_text_matches_serde_to_vec() { + // The text path is byte-for-byte the original `serde_json::to_vec`. + let item = json!({ "id": "1", "count": 7, "tags": ["a", "b"] }); + let body = serialize_item_body(&item, false).unwrap(); + assert_eq!(body, serde_json::to_vec(&item).unwrap()); + } + + #[test] + fn serialize_item_body_binary_round_trips() { + // The binary path begins with the `0x80` preamble and decodes back to + // the same value the text path would have serialized. + let item = json!({ "id": "doc-1", "count": 42, "nested": { "ok": true } }); + let body = serialize_item_body(&item, true).unwrap(); + assert_eq!(body.first(), Some(&0x80)); + let decoded: serde_json::Value = + azure_data_cosmos_driver::binary_json::decode(&body).unwrap(); + assert_eq!(decoded, item); + } + + #[test] + fn serialize_item_body_binary_differs_from_text() { + // Sanity check that the two paths actually produce different bytes. + let item = json!({ "id": "x" }); + let text = serialize_item_body(&item, false).unwrap(); + let binary = serialize_item_body(&item, true).unwrap(); + assert_ne!(text, binary); + assert_ne!(text.first(), Some(&0x80)); + } + + #[test] + fn resolve_binary_encoding_uses_client_default_when_operation_unset() { + // No per-operation value: the client-level default applies. Enabled ⇒ + // the driver option is set. + let client = BinaryEncodingOptions::new().with_enabled(true); + let (options, effective) = resolve_binary_encoding(OperationOptions::default(), &client); + assert!(effective.enabled); + assert_eq!(options.binary_encoding, Some(client)); + } + + #[test] + fn resolve_binary_encoding_carries_request_text_response() { + // Binary on with request_text_response: the driver keeps the wire binary + // and transcodes the response to text. Both flags carry through. + let client = BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true); + let (options, effective) = resolve_binary_encoding(OperationOptions::default(), &client); + assert!(effective.enabled); + assert!(effective.request_text_response); + let resolved = options.binary_encoding.expect("binary encoding set"); + assert!(resolved.enabled); + assert!(resolved.request_text_response); + } + + #[test] + fn resolve_binary_encoding_omits_option_when_disabled() { + // Disabled default with no per-op value is preserved as `Some(false)`, + // not erased to `None`, so it overrides driver-layer defaults. + let client = BinaryEncodingOptions::new().with_enabled(false); + let (options, effective) = resolve_binary_encoding(OperationOptions::default(), &client); + assert!(!effective.enabled); + assert_eq!( + options.binary_encoding.map(|b| b.enabled), + Some(false), + "resolved disable must be preserved as Some(false) to override driver defaults" + ); + } + + #[test] + fn resolve_binary_encoding_operation_disable_overrides_enabled_client() { + // A per-operation disable wins over an enabled client default. + let client = BinaryEncodingOptions::new().with_enabled(true); + let mut operation = OperationOptions::default(); + operation.binary_encoding = Some(BinaryEncodingOptions::new().with_enabled(false)); + let (options, effective) = resolve_binary_encoding(operation, &client); + assert!(!effective.enabled); + assert_eq!( + options.binary_encoding.map(|b| b.enabled), + Some(false), + "per-operation disable must be preserved as Some(false)" + ); + } + + #[test] + fn resolve_binary_encoding_operation_enable_overrides_disabled_client() { + // Client disabled, but the caller enabled binary for this operation: + // the per-operation value wins, so binary is negotiated for this request. + let client = BinaryEncodingOptions::new().with_enabled(false); + let operation_be = BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true); + let mut operation = OperationOptions::default(); + operation.binary_encoding = Some(operation_be.clone()); + let (options, effective) = resolve_binary_encoding(operation, &client); + assert!(effective.enabled); + assert_eq!(options.binary_encoding, Some(operation_be)); + } #[test] fn feed_ranges_refreshes_missing_or_empty_initial_resolution() { diff --git a/sdk/cosmos/azure_data_cosmos/src/clients/cosmos_client_builder.rs b/sdk/cosmos/azure_data_cosmos/src/clients/cosmos_client_builder.rs index e68479e7fad..426615c2243 100644 --- a/sdk/cosmos/azure_data_cosmos/src/clients/cosmos_client_builder.rs +++ b/sdk/cosmos/azure_data_cosmos/src/clients/cosmos_client_builder.rs @@ -7,9 +7,9 @@ use std::sync::Arc; use crate::{ - clients::ClientContext, + clients::{resolve_binary_encoding, ClientContext}, options::{ - CosmosClientOptions, OperationOptions, PartitionFailoverOptions, + BinaryEncodingOptions, CosmosClientOptions, OperationOptions, PartitionFailoverOptions, ThroughputControlGroupOptions, UserAgentSuffix, }, AccountReference, CosmosClient, CosmosCredential, CosmosRuntime, RoutingStrategy, @@ -173,6 +173,22 @@ impl CosmosClientBuilder { self } + /// Sets the Cosmos binary JSON encoding options for this client. + /// + /// Binary encoding governs two things together: encoding item write bodies + /// as binary and advertising that the client accepts binary responses via + /// the response-format negotiation header. The options are resolved once at + /// [`build()`](Self::build) time. + /// + /// When this setter is **not** called, enablement falls back to the + /// `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment variable (truthy + /// values `1` / `true` / `yes` / `on`, case-insensitive, trimmed). Passing + /// explicit options here takes precedence over that variable. + pub fn with_binary_encoding_options(mut self, options: BinaryEncodingOptions) -> Self { + self.options.binary_encoding = Some(options); + self + } + /// Configures fault injection for testing. /// /// Accepts a vector of [`FaultInjectionRule`](crate::fault_injection::FaultInjectionRule) @@ -289,7 +305,10 @@ impl CosmosClientBuilder { let driver = runtime.into_inner().create_driver(driver_options).await?; Ok(CosmosClient { - context: ClientContext { driver }, + context: ClientContext { + driver, + binary_encoding: resolve_binary_encoding(self.options.binary_encoding), + }, }) } } diff --git a/sdk/cosmos/azure_data_cosmos/src/clients/mod.rs b/sdk/cosmos/azure_data_cosmos/src/clients/mod.rs index 0ef8a892932..4a1291cf3fd 100644 --- a/sdk/cosmos/azure_data_cosmos/src/clients/mod.rs +++ b/sdk/cosmos/azure_data_cosmos/src/clients/mod.rs @@ -40,6 +40,8 @@ use std::sync::Arc; use azure_data_cosmos_driver::CosmosDriver; +use crate::options::BinaryEncodingOptions; + /// Shared infrastructure threaded from [`CosmosClient`](super::CosmosClient) /// through [`DatabaseClient`](super::DatabaseClient) to /// [`ContainerClient`](super::ContainerClient). @@ -49,4 +51,112 @@ use azure_data_cosmos_driver::CosmosDriver; #[derive(Clone, Debug)] pub(crate) struct ClientContext { pub(crate) driver: Arc, + pub(crate) binary_encoding: BinaryEncodingOptions, +} + +/// The environment variable that enables Cosmos binary JSON encoding when no +/// explicit client option is supplied. +pub(crate) const BINARY_ENCODING_ENV_VAR: &str = "AZURE_COSMOS_BINARY_ENCODING_ENABLED"; + +/// Resolves the client's [`BinaryEncodingOptions`] from an explicit client +/// option, falling back to the environment when the option is unset. +/// +/// Resolution happens **once** at client construction. An explicit client +/// option (see +/// [`CosmosClientBuilder::with_binary_encoding_options`](crate::CosmosClientBuilder::with_binary_encoding_options)) +/// takes precedence; when the caller leaves it unset (`None`), enablement falls +/// back to the [`BINARY_ENCODING_ENV_VAR`] environment variable (truthy values +/// `1` / `true` / `yes` / `on`, case-insensitive, trimmed). The resolved +/// options are the single source of truth for both encoding item bodies and +/// advertising binary-response negotiation. Binary encoding is in preview. +pub(crate) fn resolve_binary_encoding( + explicit: Option, +) -> BinaryEncodingOptions { + resolve_binary_encoding_with(explicit, |name| std::env::var(name).ok()) +} + +/// Resolves [`BinaryEncodingOptions`] using an injected environment reader. +/// +/// The `get_env` closure mirrors the driver's config tooling so callers (and +/// tests) can supply environment values without touching the real process +/// environment via `std::env::set_var` (which is `unsafe` in recent `std`). +fn resolve_binary_encoding_with( + explicit: Option, + get_env: impl Fn(&str) -> Option, +) -> BinaryEncodingOptions { + explicit.unwrap_or_else(|| { + let enabled = get_env(BINARY_ENCODING_ENV_VAR) + .as_deref() + .map(flag_value_is_truthy) + .unwrap_or(false); + BinaryEncodingOptions::new().with_enabled(enabled) + }) +} + +/// Returns `true` if `value` is one of the accepted truthy spellings +/// (`1` / `true` / `yes` / `on`), case-insensitive and trimmed. +fn flag_value_is_truthy(value: &str) -> bool { + matches!( + value.trim().to_ascii_lowercase().as_str(), + "1" | "true" | "yes" | "on" + ) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn truthy_flag_values_are_accepted() { + for v in ["1", "true", "TRUE", "Yes", "on", " On ", "\ttrue\n"] { + assert!(flag_value_is_truthy(v), "{v:?} should be truthy"); + } + } + + #[test] + fn non_truthy_flag_values_are_rejected() { + for v in ["", "0", "false", "no", "off", "2", "enabled", "y"] { + assert!(!flag_value_is_truthy(v), "{v:?} should not be truthy"); + } + } + + #[test] + fn explicit_options_win_over_environment() { + // An explicit disabled option must beat a truthy environment value, and + // vice versa — the injected env reader is never consulted when the + // caller supplied options. + let disabled = resolve_binary_encoding_with( + Some(BinaryEncodingOptions::new().with_enabled(false)), + |_| Some("true".to_owned()), + ); + assert!(!disabled.enabled); + + let enabled = resolve_binary_encoding_with( + Some(BinaryEncodingOptions::new().with_enabled(true)), + |_| Some("false".to_owned()), + ); + assert!(enabled.enabled); + } + + #[test] + fn environment_enables_when_option_unset() { + // With no explicit option, a truthy injected env value enables binary + // encoding — exercised without touching the real process environment. + let resolved = resolve_binary_encoding_with(None, |name| { + assert_eq!(name, BINARY_ENCODING_ENV_VAR); + Some("on".to_owned()) + }); + assert!(resolved.enabled); + } + + #[test] + fn environment_disabled_by_default_when_unset() { + // No explicit option and no env value ⇒ disabled. + let resolved = resolve_binary_encoding_with(None, |_| None); + assert!(!resolved.enabled); + + // A non-truthy env value also resolves to disabled. + let resolved = resolve_binary_encoding_with(None, |_| Some("nope".to_owned())); + assert!(!resolved.enabled); + } } diff --git a/sdk/cosmos/azure_data_cosmos/src/error.rs b/sdk/cosmos/azure_data_cosmos/src/error.rs index 24d55c86d1c..479647ee375 100644 --- a/sdk/cosmos/azure_data_cosmos/src/error.rs +++ b/sdk/cosmos/azure_data_cosmos/src/error.rs @@ -114,6 +114,27 @@ impl From for CosmosError { } } +/// Converts a binary-JSON encode error into a [`CosmosError`]. +/// +/// This is deliberately a call-site helper rather than a `From` impl: the +/// binary codec is only ever encoded on the item **write** path +/// (`binary_json::to_vec(item)`), so a failure is always a request-body +/// (encode) error, never a response-body (decode) error — binary **response** +/// decoding is mapped inside the driver. A blanket `From` would invite `?` to +/// mislabel a future decode failure as a request-body error, so the mapping is +/// kept explicit at the one call site that needs it. +pub(crate) fn convert_binary_encode_error( + error: azure_data_cosmos_driver::binary_json::BinaryError, +) -> CosmosError { + CosmosError( + DriverCosmosError::builder() + .with_status(CosmosStatus::SERIALIZATION_REQUEST_BODY_INVALID) + .with_message("failed to serialize item to Cosmos binary JSON") + .with_source(error) + .build(), + ) +} + /// Per Azure SDK for Rust guideline: every service-crate error type provides a /// [`From`] impl into [`azure_core::Error`] so callers using the foundation /// error type via `?`/`From` continue to compose. @@ -204,7 +225,8 @@ fn classify_for_azure_core(err: &CosmosError) -> azure_core::error::ErrorKind { | Some(SubStatusCode::CLIENT_GENERATED_401) => CoreKind::Credential, // Serialization boundary - Some(SubStatusCode::SERIALIZATION_RESPONSE_BODY_INVALID) => CoreKind::DataConversion, + Some(SubStatusCode::SERIALIZATION_RESPONSE_BODY_INVALID) + | Some(SubStatusCode::SERIALIZATION_REQUEST_BODY_INVALID) => CoreKind::DataConversion, // Request provably NEVER reached the wire — safe to retry non-idempotent writes // (matches `azure_core::ErrorKind::Connection` semantics). diff --git a/sdk/cosmos/azure_data_cosmos/src/options/client.rs b/sdk/cosmos/azure_data_cosmos/src/options/client.rs index 9b4bab4d324..fcbadf2150a 100644 --- a/sdk/cosmos/azure_data_cosmos/src/options/client.rs +++ b/sdk/cosmos/azure_data_cosmos/src/options/client.rs @@ -3,7 +3,7 @@ //! [`CosmosClientOptions`] — options for [`CosmosClient`](crate::CosmosClient) construction. -use azure_data_cosmos_driver::options::{OperationOptions, UserAgentSuffix}; +use azure_data_cosmos_driver::options::{BinaryEncodingOptions, OperationOptions, UserAgentSuffix}; /// Options used when creating a [`CosmosClient`](crate::CosmosClient). /// @@ -17,6 +17,8 @@ pub struct CosmosClientOptions { /// unless overridden by per-request options. pub operation: OperationOptions, pub(crate) user_agent_suffix: Option, + /// Options to control binary encoding. + pub(crate) binary_encoding: Option, } impl CosmosClientOptions { diff --git a/sdk/cosmos/azure_data_cosmos/src/options/mod.rs b/sdk/cosmos/azure_data_cosmos/src/options/mod.rs index 688af80d6ea..cc357d7c938 100644 --- a/sdk/cosmos/azure_data_cosmos/src/options/mod.rs +++ b/sdk/cosmos/azure_data_cosmos/src/options/mod.rs @@ -13,14 +13,15 @@ pub use azure_data_cosmos_driver::models::{ }; #[doc(inline)] pub use azure_data_cosmos_driver::options::{ - AvailabilityStrategy, ConnectionPoolOptions, ConnectionPoolOptionsBuilder, - ContentResponseOnWrite, DiagnosticsOptions, DiagnosticsOptionsBuilder, DiagnosticsVerbosity, - EndToEndOperationLatencyPolicy, ExcludedRegions, HedgeThreshold, HedgingStrategy, - OperationOptions, OperationOptionsBuilder, OperationOptionsView, PartitionFailoverOptions, - PartitionFailoverOptionsBuilder, PriorityLevel, ReadConsistencyStrategy, Region, - ServerCertificateValidation, ThrottlingRetryOptions, ThrottlingRetryOptionsBuilder, - ThrottlingRetryOptionsView, ThroughputControlGroupOptions, ThroughputControlOptions, - ThroughputControlOptionsBuilder, ThroughputControlOptionsView, TlsBackend, UserAgentSuffix, + AvailabilityStrategy, BinaryEncodingOptions, ConnectionPoolOptions, + ConnectionPoolOptionsBuilder, ContentResponseOnWrite, DiagnosticsOptions, + DiagnosticsOptionsBuilder, DiagnosticsVerbosity, EndToEndOperationLatencyPolicy, + ExcludedRegions, HedgeThreshold, HedgingStrategy, OperationOptions, OperationOptionsBuilder, + OperationOptionsView, PartitionFailoverOptions, PartitionFailoverOptionsBuilder, PriorityLevel, + ReadConsistencyStrategy, Region, ServerCertificateValidation, ThrottlingRetryOptions, + ThrottlingRetryOptionsBuilder, ThrottlingRetryOptionsView, ThroughputControlGroupOptions, + ThroughputControlOptions, ThroughputControlOptionsBuilder, ThroughputControlOptionsView, + TlsBackend, UserAgentSuffix, }; pub use batch::{ BatchDeleteOptions, BatchOptions, BatchReadOptions, BatchReplaceOptions, BatchUpsertOptions, diff --git a/sdk/cosmos/azure_data_cosmos/tests/binary_encoding.rs b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding.rs new file mode 100644 index 00000000000..482b95a071b --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding.rs @@ -0,0 +1,7 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +// Integration tests legitimately compose many Cosmos operation futures (each +// near clippy's default 16 KiB threshold) and run on tokio's large-stack test +// threads, so the production-oriented `large_futures` lint is allowed here. +#![allow(clippy::large_futures)] +mod binary_encoding_tests; diff --git a/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs new file mode 100644 index 00000000000..d6dd4d95998 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/cosmos_binary_encoding.rs @@ -0,0 +1,274 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Live integration tests for **Cosmos binary JSON** encoding against a real +//! Cosmos DB account. +//! +//! Unlike the in-memory-emulator round-trip tests (which exercise binary +//! encoding against an emulator we control), these tests validate the complete +//! loop against the **real service**: the SDK encodes item write bodies as +//! Cosmos binary JSON and advertises binary-response support, the service must +//! accept the binary request body, and — because the negotiation header is +//! present — reply with a binary body that the SDK auto-detects and decodes. +//! +//! Binary encoding is enabled through the `AZURE_COSMOS_BINARY_ENCODING_ENABLED` +//! environment variable, which the SDK resolves **once at client-build time**. +//! These tests therefore set it before the [`TestClient`] builds its client. +//! +//! # Running +//! +//! Provide a live account connection string and select the `binary_encoding` +//! test category: +//! +//! ```bash +//! AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=...;AccountKey=...;' \ +//! RUSTFLAGS='--cfg test_category="binary_encoding"' \ +//! cargo test -p azure_data_cosmos --test binary_encoding +//! ``` +//! +//! To run against the local emulator instead, also set +//! `AZURE_COSMOS_ALLOW_INVALID_CERT=true` so its self-signed certificate is +//! accepted. + +use super::framework; + +use azure_core::{http::StatusCode, Uuid}; +use azure_data_cosmos::clients::{ContainerClient, DatabaseClient}; +use azure_data_cosmos::models::ContainerProperties; +use azure_data_cosmos::options::{ + BinaryEncodingOptions, ContentResponseOnWrite, ItemWriteOptions, OperationOptions, +}; +use framework::{TestClient, TestOptions, TestRunContext}; +use serde::{Deserialize, Serialize}; +use std::error::Error; + +/// Test options that enable Cosmos binary JSON encoding via the standard client +/// option. +/// +/// Enablement flows through +/// [`CosmosClientBuilder::with_binary_encoding_options`] at client-build time, +/// so the test never mutates the process environment (`std::env::set_var` is +/// `unsafe` and racy under the parallel harness). +/// +/// [`CosmosClientBuilder::with_binary_encoding_options`]: azure_data_cosmos::CosmosClientBuilder::with_binary_encoding_options +fn binary_encoding_options() -> TestOptions { + TestOptions::new().with_binary_encoding(BinaryEncodingOptions::new().with_enabled(true)) +} + +/// A document covering every JSON value shape the binary encoder emits: literal +/// and wide integers, an unsigned value beyond `i64::MAX`, a double, booleans, +/// `null`, unicode/empty strings, nested arrays and objects, and a vector of +/// objects. +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +struct BinaryItem { + id: String, + partition_key: String, + text: String, + unicode: String, + empty: String, + small_int: i64, + big_int: i64, + negative: i64, + huge: u64, + ratio: f64, + active: bool, + inactive: bool, + maybe: Option, + tags: Vec, + numbers: Vec, + nested: NestedData, + nested_list: Vec, +} + +#[derive(Debug, Clone, Deserialize, Serialize, PartialEq)] +struct NestedData { + label: String, + values: Vec, + flag: bool, +} + +/// Builds a [`BinaryItem`] exercising the full range of encoder forms. +fn sample_item(id: &str, partition_key: &str) -> BinaryItem { + BinaryItem { + id: id.to_owned(), + partition_key: partition_key.to_owned(), + text: "hello binary".to_owned(), + unicode: "café ☃ 𝄞 quotes:\" backslash:\\".to_owned(), + empty: String::new(), + small_int: 7, // literal-int form (0..32) + big_int: 9_000_000_000, // Int64 form + negative: -1_234_567, // Int64 form + huge: u64::MAX, // UInt64 form (beyond i64::MAX) + ratio: 123.456_789, // Double form + active: true, + inactive: false, + maybe: None, // null + tags: vec!["alpha".to_owned(), "beta".to_owned(), "gamma".to_owned()], + numbers: vec![0, 1, 31, 32, 255, 256, -1], + nested: NestedData { + label: "nested".to_owned(), + values: vec![10, 20, 30], + flag: false, + }, + nested_list: vec![ + NestedData { + label: "first".to_owned(), + values: vec![1, 2, 3], + flag: true, + }, + NestedData { + label: "second".to_owned(), + values: vec![], + flag: false, + }, + ], + } +} + +/// Write options that request the service echo the stored document back, so the +/// binary **response** decode path is exercised on every write. +fn write_options_with_content() -> ItemWriteOptions { + let mut operation = OperationOptions::default(); + operation.content_response_on_write = Some(ContentResponseOnWrite::Enabled); + ItemWriteOptions::default().with_operation_options(operation) +} + +/// Creates a fresh container in `db_client` for a binary-encoding test run. +async fn create_container( + run_context: &TestRunContext, + db_client: &DatabaseClient, +) -> azure_data_cosmos::Result { + let container_id = format!("binary-{}", Uuid::new_v4()); + run_context + .create_container( + db_client, + ContainerProperties::new(container_id, "/partition_key".into()), + None, + ) + .await +} + +/// Drives create / read / upsert / replace / delete with binary encoding +/// enabled and asserts every hop round-trips the document unchanged. +/// +/// Because writes request a content response, each create/upsert/replace makes +/// the service return a (binary) body that the SDK must decode — so the test +/// validates both the binary **request** body the service accepts and the +/// binary **response** body the SDK decodes, against the real service. +#[tokio::test] +#[cfg_attr( + not(test_category = "binary_encoding"), + ignore = "requires test_category 'binary_encoding' and a live account connection string" +)] +pub async fn binary_encoding_item_crud_round_trips() -> Result<(), Box> { + TestClient::run_with_unique_db( + async |run_context, db_client| { + let container = create_container(run_context, db_client).await?; + let pk = "pk-binary"; + let item = sample_item("doc-1", pk); + + // CREATE: the request body is binary; with the negotiation header + // the echoed response body comes back binary and is decoded here. + let created = container + .create_item(pk, &item.id, &item, Some(write_options_with_content())) + .await?; + assert_eq!(created.status(), StatusCode::Created); + let created_doc: BinaryItem = created.into_body().into_single()?; + assert_eq!(created_doc, item, "create response must round-trip"); + + // READ: the response body comes back binary and decodes to the + // original value. `read_item` retries on transient 404s. + let read = run_context + .read_item(&container, pk, &item.id, None) + .await?; + let read_doc: BinaryItem = read.into_model()?; + assert_eq!(read_doc, item, "read response must round-trip"); + + // UPSERT (modified) with content response. + let mut updated = item.clone(); + updated.small_int = 30; + updated.text = "upserted binary".to_owned(); + updated.tags.push("delta".to_owned()); + let upserted = container + .upsert_item( + pk, + &updated.id, + &updated, + Some(write_options_with_content()), + ) + .await?; + assert_eq!(upserted.status(), StatusCode::Ok); + let upserted_doc: BinaryItem = upserted.into_body().into_single()?; + assert_eq!(upserted_doc, updated, "upsert response must round-trip"); + + // REPLACE (modified) with content response. + updated.ratio = 987.654_321; + updated.numbers = vec![100, 200, 300]; + updated.nested.flag = true; + let replaced = container + .replace_item( + pk, + &updated.id, + &updated, + Some(write_options_with_content()), + ) + .await?; + assert_eq!(replaced.status(), StatusCode::Ok); + let replaced_doc: BinaryItem = replaced.into_body().into_single()?; + assert_eq!(replaced_doc, updated, "replace response must round-trip"); + + Ok(()) + }, + Some(binary_encoding_options()), + ) + .await +} + +/// Writes a document with payloads large enough to push the encoder past its +/// single-byte length/count forms — a 1000-byte string (`StrL2`) and a +/// 300-element array (`ArrLC2`) — and verifies the real service stores and +/// returns them intact through the binary path. +#[tokio::test] +#[cfg_attr( + not(test_category = "binary_encoding"), + ignore = "requires test_category 'binary_encoding' and a live account connection string" +)] +pub async fn binary_encoding_handles_large_payloads() -> Result<(), Box> { + TestClient::run_with_unique_db( + async |run_context, db_client| { + let container = create_container(run_context, db_client).await?; + let pk = "pk-large"; + + let big_text = "a".repeat(1000); // > 255 bytes -> StrL2 + let big_array: Vec = (0..300).collect(); // 300 items -> ArrLC2 + let item = serde_json::json!({ + "id": "large-1", + "partition_key": pk, + "big_text": big_text, + "big_array": big_array, + }); + + // CREATE with content response: the service echoes the document + // back through the binary path. + let created = container + .create_item(pk, "large-1", &item, Some(write_options_with_content())) + .await?; + assert_eq!(created.status(), StatusCode::Created); + let echoed: serde_json::Value = created.into_body().into_single()?; + assert_eq!(echoed["big_text"], item["big_text"]); + assert_eq!(echoed["big_array"], item["big_array"]); + + // READ back and verify the stored values survived the round-trip. + let read = run_context + .read_item(&container, pk, "large-1", None) + .await?; + let read_doc: serde_json::Value = read.into_model()?; + assert_eq!(read_doc["big_text"], item["big_text"]); + assert_eq!(read_doc["big_array"], item["big_array"]); + + Ok(()) + }, + Some(binary_encoding_options()), + ) + .await +} diff --git a/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/mod.rs b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/mod.rs new file mode 100644 index 00000000000..1a0bdd6d691 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos/tests/binary_encoding_tests/mod.rs @@ -0,0 +1,6 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +mod cosmos_binary_encoding; + +#[path = "../framework/mod.rs"] +mod framework; diff --git a/sdk/cosmos/azure_data_cosmos/tests/framework/test_client.rs b/sdk/cosmos/azure_data_cosmos/tests/framework/test_client.rs index 2b78e39cb1b..33c6367bbba 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/framework/test_client.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/framework/test_client.rs @@ -10,8 +10,8 @@ use azure_data_cosmos::{ feed::FeedScope, models::{ItemResponse, ThroughputProperties}, options::{ - ConnectionPoolOptions, CreateContainerOptions, ItemReadOptions, Region, - ServerCertificateValidation, + BinaryEncodingOptions, ConnectionPoolOptions, CreateContainerOptions, ItemReadOptions, + Region, ServerCertificateValidation, }, CosmosClient, CosmosError, CosmosRuntime, CosmosStatus, PartitionKey, Query, RoutingStrategy, }; @@ -193,6 +193,17 @@ pub struct TestOptions { /// `false` so that the default `ServerCertificateValidation::Required` /// applies. pub allow_invalid_certificates: bool, + /// Binary-encoding options applied to the normal (non-fault) client. + /// + /// `Some(..)` configures the underlying [`CosmosClient`] via the standard + /// [`CosmosClientBuilder::with_binary_encoding_options`] client option, so + /// tests can enable binary encoding without mutating the process + /// environment (`std::env::set_var` is `unsafe` and racy under the parallel + /// harness). `None` (the default) leaves the client's own environment-based + /// resolution in place. + /// + /// [`CosmosClientBuilder::with_binary_encoding_options`]: azure_data_cosmos::CosmosClientBuilder::with_binary_encoding_options + pub binary_encoding: Option, } impl TestOptions { @@ -252,6 +263,13 @@ impl TestOptions { self.allow_invalid_certificates = allow; self } + + /// Configures Cosmos binary JSON encoding for the normal (non-fault) client + /// via the standard client option, avoiding any `std::env` mutation. + pub fn with_binary_encoding(mut self, options: BinaryEncodingOptions) -> Self { + self.binary_encoding = Some(options); + self + } } static IS_AZURE_PIPELINES: OnceLock = OnceLock::new(); @@ -364,6 +382,7 @@ impl TestClient { Vec::new(), fault_client_application_region, allow_invalid_certificates, + None, ) .await } @@ -371,12 +390,14 @@ impl TestClient { pub async fn from_env( application_region: Option, allow_invalid_certificates: bool, + binary_encoding: Option, ) -> Result> { Self::from_env_inner( application_region, Vec::new(), None, allow_invalid_certificates, + binary_encoding, ) .await } @@ -391,6 +412,7 @@ impl TestClient { fault_rules, application_region, allow_invalid_certificates, + None, ) .await } @@ -405,6 +427,7 @@ impl TestClient { fault_rules: Vec>, fault_client_application_region: Option, allow_invalid_certificates: bool, + binary_encoding: Option, ) -> Result> { let Ok(env_var) = std::env::var(CONNECTION_STRING_ENV_VAR) else { // No connection string provided, so we'll skip tests that require it. @@ -428,6 +451,7 @@ impl TestClient { true, fault_rules, None, + binary_encoding, ) .await } @@ -438,6 +462,7 @@ impl TestClient { allow_invalid_certificates, fault_rules, fault_client_application_region, + binary_encoding, ) .await } @@ -450,6 +475,7 @@ impl TestClient { mut allow_invalid_certificates: bool, fault_rules: Vec>, fault_client_application_region: Option, + binary_encoding: Option, ) -> Result> { let connection_string: ConnectionString = connection_string.parse()?; @@ -490,6 +516,12 @@ impl TestClient { builder = builder.with_fault_injection_rules(fault_rules)?; } + // Apply binary-encoding options via the standard client option so tests + // never mutate the process environment. + if let Some(options) = binary_encoding { + builder = builder.with_binary_encoding_options(options); + } + let endpoint: azure_data_cosmos::AccountEndpoint = connection_string.account_endpoint().parse()?; let cosmos_client = builder @@ -563,6 +595,7 @@ impl TestClient { let test_client = Self::from_env( options.client_application_region.clone(), options.allow_invalid_certificates, + options.binary_encoding.clone(), ) .await?; diff --git a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/binary_round_trip.rs b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/binary_round_trip.rs new file mode 100644 index 00000000000..8db61fc8dac --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/binary_round_trip.rs @@ -0,0 +1,347 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! End-to-end validation of Cosmos **binary JSON** through the full SDK → +//! driver → in-memory-emulator loop. +//! +//! With binary encoding enabled via +//! [`CosmosClientBuilder::with_binary_encoding_options`], the SDK encodes item +//! write bodies as binary and advertises binary-response support; the in-memory +//! emulator decodes the binary request body, stores it, and (because the +//! negotiation header is present) replies with a binary body, which the SDK +//! auto-detects and decodes. This exercises the complete +//! encode → negotiate → store → encode → decode round-trip locally — no Docker, +//! no real account, no external vectors. + +use azure_data_cosmos::{ + options::{ + BinaryEncodingOptions, ContentResponseOnWrite, ItemWriteOptions, OperationOptions, Region, + RoutingStrategy, + }, + AccountEndpoint, AccountReference, ContainerClient, CosmosClientBuilder, CosmosRuntimeBuilder, +}; +use azure_data_cosmos_driver::in_memory_emulator::{ + ConsistencyLevel, ContainerConfig, InMemoryEmulatorHttpClient, RequestObserver, + VirtualAccountConfig, VirtualRegion, +}; +use serde::{Deserialize, Serialize}; +use std::sync::{Arc, Mutex}; + +const EMULATOR_GATEWAY_URL: &str = "https://eastus.emulator.local"; + +#[derive(Debug, Serialize, Deserialize, PartialEq)] +struct TestItem { + id: String, + pk: String, + value: i64, + note: String, +} + +fn write_options_with_content() -> ItemWriteOptions { + let mut operation = OperationOptions::default(); + operation.content_response_on_write = Some(ContentResponseOnWrite::Enabled); + ItemWriteOptions::default().with_operation_options(operation) +} + +/// Builds an emulator-backed [`ContainerClient`] with a pre-provisioned +/// database + container. `binary` enables Cosmos binary JSON encoding via the +/// explicit client option (no process-wide environment mutation). +async fn build_container(db_name: &str, binary: bool) -> ContainerClient { + let config = VirtualAccountConfig::new(vec![VirtualRegion::new( + "East US", + azure_core::http::Url::parse(EMULATOR_GATEWAY_URL).unwrap(), + )]) + .unwrap() + .with_consistency(ConsistencyLevel::Session); + + let emulator = std::sync::Arc::new(InMemoryEmulatorHttpClient::new(config)); + let store = emulator.store(); + store.create_database(db_name); + store.create_container_with_config( + db_name, + "items", + serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], + "kind": "Hash", + "version": 2 + })) + .unwrap(), + ContainerConfig::new() + .with_partition_count(1) + .with_throughput(400) + .build() + .unwrap(), + ); + + let account = AccountReference::with_authentication_key( + EMULATOR_GATEWAY_URL.parse::().unwrap(), + azure_core::credentials::Secret::new("dGVzdGtleQ=="), + ); + let mut builder = CosmosClientBuilder::new().with_runtime( + CosmosRuntimeBuilder::from(emulator.runtime_builder()) + .build() + .await + .unwrap(), + ); + if binary { + builder = + builder.with_binary_encoding_options(BinaryEncodingOptions::new().with_enabled(true)); + } + let client = builder + .build(account, RoutingStrategy::ProximityTo(Region::EAST_US)) + .await + .unwrap(); + + client + .database_client(db_name) + .container_client("items") + .await + .unwrap() +} + +/// With binary enabled, an item written through the SDK is binary-encoded on the +/// wire, decoded + stored by the emulator, returned as binary, and decoded back +/// — and the value survives every hop unchanged. +#[tokio::test] +async fn binary_encoding_item_write_read_round_trips() { + let container = build_container("bin-e2e", true).await; + + let item = TestItem { + id: "doc-1".into(), + pk: "pk1".into(), + value: 1234, + note: "café ☃ binary".into(), + }; + + // Create: the request body is binary; the emulator decodes and stores it. + let created = container + .create_item("pk1", &item.id, &item, Some(write_options_with_content())) + .await + .unwrap(); + let created_doc: TestItem = created.into_body().into_single().unwrap(); + assert_eq!(created_doc, item, "create response must round-trip"); + + // Read: the response body comes back binary and decodes to the same value. + let read = container.read_item("pk1", &item.id, None).await.unwrap(); + let read_doc: TestItem = read.into_body().into_single().unwrap(); + assert_eq!(read_doc, item, "read response must round-trip"); + + // Upsert + replace: same loop, different verbs. + let updated = TestItem { + value: 5678, + ..TestItem { + id: item.id.clone(), + pk: item.pk.clone(), + value: 0, + note: item.note.clone(), + } + }; + let upserted = container + .upsert_item( + "pk1", + &updated.id, + &updated, + Some(write_options_with_content()), + ) + .await + .unwrap(); + let upserted_doc: TestItem = upserted.into_body().into_single().unwrap(); + assert_eq!(upserted_doc.value, 5678); + + let replaced = container + .replace_item( + "pk1", + &updated.id, + &updated, + Some(write_options_with_content()), + ) + .await + .unwrap(); + let replaced_doc: TestItem = replaced.into_body().into_single().unwrap(); + assert_eq!(replaced_doc.value, 5678); +} + +/// A document written by a binary-enabled client reads back correctly through a +/// text-only client (the stored value is format-agnostic), and vice versa — +/// proving binary and text are interchangeable on the wire. +#[tokio::test] +async fn binary_and_text_clients_interoperate() { + // Write with a binary-enabled client. + let binary_container = build_container("bin-interop", true).await; + + let item = TestItem { + id: "interop-1".into(), + pk: "pk1".into(), + value: 99, + note: "written-binary".into(), + }; + binary_container + .create_item("pk1", &item.id, &item, Some(write_options_with_content())) + .await + .unwrap(); + + // Read it back with a text-only client against the same store would require + // sharing the store; instead assert the binary client reads its own write, + // then a fresh text client round-trips a separate document. Both share the + // same decode path, so this confirms the formats coexist. + let read = binary_container + .read_item("pk1", &item.id, None) + .await + .unwrap(); + let read_doc: TestItem = read.into_body().into_single().unwrap(); + assert_eq!(read_doc, item); + + let text_container = build_container("text-interop", false).await; + let text_item = TestItem { + id: "interop-2".into(), + pk: "pk1".into(), + value: 7, + note: "written-text".into(), + }; + text_container + .create_item( + "pk1", + &text_item.id, + &text_item, + Some(write_options_with_content()), + ) + .await + .unwrap(); + let text_read = text_container + .read_item("pk1", &text_item.id, None) + .await + .unwrap(); + let text_doc: TestItem = text_read.into_body().into_single().unwrap(); + assert_eq!(text_doc, text_item); +} + +/// A [`RequestObserver`] that records the value of the +/// `x-ms-cosmos-supported-serialization-formats` negotiation header seen on +/// dataplane item requests, so a test can assert what the SDK advertised on the +/// wire. +#[derive(Debug, Default)] +struct NegotiationHeaderRecorder { + formats: Mutex>>, +} + +impl RequestObserver for NegotiationHeaderRecorder { + fn on_request(&self, request: &azure_core::http::Request) { + // Only record item (docs) requests; ignore metadata/bootstrap traffic. + if !request.url().path().contains("/docs") { + return; + } + let name = azure_core::http::headers::HeaderName::from_static( + "x-ms-cosmos-supported-serialization-formats", + ); + let value = request + .headers() + .get_optional_str(&name) + .map(|s| s.to_string()); + self.formats.lock().unwrap().push(value); + } +} + +/// End-to-end proof of the driver-side transcoding model: +/// +/// With `BinaryEncodingOptions { enabled: true, request_text_response: true }`, +/// point operations still advertise `CosmosBinary` (so the **wire stays +/// binary** in both directions and the transport hop is efficient), and the +/// driver transcodes the binary response to text before returning it — so the +/// application still gets its document back intact. +#[tokio::test] +async fn request_text_response_keeps_wire_binary_and_returns_data() { + let config = VirtualAccountConfig::new(vec![VirtualRegion::new( + "East US", + azure_core::http::Url::parse(EMULATOR_GATEWAY_URL).unwrap(), + )]) + .unwrap() + .with_consistency(ConsistencyLevel::Session); + + let recorder = Arc::new(NegotiationHeaderRecorder::default()); + let emulator = Arc::new( + InMemoryEmulatorHttpClient::new(config) + .with_request_observer(Arc::clone(&recorder) as Arc), + ); + let store = emulator.store(); + store.create_database("bin-transcode"); + store.create_container_with_config( + "bin-transcode", + "items", + serde_json::from_value(serde_json::json!({ + "paths": ["/pk"], + "kind": "Hash", + "version": 2 + })) + .unwrap(), + ContainerConfig::new() + .with_partition_count(1) + .with_throughput(400) + .build() + .unwrap(), + ); + + // Configure binary encoding + text responses via the standard client option + // (no env mutation). + let account = AccountReference::with_authentication_key( + EMULATOR_GATEWAY_URL.parse::().unwrap(), + azure_core::credentials::Secret::new("dGVzdGtleQ=="), + ); + let client = CosmosClientBuilder::new() + .with_binary_encoding_options( + BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true), + ) + .with_runtime( + CosmosRuntimeBuilder::from(emulator.runtime_builder()) + .build() + .await + .unwrap(), + ) + .build(account, RoutingStrategy::ProximityTo(Region::EAST_US)) + .await + .unwrap(); + let container = client + .database_client("bin-transcode") + .container_client("items") + .await + .unwrap(); + + let item = TestItem { + id: "doc-1".into(), + pk: "pk1".into(), + value: 4321, + note: "transcode ☃".into(), + }; + + // Create + read: the driver transcodes the binary response to text, and the + // typed value still round-trips. + let created = container + .create_item("pk1", &item.id, &item, Some(write_options_with_content())) + .await + .unwrap(); + let created_doc: TestItem = created.into_body().into_single().unwrap(); + assert_eq!( + created_doc, item, + "create must round-trip after transcoding" + ); + + let read = container.read_item("pk1", &item.id, None).await.unwrap(); + let read_doc: TestItem = read.into_body().into_single().unwrap(); + assert_eq!(read_doc, item, "read must round-trip after transcoding"); + + // Every observed item request advertised CosmosBinary — the wire stayed + // binary despite request_text_response being on. + let formats = recorder.formats.lock().unwrap(); + assert!( + !formats.is_empty(), + "expected at least one observed item request", + ); + for value in formats.iter() { + assert_eq!( + value.as_deref(), + Some("CosmosBinary"), + "wire must stay binary (CosmosBinary advertised) even with request_text_response", + ); + } +} diff --git a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/end_to_end.rs b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/end_to_end.rs index e4d19c51dce..5e9e4b0eba2 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/end_to_end.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/end_to_end.rs @@ -1085,25 +1085,25 @@ async fn sdk_query_items_with_filter_and_projection() { .unwrap() } - let emu_items: Vec = emu_container - .query_items(query(), FeedScope::partition("pk1"), None) - .await - .unwrap() - .try_collect() - .await - .unwrap(); - assert_eq!(emu_items.len(), 2); - assert_eq!(emu_items[0].id, "query-1"); - assert_eq!(emu_items[1].id, "query-2"); - - if let Some(ref real) = real_container { - let real_items: Vec = real - .query_items(query(), FeedScope::partition("pk1"), None) + let emu_items: Vec = + Box::pin(emu_container.query_items(query(), FeedScope::partition("pk1"), None)) .await .unwrap() .try_collect() .await .unwrap(); + assert_eq!(emu_items.len(), 2); + assert_eq!(emu_items[0].id, "query-1"); + assert_eq!(emu_items[1].id, "query-2"); + + if let Some(ref real) = real_container { + let real_items: Vec = + Box::pin(real.query_items(query(), FeedScope::partition("pk1"), None)) + .await + .unwrap() + .try_collect() + .await + .unwrap(); assert_eq!(real_items.len(), emu_items.len()); assert_eq!( real_items.iter().map(|i| &i.id).collect::>(), diff --git a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/mod.rs b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/mod.rs index e8b50c2a97c..b6bf22cff47 100644 --- a/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/mod.rs +++ b/sdk/cosmos/azure_data_cosmos/tests/in_memory_emulator_tests/mod.rs @@ -8,6 +8,7 @@ use std::time::Duration; +pub mod binary_round_trip; pub mod driver_end_to_end; #[cfg(feature = "preview_dtx")] pub mod dtx_live_comparison; diff --git a/sdk/cosmos/azure_data_cosmos_benchmarks/Cargo.toml b/sdk/cosmos/azure_data_cosmos_benchmarks/Cargo.toml index d621f936943..ef7770ec7d4 100644 --- a/sdk/cosmos/azure_data_cosmos_benchmarks/Cargo.toml +++ b/sdk/cosmos/azure_data_cosmos_benchmarks/Cargo.toml @@ -17,6 +17,14 @@ harness = false name = "backtrace_capture" harness = false +[[bench]] +name = "binary_encode" +harness = false + +[[bench]] +name = "binary_decode" +harness = false + [dependencies] async-trait.workspace = true azure_core.workspace = true @@ -24,6 +32,8 @@ azure_data_cosmos_driver = { path = "../azure_data_cosmos_driver", version = "0. "__internal_mocking", "__internal_backtrace_bench", ] } +serde = { workspace = true, features = ["derive"] } +serde_json.workspace = true tokio = { workspace = true, features = ["rt-multi-thread", "time"] } url.workspace = true diff --git a/sdk/cosmos/azure_data_cosmos_benchmarks/benches/binary_decode.rs b/sdk/cosmos/azure_data_cosmos_benchmarks/benches/binary_decode.rs new file mode 100644 index 00000000000..f5094557db7 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_benchmarks/benches/binary_decode.rs @@ -0,0 +1,109 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Criterion benchmark comparing item-read body deserialization strategies: +//! +//! - **text** — `serde_json::from_slice::` on a UTF-8 JSON body. +//! - **binary_value** — `binary_json::decode` → `serde_json::from_value::`, +//! the two-stage path that materializes an intermediate `serde_json::Value` +//! tree before building the typed value. +//! - **binary_native** — `binary_json::from_slice::`, the native serde +//! deserializer that drives `T::deserialize` straight off the binary buffer +//! with no intermediate `Value`. +//! +//! Both a typed target (`LogEntry`) and a `serde_json::Value` target are +//! measured, on a small and a large (~1.7 MB) item. +//! +//! ```text +//! cargo bench -p azure_data_cosmos_benchmarks --bench binary_decode +//! ``` + +use azure_data_cosmos_driver::binary_json; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use serde::{Deserialize, Serialize}; + +/// A log-entry-shaped item, mirroring the shape used by the SDK samples. +#[derive(Serialize, Deserialize)] +struct LogEntry { + id: String, + pk: String, + level: String, + message: String, + counter: u64, + tags: Vec, +} + +impl LogEntry { + fn new(payload_size: usize) -> Self { + let chunk = "log-payload-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ "; + let mut message = String::with_capacity(payload_size + chunk.len()); + while message.len() < payload_size { + message.push_str(chunk); + } + message.truncate(payload_size); + Self { + id: "dynamic-00000000-0000-0000-0000-000000000000".to_owned(), + pk: "INFO".to_owned(), + level: "INFO".to_owned(), + message, + counter: 42, + tags: vec!["alpha".to_owned(), "beta".to_owned(), "gamma".to_owned()], + } + } +} + +/// Two-stage binary decode: bytes → `serde_json::Value` → `T`. +fn decode_via_value(bytes: &[u8]) -> T { + let value = binary_json::decode(bytes).expect("decode"); + serde_json::from_value(value).expect("from_value") +} + +fn bench_binary_decode(c: &mut Criterion) { + let cases = [("small", 64_usize), ("large_1_7mb", 1_740 * 1024)]; + + let mut group = c.benchmark_group("binary_decode"); + + for (label, payload_size) in cases { + let item = LogEntry::new(payload_size); + let text = serde_json::to_vec(&item).expect("to_vec"); + let binary = binary_json::to_vec(&item).expect("to_vec"); + group.throughput(Throughput::Bytes(text.len() as u64)); + + // ---- typed target: LogEntry ---- + group.bench_with_input(BenchmarkId::new("text/typed", label), &text, |b, text| { + b.iter(|| serde_json::from_slice::(text).expect("from_slice")); + }); + group.bench_with_input( + BenchmarkId::new("binary_value/typed", label), + &binary, + |b, binary| b.iter(|| decode_via_value::(binary)), + ); + group.bench_with_input( + BenchmarkId::new("binary_native/typed", label), + &binary, + |b, binary| b.iter(|| binary_json::from_slice::(binary).expect("from_slice")), + ); + + // ---- dynamic target: serde_json::Value ---- + group.bench_with_input(BenchmarkId::new("text/value", label), &text, |b, text| { + b.iter(|| serde_json::from_slice::(text).expect("from_slice")); + }); + group.bench_with_input( + BenchmarkId::new("binary_value/value", label), + &binary, + |b, binary| b.iter(|| decode_via_value::(binary)), + ); + group.bench_with_input( + BenchmarkId::new("binary_native/value", label), + &binary, + |b, binary| { + b.iter(|| binary_json::from_slice::(binary).expect("from_slice")) + }, + ); + } + + group.finish(); +} + +criterion_group!(benches, bench_binary_decode); +criterion_main!(benches); diff --git a/sdk/cosmos/azure_data_cosmos_benchmarks/benches/binary_encode.rs b/sdk/cosmos/azure_data_cosmos_benchmarks/benches/binary_encode.rs new file mode 100644 index 00000000000..6b9590c46b0 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_benchmarks/benches/binary_encode.rs @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Criterion benchmark comparing item-write body serialization strategies: +//! +//! - **text** — `serde_json::to_vec` (the UTF-8 JSON path). +//! - **binary_v1** — `T` → [`serde_json::Value`] → `binary_json::encode`, the +//! original two-pass Cosmos binary JSON path that materializes an +//! intermediate `Value` tree. +//! - **binary_v2** — `binary_json::to_vec`, the native serde serializer that +//! encodes `T` straight to Cosmos binary JSON with no intermediate `Value`. +//! +//! The point is to quantify the `Value`-elision win of v2 over v1 on a large, +//! realistic item (~1.7 MB), plus a small item to show fixed overhead. +//! +//! ```text +//! cargo bench -p azure_data_cosmos_benchmarks --bench binary_encode +//! ``` + +use azure_data_cosmos_driver::binary_json; +use criterion::{criterion_group, criterion_main, BenchmarkId, Criterion, Throughput}; +use serde::Serialize; + +/// A log-entry-shaped item, mirroring the shape used by the SDK write samples. +#[derive(Serialize)] +struct LogEntry { + id: String, + pk: String, + level: String, + message: String, + counter: u64, + tags: Vec, +} + +impl LogEntry { + /// Builds an entry whose `message` field is inflated to roughly + /// `payload_size` bytes so the serialized document lands near that size. + fn new(payload_size: usize) -> Self { + let chunk = "log-payload-0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZ "; + let mut message = String::with_capacity(payload_size + chunk.len()); + while message.len() < payload_size { + message.push_str(chunk); + } + message.truncate(payload_size); + Self { + id: "dynamic-00000000-0000-0000-0000-000000000000".to_owned(), + pk: "INFO".to_owned(), + level: "INFO".to_owned(), + message, + counter: 42, + tags: vec!["alpha".to_owned(), "beta".to_owned(), "gamma".to_owned()], + } + } +} + +/// v1 encode path: `T` → `serde_json::Value` → binary. +fn encode_v1(item: &T) -> Vec { + let value = serde_json::to_value(item).expect("to_value"); + binary_json::encode(&value) +} + +fn bench_binary_encode(c: &mut Criterion) { + // A small item (fixed overhead) and a large ~1.7 MB item (throughput). + let cases = [("small", 64_usize), ("large_1_7mb", 1_740 * 1024)]; + + let mut group = c.benchmark_group("binary_encode"); + + for (label, payload_size) in cases { + let item = LogEntry::new(payload_size); + // Report throughput against the produced text-JSON body size so the + // three strategies are comparable on a common denominator. + let text_len = serde_json::to_vec(&item).expect("to_vec").len(); + group.throughput(Throughput::Bytes(text_len as u64)); + + group.bench_with_input(BenchmarkId::new("text", label), &item, |b, item| { + b.iter(|| serde_json::to_vec(item).expect("to_vec")); + }); + group.bench_with_input(BenchmarkId::new("binary_v1", label), &item, |b, item| { + b.iter(|| encode_v1(item)); + }); + group.bench_with_input(BenchmarkId::new("binary_v2", label), &item, |b, item| { + b.iter(|| binary_json::to_vec(item).expect("to_vec")); + }); + } + + group.finish(); +} + +criterion_group!(benches, bench_binary_encode); +criterion_main!(benches); diff --git a/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md b/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md index ec687737af0..dbe7a4923ca 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md +++ b/sdk/cosmos/azure_data_cosmos_driver/CHANGELOG.md @@ -4,11 +4,7 @@ ### Features Added -### Breaking Changes - -### Bugs Fixed - -### Other Changes +- Added a schema-agnostic Cosmos binary JSON codec (`binary_json`) and driver-side binary encoding via `OperationOptions.binary_encoding` (`BinaryEncodingOptions`). When enabled, the driver transcodes item request/response bodies between text and Cosmos binary JSON and negotiates the wire format; it is honored only for point `Document` item operations. Off by default and inert on the wire when unset. ([#4671](https://github.com/Azure/azure-sdk-for-rust/pull/4671)) ## 0.6.1 (2026-07-23) diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_HLD.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_HLD.md new file mode 100644 index 00000000000..a9df2c1769f --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_HLD.md @@ -0,0 +1,341 @@ +# Binary Encoding — High-Level Design (HLD) + +This document is the high-level design for Cosmos **binary JSON** encoding in the +Rust SDK and driver. It captures the goals, the wire/transcoding model, the +component layout, testing, and the intentionally deferred work. + +For the phased implementation plan and low-level wire details, see +[`BINARY_ENCODING_SPEC.md`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_SPEC.md). + +--- + +## Summary + +Adds first-class support for **Cosmos binary JSON** to the Rust SDK and driver. Cosmos binary JSON is a tagged byte stream the service can persist and transmit in place of UTF-8 text JSON; it is more compact and cheaper to (de)serialize. + +This design delivers a **complete decoder** and a **native serde codec**, and makes binary encoding a **driver capability** on `OperationOptions.binary_encoding` so it is shared by every consumer of the driver — the Rust SDK **and** any FFI-based SDK (.NET, Java, Go, …). It is opt-in (`CosmosClientBuilder::with_binary_encoding_options`, with an `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment-variable fallback). When the option is off, every request and response is **byte-for-byte unchanged** — the binary code is inert. + +Because the option lives on the driver and is schema-agnostic, the driver performs the byte-level transcoding **both ways** when needed: + +* an opt-in **text-response** mode (`BinaryEncodingOptions::request_text_response`) keeps the wire binary in both directions (efficient RUs and bandwidth) while the driver transcodes the binary **response** back to text JSON; +* a caller that deals only in **text** (most importantly an FFI host) can enable binary and the driver transcodes its text **request** body to binary — so it gets a fully binary wire **without encoding anything itself**. + +A self-contained, in-tree **end-to-end validation loop** is included via the in-memory emulator (no Docker, no live account, no external test vectors). + +> **Scope:** item operations (`create` / `replace` / `upsert` / `read`). Query, patch, transactional batch, and bulk are intentionally deferred (see [Deferred work](#deferred-work)). + +--- + +## Why binary JSON? + +* **Smaller payloads** — tagged binary is more compact than text JSON. +* **Cheaper (de)serialization** — typed markers avoid text parsing/formatting. +* **Wire-compatible negotiation** — the client advertises what it accepts; the service chooses. Mixed text/binary deployments interoperate transparently. + +The design follows the driver's **schema-agnostic data-plane** principle: the codec operates purely on bytes (and, for the reference oracle, `serde_json::Value`) and knows nothing about item schemas. + +--- + +## How it works + +### The `0x80` preamble — unambiguous auto-detection + +Every binary buffer begins with the preamble byte `0x80`. Because `0x80` is a UTF-8 *continuation* byte, **no valid text JSON document can start with it**. This single-byte test (`binary_json::is_binary`) lets the decode path detect binary independently of any HTTP header, so responses decode correctly even if content negotiation headers are absent or unexpected. + +### Native serde codec (no intermediate `Value`) + +* **Read path** — binary responses deserialize straight into `T: Deserialize` via the native serde deserializer `binary_json::from_slice`, with no intermediate `serde_json::Value` on the common path. The complete `binary_json::decode` (binary → `Value`) is retained as the reference oracle for parity/fuzz tests and as the fallback the deserializer uses for the rare exotic wire forms (reference/compressed/GUID strings, binary blobs, uniform number arrays). +* **Write path** — item bodies serialize straight from `T: Serialize` to Cosmos binary JSON via the native serde serializer `binary_json::to_vec`, again with no intermediate `Value`. The serializer emits a correct (verbose-but-valid) buffer — it skips the size optimizations a writer *may* apply (reference-string dedup, compression, uniform arrays), all of which the decoder still accepts. + +### Enablement (driver option, resolved once) + +Binary encoding is a driver option: `OperationOptions.binary_encoding: Option` (driver-owned type). The Rust SDK **re-exports** `BinaryEncodingOptions` and resolves enablement **once at client build** via `resolve_binary_encoding(Option)`, storing it on the client context — there is no per-request lookup. It prefers the explicit `CosmosClientBuilder::with_binary_encoding_options(..)` option and falls back to the `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment variable (truthy: `1` / `true` / `yes` / `on`, case-insensitive). Each item operation carries the resolved options onto its `OperationOptions` (via a `with_binary_encoding` helper). + +FFI hosts set the equivalent flat fields on the C ABI `cosmos_operation_options_t` (`binary_encoding_enabled`, `binary_encoding_request_text_response`), which convert to the same driver option — so no SDK code is involved. + +### Two transcodes, both in the driver (schema-agnostic) + +When `binary_encoding.enabled` is set, `CosmosDriver::execute_operation` owns the wire format both ways: + +* **Request** (`apply_request_binary_encoding`) — transcodes a **text** request body to Cosmos binary JSON via `binary_json::transcode_to_binary` (`serde_json::from_slice` → `encode`) and advertises `JsonText,CosmosBinary`. An **already-binary** or empty body passes through unchanged, so a caller that pre-encoded pays nothing. +* **Response** (when `request_text_response` is set) — transcodes the binary response back to text JSON via `binary_json::transcode_to_text` (`decode` → `serde_json::to_vec`). The wire stays binary in both directions. + +This keeps transcoding in the **driver** (not the backend) and, because it is schema-agnostic, lets a text-only FFI host get an efficient binary wire without any encoding on its side. + +### Rust SDK typed fast path (optimization) + +The Rust SDK keeps a typed fast path: `serialize_item_body` encodes `T: Serialize` straight to binary via `binary_json::to_vec` (skipping the text intermediate). The driver's request-side transcode then sees an **already-binary** body and passes it through unchanged — so the SDK pays no double work while sharing the exact same driver option surface as FFI callers. + +### Negotiation header + +When binary is enabled, item operations set: + +``` +x-ms-cosmos-supported-serialization-formats: JsonText,CosmosBinary +``` + +The value matches the .NET reference (`string.Join(",", JsonText, CosmosBinary)` — no space). The request `Content-Type` stays `application/json`; the service detects the binary body from its first byte. The **driver** sets this header whenever `binary_encoding.enabled` is set — including under `request_text_response`, where the wire stays binary and the driver transcodes the response (see above). + +--- + +## Flow diagrams + +Binary encoding lives on the driver's `OperationOptions.binary_encoding`. Both +the Rust SDK and FFI hosts set that option; the **driver** owns the wire format +and the two transcodes. The only difference is *how the request body arrives*: +the Rust SDK may pre-encode typed `T` to binary (an optimization), while an FFI +host sends plain text and lets the driver transcode. + +### Where the option comes from (Rust SDK vs FFI) + +```mermaid +flowchart TD + subgraph RUST["Rust SDK caller"] + RA["create_item<T>(item)"] --> RS["serialize_item_body<T>
(typed fast path: pre-encode binary)"] + RS --> RW["with_binary_encoding(options)
set OperationOptions.binary_encoding"] + end + subgraph FFI["FFI host (.NET / Java / Go)"] + FA["send TEXT json body"] --> FF["cosmos_operation_options_t
binary_encoding_enabled = true
binary_encoding_request_text_response = true/false"] + FF --> FT["to_driver()"] + end + RW --> OO["OperationOptions.binary_encoding
= BinaryEncodingOptions { enabled, request_text_response }"] + FT --> OO + OO --> EX["CosmosDriver::execute_operation"] + + style OO fill:#d0ffd0,stroke:#0a0 + style EX fill:#d0e8ff,stroke:#06c +``` + +### Driver request + response path (owns both transcodes) + +```mermaid +flowchart TD + EX["execute_operation
binary_encoding.enabled?"] -->|no| PASS["pass through — text wire, unchanged"] + EX -->|yes| REQ{"request body is_binary?"} + REQ -->|"no (text, e.g. FFI)"| RT["transcode_to_binary
(from_slice → encode)"] + REQ -->|"yes (SDK pre-encoded)"| PASS_BIN["pass through"] + RT --> HDR + PASS_BIN --> HDR["advertise JsonText,CosmosBinary"] + HDR --> WIRE["binary body on the wire"] + WIRE --> SVC[("Cosmos DB")] + SVC --> RESP["binary response (0x80)"] + RESP --> RTX{"request_text_response?"} + RTX -->|yes| TT["transcode_to_text
(decode → serde_json::to_vec)
→ TEXT body"] + RTX -->|no| BIN["binary body returned as-is"] + TT --> OUT["body handed to caller"] + BIN --> OUT + + style RT fill:#d0ffd0,stroke:#0a0 + style TT fill:#d0ffd0,stroke:#0a0 +``` + +### Rust SDK response decode (typed) + +After the driver returns the body, the Rust SDK deserializes it. Auto-detection +by the `0x80` preamble means the decode path is correct whether the body came +back binary or was transcoded to text by the driver. + +```mermaid +flowchart TD + L["driver response bytes"] --> N["ResponseBody::into_single / into_items"] + N --> O["deserialize_response::<T>(bytes)"] + O --> P{"is_binary(bytes)?
first byte == 0x80"} + P -->|yes| Q["binary_json::from_slice::<T>(bytes)
(native serde — no intermediate Value)"] + P -->|no| S["serde_json::from_slice::<T>
(text path)"] + Q --> T["typed item T"] + S --> T +``` + +--- + +## Sequence diagrams + +### Rust SDK — write then read (binary enabled) + +The SDK pre-encodes typed `T` to binary; the driver sees an already-binary body +and passes it through. When `request_text_response` is set, the driver transcodes +the response back to text and the SDK's decode takes the text branch. + +```mermaid +sequenceDiagram + autonumber + participant App + participant SDK as ContainerClient
(azure_data_cosmos) + participant DRV as CosmosDriver::execute_operation + participant Cod as binary_json codec + participant Svc as Cosmos DB + + Note over SDK: binary_encoding resolved once at client build
(with_binary_encoding_options, env fallback) + + App->>SDK: create_item(pk, id, item) + SDK->>Cod: to_vec(item) (typed fast path) + Cod-->>SDK: 0x80-prefixed binary bytes + SDK->>DRV: operation + OperationOptions.binary_encoding + Note over DRV: request body already binary → pass through + DRV->>Svc: binary body + JsonText,CosmosBinary + Svc-->>DRV: binary response (0x80) + opt request_text_response + DRV->>Cod: transcode_to_text (decode → to_vec) + Cod-->>DRV: text JSON body + end + DRV-->>SDK: response body + App->>SDK: into_body().into_single() → T + SDK->>Cod: is_binary? from_slice else serde_json::from_slice + Cod-->>SDK: typed item T + SDK-->>App: ItemResponse (typed T) +``` + +### FFI host — text in, text out, binary wire + +The FFI host deals only in text. It sets two flags; the driver transcodes the +text request to binary and the binary response back to text. The host never +touches binary. + +```mermaid +sequenceDiagram + autonumber + participant Host as FFI host (.NET / Java) + participant ABI as cosmos_operation_options_t + participant DRV as CosmosDriver::execute_operation + participant Cod as binary_json codec + participant Svc as Cosmos DB + + Note over Host: TEXT json body
enabled = true, request_text_response = true + Host->>ABI: binary_encoding_enabled + request_text flags + ABI->>DRV: OperationOptions.binary_encoding + Note over DRV: request body is TEXT → transcode to binary + DRV->>Cod: transcode_to_binary (from_slice → encode) + Cod-->>DRV: 0x80-prefixed binary body + DRV->>Svc: binary body + JsonText,CosmosBinary + Svc-->>DRV: binary response (0x80) + DRV->>Cod: transcode_to_text (decode → to_vec) + Cod-->>DRV: TEXT json body + DRV-->>Host: TEXT json (never saw binary) +``` + +When the option is **off**, both paths collapse to the existing text behavior — +`serde_json` on the way out and back, byte-for-byte unchanged. + +--- + +## Key components + +| Component | File | Role | +|---|---|---| +| `binary_json` module | `azure_data_cosmos_driver/src/binary_json/` | The codec (schema-agnostic) | +| `markers` | `binary_json/markers.rs` | Type-marker constants, transcribed byte-for-byte from .NET `JsonBinaryEncoding.TypeMarker.cs` | +| `system_strings` | `binary_json/system_strings.rs` | 32-entry system-string dictionary (byte-exact vs .NET) | +| `decode` / `Reader` | `binary_json/reader.rs` | **Complete** decoder → `serde_json::Value` (reference oracle + exotic-form fallback) | +| `from_slice` / `de` | `binary_json/de.rs` | Native serde **deserializer** (`binary` → `T`), the production read path | +| `to_vec` / `ser` | `binary_json/ser.rs` | Native serde **serializer** (`T` → `binary`), the production write path | +| `encode` | `binary_json/writer.rs` | `Value` → `binary` reference encoder (parity oracle + shared emit helpers) | +| `is_binary` / `PREAMBLE` / `transcode_to_text` / `transcode_to_binary` | `binary_json/mod.rs` | First-byte (`0x80`) auto-detection + the two schema-agnostic transcoding primitives (binary→text, text→binary) | +| `deserialize_response` / `ResponseBody::transcode_to_text` | `models/response_body.rs` | Decode choke point for `into_single` / `into_items`; in-place binary→text conversion | +| `BinaryEncodingOptions` | `azure_data_cosmos_driver/src/options/binary_encoding.rs` | **Driver-owned** options (`enabled`, `request_text_response`), on `OperationOptions.binary_encoding`; the SDK re-exports it | +| `OperationOptions.binary_encoding` | `driver/options/operation_options.rs` | Layered option carrying binary encoding to every consumer (SDK + FFI) | +| `execute_operation` / `apply_request_binary_encoding` | `driver/cosmos_driver.rs` | Driver owns the wire: transcodes text→binary request, advertises `CosmosBinary`, transcodes binary→text response | +| `CosmosResponse::transcode_body_to_text` | `models/cosmos_response.rs` | Applies driver-side response transcoding | +| `resolve_binary_encoding` / `with_binary_encoding` | `azure_data_cosmos/src/clients/{mod,container_client}.rs` | SDK: resolve enablement once; set the option on `OperationOptions` per item op | +| `serialize_item_body` | `azure_data_cosmos/src/clients/container_client.rs` | SDK typed fast path: pre-encode `T` to binary (driver passes it through) | +| `cosmos_operation_options_t.binary_encoding_*` | `driver_native/src/op_request.rs` | FFI: flat `binary_encoding_enabled` / `binary_encoding_request_text_response` flags → driver option | +| In-memory emulator binary support | `in_memory_emulator/{dispatch,response,operations}.rs` | Decodes binary requests, replies binary on negotiation — enables the E2E loop | + +--- + +## Testing + +* **Decoder** — golden-vector parity corpus; per-form unit tests across every marker family. +* **Encoder** — `encode → decode` round-trip tests (numbers across all widths, strings incl. unicode/escapes/boundary lengths, nested containers, empties). +* **Robustness (P4)** — deterministic fuzz suite asserting the decoder *always* terminates with `Ok | Err` on untrusted input: never panics on random / truncated / single-byte-corrupted buffers, never over-allocates on adversarial length prefixes (`u32::MAX` errors in O(1)), and rejects deep nesting with `DepthLimitExceeded` instead of a stack overflow. +* **End-to-end (in-memory emulator)** — `binary_round_trip.rs`: + * `binary_encoding_item_write_read_round_trips` — create / read / upsert / replace through the full SDK → driver → emulator binary loop, including a unicode payload (`"café ☃ binary"`). + * `binary_and_text_clients_interoperate` — a binary-written document reads back through a text client and vice-versa. + * `request_text_response_keeps_wire_binary_and_returns_data` — with `request_text_response`, a `RequestObserver` confirms every item request advertised `CosmosBinary` (wire stayed binary) while the typed document still round-trips (driver transcoded to text). +* **Response-format negotiation** — `binary_response_format.rs` sends a binary request through the emulator and inspects the **raw** response bytes for each negotiation (`CosmosBinary` → binary, `JsonText` → text, none → text); `dispatch.rs` unit tests cover the emulator's `binary_response` decision. +* **Transcoding** — `binary_json::transcode_to_text` **and** `transcode_to_binary` unit tests (round-trip equivalence, binary/text/empty passthrough, malformed-input errors); `ResponseBody::transcode_to_text` tests. +* **Driver option + request encode** — `OperationOptions.binary_encoding` builder/layered-resolution tests; `apply_request_binary_encoding` tests (text→binary + header, already-binary passthrough, invalid-text error). +* **FFI** — `cosmos_operation_options_t` conversion tests: `binary_encoding_enabled` + `request_text_response` build the driver option (enabled+text, enabled-only, disabled-yields-none). +* **Benchmarks** — `azure_data_cosmos_benchmarks`'s `binary_encode` / `binary_decode` compare text, the retired via-`Value` path, and the shipped native codec on small and ~1.7 MB items. + +Validation sweep (per the cosmos contributing guidelines): `cargo fmt`, `clippy` (driver `--all-features`; SDK default features), `cargo doc -D warnings` (driver), `cspell`, and the driver lib + SDK test suites — all clean. + +Run the E2E loop locally: + +```bash +cargo test -p azure_data_cosmos --features __internal_in_memory_emulator \ + --test in_memory_emulator binary_round_trip +``` + +--- + +## Enabling binary encoding + +Binary is **off by default**. To opt in for item operations, set it on the client builder: + +```rust +use azure_data_cosmos::options::BinaryEncodingOptions; + +let client = CosmosClientBuilder::new() + .with_binary_encoding_options(BinaryEncodingOptions::new().with_enabled(true)) + .build(account, routing_strategy) + .await?; +``` + +To keep the wire binary but receive text-JSON responses (driver transcodes): + +```rust +let options = BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true); +let client = CosmosClientBuilder::new() + .with_binary_encoding_options(options) + .build(account, routing_strategy) + .await?; +``` + +As a fallback (e.g. for enabling encoding without a code change), the same enablement is read from an environment variable when the builder option is not set: + +```bash +AZURE_COSMOS_BINARY_ENCODING_ENABLED=true +``` + +The explicit builder option takes precedence; the flag is resolved once at client build. + +### From an FFI host (text in, text out) + +An FFI-based SDK sets two flat flags on `cosmos_operation_options_t` — no +encoding on its side. It sends a plain **text** JSON body; the driver puts binary +on the wire and (with `request_text_response`) transcodes the response back to +text: + +```c +cosmos_operation_options_t opts = cosmos_operation_options_default(); +opts.binary_encoding_enabled = 2; /* tri-state: 2 = true */ +opts.binary_encoding_request_text_response = 2; /* 2 = true */ +/* request.body = plain TEXT JSON bytes; request.options = &opts */ +``` + +--- + +## Backward compatibility & safety + +* **Off by default.** With the flag unset, requests and responses are byte-for-byte identical to current behavior; the text path is unchanged. +* **Response decode is always on but inert.** `is_binary` only triggers on a `0x80` first byte, which the service emits solely when it has negotiated binary — so enabling decode cannot affect existing text responses. +* **No model sharing across crates.** The SDK consumes the driver's codec via its public `binary_json` API. `BinaryEncodingOptions` is a **driver** type the SDK re-exports (like `Region` / `ConsistencyLevel`), because binary encoding is a wire/driver concern shared with FFI hosts; no item/document models cross the boundary. +* **No `CHANGELOG` entry yet** — this is a gated feature with no user-facing default change. The entry lands when the feature graduates. + +--- + +## Deferred work + +* **Query binary negotiation** — the request-body encoding + negotiation for query pages is still deferred. Binary encoding now lives on `OperationOptions.binary_encoding`, so a driver-minted page operation *can* carry it; what remains is confirming query-body semantics (`application/query+json`) and the native cross-partition query engine's handling of binary item bytes. The query *response* decode already works via the shared choke point. +* **Binary feed responses** — the feed splitter scans **text** JSON, so binary `Documents` envelopes cannot be sliced yet; making it binary-aware is a prerequisite for any feed/query binary negotiation. +* **`patch`** — excluded from binary encoding for now (the driver's request-side encode intentionally skips patch); transactional `batch` / `bulk` are deferred by spec. +* **Cross-implementation vectors** — validate against captured real .NET / Java binary output. + +--- + +## Reference + +* Design + phased plan: [`BINARY_ENCODING_SPEC.md`](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_SPEC.md) +* Wire constants transcribed from .NET `Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.*` diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md new file mode 100644 index 00000000000..b7731e24c6e --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_RFC.md @@ -0,0 +1,583 @@ +# Cosmos Binary JSON Encoding — Wire Format Specification + +**Status:** Draft · **Version:** 0.1 · **Audience:** SDK / codec implementers (Rust, .NET, C++, Java, Go, Python) + +> This document is a **normative, self-contained** description of the Cosmos +> Binary JSON wire format. A conforming encoder/decoder can be implemented from +> this document alone, without reference to any SDK source. It is modeled on the +> style of [RFC 8949 (CBOR)](https://datatracker.ietf.org/doc/html/rfc8949) and +> the [Protocol Buffers encoding guide](https://protobuf.dev/programming-guides/encoding/). +> +> **Provenance.** The wire constants are transcribed from the .NET reference +> implementation (`Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs`) +> and cross-checked against the Rust codec (`azure_data_cosmos_driver::binary_json`). +> Details that could not be confirmed from the Rust implementation alone are +> tagged **`[CROSS-VERIFY: .NET/C++]`** and MUST be validated against the .NET +> and C++ sources before this draft is promoted to a stable version. + +--- + +## 1. Introduction + +### 1.1 Purpose + +Cosmos Binary JSON is a compact, self-describing binary serialization of the +JSON data model used by Azure Cosmos DB for item request and response bodies. It +is **information-preserving with respect to the JSON value model** (null, +boolean, number, string, array, object) while being smaller and faster to +parse than UTF-8 JSON text. The service and every language SDK MUST agree on +this format byte-for-byte. + +### 1.2 Scope + +This specification defines: + +- the byte-level layout of every value kind (§3–§6), +- the **canonical** encoding a conforming encoder emits when multiple encodings + are valid (§7), +- decoder conformance requirements, including bounds and resource limits (§8), +- security considerations for decoding untrusted input (§9). + +It does **not** define: transport framing, HTTP/RNTBD negotiation headers, +per-account dictionary (user-string) construction policy, or the query wire +protocol. Those are layered above this format. + +### 1.3 Requirements language + +The key words **MUST**, **MUST NOT**, **SHOULD**, **SHOULD NOT**, and **MAY** are +to be interpreted as described in [RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119). + +### 1.4 Relationship to the other binary-encoding artifacts + +This RFC is the **source of truth** for the wire format. Several other artifacts +either derive from it or validate against it — they do not redefine it: + +```mermaid +flowchart TD + RFC["BINARY_ENCODING_RFC.md
(normative spec: what correct means)"] + RFC -->|"§7 canonical encoding
§8 decoder conformance"| CONF["binary_json/conformance.rs
(encoder byte-exactness)"] + RFC -->|"Appendix A golden vectors"| CORPUS["testdata/binary_json_vectors.json
(fixed decode+encode oracle)"] + RFC -->|"value model + round-trip invariant"| FUZZ["BINARY_ENCODING_ROUNDTRIP_FUZZER
(random JSON, live service, at scale)"] + RFC -->|"§8 decoder + §9 security"| IN_FUZZ["binary_json/fuzz_tests.rs
(malformed/truncated buffers)"] + CORPUS --> CONF + CORPUS --> FUZZ + CORPUS --> IN_FUZZ +``` + +| Artifact | Role relative to this RFC | RFC sections it enforces | +| -------- | ------------------------- | ------------------------ | +| `testdata/binary_json_vectors.json` | Machine-readable golden corpus (this RFC's Appendix A) | Appendix A | +| `binary_json/conformance.rs` | Encoder byte-exactness + canonical-form snapshots | §3.1, §7 | +| `binary_json/reader.rs`, `de.rs` (tests) | Decoder conformance per form | §4–§6, §8 | +| `binary_json/fuzz_tests.rs` | Decoder never panics/hangs/over-allocates on malformed input | §8, §9 | +| [`BINARY_ENCODING_ROUNDTRIP_FUZZER.md`](BINARY_ENCODING_ROUNDTRIP_FUZZER.md) + harness | End-to-end **round-trip invariant** on random JSON against the live service, at volume | §7 (round-trip), value model (§2) | + +Two connections are worth calling out explicitly: + +- **The round-trip fuzzer validates this RFC's round-trip invariant (§7), but + end-to-end rather than byte-level.** Where `conformance.rs` checks + `decode(encode(v)) == v` in-process on fixed vectors, the fuzzer checks the + *same* invariant across the full pipeline (Rust encode → wire → backend + store/rewrite → wire → Rust decode) on millions of random documents. +- **The fuzzer is the instrument that closes this RFC's open number-format + items.** The `[CROSS-VERIFY: .NET/C++]` tags and §7 canonical rules leave the + encoder-vs-backend number-normalization under-specified; the fuzzer's + canonicalization/calibration surface (its design doc §3.1) empirically + discovers the backend's actual number rewrite, and those findings feed back + into §7 here. + +--- + +## 2. Notation and terminology + +- **byte** — an 8-bit octet, written as two hexadecimal digits, e.g. `C8`. +- **marker** (or **type marker**) — the single leading byte that selects how the + following bytes are interpreted. Every encoded value begins with exactly one + marker (except literal small integers, where the marker byte *is* the value — + see §4.1). +- **spaced-hex** — the human-reviewable notation for a byte sequence used + throughout this document and in the shared test corpus, e.g. `80 CC 00 00 00 + 00 00 00 0C 40`. +- **preamble** — the single byte `0x80` that MUST prefix a complete buffer (§3.1). +- **little-endian (LE)** — multi-byte integers and floats are serialized + least-significant-byte first. This is **normative and independent of host + architecture**: an encoder on a big-endian host MUST still emit LE. +- **value model** — the abstract JSON value: `null | bool | number | string | + array | object`. Numbers are IEEE-754 doubles or integers in the ranges + encodable by the number markers in §4. + +--- + +## 3. Structure of an encoded value + +### 3.1 Buffer preamble and auto-detection + +A complete Cosmos Binary JSON buffer MUST begin with the **preamble byte** +`0x80`, followed by exactly one encoded value: + +``` +buffer = 0x80 value +``` + +Because `0x80` is also the marker for a zero-length encoded string (§4.4), the +preamble is **always consumed first**; a top-level empty string is therefore +`80 80` (preamble + zero-length-string marker). + +Consumers distinguish binary from UTF-8 JSON text by inspecting the **first +byte**: a payload whose first byte is `0x80` is Cosmos Binary JSON; any other +first byte (`{`, `[`, `"`, digit, `t`, `f`, `n`, whitespace, …) is UTF-8 text. +This is the `is_binary` predicate. + +> **Note.** UTF-8 text can never legitimately begin with `0x80` (a continuation +> byte), so the discriminator is unambiguous. + +### 3.2 Marker byte taxonomy + +The 256 marker values are partitioned into contiguous ranges. Ranges are written +`[MIN, MAX)` — MIN inclusive, MAX exclusive. + +| Range | Meaning | Section | +| ------------- | ------------------------------------------------------------- | ------- | +| `[0x00,0x20)` | Literal small integer (`value == marker`, range 0–31) | §4.1 | +| `[0x20,0x40)` | 1-byte **system** string (index into fixed dictionary) | §4.4.3 | +| `[0x40,0x60)` | 1-byte **user** string (index into per-buffer dictionary) | §4.4.4 | +| `[0x60,0x68)` | 2-byte user string | §4.4.4 | +| `[0x68,0x80)` | base64 / GUID-string / compressed-string forms | §4.4.5–6| +| `[0x80,0xC0)` | Encoded-length string (`len == marker & 0x7F`, 0–63) | §4.4.1 | +| `[0xC0,0xC8)` | Length-prefixed strings, reference strings, `NumberUInt64` | §4.4.1, §6, §4.2 | +| `[0xC8,0xD0)` | Fixed-width numbers (`UInt8`,`Int16/32/64`,`Double`,`Float*`) | §4.2–3 | +| `[0xD0,0xE0)` | null, bool, GUID value, extended sized ints, binary blobs | §4.1, §4.2, §4.7 | +| `[0xE0,0xE8)` | Arrays | §5.1 | +| `[0xE8,0xF0)` | Objects | §5.2 | +| `[0xF0,0xF8)` | Uniform (typed) number arrays | §5.3 | +| `[0xF8,0xFF]` | Reserved; `0xFF` == Invalid | §8 | + +The complete marker constant table is given in Appendix C. + +--- + +## 4. Scalars + +### 4.1 Null, boolean, and literal small integers + +| Value | Marker | Sample (with preamble) | +| --------------- | ------- | ---------------------- | +| `null` | `D0` | `80 D0` | +| `false` | `D1` | `80 D1` | +| `true` | `D2` | `80 D2` | +| integer `0`–`31`| `00`–`1F` | `0` → `80 00`; `31` → `80 1F` | + +For an integer `n` in `[0, 31]`, the marker byte itself is the value: the encoded +form is the single byte `n`. This is the most compact integer encoding and is the +canonical form for that range (§7). + +### 4.2 Integers + +Two families of integer markers exist, distinguished only by their historical +range. Both are LE. + +| Marker | Byte | Width | Signedness | Sample | +| ----------------- | ---- | ----- | ---------- | ----------------------------------- | +| `NumberUInt8` | `C8` | 1 | unsigned | `200` → `80 C8 C8` | +| `NumberInt16` | `C9` | 2 | signed | `-1000` → `80 C9 18 FC` | +| `NumberInt32` | `CA` | 4 | signed | `70000` → `80 CA 70 11 01 00` | +| `NumberInt64` | `CB` | 8 | signed | `-5000000000` → `80 CB 00 0E FA D5 FE FF FF FF` | +| `NumberUInt64` | `C7` | 8 | unsigned | `18446744073709551614` → `80 C7 FE FF FF FF FF FF FF FF` | +| `UInt8` (ext.) | `D7` | 1 | unsigned | `[CROSS-VERIFY: .NET/C++]` | +| `Int8` (ext.) | `D8` | 1 | signed | `-5` → `80 D8 FB` | +| `Int16` (ext.) | `D9` | 2 | signed | `-1000` → `80 D9 18 FC` | +| `Int32` (ext.) | `DA` | 4 | signed | `-70000` → `80 DA 90 EE FE FF` | +| `Int64` (ext.) | `DB` | 8 | signed | `-5000000000` → `80 DB 00 0E FA D5 FE FF FF FF` | +| `UInt32` (ext.) | `DC` | 4 | unsigned | `4294967294` → `80 DC FE FF FF FF` | + +Decoders MUST accept **both** families. Encoders emit the canonical family per +§7. The `NumberUInt64` marker (`C7`) is the only encoding able to carry +unsigned 64-bit values above `i64::MAX`. + +### 4.3 Floating-point numbers + +| Marker | Byte | Width | Encoding | Sample | +| -------------- | ---- | ----- | ---------------------------- | ------------------------------- | +| `NumberDouble` | `CC` | 8 | IEEE-754 binary64 (LE) | `3.5` → `80 CC 00 00 00 00 00 00 0C 40` | +| `Float32` | `CD` | 4 | IEEE-754 binary32 (LE) | `1.5` → `80 CD 00 00 C0 3F` | +| `Float64` | `CE` | 8 | IEEE-754 binary64 (LE) | `-2.25` → `80 CE 00 00 00 00 00 00 02 C0` | +| `Float16` | `CF` | 2 | IEEE-754 binary16 (LE) | `[CROSS-VERIFY: .NET/C++]` | + +`NumberDouble` (`CC`) is the canonical JSON-number float form (§7). + +**Non-finite values.** JSON has no representation for `NaN` or `±Infinity`. +A conforming encoder MUST NOT emit a non-finite double; instead it MUST encode +`null` (`D0`), mirroring `serde_json` / JavaScript `JSON.stringify`. A conforming +decoder MUST reject a non-finite `NumberDouble`/`Float*` payload as an invalid +number. `[CROSS-VERIFY: .NET/C++]` — confirm the .NET encoder/decoder policy is +identical. + +### 4.4 Strings + +JSON strings are UTF-8. Several encodings exist; a decoder MUST accept all of +them, and an encoder selects the canonical one per §7. + +#### 4.4.1 Length-framed strings + +| Form | Marker | Length field | Sample | +| ------------------ | ------ | ------------------ | -------------------------- | +| Encoded-length | `80`–`BF` | in marker: `len = marker & 0x7F` (0–63) | `""` → `80 80`; `"hi"` → `80 82 68 69` | +| `StrL1` | `C0` | u8 | `"hello"` → `80 C0 05 68 65 6C 6C 6F` | +| `StrL2` | `C1` | u16 LE | 300×`"a"` → `80 C1 2C 01 …` | +| `StrL4` | `C2` | u32 LE | (large strings) | + +The string's UTF-8 bytes follow the length field verbatim. + +#### 4.4.2 GUID strings + +| Marker | Byte | Meaning | Sample | +| ------ | ---- | ------------------------------------ | ------ | +| lower | `75` | 36-char lowercase GUID string | `80 75 00 01 … 0F` → `"00010203-0405-0607-0809-0a0b0c0d0e0f"` | +| upper | `76` | 36-char uppercase GUID string | `80 76 …` → uppercase | +| quoted | `77` | double-quoted lowercase GUID string | `80 77 …` → `"\"…\""` | + +The 16 raw GUID bytes follow the marker; the decoder formats them as a hyphenated +GUID string (byte order per Appendix B). This is distinct from a **GUID value** +(§4.7). + +#### 4.4.3 System strings + +Markers `[0x20, 0x40)` encode a string by **index into a fixed, well-known +dictionary** of common Cosmos property names (`id`, `_rid`, `_etag`, …). The +encoded form is a single byte; `marker - 0x20` is the dictionary index. + +Example: `"id"` has system index `0x0C`, so `"id"` → `80 2C` (`0x20 + 0x0C`). + +The full system-string table is normative and given in Appendix D. +`[CROSS-VERIFY: .NET/C++]` — the index assignments MUST match `JsonBinaryEncoding`. + +#### 4.4.4 User (per-buffer dictionary) strings + +Markers `[0x40, 0x60)` (1-byte) and `[0x60, 0x68)` (2-byte) encode a string by +index into a **per-buffer user-string dictionary**. The dictionary and its +construction policy are out of scope for this document (a decoder receives the +dictionary alongside the buffer, or the buffer contains no user-dictionary +references). `[CROSS-VERIFY: .NET/C++]` + +#### 4.4.5 base64 strings + +| Marker | Byte | Alphabet | Length field | Sample | +| ------ | ---- | ------------ | ------------ | ------ | +| `Base64Len1` | `71` | standard | u8 | `"Zm9v"` → `80 71 01 00 66 6F 6F` | +| `Base64Len2` | `72` | standard | u16 | `"Zm9vYmFy"` → `80 72 02 00 00 66 6F 6F 62 61 72` | +| `Base64UrlLen1` | `73` | URL-safe | u8 | `"-__-"` → `80 73 01 00 FB FF FE` | +| `Base64UrlLen2` | `74` | URL-safe | u16 | (as above, 2-byte length) | + +The payload is the **decoded** bytes; the decoder re-encodes them to a base64 +string using the marker's alphabet. Padding handling (`=`) and the "omitted +padding" length-field convention are illustrated in Appendix B. +`[CROSS-VERIFY: .NET/C++]` for the exact padding/length-offset encoding. + +#### 4.4.6 Compressed strings + +Restricted-alphabet strings are bit-packed relative to a base character: + +| Marker | Byte | Packing | Sample | +| ------ | ---- | ------------------------------- | ------ | +| lower-hex | `78` | 4-bit hex digits (lowercase) | `"1a2b"` → `80 78 04 A1 B2` | +| upper-hex | `79` | 4-bit hex digits (uppercase) | `"1A2B"` → `80 79 04 A1 B2` | +| date-time | `7A` | 4-bit date-time charset | `"2024-01"` → `80 7A 07 13 53 1C 02` | +| packed-4bit | `7B` | 4 bits/char + base | `"0123"` → `80 7B 04 30 10 32` | +| packed-5bit | `7C` | 5 bits/char + base | `"abc"` → `80 7C 03 61 20 08` | +| packed-6bit | `7D` | 6 bits/char + base | `"abcd"` → `80 7D 04 61 40 20 0C` | +| packed-7bit-L1 | `7E` | 7 bits/char, u8 length | `"Hi"` → `80 7E 02 C8 34` | +| packed-7bit-L2 | `7F` | 7 bits/char, u16 length | `"Hi"` → `80 7F 02 00 C8 34` | + +The byte immediately after the marker is the **character count** (for L1/L2 forms +a 1-/2-byte length), followed by the packed bits. Appendix B gives a worked +unpacking example. `[CROSS-VERIFY: .NET/C++]` for the exact base/charset tables. + +### 4.7 GUID value and binary blobs + +| Marker | Byte | Meaning | Sample | +| ------ | ---- | -------------------------------- | ------ | +| `Guid` | `D3` | raw 16-byte GUID **value** | `80 D3 00 01 … 0F` → `"03020100-0504-0706-0809-0a0b0c0d0e0f"` | +| `Binary1` | `DD` | blob, u8 length prefix | `0xDEADBEEF` → `80 DD 04 DE AD BE EF` → `"3q2+7w=="` | +| `Binary2` | `DE` | blob, u16 length prefix | `80 DE 03 00 01 02 03` → `"AQID"` | +| `Binary4` | `DF` | blob, u32 length prefix | (large blobs) | + +A **binary blob** decodes to a **standard base64 string** in the JSON value +model. Note the byte order of a GUID *value* (`D3`) differs from a GUID *string* +(§4.4.2) — see Appendix B. `[CROSS-VERIFY: .NET/C++]` + +--- + +## 5. Containers + +### 5.1 Arrays + +| Marker | Byte | Framing | Sample | +| ------- | ---- | ---------------------------------------- | ------ | +| `Arr0` | `E0` | empty | `[]` → `80 E0` | +| `Arr1` | `E1` | exactly one element, no length/count | `[true]` → `80 E1 D2` | +| `ArrL1` | `E2` | u8 **byte-length** prefix | `[0,1,null]` → `80 E2 03 00 01 D0` | +| `ArrL2` | `E3` | u16 byte-length | | +| `ArrL4` | `E4` | u32 byte-length | | +| `ArrLC1`| `E5` | u8 byte-length **+** u8 item-count | `[0,1,null]` → `80 E5 03 03 00 01 D0` | +| `ArrLC2`| `E6` | u16 byte-length + u16 count | | +| `ArrLC4`| `E7` | u32 byte-length + u32 count | | + +For `L*` forms, the length is the **byte length of the element region** (not the +element count); the decoder reads elements until it has consumed exactly that +many bytes. For `LC*` forms, both the byte length **and** the element count are +given, and a conforming decoder MUST verify that reading `count` elements +consumes exactly the declared byte length (§8). + +### 5.2 Objects + +| Marker | Byte | Framing | Sample | +| ------- | ---- | ----------------------------------- | ------ | +| `Obj0` | `E8` | empty | `{}` → `80 E8` | +| `Obj1` | `E9` | exactly one name/value pair | `{"id":true}` → `80 E9 2C D2` | +| `ObjL1` | `EA` | u8 byte-length | `{"id":0,"type":1}` → `80 EA 04 2C 00 3B 01` | +| `ObjL2` | `EB` | u16 byte-length | | +| `ObjL4` | `EC` | u32 byte-length | | +| `ObjLC1`| `ED` | u8 byte-length + u8 pair-count | `{"id":0,"type":1}` → `80 ED 04 02 2C 00 3B 01` | +| `ObjLC2`| `EE` | u16 byte-length + u16 count | | +| `ObjLC4`| `EF` | u32 byte-length + u32 count | | + +Members are encoded as **name, value, name, value, …**. Each **name** is itself +an encoded string (any string form, including system strings — note `2C` = system +`"id"` in the samples). Member ordering is preserved as encoded. + +### 5.3 Uniform (typed) number arrays + +A homogeneous array of numbers of a single width is encoded compactly by writing +the item type marker once, then the raw item bytes with no per-item markers. + +| Marker | Byte | Layout | Sample | +| ----------- | ---- | -------------------------------------------------------- | ------ | +| `ArrNumC1` | `F0` | `itemMarker`, u8 count, then `count` bare numbers | `[1,2,3]` (Int32) → `80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00` | +| `ArrNumC2` | `F1` | `itemMarker`, u16 count, then bare numbers | `[-1,0,1000]` (Int16) → `80 F1 D9 03 00 FF FF 00 00 E8 03` | +| `ArrArrNumC1C1` | `F2` | innerMarker, itemMarker, u8 inner-count, u8 outer-count, then inner arrays | `[[1,2],[3,4]]` → `80 F2 F0 DA 02 02 01 00 00 00 …` | +| `ArrArrNumC2C2` | `F3` | as above with u16 counts | | + +Further examples: `[10,20,30]` (UInt8) → `80 F0 D7 03 0A 14 1E`; empty uniform +array → `80 F0 DA 00`; `[1.5,-0.25]` (Float32) → `80 F0 CD 02 00 00 C0 3F 00 00 +80 BE`. + +**Item markers.** The uniform-array item type MUST be one of the **extended** +number markers (`Int8`/`UInt8`/`Int16`/`Int32`/`Int64`/`UInt32`/`Float32`/ +`Float64`, i.e. `D7`–`DC`, `CD`, `CE`). The self-describing `Number*` markers +(`C7`–`CC`) MUST NOT appear as a uniform-array item type and a conforming decoder +MUST reject them there. `[CROSS-VERIFY: .NET/C++]` + +--- + +## 6. Reference strings + +A string that already appeared earlier in the same buffer MAY be encoded as a +**back-reference** to its byte offset, saving space for repeated keys/values. + +| Marker | Byte | Offset field | +| ------- | ---- | ------------ | +| `StrR1` | `C3` | u8 offset | +| `StrR2` | `C4` | u16 offset | +| `StrR3` | `C5` | u24 offset (3 bytes LE) | +| `StrR4` | `C6` | u32 offset | + +The offset is an **absolute byte offset into the buffer**, measured in the same +frame as the preamble (the preamble is offset `0`). The referenced offset MUST +hold a non-reference string; reference-to-reference chains are prohibited, which +makes cycles impossible and bounds resolution without recursion. + +**Decoder resource bound (normative).** Because many references can point at one +large string, a naïve decoder can be forced into O(S²) output for a size-`S` +buffer. A conforming decoder MUST bound total materialized reference bytes by a +budget proportional to the input size (e.g. `max(16 × buffer_len, 64 KiB)`) and +fail with an invalid-length error once exceeded (§9). + +--- + +## 7. Canonical encoding + +Multiple valid encodings exist for the same value (e.g. the integer `5` can be a +literal `05`, `NumberUInt8`, `Int8`, `Int16`, …; the string `"id"` can be a +system string, an encoded-length string, `StrL1`, …). A **decoder MUST accept +all valid encodings**. An **encoder MUST be deterministic**: for a given value it +MUST emit exactly one, canonical encoding. The canonical rules are: + +1. **Integers in `[0,31]`** → literal small integer (single byte). +2. **Other integers** → the narrowest fixed-width `Number*` marker that holds the + value (`NumberUInt8` → `NumberInt16` → `NumberInt32` → `NumberInt64`; values + above `i64::MAX` use `NumberUInt64`). `[CROSS-VERIFY: .NET/C++]` — confirm the + .NET encoder prefers `Number*` over the extended `D7`–`DC` markers. +3. **Floating-point (non-integer) numbers** → `NumberDouble` (`CC`). +4. **Non-finite floats** → `null` (§4.3). +5. **Strings** → system string if the value is in the system dictionary; + otherwise encoded-length (`< 64` bytes), `StrL1`, `StrL2`, or `StrL4` by + length. Compressed/base64/GUID/reference forms are **decoder-accepted but + encoder-optional** optimizations; a minimal conforming encoder need not emit + them. `[CROSS-VERIFY: .NET/C++]` — enumerate which optimizations the .NET + encoder applies by default. +6. **Containers** → `LC*` (length-and-count) framing at the narrowest width that + fits. `[CROSS-VERIFY: .NET/C++]` + +> **Why canonical encoding matters.** A snapshot / golden-vector test asserts +> `encode(value) == expected_bytes`. Without a fixed canonical rule, two +> conforming encoders could produce different (both valid) buffers and the test +> would be meaningless. §7 is the contract that makes cross-SDK byte-equality +> tests possible. + +--- + +## 8. Decoder conformance requirements + +A conforming decoder MUST: + +1. **Reject a missing preamble.** The first byte MUST be `0x80` (§3.1). +2. **Reject trailing bytes.** After decoding the single top-level value, no bytes + may remain. +3. **Bounds-check every read.** A length/offset field that would read past the + end of the buffer MUST fail with an unexpected-EOF / invalid-length error, not + panic or read out of bounds. +4. **Enforce a maximum nesting depth** to prevent stack exhaustion on deeply + nested containers. The reference limit is `256`. Both the reference (`Value`) + decoder and any streaming decoder MUST reject at the **same** depth. +5. **Validate `LC*` containers.** After reading `count` elements/members, the + cursor MUST be exactly at the declared byte-length boundary; a mismatch MUST + fail (this catches malformed length+count buffers that a count-only decoder + would silently under-read). +6. **Enforce the reference-string budget** (§6). +7. **Bound uniform-array output.** For `ArrArrNum*` with a zero inner count, the + outer count MUST NOT exceed the remaining buffer bytes (else a few bytes could + materialize `u16::MAX` empty arrays). +8. **Reject the `Invalid` marker (`0xFF`)** and any unassigned marker with an + invalid-marker error. +9. **Reject non-finite numbers** (§4.3). + +A conforming decoder MUST NOT panic, hang, or allocate unboundedly on any input, +because it parses **untrusted** service/network bytes (§9). + +--- + +## 9. Security considerations + +Decoders process bytes that may originate from a compromised or buggy service, a +MITM, or a corrupted cache. The threats and required mitigations: + +| Threat | Mitigation (normative) | +| ---------------------------------------- | ---------------------------------------------- | +| Out-of-bounds read via a large length | Bounds-check every read against buffer end (§8.3) | +| Stack exhaustion via deep nesting | Max depth limit, rejected identically by all decoders (§8.4) | +| O(S²) memory via many back-references | Per-decode reference-expansion budget (§6) | +| Output amplification via empty uniform arrays | Bound outer count by remaining bytes (§8.7) | +| Malformed length+count under-read | Assert cursor == declared end (§8.5) | + +Encoders SHOULD reject values that cannot be represented (e.g. integer widths +beyond `u32::MAX` container framing) rather than silently truncating. + +--- + +## Appendix A — Golden test vectors (shared corpus) + +The normative, cross-SDK test corpus lives in machine-readable form at +`azure_data_cosmos_driver/testdata/binary_json_vectors.json`. Each entry pairs a +`name`, a spaced-hex `binary` buffer (including the `0x80` preamble), and the +`json` value it decodes to. A conforming decoder MUST reproduce every `json` from +its `binary`; a conforming encoder MUST reproduce the canonical `binary` for +every `json` that is in canonical form (§7). + +A representative subset (see the file for the full set): + +| name | binary | json | +| ---- | ------ | ---- | +| null | `80 D0` | `null` | +| true | `80 D2` | `true` | +| literal_int_max | `80 1F` | `31` | +| uint8 | `80 C8 C8` | `200` | +| double | `80 CC 00 00 00 00 00 00 0C 40` | `3.5` | +| system_string_id | `80 2C` | `"id"` | +| str_l1_hello | `80 C0 05 68 65 6C 6C 6F` | `"hello"` | +| binary_deadbeef | `80 DD 04 DE AD BE EF` | `"3q2+7w=="` | +| uniform_int32 | `80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00` | `[1,2,3]` | +| object_lc1 | `80 ED 04 02 2C 00 3B 01` | `{"id":0,"type":1}` | +| nested_containers | `80 E2 05 E1 00 E9 2C 01` | `[[0],{"id":1}]` | + +## Appendix B — Worked examples (byte-by-byte) + +**`{"id":0,"type":1}` as `ObjLC1` (`80 ED 04 02 2C 00 3B 01`):** + +``` +80 preamble +ED ObjLC1 marker (u8 byte-length + u8 count) +04 byte-length of member region = 4 +02 member (pair) count = 2 +2C name: system string 0x0C ("id") ← 0x20 + 0x0C +00 value: literal small int 0 +3B name: system string 0x1B ("type") ← 0x20 + 0x1B [CROSS-VERIFY] +01 value: literal small int 1 +``` + +**`[1,2,3]` as a uniform Int32 array (`80 F0 DA 03 01 00 00 00 …`):** + +``` +80 preamble +F0 ArrNumC1 (uniform number array, u8 count) +DA item type marker = Int32 +03 item count = 3 +01 00 00 00 1 (Int32 LE) +02 00 00 00 2 +03 00 00 00 3 +``` + +**base64 with omitted padding (`80 71 01 FD 41` → `"QQ"`):** the length field +uses a signed offset convention to signal padding omission; see the codec's +base64 decoder for the exact rule. `[CROSS-VERIFY: .NET/C++]` + +## Appendix C — Complete marker constant table + +*(Authoritative values; transcribed from `markers.rs` / `JsonBinaryEncoding.TypeMarker.cs`.)* + +``` +Literal int 0x00–0x1F +System string 1B 0x20–0x3F +User string 1B 0x40–0x5F +User string 2B 0x60–0x67 +Base64Len1 0x71 Base64Len2 0x72 +Base64UrlLen1 0x73 Base64UrlLen2 0x74 +GuidLower 0x75 GuidUpper 0x76 GuidQuoted 0x77 +CompLowerHex 0x78 CompUpperHex 0x79 CompDateTime 0x7A +Packed4/5/6bit 0x7B/0x7C/0x7D +Packed7bitL1/L2 0x7E/0x7F +Encoded-len str 0x80–0xBF (len = marker & 0x7F) +StrL1/L2/L4 0xC0/0xC1/0xC2 +StrR1/R2/R3/R4 0xC3/0xC4/0xC5/0xC6 +NumberUInt64 0xC7 +NumberUInt8 0xC8 NumberInt16 0xC9 NumberInt32 0xCA NumberInt64 0xCB +NumberDouble 0xCC Float32 0xCD Float64 0xCE Float16 0xCF +Null 0xD0 False 0xD1 True 0xD2 Guid 0xD3 +UInt8 0xD7 Int8 0xD8 Int16 0xD9 Int32 0xDA Int64 0xDB UInt32 0xDC +Binary1/2/4 0xDD/0xDE/0xDF +Arr0..ArrLC4 0xE0–0xE7 +Obj0..ObjLC4 0xE8–0xEF +ArrNumC1/C2 0xF0/0xF1 +ArrArrNumC1C1/C2C2 0xF2/0xF3 +Invalid 0xFF +``` + +## Appendix D — System string dictionary + +The fixed system-string table (index → string) is normative and MUST match +`JsonBinaryEncoding` across SDKs. It is defined in +`azure_data_cosmos_driver/src/binary_json/system_strings.rs`. +`[CROSS-VERIFY: .NET/C++]` — reproduce the full table here once confirmed against +the .NET source (only `id` = index `0x0C` and `type` = index `0x1B` are shown +inline in this draft's examples). + +--- + +## Open items before promotion to stable + +- Resolve every `[CROSS-VERIFY: .NET/C++]` tag against the .NET + (`Microsoft.Azure.Cosmos/src/Json/`) and C++ reference implementations. +- Complete Appendix D (full system-string table). +- Confirm the canonical-encoding rules in §7 match the .NET encoder's actual + output (needed for cross-SDK byte-equality snapshot tests). +- Specify the exact base64 padding/length-offset convention (Appendix B). +- Specify the compressed-string base/charset tables (§4.4.6). diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md new file mode 100644 index 00000000000..f539eee5a82 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md @@ -0,0 +1,529 @@ +# Binary-Encoding Round-Trip Fuzzer — Design + +**Status:** Draft · **Companion harness:** `azure_data_cosmos_perf/tests/binary_roundtrip_fuzzer.rs` + +## 1. Goal + +Validate that **arbitrary JSON survives a full Cosmos round-trip unchanged**, +across binary-encoding configurations, at high volume. Where the deterministic +golden vectors ([`binary_json_vectors.json`](../testdata/binary_json_vectors.json)) +prove specific byte layouts and the in-tree fuzz suite hammers the *decoder* with +malformed input, this harness exercises the **end-to-end path**: + +``` +generate JSON → Rust encode → wire → backend store+rewrite → wire → Rust decode → compare +``` + +A single machine can validate **millions of distinct JSON structures over a few +days**, which no hand-written test set can approach. + +This directly implements the reviewer request (FabianMeiswinkel, PR #4671): + +> a tool that can produce random json structures and does e2e validation (via +> canonicalization + hash) — this would allow us to test millions of differently +> structured json objects and increase confidence level. + +## 2. Core idea: canonicalize + compare + +For each generated document `D`: + +1. `H0 = hash(canonicalize(D))` +2. For each config `C` (binary on/off, text-response on/off, …): + - drive every **body-carrying point op** — `create` → `read` → `replace` → + `upsert` — each returning a document `R` (writes use content-response so the + response decode path is exercised too); + - `Hc = hash(canonicalize(project(R, keys(D))))` for each op's `R`; + - **assert `Hc == H0`** for every op — otherwise dump the seed + both + canonical forms. + +These are exactly the four point operations for which binary encoding is honored +(`create` / `read` / `replace` / `upsert`); `delete` carries no body, and +`patch` / transactional batch / bulk are deferred (see the SPEC/HLD), so they are +intentionally excluded. + +`project(R, keys(D))` strips the service-added system fields (`_rid`, `_etag`, +`_ts`, `_self`, `_attachments`) so only the fields we control are compared. + +The harness compares **canonical strings directly** (strongest signal — it can +print the exact diff) *and* logs a **SHA-256** digest so the "store `H0` once, +compare later" workflow is available for a persistent corpus. SHA-256 is stable +across runs and platforms; we are detecting *differences*, so the digest's role +is a compact, durable corpus key rather than collision defense. (The canonical +string is produced by RFC 8785 / `json_canon` over the number-normalized value — +see §9.) + +### 2.1 How this detects codec gaps + +The fuzzer needs no knowledge of *correct bytes* — it exploits one invariant: +**a value stored and read back must be identical.** It runs each generated +document through **three configs** (text control, binary, binary+text-response), +which is what lets a mismatch **localize the broken layer**: + +| Symptom in the mismatch dump | Where the bug is | +| ---------------------------- | ---------------- | +| Text config passes, **binary** config fails | The **encoder** (`ser.rs` / `writer.rs`) emitted wrong bytes for some value. | +| Binary write succeeds but the **read decodes wrong** | The **decoder** (`de.rs` / `reader.rs`) mishandles a wire form. | +| **binary+text-response** fails but plain binary passes | The **driver transcode** (`transcode_to_text`) loses something on binary→text. | +| **All configs fail identically** | Likely a backend rewrite the canonicalizer doesn't model yet → a calibration gap (tune `canonicalize_number`, §3.1), or a genuine service behavior to escalate. | + +The specific classes of gap it is built to surface — the ones curated tests miss +because no human authored the triggering input: + +- **Encoder ↔ decoder disagreement on a wire form the *backend* emits.** The Rust + encoder emits only a *subset* of wire forms; the decoder accepts *all* of them + (system strings, compressed 4/5/6/7-bit strings, GUID/base64 forms, uniform + number arrays, reference strings). If the decoder mishandles a compact form the + **backend** produces but the Rust encoder never does, no unit test exercises + it — only a live round-trip does. +- **Number precision/representation edges** — exactly what calibration surfaces: + non-finite handling, `-0`, integers above `i64::MAX` (backend stores as + doubles), high-precision floats. +- **Unicode / string escaping** — astral code points, control characters, + characters needing JSON escaping; a mismatch here is an encoder/decoder + UTF-8/length bug. +- **Container framing** — deep nesting, mixed vs. uniform arrays, empty + containers, arrays-of-objects; catches off-by-one length/count bugs. +- **The transcode path specifically** — the `binary+text-response` config is the + *only* end-to-end exercise of `transcode_to_text` against real binary the + backend produced. + +**The debugging loop:** a failure prints the exact document, the config, and the +seed. Re-run with `AZURE_COSMOS_FUZZ_SEED=` to reproduce deterministically, +reduce to the minimal triggering value, add it as a golden vector, and fix the +codec — then the new vector guards against regression. + +**Limitations to keep in mind:** the fuzzer is only as good as its calibration +and its generator's range. Under-calibrated `canonicalize_number` → false +positives (noise); a form the generator never emits → false negatives (blind +spots). Calibrate first (§3.1), then widen coverage progressively with +`--wide-numbers` / `max_depth` / `unicode`. + +### 2.2 How it works, visualized + +**The per-document loop.** Every generated document is canonicalized once to get +the expected hash `H0`, then stored + read back under each config and compared: + +```mermaid +flowchart TD + SEED["Seed (SplitMix64)\nAZURE_COSMOS_FUZZ_SEED"] --> GEN + GEN["arbitrary-json\ngenerate random Value D"] --> BOUND["bound_value\nclamp numbers/strings\nto calibrated envelope"] + BOUND --> NORM0["normalize_numbers(D)\nCosmos number rewrite"] + NORM0 --> CANON0["json-canon (RFC 8785)\ncanonical string"] + CANON0 --> HASH0["SHA-256 -> H0\n(expected)"] + + BOUND --> STORE["for each config:\ncreate -> read -> replace -> upsert\n(each returns R)"] + STORE --> PROJ["project(R, keys(D))\nstrip _rid/_etag/_ts/..."] + PROJ --> NORM1["normalize_numbers"] + NORM1 --> CANON1["json-canon"] + CANON1 --> HASH1["SHA-256 -> Hc\n(actual)"] + + HASH0 --> CMP{"Hc == H0 ?"} + HASH1 --> CMP + CMP -->|yes| OK["round-trip OK\nnext doc"] + CMP -->|no| FAIL["MISMATCH\ndump seed + both canonical forms"] +``` + +**Three configs localize the broken layer.** The same document `D` runs through +three client configurations; because only one variable differs, *which* config +fails points at *which* layer is broken: + +```mermaid +flowchart LR + D["Document D"] --> C1 & C2 & C3 + + subgraph C1["Config A - text control"] + A1["binary = off\nwire = text JSON"] + end + subgraph C2["Config B - binary"] + B1["binary = on\nwire = binary both ways"] + end + subgraph C3["Config C - binary + text-response"] + G1["binary = on\nrequest_text_response = on\ndriver transcodes response to text"] + end + + C1 --> R{"compare canonical(sent)\nvs canonical(returned)"} + C2 --> R + C3 --> R +``` + +| What fails | Where the bug is | +| ---------- | ---------------- | +| **Text (A) fails** | Not the codec — a canonicalization gap or a real backend rewrite; escalate. | +| **Binary (B) fails, text (A) passes** | The **encoder** (`ser.rs`/`writer.rs`) emitted wrong bytes. | +| Binary write OK but **read decodes wrong** | The **decoder** (`de.rs`/`reader.rs`) mishandles a wire form. | +| **C fails, B passes** | The **driver transcode** (`transcode_to_text`) loses something binary to text. | +| **All three fail identically** | A backend rewrite the canonicalizer doesn't model yet (tune `normalize_number`), or a genuine service behavior to escalate. | + +**The debugging loop.** Every failure is deterministically reproducible and +reduces to a permanent regression guard: + +```mermaid +flowchart LR + F["MISMATCH\nprints seed + config\n+ both canonical forms"] --> REPRO["Reproduce:\nAZURE_COSMOS_FUZZ_SEED=\ndeterministic replay"] + REPRO --> REDUCE["Reduce to the minimal\ntriggering value"] + REDUCE --> CLASS{"Classify"} + CLASS -->|codec bug| FIX["Fix ser/de + add a\ngolden vector (regression)"] + CLASS -->|canonicalization gap| TUNE["Tune normalize_number\n(re-run calibration -> MATCH)"] + CLASS -->|backend rewrite| ESC["Escalate - genuine\nservice behavior difference"] +``` + +**Calibration is the safety valve.** Before a soak is trustworthy, calibration +proves the number model matches *this* account, so a mismatch is a real bug and +not modeling noise (it prints a table; it does not assert — see §3.1): + +```mermaid +flowchart LR + P["NUMBER_PROBES\n(1e20, i64::MAX, u64-1, -0, 1.0, ...)"] --> ST["store via binary\n-> read back"] + ST --> T{"our-canonical ==\nbackend-returned ?"} + T -->|MATCH| GOOD["number model correct\n-> soak results are trustworthy"] + T -->|DIFF| BAD["tune normalize_number\nbefore soaking"] +``` + +## 3. Canonicalization (the hard part) + +Two JSON texts are "the same value" if they canonicalize identically. Rules: + +| Aspect | Rule | +| ----------- | ---- | +| Whitespace | removed entirely | +| Object keys | sorted lexicographically (by UTF-8 code unit) | +| Strings | minimally JSON-escaped (via `serde_json`) | +| Arrays | order preserved | +| **Numbers** | **Cosmos-compatible normalization — see §3.1** | + +### 3.1 Number normalization (the tuning surface) + +The subtlety the reviewer flagged: **the backend rewrites numbers on store**, and +its rewrite is *not* identical to a strict canonicalizer like +[JCS / RFC 8785](https://datatracker.ietf.org/doc/html/rfc8785). If we +canonicalized with JCS but the backend renders `1.0` as `1`, a faithful +round-trip would *falsely* report a mismatch. + +The harness therefore uses a **Cosmos-compatible** number canonicalizer, not JCS: + +- **Integers up to `i64::MAX`**: emit plain decimal, no decimal point, no + exponent, no leading zeros. `-0` → `0`. +- **Integers above `i64::MAX`**: routed through `f64` (see the calibration + finding below) — the backend stores them as doubles, so a sent u64 and its + returned double must canonicalize identically. +- **Integral-valued floats** (e.g. `1.0`, `2.0e1`): normalized to their integer + form (`1`, `20`) — this mirrors the observed backend rewrite where a trailing + `.0` is dropped. +- **Non-integral floats**: shortest round-trippable decimal (Rust's `ryu`, via + `serde_json`'s `f64` formatting). + +> **Calibrated against a live account (§6).** The first calibration run +> (18 probes) confirmed **16/18 forms already match**, including the tricky ones: +> integral floats and integral exponents collapse to integers (`1.0`, `2e1` → +> `1`, `20`), `-0` → `0`, trailing zeros are dropped (`1.2300` → `1.23`), +> repeating/high-precision floats and `0.1 + 0.2` round-trip exactly, and the +> backend renders large/small exponents in scientific notation (`1e20`, +> `1e-20`) which reparses to the same `f64`. The **two DIFFs** were integers +> above `i64::MAX`: the backend stores them as IEEE-754 doubles (lossy) and +> returns scientific notation — `18446744073709551614` → `1.8446744073709552e+19` +> and `2^63` → `9.223372036854776e+18`. `canonicalize_number` now models this by +> routing `u64`-above-`i64::MAX` through `f64`, so both sides canonicalize to the +> same double form. Re-running calibration after this change yields all `MATCH`. +> +> To re-calibrate after any change (or against a different account/config), run +> calibration mode (`AZURE_COSMOS_FUZZ_CALIBRATE=true`, see §6): it stores each +> probe in `NUMBER_PROBES` through the binary path, reads it back, and prints a +> table comparing `canonicalize_number`'s rendering against the backend's +> returned form. Every `DIFF` is a form to model; calibration is a **diagnostic** +> (prints the table, does not assert), since a `DIFF` is the signal to tune, not +> a failure. + +### 3.2 Generator stays inside the calibrated envelope + +To avoid false positives from *un-calibrated* number forms, the generator emits + +### 3.2 Generator stays inside the calibrated envelope + +To avoid false positives from *un-calibrated* number forms, the generator emits +numbers in **backend-safe ranges by default** (bounded integers, bounded-precision +floats). A `--wide-numbers` flag widens the range once the canonicalizer is +calibrated for those forms — this is how you progressively expand coverage. + +## 4. Generator + +A seeded PRNG (`SplitMix64`, seed logged for exact reproduction) produces a +random JSON **object** (Cosmos items are objects) using a **hybrid** strategy: + +- a **depth-controlled skeleton** builds a nested container *spine* to a target + depth drawn from `[1, max_depth]`, guaranteeing the document actually reaches + that depth (each level is randomly an object or an array, with a few irregular + filler siblings); +- every **leaf and filler branch** is irregular JSON from + [`arbitrary-json`](https://docs.rs/arbitrary-json) — random keys (incl. empty, + control-char, and Unicode), mixed-type arrays, nested sub-objects and + arrays-of-objects, occasional homogeneous number arrays, and the scalars + `null` / `true` / `false`; +- numbers are clamped to the calibrated envelope (§3.2) unless `--wide-numbers`; + strings to ASCII unless `unicode`. + +> **Why hybrid?** `arbitrary-json`'s `arbitrary_iter` decides whether to recurse +> from *remaining bytes* and stops almost immediately, so an `arbitrary-json`-only +> generator produced near-flat documents (avg depth ≈ 1.3, unchanged by +> `max_depth`). The explicit skeleton restores real depth: measured average depth +> now scales with the knob (≈ 3.9 at `max_depth=3`, ≈ 8.5 at `max_depth=12`, with +> deepest docs reaching 11–17 levels), which is what exercises the codec's +> container framing (length/count prefixes, nested arrays-of-objects) and the +> decoder's `MAX_DEPTH` guard. A `generator_depth_scales_with_max_depth` offline +> test locks this in. + +Every run prints its seed; a failing run is reproduced exactly with +`AZURE_COSMOS_FUZZ_SEED=`. + +## 5. Configurations exercised + +Each config is a separate `CosmosClient` (binary encoding is resolved once at +build time): + +| Config | `enabled` | `request_text_response` | +| ------ | --------- | ----------------------- | +| control (text) | false | — | +| binary | true | false | +| binary + text response | true | true | + +Extend with two accounts (dictionary encoding on/off) by pointing +`AZURE_COSMOS_FUZZ_CONNECTION_STRING_2` at a second account — the harness runs +every generated doc through both. + +## 6. Running the harness + +```bash +# One-shot smoke run (a few hundred docs), local emulator: +AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=...;AccountKey=...;' \ +AZURE_COSMOS_ALLOW_INVALID_CERT=true \ +RUSTFLAGS='--cfg test_category="binary_encoding"' \ + cargo test -p azure_data_cosmos_perf --test binary_roundtrip_fuzzer -- --nocapture + +# Multi-day soak (millions of docs): +AZURE_COSMOS_CONNECTION_STRING='...' \ +AZURE_COSMOS_FUZZ_ITERATIONS=5000000 \ +AZURE_COSMOS_FUZZ_MAX_DEPTH=6 \ +RUSTFLAGS='--cfg test_category="binary_encoding"' \ + cargo test -p azure_data_cosmos_perf --test binary_roundtrip_fuzzer --release -- --nocapture + +# Reproduce a failure: +AZURE_COSMOS_FUZZ_SEED=12345678901234567890 ... cargo test ... + +# Calibrate number canonicalization against the account (prints a table, no assert): +AZURE_COSMOS_CONNECTION_STRING='...' AZURE_COSMOS_FUZZ_CALIBRATE=true \ +RUSTFLAGS='--cfg test_category="binary_encoding"' \ + cargo test -p azure_data_cosmos_perf --test binary_roundtrip_fuzzer -- --nocapture +``` + +### Environment knobs + +| Variable | Default | Meaning | +| -------- | ------- | ------- | +| `AZURE_COSMOS_CONNECTION_STRING` | — (required) | live account (endpoint + key) | +| `AZURE_COSMOS_ALLOW_INVALID_CERT` | false | accept emulator cert | +| `AZURE_COSMOS_FUZZ_ITERATIONS` | 200 | number of generated docs | +| `AZURE_COSMOS_FUZZ_SEED` | random | PRNG seed (for reproduction) | +| `AZURE_COSMOS_FUZZ_MAX_DEPTH` | 5 | max JSON nesting depth | +| `AZURE_COSMOS_FUZZ_WIDE_NUMBERS` | false | widen numeric range (post-calibration) | +| `AZURE_COSMOS_FUZZ_UNICODE` | true | include Unicode strings | +| `AZURE_COSMOS_FUZZ_CALIBRATE` | false | number-calibration mode (§3.1) | +| `AZURE_COSMOS_BINARY_TEST_DATABASE` / `_CONTAINER` | `binary-fuzz-*` | target names | + +## 7. What a failure tells you + +A mismatch is one of: + +1. **A real codec bug** — Rust encoded or decoded a value wrong. (The golden + vectors + in-tree fuzz should also then be extended with the reduced case.) +2. **A canonicalization gap** — the backend rewrote a number/string in a form the + canonicalizer doesn't yet model. Fix `canonicalize_number` (§3.1) and, if the + form is legitimately out of scope, narrow the generator. +3. **A backend rewrite difference** — genuinely different value after store; this + is the highest-value finding and should be escalated. + +The harness prints the seed, the config, and both canonical forms so the case is +immediately reproducible and reducible. + +## 8. Relationship to the other test layers + +The normative wire format is defined by +[`BINARY_ENCODING_RFC.md`](BINARY_ENCODING_RFC.md); this harness is one of the +mechanisms that **validates Rust against that spec** — specifically the RFC's §7 +round-trip invariant, exercised end-to-end at scale (see the RFC's §1.4 diagram +for how all the artifacts relate). + +| Layer | Input | Checks | Location | +| ----- | ----- | ------ | -------- | +| RFC | — | normative wire spec (source of truth) | `docs/BINARY_ENCODING_RFC.md` | +| Golden vectors | fixed corpus | exact byte layout, decode parity | `binary_json/vectors.rs` | +| Encoder conformance | fixed corpus | encode byte-exactness, canonical form | `binary_json/conformance.rs` | +| In-tree fuzz | random/truncated buffers | decoder never panics; `decode` ≡ `from_slice` | `binary_json/fuzz_tests.rs` | +| Coverage-guided codec fuzz | mutated **bytes** | decoder/serde/transcode no-crash + `decode∘encode` idempotence | `fuzz/` (cargo-fuzz) | +| **Round-trip fuzzer** | **random JSON** | **end-to-end value fidelity across configs** | **this harness** | + +These are complementary: golden vectors + conformance pin *the format*, in-tree +fuzz and the coverage-guided `cargo-fuzz` targets harden *the decoder* against +mis-encoded bytes (the latter mutating outward from valid frames with libFuzzer, +offline), and this harness validates *the whole pipeline against the live +service* — and its number-canonicalization calibration (§3.1) feeds the +backend's observed rewrite rules back into the RFC's §7. + +## 9. Planned evolution — `arbitrary-json` + `json-canon` + `sha2` + +This section captures an agreed enhancement plan for the harness. The current harness uses a hand-rolled seeded generator (§4) and a hand-rolled canonicalizer (§3). Three well-maintained crates can replace the parts of that machinery that are pure boilerplate, while we **keep** the one part that is genuinely Cosmos-specific. + +### 9.1 The crates and what each replaces + +| Crate | Role | Replaces | +| ----- | ---- | -------- | +| [`arbitrary-json`](https://docs.rs/arbitrary-json) | Turns raw fuzzer/PRNG bytes into a random, structurally-valid `serde_json::Value` (via the `arbitrary` crate). | Our hand-rolled `gen_object` / `gen_value` / `gen_array` generator (§4). | +| [`json-canon`](https://docs.rs/json-canon) | RFC 8785 (JCS) canonical serialization — object-key sort, whitespace removal, string escaping. | The **structural** part of our `canonicalize` (§3): keys, whitespace, strings, array order. | +| [`sha2`](https://docs.rs/sha2) | SHA-256 over the canonical string, enabling a durable cross-run corpus of `H0` hashes. | Our `DefaultHasher` (SipHash) 64-bit hash. | + +### 9.2 The critical constraint — keep Cosmos number canonicalization + +**JCS number formatting is *not* Cosmos number formatting.** This is the whole reason the harness exists (§3.1). RFC 8785 uses ES6 `Number.prototype.toString` (shortest round-trippable), which differs from the backend's observed store-time rewrite: + +- the backend stores integers above `i64::MAX` as IEEE-754 **doubles** and returns scientific notation (`18446744073709551614` → `1.8446744073709552e+19`); +- integral floats/exponents collapse to integers (`2e1` → `20`). + +If we canonicalized numbers with raw JCS, a *faithful* round-trip would report **false-positive** mismatches on exactly the number edges we most want to test. So the plan is a **hybrid**, not a wholesale swap: + +> **Normalize numbers with our calibrated `canonicalize_number` first (produce a number-normalized `Value`), then run that `Value` through `json_canon` for the structural pass, then `sha2` the result.** + +`json-canon`'s own docs also note it emits `null` for `NaN`/`Inf` — incidentally aligned with Cosmos, but we do not want to rely on that incidentally, so number handling stays under our control. + +### 9.3 Target pipeline + +``` +generate: bytes ──arbitrary-json──▶ Value +normalize: Value ──our normalize_numbers (calibrated §3.1)──▶ Value′ +canonicalize: Value′ ──json_canon (RFC 8785 structural)──▶ canonical String +hash: String ──sha2 (SHA-256)──▶ H +compare: H(sent) == H(project(returned, keys(sent))) +``` + +Only **step 2** is Cosmos-specific and stays in our code; steps 1, 3, 4 become library calls. + +### 9.4 Two harness shapes (we will land both, in order) + +1. **Live-service round-trip (this harness, evolved).** Keep the `#[tokio::test]` soak driven by a seeded PRNG, but feed the PRNG bytes into `arbitrary-json` for generation and swap the structural canonicalizer to `json_canon` + `sha2`. This is the primary deliverable — it validates the whole pipeline against a real account, which per-doc network I/O makes unsuitable for a coverage-guided engine. +2. **Offline codec fuzzer (new, no account).** A `cargo-fuzz` crate that feeds **raw/mutated bytes** straight into the decoder with **no network**, so libfuzzer's coverage guidance and speed apply. As landed it goes *beyond* the originally-planned round-trip check: it fuzzes the **binary protocol itself** (mis-encoded frames the encoder never produces), not just encoder-produced buffers. See §9.8 for the target set. This complements the decoder-only `fuzz_tests.rs` with coverage-guided, byte-level hardening. + +### 9.5 Work plan + +1. Add `arbitrary`, `arbitrary-json`, `json-canon`, and `sha2` as **dev-dependencies** of `azure_data_cosmos_perf` (test-only; not shipped in the SDK). +2. Extract the current number logic into a standalone `normalize_numbers(&Value) -> Value` that applies the calibrated `canonicalize_number` rules and leave calibration mode (§6) pointing at it. +3. Replace `canonicalize` internals with: `normalize_numbers` → `json_canon::to_string` → `sha2` digest. Keep the `project_to_sent_keys` step (§2) unchanged. +4. Replace `gen_object`/`gen_value` with an `arbitrary-json`-backed generator seeded from the existing `SplitMix64` byte stream (so runs stay reproducible via `AZURE_COSMOS_FUZZ_SEED`). +5. Re-run **calibration** (§6) against a live account to confirm `normalize_numbers` still yields all `MATCH` after the refactor; fold any new `DIFF` back in. +6. (Separate change) Add the `cargo-fuzz` offline codec target from §9.4(2). *(Landed — see §9.8; scope broadened to byte-level protocol fuzzing.)* +7. Update §3, §4, and the layer table (§8) to reference the crates once landed. + +### 9.6 Acceptance + +- [x] Live harness produces identical pass/fail decisions to the pre-refactor version on a fixed seed set (no behavior regression), with cleaner internals. +- [x] Number edges (`> i64::MAX`, integral floats, `-0`, high-precision) still round-trip without false positives — verified by calibration `MATCH`. +- [x] `sha2` hashes are stable across runs for the same canonical input (enables a persistent corpus). +- [x] Offline `cargo-fuzz` codec crate lands (byte-level decoder/serde/transcode no-crash + `decode∘encode` idempotence). See §9.8. + +### 9.7 Implementation status (landed) + +Phases 1–4 of §9.5 are implemented in `binary_roundtrip_fuzzer.rs` across four commits: + +| Phase | Change | Status | +| ----- | ------ | ------ | +| 1 | Dev-deps `arbitrary`, `arbitrary-json`, `json-canon`, `sha2` wired into `azure_data_cosmos_perf`. | ✅ landed | +| 2 | `normalize_number` / `normalize_numbers` extracted as the sole Cosmos-specific number transform (behavior-preserving). | ✅ landed | +| 3 | `canonicalize` now = `normalize_numbers` → `json_canon::to_string` (RFC 8785); differential hash switched to SHA-256. | ✅ landed | +| 4 | Generator replaced with `arbitrary-json`, seeded from `SplitMix64` (deterministic per `AZURE_COSMOS_FUZZ_SEED`); a `bound_value` pass keeps the `wide_numbers`/`unicode` envelope contract. | ✅ landed | +| 5 | Live re-calibration + soak against a real account. | ✅ landed (see below) | +| 6 | Offline `cargo-fuzz` codec crate (byte-level protocol fuzzing). | ✅ landed (see §9.8) | + +#### Key finding — `json-canon` rejects integers ≥ 2⁵³ + +RFC 8785 / `json-canon` refuses to serialize any integer at or beyond the JSON +"max safe integer" (`2^53`), returning `Error("u64 must be less than JSON max +safe integer")`. Cosmos, however, **preserves `i64` integers exactly** and stores +`u64` above `i64::MAX` as lossy IEEE-754 doubles. To bridge this, +`normalize_number` maps JCS-unsafe numbers to **stable string tokens** (an exact +decimal for large `i64`, or the `f64` form for the lossy-double case), which are +only ever compared for equality — never parsed back. This keeps the sent and +round-tripped values comparable without tripping the JCS safe-integer guard, and +is the concrete realization of the §9.2 "keep Cosmos number canonicalization" +constraint. JCS-safe numbers still serialize as bare JSON numbers. + +#### Remaining manual step (Phase 5) + +Run calibration and a short soak against a live account to confirm the refactor +did not regress the number model (all `MATCH`) — see §6 for the commands. This is +the one step that cannot run in CI or offline because it requires a real Cosmos +endpoint. + +#### Live validation (Phase 5, recorded) + +Run against a real Cosmos account after the crate refactor: + +- **Calibration:** all **18/18** number probes `MATCH` — every JCS-unsafe edge + (`1e20`, `-1.5e18`, `i64::MAX`/`MIN`, `u64::MAX-1`, `2^63`) canonicalizes to the + same stable string token on both the sent and backend-returned sides, and the + integral-float/`-0`/trailing-zero rewrites all match. +- **Soak (initial, create + read):** 500 documents × 3 configs = 1500 + round-trips, all canonical-equal (seed `1784934026943565900`). +- **Soak (all four point ops):** **1000 documents × 3 configs × 4 point ops + (create/read/replace/upsert) = 12,000 round-trips, all canonical-equal** + (seed `1784944014111583800`), plus the offline unit tests. No mismatches. + +This confirms no behavior regression from the `arbitrary-json` + `json-canon` + +SHA-256 refactor, and that the request-encode + response-decode paths for +`replace` and `upsert` round-trip identically to `create`/`read`. Closes §9.6's +first three acceptance items. + +## 9.8 Offline codec fuzzer (`cargo-fuzz`) — landed + +Phase 6 lands as a self-contained `cargo-fuzz` crate at +[`azure_data_cosmos_driver/fuzz/`](../fuzz/) (see its +[`README.md`](../fuzz/README.md)). It closes a gap this live harness structurally +cannot: because the round-trip fuzzer generates random *JSON values* and only +ever feeds the decoder **encoder-produced** (well-formed) bytes, it exercises +happy-path encode/decode symmetry but never the decoder's defensive paths. The +`cargo-fuzz` crate feeds **arbitrary and mutated bytes** straight into the codec, +so it fuzzes the **binary protocol itself** — truncated buffers, bad length +prefixes, unknown/misused markers, reference and depth bombs, non-UTF-8 string +payloads, and trailing bytes. + +### Why a separate crate + +`cargo-fuzz` builds targets on **nightly** with **libFuzzer**, whereas the repo +workspace builds on stable. The crate therefore carries its own empty +`[workspace]` table so it is *excluded* from the parent workspace; it is not a +workspace member and does not affect stable builds. It depends on the driver by +path and is **offline** (no Cosmos account), so unlike this live harness it is +cheap enough for a CI nightly or a time-boxed PR smoke run. + +### Targets + +| Target | Entry point | Oracle | +| ------ | ----------- | ------ | +| `decode` | `binary_json::decode` | no panic/hang/over-alloc on any bytes | +| `from_slice` | `binary_json::from_slice::` | same, for the native serde streaming path | +| `transcode_to_text` | `binary_json::transcode_to_text` | same, for the driver-side binary→text response path | +| `decode_reencode_roundtrip` | `decode` + `encode` | **differential**: `decode(encode(decode(x))) == decode(x)` on decoder-accepted input | + +The first three assert the robustness oracle (terminate with `Ok`/`Err`, never +crash); the fourth adds a semantic oracle catching reader/writer disagreements +that fixed golden vectors don't enumerate. + +### Corpus seeding + +libFuzzer starts from a corpus of **valid** frames so it mutates outward from +real wire shapes. The [golden vectors](../testdata/binary_json_vectors.json) +(every marker family, as hex) seed it directly — the crate README carries the +one-liner (PowerShell / jq+xxd) that materializes them into `fuzz/corpus/decode`. + +### Relationship to the other layers + +This `cargo-fuzz` crate and `fuzz_tests.rs` both harden the *decoder* against +mis-encoded bytes; the difference is coverage-guided mutation and scale +(libFuzzer) versus a fixed in-tree sweep. Neither replaces this live harness, +which is the only layer that validates the **whole pipeline against the real +service**. See the layer table in §8. diff --git a/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_SPEC.md b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_SPEC.md new file mode 100644 index 00000000000..462001d07cd --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/docs/BINARY_ENCODING_SPEC.md @@ -0,0 +1,568 @@ +# Binary Encoding Specification + +This document describes the design and phased implementation plan for **Cosmos +binary JSON encoding** in the Rust Cosmos DB stack (`azure_data_cosmos` and +`azure_data_cosmos_driver`). + +> **Status:** Implemented (encode + decode). The **native serde serializer** +> ([`binary_json::to_vec`]) and the **native serde deserializer** +> ([`binary_json::from_slice`]) are the two production paths and are wired in: +> item writes serialize `T: Serialize` straight to binary via `to_vec` with no +> intermediate `serde_json::Value`, and responses auto-detect the `0x80` +> preamble and deserialize straight into `T` via `from_slice` (again with no +> intermediate `Value` on the common path). The deferred items (patch / batch / +> bulk) are noted inline. Open questions are tracked in +> [§12](#12-open-questions). +> +> **Note on `encode(&Value)` / `decode(&[u8]) -> Value`.** Earlier prototypes +> routed through `serde_json::Value` on both sides (`T → Value → encode` on +> write, `decode → Value → from_value` on read). Neither is on the hot path +> anymore — the SDK calls the native [`binary_json::to_vec`] / +> [`binary_json::from_slice`] directly. The `Value`-based +> [`encode`](#81-serializer-native-minimal-valid--t-serialize--binary) and +> [`decode`](#82-decoder-complete--binary--value-reference-oracle--fallback) +> functions are retained as reference oracles (parity tests, fuzzing corpus) +> and, for `decode`, as the fallback the deserializer uses for rare exotic wire +> forms (see [§8.3](#83-deserializer-native--binary--t)). + +## 1. Overview + +Binary encoding transmits the request payload as **Cosmos binary JSON** (a +tagged byte stream whose first byte is `0x80`) instead of UTF-8 text JSON, and +accepts binary response bodies, decoding them back to text/typed values on the +response path. The primary benefit is reduced backend storage cost (COGS), +since the service persists the binary form directly. A secondary benefit is +faster serialization/deserialization when the typed path reads and writes the +binary form natively. + +The feature is **opt-in** and **transparent**: when enabled, callers use the +same `create_item` / `read_item` / `query_items` / etc. APIs and observe +text-equivalent results. + +### Goals + +- Encode request bodies as Cosmos binary JSON for **writes** and **query**. +- Decode binary response bodies for **reads**, **write responses**, and + **query** result envelopes. +- Keep the data-plane driver **schema-agnostic** — it never parses item bodies + (see [ARCHITECTURE.md](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/ARCHITECTURE.md)). +- Make decoding robust via **first-byte auto-detection**, independent of + header negotiation. +- Offer an opt-in **text-response** mode via + `BinaryEncodingOptions::request_text_response` that keeps the wire binary in + both directions but has the **driver transcode** the binary response to text + JSON, so an application can deal only in text while still benefiting from the + efficient binary transport (see [§9.1](#91-driver-side-binary-encoding-and-transcoding)). + +### Non-goals (deferred) + +- **Patch**, **transactional batch**, and **bulk** operations. These mirror the + .NET out-of-scope set. Patch in particular is the only driver code path that + decodes-merges-re-encodes a body (see + [PATCH_HANDLER_SPEC.md](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/PATCH_HANDLER_SPEC.md)), so it needs the codec but + is sequenced after the core read/write/query path lands. + +## 2. Scope + +| Operation | Request body | Response body | Status | +| ------------------------------------------- | :----------: | :-----------: | --------- | +| `read_item` | — | decode | ✅ done | +| `create_item` / `upsert_item` / `replace_item` | encode | decode | ✅ done | +| `query_items` | deferred | decode | response done; request-encode deferred | +| `delete_item` | — | — | n/a | +| `patch_item` | deferred | deferred | deferred | +| transactional batch / bulk | deferred | deferred | deferred | + +The response-decode boundary is shared, so `query_items` already decodes binary +response envelopes; only its *request-body* encoding + negotiation header remain. + +## 3. Background: the .NET reference + +.NET PR [#4652](https://github.com/Azure/azure-cosmos-dotnet-v3/pull/4652) +introduced binary encoding for point operations: + +- Opt-in via the `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment variable. +- Typed (`ItemAsync`) APIs: the serializer was refactored to read and write the + binary bits directly into the stream (no intermediate text conversion). +- Stream (`ItemStreamAsync`) APIs: a text stream is transcoded to binary on the + request path and back to text on the response path. Output streams are always + text unless the caller explicitly opts into raw binary via the internal + `EnableBinaryResponseOnPointOperations` request option. +- Patch, batch, and bulk were explicitly **out of scope**. + +The Rust design adopts the same enablement model but goes **straight to native +serde codecs** on both sides (rather than text↔binary transcoders): +`T: Serialize` is encoded directly to Cosmos binary JSON via +[`binary_json::to_vec`], and binary responses are deserialized straight into +`T: Deserialize` via [`binary_json::from_slice`], with no intermediate text or +`serde_json::Value` on the common path. Scope is also extended to query. + +## 4. The Cosmos binary JSON format + +The format is a tagged byte stream. A buffer begins with the preamble byte +`0x80`; because no valid UTF-8 text JSON document starts with `0x80`, the first +byte unambiguously distinguishes binary from text. Each value is introduced by +a **type-marker** byte that selects how the following bytes are interpreted. + +### 4.1 Type-marker map + +| Range | Meaning | +| -------------- | ------------------------------------------------------------------------------------ | +| `0x00`–`0x1F` | Literal integer — the value *is* encoded in the marker (`value = marker`). | +| `0x20`–`0x3F` | 1-byte **system string** — index into a fixed built-in dictionary. | +| `0x40`–`0x5F` | 1-byte **user string** — index into the per-buffer string dictionary. | +| `0x60`–`0x67` | 2-byte **user string**. | +| `0x68`–`0x7F` | base64 / GUID / **compressed** strings (hex, datetime, packed 4/5/6/7-bit). | +| `0x80`–`0xBF` | Encoded-length string — `length = marker & 0x7F` (and `0x80` is the buffer preamble). | +| `0xC0`–`0xC7` | `StrL1/2/4` (length-prefixed strings), `StrR1`–`StrR4` (**reference** strings), `NumberUInt64`. | +| `0xC8`–`0xCF` | Numbers: `UInt8`, `Int16`, `Int32`, `Int64`, `Double`, `Float16`, `Float32`, `Float64`. | +| `0xD0`–`0xDF` | `Null` (`0xD0`), `False` (`0xD1`), `True` (`0xD2`), `Guid` (`0xD3`), sized signed/unsigned ints, `Binary1/2/4ByteLength`. | +| `0xE0`–`0xE7` | Arrays: `Arr0`, `Arr1`, `ArrL1/2/4` (length-prefixed), `ArrLC1/2/4` (length + item count). | +| `0xE8`–`0xEF` | Objects: `Obj0`, `Obj1`, `ObjL1/2/4` (length-prefixed), `ObjLC1/2/4` (length + property count). | +| `0xF0`–`0xF7` | Uniform / typed number arrays (analytics-oriented). | +| `0xFF` | `Invalid` (reserved to flag an invalid marker). | + +The authoritative source is the .NET file +`Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs`. + +### 4.2 The system-string dictionary + +System strings (`0x20`–`0x3F` for 1-byte, plus 2-byte forms) are a **fixed, +hardcoded dictionary** of ~128 common Cosmos property names (`id`, `_rid`, +`_etag`, `_ts`, `_self`, `_attachments`, …). The table must match the service's +ordering **byte-for-byte**; an off-by-one produces silently wrong keys. The Rust +implementation embeds this table as a `const` array in `binary_json::system_strings`, +cross-checked against the .NET source and against captured service vectors. + +### 4.3 Reference strings (dedup) + +`StrR1`–`StrR4` encode a back-reference (by byte offset) to a string that +already appeared earlier in the buffer. The **decoder must resolve these**; the +**encoder may ignore them** (always emit the string inline). This is the core of +the encode/decode asymmetry. + +## 5. Encode/decode asymmetry — the key design lever + +> **The decoder must be complete; the serializer can be minimal-but-valid.** + +- **Decoder (complete).** The service may emit *any* form: literal ints, system + **and** user strings, reference strings, base64/GUID/compressed strings, every + number width, and uniform number arrays. All branches are mandatory — the + decoder parses untrusted service output and must handle everything. + +- **Serializer (minimal valid).** To produce a *correct* (not size-optimal) + buffer the native serde serializer only needs: + - strings → encoded-length or `StrL1/2/4`, + - numbers → literal int / `Int64` / `UInt64` / `Double`, + - containers → `ObjLC*` / `ArrLC*` (length + count), + - `Null` / `False` / `True`. + + It **skips** reference-string dedup, compressed strings, and uniform arrays. + The service accepts the verbose-but-valid form. + +Consequence: the heavy implementation lift is the decoder and the system-string +table; the serializer is comparatively small. + +## 6. Rust ser/de architecture + +The write path serializes `T` straight to binary via the native serde +serializer; the response path auto-detects the `0x80` preamble and deserializes +straight into `T` via the native serde deserializer. The schema-agnostic driver +stays a byte passthrough in both directions — it only emits the negotiation +header. + +```mermaid +flowchart LR + subgraph SDK["azure_data_cosmos (schema-aware)"] + CI["clients/container_client.rs
serialize_item_body:
binary_json::to_vec(&T) → with_body"] + RB["models/response_body.rs
into_single / into_items
+ feed FeedBody<T> envelope"] + end + subgraph DRV["azure_data_cosmos_driver (schema-agnostic)"] + OP["models/cosmos_operation.rs
body: Vec<u8> (0x80…)"] + HDR["driver/transport/cosmos_headers.rs
x-ms-cosmos-supported-serialization-formats"] + DRB["models/response_body.rs
deserialize_response:
is_binary? from_slice::<T> : serde_json::from_slice"] + end + subgraph CODEC["binary_json codec"] + SER["ser::to_vec
(serde::Serializer)"] + DE["de::from_slice
(serde::Deserializer)"] + end + CI --> SER --> OP --> HDR + HDR -->|HTTP| SVC[(Cosmos DB)] + SVC --> DRB --> DE --> RB +``` + +Key facts (verified against the current tree): + +- **One serialize choke point.** `clients/container_client.rs::serialize_item_body` + calls `binary_json::to_vec(item)` when binary is enabled and + `serde_json::to_vec(item)` otherwise. `create_item` / `replace_item` / + `upsert_item` all route through it. +- **One deserialize choke point.** Reads, write responses, **and** query all + funnel through `models/response_body.rs::deserialize_response`, which inspects + the first byte (`is_binary`) and routes binary buffers through the native + `binary_json::from_slice::` and text through `serde_json::from_slice`. + Query parses the whole `{"Documents":[…]}` envelope as `FeedBody`, which + itself lands on the same boundary — so all three response shapes are covered + at once. +- **Driver stays (mostly) passthrough.** The schema-agnostic driver never + *parses* item bodies; its encode-side change is emitting the negotiation + header. The one schema-agnostic transform it performs is **binary→text + transcoding** when the resolved `BinaryEncodingOptions::request_text_response` + is set — a byte-level `decode → serde_json::to_vec` that + needs no type knowledge (see [§9.1](#91-driver-side-binary-encoding-and-transcoding)). The lone + body-*parsing* exception is the patch handler — and patch is deferred. + +### 6.1 Sequence — write then read (binary enabled) + +```mermaid +sequenceDiagram + participant App as Application + participant CC as ContainerClient + participant SER as Serializer to_vec + participant DRV as Driver + participant SVC as Cosmos DB + participant DE as Deserializer from_slice + + App->>CC: create_item(pk, id, item) + CC->>SER: to_vec(item) + Note over SER: T serialize drives the binary serializer straight to 0x80 bytes, no Value + SER-->>CC: binary body bytes + CC->>DRV: with_body(bytes) plus serialization-format header + DRV->>SVC: HTTP POST, content type application json + SVC-->>DRV: response body, text or 0x80 binary + DRV-->>CC: raw body bytes + + App->>CC: read_item(pk, id) then into_model + CC->>DE: deserialize_response(bytes) + Note over DE: if 0x80 use native from_slice (no Value, exotic forms via decode fallback) else use serde_json from_slice + DE-->>CC: typed value T + CC-->>App: ItemResponse +``` + +## 7. Design decisions + +1. **Self-contained codec module.** The codec is schema-agnostic and lives in + `azure_data_cosmos_driver::binary_json`. It operates directly on + `T: Serialize` (encode) and `T: Deserialize` (decode). The binary format is a + stable wire format (algorithm plus constants), which the cosmos `AGENTS.md` + permits sharing rather than duplicating. + +2. **Native serde serializer on the write path.** `BinarySerializer: + serde::Serializer` (module `binary_json::ser`, entry point + [`binary_json::to_vec`]) encodes `T` straight to binary — zero intermediate + `serde_json::Value`, one fewer allocation and traversal than a + transcode-through-`Value` approach. This mirrors .NET's refactored + typed-serializer path and is the **only** encode strategy shipped. + +3. **Native serde deserializer on the read path.** `BinaryDeserializer: + serde::Deserializer` (module `binary_json::de`, entry point + [`binary_json::from_slice`]) drives `T::deserialize` straight off the bytes. + `deserialize_response` inspects the first byte; `0x80` ⇒ binary ⇒ + `from_slice::`; anything else ⇒ `serde_json::from_slice`. Robust even if + header negotiation changes, and uniformly covers reads / write responses / + query. Objects, arrays, and plain scalars stream natively with no + intermediate `Value`; the rare exotic wire forms fall back to the reference + [`decode`] reader for a single value (see [§8.3](#83-deserializer-native--binary--t)). + +4. **Encode at the SDK call sites,** gated by an enablement flag. The path is + `binary_json::to_vec(item)` → `with_body`, chosen in `serialize_item_body`. + +5. **Negotiation + enablement.** The driver advertises + `x-ms-cosmos-supported-serialization-formats: CosmosBinary` on point + operations whenever binary encoding is enabled (matching the .NET SDK's + point-op default), so the service is required to reply in binary and the + **wire stays binary in both directions**. When the caller opts into text + responses via `BinaryEncodingOptions::request_text_response`, the negotiation + header is unchanged (still `CosmosBinary`); the **driver** converts the binary + response to text JSON before returning it (see + [§9.1](#91-driver-side-binary-encoding-and-transcoding)). Enablement resolves once at client + construction, preferring the explicit + `CosmosClientBuilder::with_binary_encoding_options` / + `with_binary_encoding_enabled` option and falling back to the + `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment variable. + +## 8. The codec layer in detail + +### 8.1 Serializer (native, minimal valid) — `T: Serialize → binary` + +`binary_json::ser` implements `serde::Serializer` and is exposed as +[`binary_json::to_vec`]. It drives a value's own `Serialize` impl straight to +Cosmos binary JSON: + +- Prepend the `0x80` preamble. +- Scalars: `bool → False/True`, integers → literal int / `Int64` / `UInt64`, + floats → `Double`, strings/chars → encoded-length or `StrL1/2/4`, + `None`/`unit` → `Null`. +- Containers: objects/maps/structs → `ObjLC*`, arrays/tuples/seqs → `ArrLC*`, + each written as `marker + byte-length + element-count + body`. +- Enums: serde's externally-tagged convention (unit → name string, others → + `{ "Variant": }`), matching `serde_json`. +- **No** reference-string dedup, compression, or uniform arrays. + +**The length-prefix problem.** The `ObjLC*` / `ArrLC*` markers place the payload +byte length and element count *before* the body, but serde drives serialization +sequentially and supplies neither up front. Each compound serializer therefore +buffers its children into a scratch `Vec` and frames them with the narrowest +fitting `LC` marker on `end()`. This is one scratch allocation per nesting level +— the only allocation cost — with **no** materialized `serde_json::Value` tree. + +**Field ordering.** Typed structs preserve field *declaration* order (like +`serde_json::to_vec`); the alphabetized key order of `serde_json::to_value` is +irrelevant because the serializer never builds a `Value`. + +### 8.2 Decoder (complete) — `binary → Value` (reference oracle + fallback) + +`binary_json::reader::decode` is a reader over `&[u8]`: + +- Reads the type marker, dispatches to the matching parser. +- Implements **every** branch: literal ints; system strings (table lookup); user + strings (track the per-buffer dictionary; resolve `StrR*` back-references by + offset); all string forms incl. base64/GUID/compressed (hex, datetime, packed + N-bit); all number widths; null/bool/guid; arrays and objects with 1/2/4-byte + length and optional count; uniform number arrays. +- Output is a `serde_json::Value`. + +`decode` is no longer on the SDK read hot path. It remains as (a) the reference +oracle for parity tests and the fuzzing target, and (b) the **fallback** the +native deserializer ([§8.3](#83-deserializer-native--binary--t)) invokes for the +rare exotic wire forms. + +### 8.3 Deserializer (native) — `binary → T` + +`binary_json::de` implements `serde::Deserializer` and is exposed as +[`binary_json::from_slice`]. It drives a target type's own `Deserialize` impl +straight off the buffer: + +- **Objects / arrays** stream through `MapAccess` / `SeqAccess`, deserializing + each key/value or element in place — **no** intermediate `serde_json::Value` + for the container structure. The container's declared count (or payload end + offset) frames the stream, mirroring the reference decoder's bounds checks. +- **Common scalars** (null, booleans, every literal/fixed-width/extended number, + and plain UTF-8 strings — system, encoded-length, and `StrL1/2/4`) feed the + visitor directly. Plain strings are handed over as **borrowed** slices + (`visit_borrowed_str`) pointing into the response buffer, so no per-string + allocation is needed for types that accept borrowed data. +- **`Option` / newtype structs** are handled explicitly (`null` ⇒ `None`; + newtype ⇒ transparent inner value). +- **Exotic wire forms** — GUID / base64 / compressed / reference strings, binary + blobs, uniform number arrays — and **Rust enums** (serde's externally-tagged + shape) fall back to the reference [`decode`] reader for a single value, which + is then forwarded through `serde_json::Value`'s own deserializer. This keeps + the native fast path small while inheriting the decoder's completeness. + +Real Cosmos item bodies are objects of plain scalars, so the native fast path +covers them end-to-end; the `Value` fallback fires only for the uncommon forms. + +`from_slice::(buf)` is asserted to equal `decode(buf)`, and a +2000-case generative test checks native-vs-`decode` parity over random values. + +### 8.4 Performance + +Both benches live in `azure_data_cosmos_benchmarks` +(`cargo bench -p azure_data_cosmos_benchmarks --bench binary_encode` / +`--bench binary_decode`), comparing text, the retired via-`Value` path, and the +shipped native codec on a small (~64 B) and a large (~1.7 MB) item. + +**Encode** (`binary_encode`): + +| Item | text (`to_vec`) | via-`Value` (`encode`) | **native `to_vec`** | +| ---- | --------------: | ------------------: | -------------------: | +| ~64 B | ~0.33 µs | ~1.85 µs | **~0.75 µs** (~2.5× faster) | +| ~1.7 MB | ~2.33 ms | ~2.17 ms | **~1.64 ms** (~24% faster; ~1.0 GiB/s) | + +**Decode into a typed struct** (`binary_decode`, `LogEntry` target): + +| Item | text (`from_slice`) | via-`Value` (`decode`+`from_value`) | **native `from_slice`** | +| ---- | --------------: | ------------------: | -------------------: | +| ~64 B | ~0.90 µs | ~1.64 µs | **~0.90 µs** (~1.8× faster than via-`Value`) | +| ~1.7 MB | ~852 µs | ~587 µs | **~582 µs** (~32% faster than text) | + +On both sides the native codec is faster than *both* the via-`Value` path and +text JSON on large payloads, by skipping the `Value` build / extra traversal +(and, on decode, borrowing strings from the buffer). The gain is largest on +small items, where the `Value` allocation dominated the fixed overhead. + +## 9. Negotiation and enablement + +- **Request.** When binary is enabled, the **driver** sets + `x-ms-cosmos-supported-serialization-formats: CosmosBinary` on point + operations (matching the .NET SDK's point-op default) — the same value whether + or not `request_text_response` is set, so the service always replies with + binary and the wire stays binary in both directions. The request body is + Cosmos binary JSON (the driver transcodes a text body to binary when needed) + and the request `Content-Type` stays `application/json` — the service detects + the binary form from the first byte. +- **Response.** Decoding does **not** depend on negotiation: the SDK auto-detects + the `0x80` preamble. Negotiation governs whether the service *chooses* to send + binary; the driver-side transcoding directive governs whether the driver hands + the caller text or binary (see [§9.1](#91-driver-side-binary-encoding-and-transcoding)). +- **Enablement.** Resolved once at client construction, preferring the explicit + `CosmosClientBuilder::with_binary_encoding_options` / + `with_binary_encoding_enabled` option and falling back to the + `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment variable; disabled by + default. + +### 9.1 Driver-side binary encoding and transcoding + +Binary encoding is a **driver** capability, exposed as +`OperationOptions::binary_encoding: Option`. Because the +option lives on the driver and is schema-agnostic, every consumer shares it: the +Rust SDK re-exports `BinaryEncodingOptions`, and FFI hosts (.NET, Java, Go, …) +set the equivalent flat `binary_encoding_enabled` / +`binary_encoding_request_text_response` fields on the C ABI +`cosmos_operation_options_t`. The two flags mean: + +- `enabled` — **binary on the wire**. In `CosmosDriver::execute_operation`, when + enabled, the driver applies `apply_request_binary_encoding`: it transcodes a + **text** request body to Cosmos binary JSON via + `binary_json::transcode_to_binary` (`serde_json::from_slice` → `encode`) and + advertises `CosmosBinary` (the .NET point-op default). An already-binary or + empty body is passed through, so a caller that pre-encodes (see below) pays + nothing. +- `request_text_response` — **text back to the caller**. After the response is + assembled, the driver converts the binary body to text JSON via + `binary_json::transcode_to_text` (`decode` → `serde_json::to_vec`). The wire + stays binary in both directions. Text or empty bodies pass through unchanged. + +This means a caller that deals only in **text** — most importantly an FFI host — +gets a fully binary wire (efficient RUs and bandwidth) **without encoding +anything itself**: it sends text, sets the two flags, and the driver does both +transcodes. + +The Rust SDK keeps a **typed fast path** as an optimization: `serialize_item_body` +encodes `T: Serialize` straight to binary (skipping the text intermediate), and +the driver's request-side transcode then sees an already-binary body and passes +it through. It sets the option via a `with_binary_encoding` helper on +`OperationOptions` rather than stamping headers directly. + +**Option resolution + operation-type guard.** `execute_operation` resolves +`binary_encoding` through the same runtime → account → operation layered view +(`operation_options_view`) as every other option, so a default set at the +runtime/account layer is honored. Binary encoding is honored **only for point +item operations** (`OperationType::supports_binary_encoding`: create, read, +replace, upsert, delete); query, feed, batch, and stored-procedure operations +are ignored even if a caller (e.g. an FFI host) sets the flag, since those paths +remain deferred. Patch is dispatched to its own handler before this check and is +likewise excluded. + +This matches the guidance that the **driver** (not the backend) performs the +transcoding, so the backend rewrite/transport can stay binary. Feed (`Items`) +responses transcode per-slice, but binary feed negotiation itself remains +deferred (see [§2](#2-scope)). Patch is also excluded from binary encoding for +now. + +## 10. Delivery status + +All phases below are **done** except the noted follow-ups. + +| Phase | Deliverable | Status | +| ----- | ----------- | ------ | +| **P0** | Marker constants, system-string table, error types, cross-language round-trip test corpus. | ✅ done | +| **P1** | Complete decoder ([`decode`]) with first-byte auto-detect; native deserializer ([`from_slice`]) wired into `deserialize_response`. | ✅ done (binary reads + response envelopes) | +| **P2** | Native serde serializer ([`to_vec`]) wired into `create` / `upsert` / `replace`, behind the enablement flag. | ✅ done (binary writes) | +| **P3** | Negotiation header + env-var enablement; end-to-end binary round-trip via the in-memory emulator. | ✅ done | +| **P4** | Decoder fuzzing; text-vs-binary encode **and** decode benchmarks. | ✅ done | +| **P5** | Driver-side binary→text transcoding for `BinaryEncodingOptions::request_text_response`: wire stays binary both ways, driver converts the response to text ([§9.1](#91-driver-side-binary-encoding-and-transcoding)). | ✅ done | +| **P6** | Binary encoding moved to the driver: `OperationOptions::binary_encoding` (driver-owned `BinaryEncodingOptions`), request-side text→binary transcoding, and FFI `cosmos_operation_options_t` flags — so FFI hosts can request a binary wire + text response without encoding anything. | ✅ done | + +**Follow-ups / deferred:** query request-body encoding + negotiation; patch, +transactional batch, and bulk. (The native deserializer's exotic-form path still +routes through `decode` -> `Value`; extending native visitor coverage to those +rare forms is a possible future optimization.) + +## 11. Testing strategy + +- **Round-trip property tests:** `T → binary → T` on both codecs (native + `to_vec` → native `from_slice`, and `to_vec` → `decode`), plus 2000-case + generative parity tests asserting `to_vec(&value)` byte-matches the + `encode(&Value)` oracle and `from_slice::(bytes)` equals `decode(bytes)` + for arbitrary `serde_json::Value` inputs. +- **Cross-compatibility vectors:** binary buffers captured from .NET output; + decode-parity against the known text form is the correctness bar. +- **Decoder fuzzing:** malformed / truncated / adversarial buffers. The decoder + parses untrusted service bytes, so this is security-relevant (bounds checks, + no panics, no unbounded allocation from attacker-controlled length prefixes). +- **Emulator integration tests:** read / write / query with binary enabled, + asserting text-equivalent results. Gated under the existing `emulator` + test categories. +- **Benchmarks:** `azure_data_cosmos_benchmarks`'s `binary_encode` and + `binary_decode` benches compare text (`serde_json`), the retired via-`Value` + path, and the shipped native codec on small and ~1.7 MB items (see the + [§8.4](#84-performance) tables). Run with + `cargo bench -p azure_data_cosmos_benchmarks --bench binary_encode` and + `--bench binary_decode`. + +## 12. Open questions + +1. **Native query engine** — does the cross-partition path + (`query_plan_native`, `query/eval`) ever evaluate on *binary* item bytes, or + are items always decoded to values before evaluation? Determines whether + query request-body encoding touches the engine or only the envelope. +2. **Codec placement** — keep the dedicated internal `binary_json` module + (current choice) vs. a small standalone crate (e.g. `azure_data_cosmos_json`) + if other language SDKs want to share it. +3. **Encoder API shape (review follow-ups from PR #4671)** — two reviewer + suggestions on the `Value`-based reference encoder remain open: + - Rename `writer::encode(&Value)` to disambiguate it from the native + `ser::to_vec` (the reviewer suggested `to_vec`, which is already taken by + the serde path); needs a name decision before implementing. + - Change the shared emit helpers from `out: &mut Vec` to + `mut out: impl std::io::Write` so callers can pool/stream buffers. This + makes the currently-infallible helpers fallible and threads the `Write` + bound through `ser`, so it is sequenced as a follow-up. + +## 13. Change map (as implemented) + +- **`azure_data_cosmos_driver/src/binary_json/`** — codec module: `markers`, + `system_strings`, `error` (incl. `serde::ser::Error` + `serde::de::Error`), + `reader` (`decode` reference oracle + shared cursor exposed to `de`), + `writer` (`encode(&Value)` parity oracle + shared emit helpers), `ser` + (native `serde::Serializer`, `to_vec`), `de` (native `serde::Deserializer`, + `from_slice`), `vectors`, `fuzz_tests`. +- `azure_data_cosmos_driver/src/models/response_body.rs`: + `deserialize_response` auto-detects the `0x80` preamble and calls + `binary_json::from_slice::`; `ResponseBody::transcode_to_text` converts a + binary payload to text JSON in place (for the driver-side text-response path). +- `azure_data_cosmos_driver/src/binary_json/mod.rs`: `transcode_to_text(&[u8])` + converts a binary buffer to text JSON (`decode → serde_json::to_vec`); + text/empty input passes through unchanged. +- `azure_data_cosmos_driver/src/models/cosmos_response.rs`: + `CosmosResponse::transcode_body_to_text` applies the conversion to the + assembled response body. +- `azure_data_cosmos_driver/src/models/mod.rs`: + `OperationType::supports_binary_encoding` gates binary encoding to point item + operations (create/read/replace/upsert/delete). +- `azure_data_cosmos_driver/src/driver/cosmos_driver.rs`: `execute_operation` + resolves `binary_encoding` via the layered `operation_options_view`, applies + request-side transcoding for supported operation types, and transcodes the + response body to text when `request_text_response` is set. +- `azure_data_cosmos/src/clients/container_client.rs`: `serialize_item_body` + calls `binary_json::to_vec` on the binary write path; `with_binary_encoding` + sets the driver `OperationOptions::binary_encoding` so the driver negotiates + `CosmosBinary` and, for `request_text_response`, transcodes the response. +- `azure_data_cosmos/src/error.rs`: `convert_binary_encode_error` maps a + `BinaryError` from the item-write encode path to the SDK error type (a + request-body error). It is a helper called via `map_err` at the single + call site rather than a `From` impl, so a future response-side decode error + cannot be mislabeled as a request-body error. +- `azure_data_cosmos/src/clients/mod.rs`: `resolve_binary_encoding` prefers the + explicit `CosmosClientBuilder::with_binary_encoding_options` / + `with_binary_encoding_enabled` option and falls back to the + `AZURE_COSMOS_BINARY_ENCODING_ENABLED` environment variable, resolving a + `BinaryEncodingOptions`. +- `azure_data_cosmos/src/options/client.rs`: the public `BinaryEncodingOptions` + (`enabled`, `request_text_response`) exposed through `options`. +- `azure_data_cosmos_driver/src/models/cosmos_headers.rs`: the + supported-serialization-formats header field + emission. +- `azure_data_cosmos_benchmarks/benches/binary_encode.rs` / + `binary_decode.rs`: the encode and decode benchmarks. + +## 14. References + +- .NET PR #4652 — Binary Encoding for Point Operations: + +- .NET type markers — + `Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs` +- [ARCHITECTURE.md](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/ARCHITECTURE.md) — schema-agnostic data-plane principle. +- [PATCH_HANDLER_SPEC.md](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/PATCH_HANDLER_SPEC.md) — the deferred body-parsing path. +- [TRANSPORT_PIPELINE_SPEC.md](https://github.com/Azure/azure-sdk-for-rust/blob/main/sdk/cosmos/azure_data_cosmos_driver/docs/TRANSPORT_PIPELINE_SPEC.md) — header application. diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore b/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore new file mode 100644 index 00000000000..1a45eee7760 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/.gitignore @@ -0,0 +1,4 @@ +target +corpus +artifacts +coverage diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml b/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml new file mode 100644 index 00000000000..f54748f8658 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/Cargo.toml @@ -0,0 +1,60 @@ +# cargo-fuzz crate for the Cosmos binary JSON codec. +# +# This is a SEPARATE crate with its own `[workspace]` (see the empty table at +# the bottom) so it is NOT pulled into the parent stable workspace: cargo-fuzz +# builds these targets on nightly with libFuzzer, while the repo workspace +# builds on stable. +# +# Usage (from this directory or the driver crate root): +# rustup toolchain install nightly +# cargo install cargo-fuzz +# cargo +nightly fuzz run decode +# See README.md for target descriptions and corpus seeding. + +[package] +name = "azure_data_cosmos_driver-fuzz" +version = "0.0.0" +publish = false +edition = "2021" +license = "MIT" + +[package.metadata] +cargo-fuzz = true + +[dependencies] +libfuzzer-sys = "0.4" +serde_json = "1" + +[dependencies.azure_data_cosmos_driver] +path = ".." + +[[bin]] +name = "decode" +path = "fuzz_targets/decode.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "from_slice" +path = "fuzz_targets/from_slice.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "transcode_to_text" +path = "fuzz_targets/transcode_to_text.rs" +test = false +doc = false +bench = false + +[[bin]] +name = "decode_reencode_roundtrip" +path = "fuzz_targets/decode_reencode_roundtrip.rs" +test = false +doc = false +bench = false + +# Isolate this crate from the parent workspace (nightly + libFuzzer only). +[workspace] diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md new file mode 100644 index 00000000000..28285f5fe91 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/README.md @@ -0,0 +1,92 @@ +# Binary JSON codec fuzzing (`cargo-fuzz`) + +Coverage-guided, **byte-level** fuzzing for the Cosmos binary JSON codec +(`azure_data_cosmos_driver::binary_json`). Where the live +[round-trip fuzzer](../../azure_data_cosmos_perf/tests/binary_roundtrip_fuzzer.rs) +generates random JSON *values* and only ever feeds the decoder **encoder-produced** +(well-formed) bytes, these targets feed **arbitrary and mutated bytes** straight +into the decoder — so they exercise the *format*/protocol itself: truncated +buffers, bad length prefixes, unknown or misused markers, reference/depth bombs, +non-UTF-8 string payloads, and trailing bytes. + +This is a **separate crate** with its own empty `[workspace]` in `Cargo.toml`, so +it stays isolated from the stable repo workspace: cargo-fuzz builds it on nightly +with libFuzzer. + +## Prerequisites + +```bash +rustup toolchain install nightly +cargo install cargo-fuzz +``` + +## Targets + +| Target | Entry point | What it checks | +| --- | --- | --- | +| `decode` | `binary_json::decode` | `Value` decode never panics/hangs/over-allocates on any bytes. | +| `from_slice` | `binary_json::from_slice::` | Native serde streaming decode honors the same no-crash contract. | +| `transcode_to_text` | `binary_json::transcode_to_text` | Driver-side binary→text response transcode never panics on a malformed body. | +| `decode_reencode_roundtrip` | `decode` + `encode` | **Differential**: any buffer the decoder accepts must satisfy `decode(encode(decode(x))) == decode(x)` — catches reader/writer disagreements. | + +All four assert the **robustness oracle**: for *any* input the codec terminates +and returns `Ok`/`Err` — never panics, hangs, or allocates beyond the buffer. +The last one adds a **semantic** oracle on decoder-accepted inputs. + +## Running + +From this `fuzz/` directory (or the driver crate root): + +```bash +# Explore one target (Ctrl-C to stop): +cargo +nightly fuzz run decode + +# Time-boxed CI-style smoke run (60s), 4 workers: +cargo +nightly fuzz run decode -- -max_total_time=60 -workers=4 + +# Reproduce a crash from a saved artifact: +cargo +nightly fuzz run decode fuzz/artifacts/decode/crash- + +# Minimize a crashing input: +cargo +nightly fuzz tmin decode fuzz/artifacts/decode/crash- +``` + +## Seeding the corpus from the golden vectors + +Seeding libFuzzer with **valid** frames lets it mutate outward from real wire +shapes and reach the interesting error paths far faster than blind byte flips. +The [golden vectors](../testdata/binary_json_vectors.json) already contain every +marker family as space-separated hex. Materialize them into the `decode` corpus: + +PowerShell: + +```powershell +$dir = "fuzz/corpus/decode"; New-Item -ItemType Directory -Force $dir | Out-Null +(Get-Content ../testdata/binary_json_vectors.json | ConvertFrom-Json) | ForEach-Object { + $bytes = $_.binary -split '\s+' | ForEach-Object { [Convert]::ToByte($_, 16) } + [IO.File]::WriteAllBytes("$dir/$($_.name)", [byte[]]$bytes) +} +``` + +bash + jq + xxd: + +```bash +mkdir -p fuzz/corpus/decode +jq -r '.[] | "\(.name) \(.binary)"' ../testdata/binary_json_vectors.json | +while read -r name hex; do + echo "$hex" | tr -d ' ' | xxd -r -p > "fuzz/corpus/decode/$name" +done +``` + +The same corpus works for `from_slice`, `transcode_to_text`, and +`decode_reencode_roundtrip` (all consume raw binary buffers); copy or point +`--corpus` at `fuzz/corpus/decode`. + +## Notes + +- `corpus/`, `artifacts/`, and `target/` are git-ignored (regenerated locally / in CI). +- These targets are **offline** (no live account), so they are cheap enough to + run in CI as a nightly job or a time-boxed smoke check on PRs touching + `binary_json`. +- A reproducible crash should be reduced with `cargo fuzz tmin`, added as a + golden vector / unit test in `src/binary_json/`, and fixed there. diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs new file mode 100644 index 00000000000..d5d158df996 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode.rs @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the binary-JSON **decoder**. +//! +//! libFuzzer feeds arbitrary (and, once seeded, mutated-from-valid) byte +//! buffers straight into [`decode`]. This is the format fuzzer the live +//! round-trip test can't be: it explores mis-encoded frames — truncated +//! buffers, bad length prefixes, unknown/misused markers, reference and +//! depth bombs, non-UTF-8 string payloads, trailing bytes — that the encoder +//! never produces. +//! +//! Oracle: for **any** input the decoder must terminate and return either +//! `Ok(Value)` or `Err(BinaryError)` — never panic, hang, or allocate beyond +//! what the buffer can back. A crash or hang here is a decoder-hardening bug. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::decode; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = decode(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs new file mode 100644 index 00000000000..661bc1a9349 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/decode_reencode_roundtrip.rs @@ -0,0 +1,32 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Differential fuzz target: decode → encode → decode idempotence. +//! +//! Unlike the plain `decode` no-crash target, this asserts a **semantic** +//! invariant on every buffer the decoder *accepts*: re-encoding the decoded +//! value and decoding it again must reproduce the exact same value. It catches +//! the class of bug the live round-trip fuzzer cannot — a buffer the decoder +//! accepts but the encoder would round-trip to a *different* value (marker or +//! number-form disagreements between the reader and writer). libFuzzer's +//! mutation reaches decoder-accepted-but-unusual frames that hand-written +//! golden vectors don't enumerate. +//! +//! Oracle: `decode(data) = Ok(v)` ⇒ `decode(encode(v)) = Ok(v)`. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::{decode, encode}; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + if let Ok(value) = decode(data) { + let reencoded = encode(&value); + let redecoded = + decode(&reencoded).expect("re-encoding a decoded value must itself decode"); + assert_eq!( + value, redecoded, + "decode∘encode∘decode is not idempotent for a decoder-accepted buffer" + ); + } +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs new file mode 100644 index 00000000000..6c0046c654c --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/from_slice.rs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the native serde **deserializer**. +//! +//! [`from_slice`] is the zero-`Value` streaming decode path used by the SDK's +//! typed reads; it drives a different code path from [`decode`] (it streams +//! tokens into a serde visitor instead of materializing a +//! [`serde_json::Value`]). Fuzzing it independently ensures the streaming +//! deserializer honors the same no-crash contract on malformed input. +//! +//! Oracle: for any input, deserialization must terminate with `Ok`/`Err` — +//! never panic, hang, or over-allocate. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::from_slice; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = from_slice::(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs new file mode 100644 index 00000000000..199c0371196 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/fuzz/fuzz_targets/transcode_to_text.rs @@ -0,0 +1,22 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Byte-level protocol fuzz target for the driver-side response transcode. +//! +//! [`transcode_to_text`] is what the driver runs on a binary response body when +//! a text-only host asked for text back: it decodes the binary buffer and +//! re-serializes it as UTF-8 text JSON (or passes text/empty input through +//! unchanged). It sits on the FFI/text-host response path, so a panic here on a +//! malformed service body would take down the host. +//! +//! Oracle: for any input, transcoding must terminate with `Ok`/`Err` — never +//! panic, hang, or over-allocate. + +#![no_main] + +use azure_data_cosmos_driver::binary_json::transcode_to_text; +use libfuzzer_sys::fuzz_target; + +fuzz_target!(|data: &[u8]| { + let _ = transcode_to_text(data); +}); diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs new file mode 100644 index 00000000000..2f70f89b185 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/conformance.rs @@ -0,0 +1,188 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Encode-direction conformance tests for the Cosmos binary JSON codec. +//! +//! These tests implement the encoder conformance requirements of the +//! wire-format spec (`docs/BINARY_ENCODING_RFC.md` §7 "Canonical encoding" and +//! Appendix A "Golden test vectors"). Decoder conformance (§8) is covered by the +//! per-form tests in [`reader`](super::reader) and [`de`](super::de); this +//! module fills the previously-missing encode side. +//! +//! Two guarantees are asserted: +//! +//! 1. **Round-trip validity** — for every golden-corpus value, `decode(encode(v)) +//! == v`. The encoder MUST emit a valid buffer that decodes back to the input +//! (RFC §7: the encoder emits a conformant *subset* of the wire forms; the +//! decoder accepts all of them). +//! 2. **Canonical output snapshots** — the encoder is deterministic, so its exact +//! bytes for representative values are pinned as regression snapshots. This +//! documents the Rust encoder's actual canonical form, which is a valid +//! subset that does **not** use the most compact forms (system strings, +//! `Arr0`/`Arr1`, narrowest `Number*`, etc.) — see the notes below. +//! +//! The module is compiled only under `cfg(test)`. + +use super::{decode, encode, is_binary, PREAMBLE}; +use serde_json::{json, Value}; + +/// Parses a spaced-hex string (e.g. `"80 D0"`) into bytes, matching the corpus +/// notation used throughout the RFC. +fn hex(s: &str) -> Vec { + s.split_whitespace() + .map(|b| u8::from_str_radix(b, 16).expect("valid hex byte")) + .collect() +} + +/// RFC §7 (round-trip validity): every value in the shared golden corpus MUST +/// re-encode to a buffer that decodes back to the identical value. This is the +/// encode-direction counterpart to `reader::decodes_golden_corpus`. +#[test] +fn encode_round_trips_golden_corpus() { + for vector in super::vectors::golden_vectors() { + let expected: Value = + serde_json::from_str(&vector.json).expect("corpus json is valid JSON"); + let encoded = encode(&expected); + assert!( + is_binary(&encoded), + "{}: encoder output missing preamble", + vector.name + ); + let decoded = decode(&encoded) + .unwrap_or_else(|e| panic!("{}: re-encoded buffer failed to decode: {e}", vector.name)); + assert_eq!( + decoded, expected, + "{}: encode→decode did not round-trip", + vector.name + ); + } +} + +/// RFC §7 (deterministic canonical output): the encoder emits exactly these +/// bytes for representative values. These snapshots are the regression bar for +/// the Rust encoder's canonical form. +/// +/// Note the encoder deliberately emits a **valid but non-minimal subset** of the +/// wire forms (RFC §7): integers outside `[0,31]` use `Int64`/`UInt64` (never the +/// narrower `NumberUInt8`/`Int16`/`Int32`), strings use the encoded-length or +/// `StrL*` forms (never system/user/compressed strings), and containers always +/// use the `LC*` length+count framing (never `Arr0`/`Arr1`/`Obj0`/`Obj1`). The +/// decoder accepts the compact forms the service may emit; the encoder need not +/// produce them. +#[test] +fn encode_produces_expected_canonical_bytes() { + let cases: &[(Value, &str)] = &[ + // Singletons. + (json!(null), "80 D0"), + (json!(false), "80 D1"), + (json!(true), "80 D2"), + // Literal small integers (value == marker), 0..=31. + (json!(0), "80 00"), + (json!(31), "80 1F"), + // Integers outside [0,31] use Int64 (not the narrower Number* forms). + (json!(32), "80 CB 20 00 00 00 00 00 00 00"), + (json!(200), "80 CB C8 00 00 00 00 00 00 00"), + (json!(-5), "80 CB FB FF FF FF FF FF FF FF"), + // Values above i64::MAX use UInt64. + ( + json!(18446744073709551614u64), + "80 C7 FE FF FF FF FF FF FF FF", + ), + // Non-integral numbers use NumberDouble. + (json!(3.5), "80 CC 00 00 00 00 00 00 0C 40"), + // Strings ≤ 63 bytes use the encoded-length form (length baked into the + // marker), including "hello" — the corpus stores it as StrL1, but the + // encoder's canonical form is encoded-length. + (json!(""), "80 80"), + (json!("hi"), "80 82 68 69"), + (json!("hello"), "80 85 68 65 6C 6C 6F"), + // Containers always use LC* framing (byte-length + count). + (json!([]), "80 E5 00 00"), + (json!([true]), "80 E5 01 01 D2"), + (json!({}), "80 ED 00 00"), + (json!({"id": 0}), "80 ED 04 01 82 69 64 00"), + ]; + + for (value, expected_hex) in cases { + let encoded = encode(value); + let expected = hex(expected_hex); + assert_eq!( + encoded, expected, + "encoder output for {value} did not match the canonical snapshot\n expected: {expected_hex}\n actual: {}", + spaced_hex(&encoded), + ); + } +} + +/// RFC §7 (valid subset): where the golden corpus stores a **compact** wire form +/// the encoder does not emit (system strings, `Arr0`, `NumberUInt8`, …), the +/// encoder's own output differs byte-wise but still decodes to the same value. +/// This pins the intentional asymmetry so a future "make the encoder compact" +/// change is a conscious decision rather than a silent regression. +#[test] +fn encoder_emits_valid_subset_for_compact_corpus_forms() { + // (value, the corpus's compact encoding) — the encoder produces a *different* + // buffer, but both decode to `value`. + let compact_cases: &[(Value, &str)] = &[ + (json!(200), "80 C8 C8"), // corpus: NumberUInt8 + (json!("id"), "80 2C"), // corpus: system string + (json!([]), "80 E0"), // corpus: Arr0 + (json!({}), "80 E8"), // corpus: Obj0 + (json!([true]), "80 E1 D2"), // corpus: Arr1 + ( + json!([1, 2, 3]), + "80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00", + ), // uniform array + ]; + + for (value, compact_hex) in compact_cases { + let compact = hex(compact_hex); + // The compact form is valid and decodes to `value` ... + assert_eq!( + decode(&compact).unwrap(), + *value, + "compact corpus form {compact_hex} did not decode to {value}", + ); + // ... but the encoder emits a different (verbose) buffer. + let encoded = encode(value); + assert_ne!( + encoded, compact, + "encoder unexpectedly produced the compact form for {value}; update this test if the encoder was made compact", + ); + // ... which still decodes to the same value. + assert_eq!( + decode(&encoded).unwrap(), + *value, + "encoder's verbose form for {value} did not round-trip", + ); + } +} + +/// RFC §3.1: a complete buffer begins with the preamble and the encoder always +/// emits it. +#[test] +fn encoder_always_emits_preamble() { + for value in [ + json!(null), + json!(1), + json!("x"), + json!([1]), + json!({"a": 1}), + ] { + let encoded = encode(&value); + assert_eq!( + encoded.first(), + Some(&PREAMBLE), + "missing preamble for {value}" + ); + } +} + +/// Formats bytes as spaced uppercase hex for assertion messages. +fn spaced_hex(bytes: &[u8]) -> String { + bytes + .iter() + .map(|b| format!("{b:02X}")) + .collect::>() + .join(" ") +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/de.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/de.rs new file mode 100644 index 00000000000..c50236084db --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/de.rs @@ -0,0 +1,559 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Native `serde` deserializer for Cosmos binary JSON (`binary` → `T`). +//! +//! [`from_slice`] drives a target type's own [`Deserialize`](serde::Deserialize) +//! implementation directly off the binary buffer, without building an +//! intermediate [`serde_json::Value`]. Objects, arrays, and plain scalars are +//! streamed straight into the visitor, with plain strings borrowed directly +//! from the buffer. The rarer wire forms (GUID, base64, compressed, and +//! reference strings, binary blobs, and uniform number arrays) are decoded +//! through [`decode`](super::decode) and forwarded to the visitor. + +use serde::de::{DeserializeSeed, Deserializer, IntoDeserializer, MapAccess, SeqAccess, Visitor}; +use serde::forward_to_deserialize_any; + +use super::markers::NULL; +use super::reader::{ContainerHeader, Frame, Reader, ScalarToken}; +use super::{is_binary, BinaryError, Result}; + +/// Maximum container nesting depth, mirroring the reference decoder's +/// [`MAX_DEPTH`](super::reader) so both paths reject the same +/// nesting. +const MAX_DEPTH: usize = 256; + +/// Deserializes a Cosmos binary JSON buffer into a value of type `T`. +/// +/// The buffer must begin with the [`PREAMBLE`](super::PREAMBLE) byte (`0x80`); +/// the single top-level value that follows is decoded, and any trailing bytes +/// are reported as [`BinaryError::TrailingBytes`]. +/// +/// This is the native, allocation-light counterpart to +/// `decode(buf).and_then(serde_json::from_value)` — it drives `T::deserialize` +/// straight off the bytes, materializing a [`serde_json::Value`] only for the +/// rare exotic forms handled by the fallback (see the module docs). +/// +/// # Errors +/// +/// Returns a [`BinaryError`] if the buffer is not binary (missing preamble), is +/// truncated, contains an invalid marker, has trailing bytes, or if `T`'s +/// `Deserialize` implementation rejects the decoded shape. +/// +/// # Examples +/// +/// ``` +/// use azure_data_cosmos_driver::binary_json::{from_slice, to_vec}; +/// +/// let bytes = to_vec(&serde_json::json!({ "id": "1", "count": 7 })).unwrap(); +/// let value: serde_json::Value = from_slice(&bytes).unwrap(); +/// assert_eq!(value, serde_json::json!({ "id": "1", "count": 7 })); +/// ``` +pub fn from_slice<'de, T>(buffer: &'de [u8]) -> Result +where + T: serde::Deserialize<'de>, +{ + if !is_binary(buffer) { + return Err(match buffer.first() { + Some(&found) => BinaryError::MissingPreamble { found }, + None => BinaryError::UnexpectedEof { needed: 1 }, + }); + } + + let mut de = BinaryDeserializer { + reader: Reader::new(buffer, 1), + depth: 0, + }; + let value = T::deserialize(&mut de)?; + let remaining = buffer.len() - de.reader.pos; + if remaining != 0 { + return Err(BinaryError::TrailingBytes { remaining }); + } + Ok(value) +} + +/// A serde deserializer over a binary JSON buffer. +struct BinaryDeserializer<'de> { + reader: Reader<'de>, + depth: usize, +} + +impl<'de> BinaryDeserializer<'de> { + /// Reads the next value as an owned [`serde_json::Value`] (consuming its + /// bytes) and forwards it through `Value`'s deserializer. Used for the + /// exotic wire forms the native fast path does not handle. + fn deserialize_via_value(&mut self, visitor: V) -> Result + where + V: Visitor<'de>, + { + let value = self.reader.read_value(self.depth)?; + value + .deserialize_any(visitor) + .map_err(|e| BinaryError::Custom(e.to_string())) + } +} + +impl<'de> Deserializer<'de> for &mut BinaryDeserializer<'de> { + type Error = BinaryError; + + fn deserialize_any(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.depth > MAX_DEPTH { + return Err(BinaryError::DepthLimitExceeded { limit: MAX_DEPTH }); + } + + // Standard array/object container: stream it natively. + if let Some(header) = self.reader.read_container_header()? { + self.depth += 1; + let result = match header { + ContainerHeader::Array(frame) => visitor.visit_seq(SeqStream::new(self, frame)), + ContainerHeader::Object(frame) => visitor.visit_map(MapStream::new(self, frame)), + }; + self.depth -= 1; + return result; + } + + // Common scalar: feed the visitor directly, borrowing plain strings. + if let Some(token) = self.reader.try_read_native_scalar()? { + return match token { + ScalarToken::Null => visitor.visit_unit(), + ScalarToken::Bool(b) => visitor.visit_bool(b), + ScalarToken::I64(i) => visitor.visit_i64(i), + ScalarToken::U64(u) => visitor.visit_u64(u), + ScalarToken::F64(f) => { + // Reject non-finite doubles (`NaN`/`±∞`) so the native + // deserializer agrees with the reference `decode`, which + // maps a non-finite `Double` to `BinaryError::InvalidNumber` + // (JSON has no representation for these). + if !f.is_finite() { + return Err(BinaryError::InvalidNumber { + detail: "non-finite double (NaN or infinity)", + }); + } + visitor.visit_f64(f) + } + ScalarToken::Str(s) => visitor.visit_borrowed_str(s), + }; + } + + // Exotic form (guid/base64/compressed/reference string, binary blob, + // uniform number array): defer to the reference decoder + Value. + self.deserialize_via_value(visitor) + } + + fn deserialize_option(self, visitor: V) -> Result + where + V: Visitor<'de>, + { + if self.reader.peek_u8()? == NULL { + // Consume the `null` marker, then report absence. + self.reader.try_read_native_scalar()?; + visitor.visit_none() + } else { + visitor.visit_some(self) + } + } + + fn deserialize_newtype_struct(self, _name: &'static str, visitor: V) -> Result + where + V: Visitor<'de>, + { + // Newtype structs serialize transparently, so deserialize the inner + // value straight through. + visitor.visit_newtype_struct(self) + } + + fn deserialize_enum( + self, + name: &'static str, + variants: &'static [&'static str], + visitor: V, + ) -> Result + where + V: Visitor<'de>, + { + // Enums use serde's externally-tagged shape (unit → name string, + // others → single-key object). Decode the whole value and let + // `serde_json::Value`'s enum deserializer apply the matching rule. + let value = self.reader.read_value(self.depth)?; + value + .into_deserializer() + .deserialize_enum(name, variants, visitor) + .map_err(|e| BinaryError::Custom(e.to_string())) + } + + forward_to_deserialize_any! { + bool i8 i16 i32 i64 i128 u8 u16 u32 u64 u128 f32 f64 char str string + bytes byte_buf unit unit_struct seq tuple tuple_struct map struct + identifier ignored_any + } +} + +/// Streams the elements of a binary array into a [`SeqAccess`], deserializing +/// each element natively (no intermediate `Value` for the array structure). +struct SeqStream<'a, 'de> { + de: &'a mut BinaryDeserializer<'de>, + frame: Frame, + produced: usize, +} + +impl<'a, 'de> SeqStream<'a, 'de> { + fn new(de: &'a mut BinaryDeserializer<'de>, frame: Frame) -> Self { + Self { + de, + frame, + produced: 0, + } + } + + /// Whether all elements have been produced: by declared count when the + /// marker carried one, else by reaching the payload's end offset. + fn finished(&self) -> bool { + match self.frame.count { + Some(count) => self.produced >= count, + None => self.de.reader.pos >= self.frame.end, + } + } + + /// After a count-framed container is fully produced, verify the declared + /// count consumed exactly the framed byte span. A declared count smaller + /// than the bytes the container spans would otherwise let the native path + /// silently under-read and reinterpret the leftover bytes as the parent's + /// next element/member — the reference decoder rejects this with + /// `InvalidLength`, so `from_slice` must too. Skipped for `Arr1`/`Obj1` + /// (`exact_end == false`), whose `end` is a buffer-length sentinel. + fn validate_exact_end(&self) -> Result<()> { + if self.frame.exact_end && self.de.reader.pos != self.frame.end { + return Err(BinaryError::InvalidLength { + detail: "array declared count does not span its declared length", + }); + } + Ok(()) + } +} + +impl<'de> SeqAccess<'de> for SeqStream<'_, 'de> { + type Error = BinaryError; + + fn next_element_seed(&mut self, seed: T) -> Result> + where + T: DeserializeSeed<'de>, + { + if self.finished() { + self.validate_exact_end()?; + return Ok(None); + } + let value = seed.deserialize(&mut *self.de)?; + self.produced += 1; + if self.de.reader.pos > self.frame.end { + return Err(BinaryError::InvalidLength { + detail: "array element extends past the array's declared length", + }); + } + Ok(Some(value)) + } + + fn size_hint(&self) -> Option { + self.frame + .count + .map(|count| count.saturating_sub(self.produced)) + } +} + +/// Streams the members of a binary object into a [`MapAccess`], deserializing +/// each key and value natively. +struct MapStream<'a, 'de> { + de: &'a mut BinaryDeserializer<'de>, + frame: Frame, + produced: usize, +} + +impl<'a, 'de> MapStream<'a, 'de> { + fn new(de: &'a mut BinaryDeserializer<'de>, frame: Frame) -> Self { + Self { + de, + frame, + produced: 0, + } + } + + fn finished(&self) -> bool { + match self.frame.count { + Some(count) => self.produced >= count, + None => self.de.reader.pos >= self.frame.end, + } + } + + /// See [`SeqStream::validate_exact_end`]: rejects a declared member count + /// that does not span the object's framed byte length, matching the + /// reference decoder. + fn validate_exact_end(&self) -> Result<()> { + if self.frame.exact_end && self.de.reader.pos != self.frame.end { + return Err(BinaryError::InvalidLength { + detail: "object declared count does not span its declared length", + }); + } + Ok(()) + } +} + +impl<'de> MapAccess<'de> for MapStream<'_, 'de> { + type Error = BinaryError; + + fn next_key_seed(&mut self, seed: K) -> Result> + where + K: DeserializeSeed<'de>, + { + if self.finished() { + self.validate_exact_end()?; + return Ok(None); + } + // The property name is a string value; the seed (a field identifier or + // `String`) consumes it through the normal scalar path. + let key = seed.deserialize(&mut *self.de)?; + Ok(Some(key)) + } + + fn next_value_seed(&mut self, seed: V) -> Result + where + V: DeserializeSeed<'de>, + { + let value = seed.deserialize(&mut *self.de)?; + self.produced += 1; + if self.de.reader.pos > self.frame.end { + return Err(BinaryError::InvalidLength { + detail: "object member extends past the object's declared length", + }); + } + Ok(value) + } + + fn size_hint(&self) -> Option { + self.frame + .count + .map(|count| count.saturating_sub(self.produced)) + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary_json::{decode, markers, to_vec}; + use serde::{Deserialize, Serialize}; + use serde_json::json; + + #[derive(Serialize, Deserialize, PartialEq, Debug)] + struct Product { + id: String, + count: u64, + tags: Vec, + in_stock: bool, + } + + #[test] + fn typed_struct_round_trips() { + let product = Product { + id: "p1".to_owned(), + count: 7, + tags: vec!["a".to_owned(), "b".to_owned()], + in_stock: true, + }; + let bytes = to_vec(&product).unwrap(); + let decoded: Product = from_slice(&bytes).unwrap(); + assert_eq!(decoded, product); + } + + #[derive(Serialize, Deserialize, PartialEq, Debug)] + struct WithOption { + id: String, + note: Option, + } + + #[test] + fn option_fields_round_trip() { + for note in [Some("hi".to_owned()), None] { + let value = WithOption { + id: "x".to_owned(), + note, + }; + let bytes = to_vec(&value).unwrap(); + let decoded: WithOption = from_slice(&bytes).unwrap(); + assert_eq!(decoded, value); + } + } + + #[derive(Serialize, Deserialize, PartialEq, Debug)] + enum Shape { + Unit, + Newtype(u32), + Tuple(u8, u8), + Struct { width: u32, height: u32 }, + } + + #[test] + fn enum_variants_round_trip() { + for shape in [ + Shape::Unit, + Shape::Newtype(5), + Shape::Tuple(1, 2), + Shape::Struct { + width: 3, + height: 4, + }, + ] { + let bytes = to_vec(&shape).unwrap(); + let decoded: Shape = from_slice(&bytes).unwrap(); + assert_eq!(decoded, shape); + } + } + + #[test] + fn hash_map_round_trips() { + use std::collections::BTreeMap; + let mut map = BTreeMap::new(); + map.insert("alpha".to_owned(), 1u32); + map.insert("beta".to_owned(), 2u32); + let bytes = to_vec(&map).unwrap(); + let decoded: BTreeMap = from_slice(&bytes).unwrap(); + assert_eq!(decoded, map); + } + + #[test] + fn trailing_bytes_are_rejected() { + let mut bytes = to_vec(&json!(true)).unwrap(); + bytes.push(0x00); + let result: Result = from_slice(&bytes); + assert!(matches!( + result, + Err(BinaryError::TrailingBytes { remaining: 1 }) + )); + } + + #[test] + fn missing_preamble_is_rejected() { + let result: Result = from_slice(b"{}"); + assert!(matches!(result, Err(BinaryError::MissingPreamble { .. }))); + } + + #[test] + fn non_finite_double_is_rejected_like_decode() { + // A hand-crafted buffer carrying a non-finite `Double` (NaN / +∞). + // JSON cannot represent these, so the native `from_slice` path must + // reject them with the same `InvalidNumber` error as the reference + // `decode`, rather than accepting a `NaN`/`inf` the way an unguarded + // `visit_f64` would. + for bits in [f64::NAN, f64::INFINITY, f64::NEG_INFINITY] { + let mut bytes = vec![crate::binary_json::PREAMBLE, markers::NUMBER_DOUBLE]; + bytes.extend_from_slice(&bits.to_le_bytes()); + + let decoded = decode(&bytes); + assert!( + matches!(decoded, Err(BinaryError::InvalidNumber { .. })), + "decode must reject non-finite double, got {decoded:?}" + ); + + let native: Result = from_slice(&bytes); + assert!( + matches!(native, Err(BinaryError::InvalidNumber { .. })), + "from_slice must reject non-finite double, got {native:?}" + ); + } + } + + /// A tiny deterministic LCG so the generative parity test needs no external + /// RNG dependency and reproduces the same values on every run. + struct Lcg(u64); + + impl Lcg { + fn next_u64(&mut self) -> u64 { + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1); + self.0 + } + + fn below(&mut self, n: u64) -> u64 { + self.next_u64() % n + } + } + + /// Builds a random JSON value up to `depth` levels deep, exercising every + /// scalar form the native fast path handles plus nested containers. + fn random_value(rng: &mut Lcg, depth: u32) -> serde_json::Value { + let arms = if depth == 0 { 6 } else { 8 }; + match rng.below(arms) { + 0 => serde_json::Value::Null, + 1 => json!(rng.next_u64().is_multiple_of(2)), + 2 => json!(rng.below(64) as i64), + 3 => json!((rng.next_u64() as i64).wrapping_sub(i64::MAX / 2)), + 4 => json!((rng.next_u64() as f64) / 7.0), + 5 => { + let len = rng.below(80) as usize; + json!("s".repeat(len)) + } + 6 => { + let n = rng.below(5) as usize; + let items: Vec<_> = (0..n).map(|_| random_value(rng, depth - 1)).collect(); + serde_json::Value::Array(items) + } + _ => { + let n = rng.below(5) as usize; + let mut map = serde_json::Map::new(); + for i in 0..n { + map.insert(format!("k{i}"), random_value(rng, depth - 1)); + } + serde_json::Value::Object(map) + } + } + } + + #[test] + fn generative_parity_from_slice_matches_decode() { + // Property: for any value, the native `from_slice::` equals the + // reference `decode`, and both equal the original. + let mut rng = Lcg(0x0f1e_2d3c_4b5a_6978); + for _ in 0..2_000 { + let value = random_value(&mut rng, 4); + let bytes = to_vec(&value).unwrap(); + let native: serde_json::Value = from_slice(&bytes).unwrap(); + assert_eq!(native, decode(&bytes).unwrap(), "parity for {value:?}"); + assert_eq!(native, value, "round-trip for {value:?}"); + } + } + + #[test] + fn exotic_forms_decode_via_fallback() { + // A binary blob (`Binary1ByteLength`) is not one of the native fast-path + // forms, so it must route through the `Value` fallback and yield the + // same base64 string the reference decoder produces. + // cSpell:ignore AQID + let bytes = [ + crate::binary_json::PREAMBLE, + crate::binary_json::markers::BINARY_1BYTE_LENGTH, + 4, + 0xDE, + 0xAD, + 0xBE, + 0xEF, + ]; + let native: serde_json::Value = from_slice(&bytes).unwrap(); + assert_eq!(native, json!("3q2+7w==")); + assert_eq!(native, decode(&bytes).unwrap()); + } + + #[test] + fn corpus_vectors_deserialize_natively() { + // Every golden vector `decode` accepts must also deserialize through the + // native path, yielding the identical value. + for vector in crate::binary_json::vectors::golden_vectors() { + let native: serde_json::Value = from_slice(&vector.binary) + .unwrap_or_else(|e| panic!("from_slice failed for vector {}: {e}", vector.name)); + assert_eq!( + native, + decode(&vector.binary).unwrap(), + "from_slice must equal decode for vector {}", + vector.name + ); + } + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/error.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/error.rs new file mode 100644 index 00000000000..7eff258445e --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/error.rs @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Error types for the Cosmos binary JSON codec. +//! +//! [`BinaryError`] is the error type the decoder and encoder produce. The +//! decoder returns one of these variants rather than panicking, so malformed or +//! truncated buffers fail gracefully. + +use std::fmt; + +/// A specialized [`Result`](std::result::Result) for binary JSON codec +/// operations. +pub type Result = std::result::Result; + +/// An error produced while decoding or encoding Cosmos binary JSON. +#[derive(Clone, Debug, PartialEq, Eq)] +#[non_exhaustive] +pub enum BinaryError { + /// The buffer ended before a value could be fully read. + /// + /// Carries the number of additional bytes the reader needed at the point it + /// ran out of input. + UnexpectedEof { + /// How many more bytes were required to continue. + needed: usize, + }, + + /// A type-marker byte was encountered that is not valid in the position it + /// appeared (for example a reserved marker, or [`crate::binary_json::markers::INVALID`]). + InvalidMarker { + /// The offending marker byte. + marker: u8, + /// Byte offset of the marker within the buffer. + offset: usize, + }, + + /// A length or count prefix was malformed or describes a region that does + /// not fit within the remaining buffer. + InvalidLength { + /// Human-readable detail about which length was invalid. + detail: &'static str, + }, + + /// A string's bytes are not valid UTF-8. + InvalidUtf8 { + /// Byte offset of the string payload within the buffer. + offset: usize, + }, + + /// A decoded number cannot be represented as JSON (for example a non-finite + /// `double` such as NaN or infinity, which JSON does not permit). + InvalidNumber { + /// Human-readable detail about why the number is not representable. + detail: &'static str, + }, + + /// A reference string ([`StrR1`](crate::binary_json::markers::STR_R1)–[`StrR4`](crate::binary_json::markers::STR_R4)) + /// pointed at an offset that does not correspond to an earlier string. + UnresolvedReference { + /// The byte offset the reference attempted to resolve. + target: usize, + }, + + /// A user string ([`UserString1ByteLengthMin`](crate::binary_json::markers::USER_STRING_1BYTE_MIN)–`0x67`) + /// was encountered. User strings are encoded against an external string + /// dictionary that the Cosmos data plane does not provide, so the string + /// cannot be resolved. + UnsupportedUserString { + /// The decoded user-string dictionary id. + id: usize, + }, + + /// The buffer nests containers more deeply than the decoder's configured + /// limit. A depth bound prevents stack exhaustion from adversarial input. + DepthLimitExceeded { + /// The configured maximum nesting depth. + limit: usize, + }, + + /// The buffer did not begin with the expected + /// [`PREAMBLE`](crate::binary_json::PREAMBLE) byte. + MissingPreamble { + /// The first byte that was found instead. + found: u8, + }, + + /// Extra bytes remained after a complete top-level value was decoded. + TrailingBytes { + /// Number of unconsumed bytes. + remaining: usize, + }, + + /// A custom, caller-supplied error message. + /// + /// Produced on the **encode** path by the native `serde` serializer: when a + /// value's `Serialize` implementation fails, serde funnels the failure + /// through [`serde::ser::Error::custom`], which maps to this variant. + Custom(String), +} + +impl fmt::Display for BinaryError { + fn fmt(&self, f: &mut fmt::Formatter<'_>) -> fmt::Result { + match self { + BinaryError::UnexpectedEof { needed } => { + write!( + f, + "unexpected end of binary JSON buffer (needed {needed} more byte(s))" + ) + } + BinaryError::InvalidMarker { marker, offset } => { + write!( + f, + "invalid binary JSON type marker {marker:#04x} at offset {offset}" + ) + } + BinaryError::InvalidLength { detail } => { + write!(f, "invalid binary JSON length prefix: {detail}") + } + BinaryError::InvalidUtf8 { offset } => { + write!(f, "invalid UTF-8 in binary JSON string at offset {offset}") + } + BinaryError::InvalidNumber { detail } => { + write!( + f, + "binary JSON number is not representable as JSON: {detail}" + ) + } + BinaryError::UnresolvedReference { target } => { + write!( + f, + "binary JSON reference string targets unresolved offset {target}" + ) + } + BinaryError::UnsupportedUserString { id } => { + write!( + f, + "binary JSON user string (id {id}) requires a string dictionary that is not available" + ) + } + BinaryError::DepthLimitExceeded { limit } => { + write!( + f, + "binary JSON nesting exceeds the maximum depth of {limit}" + ) + } + BinaryError::MissingPreamble { found } => { + write!( + f, + "binary JSON buffer does not start with the 0x80 preamble (found {found:#04x})" + ) + } + BinaryError::TrailingBytes { remaining } => { + write!( + f, + "binary JSON buffer has {remaining} trailing byte(s) after the top-level value" + ) + } + BinaryError::Custom(message) => { + write!(f, "binary JSON serialization error: {message}") + } + } + } +} + +impl std::error::Error for BinaryError {} + +impl serde::ser::Error for BinaryError { + fn custom(msg: T) -> Self { + BinaryError::Custom(msg.to_string()) + } +} + +impl serde::de::Error for BinaryError { + fn custom(msg: T) -> Self { + BinaryError::Custom(msg.to_string()) + } +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn display_messages_are_informative() { + assert!(BinaryError::UnexpectedEof { needed: 4 } + .to_string() + .contains("needed 4")); + assert!(BinaryError::InvalidMarker { + marker: 0xFF, + offset: 7 + } + .to_string() + .contains("0xff")); + assert!(BinaryError::MissingPreamble { found: 0x7B } + .to_string() + .contains("0x7b")); + } + + #[test] + fn implements_std_error() { + // Compile-time check that BinaryError satisfies the std error trait so + // it can participate in `?`/`Box` flows. + fn assert_error(_: &E) {} + assert_error(&BinaryError::TrailingBytes { remaining: 1 }); + } + + #[test] + fn ser_error_custom_maps_to_custom_variant() { + // `serde::ser::Error::custom` is how a value's failing `Serialize` + // impl surfaces on the native binary encode path; it must land in the + // `Custom` variant and preserve the message. + use serde::ser::Error as _; + let err = BinaryError::custom("field went wrong"); + assert_eq!(err, BinaryError::Custom("field went wrong".to_owned())); + assert!(err.to_string().contains("field went wrong")); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/fuzz_tests.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/fuzz_tests.rs new file mode 100644 index 00000000000..c0e29efec83 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/fuzz_tests.rs @@ -0,0 +1,348 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Decoder robustness ("fuzz") tests for the Cosmos binary JSON codec. +//! +//! For any input buffer the decoder must terminate and either succeed or return +//! a [`BinaryError`](super::BinaryError) — never panic, never hang, and never +//! allocate based on a length prefix beyond what the buffer can back. These +//! tests assert that by throwing random, truncated, corrupted, and adversarial +//! buffers at [`decode`](super::decode). Randomness is deterministic so failures +//! reproduce exactly. + +use super::vectors::golden_vectors; +use super::{decode, encode, from_slice, markers, PREAMBLE}; +use serde_json::json; + +/// A tiny deterministic SplitMix64 PRNG. +/// +/// Dependency-free (the crate avoids `rand`); the finalizer matches the +/// SplitMix64 mixing used elsewhere in the driver. Deterministic so a failing +/// case always reproduces from the same seed. +struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next_u64(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9e37_79b9_7f4a_7c15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xbf58_476d_1ce4_e5b9); + z = (z ^ (z >> 27)).wrapping_mul(0x94d0_49bb_1331_11eb); + z ^ (z >> 31) + } + + /// Returns a `u64` in `[0, bound)` (`bound` must be non-zero). + fn below(&mut self, bound: u64) -> u64 { + self.next_u64() % bound + } + + fn byte(&mut self) -> u8 { + self.next_u64() as u8 + } +} + +/// A set of value-producing markers worth biasing random buffers toward, so the +/// generator spends time on the structurally-interesting forms (containers, +/// length-prefixed strings, numbers) rather than mostly-invalid markers. +const INTERESTING_MARKERS: &[u8] = &[ + markers::NULL, + markers::FALSE, + markers::TRUE, + markers::NUMBER_INT64, + markers::NUMBER_DOUBLE, + markers::STR_L1, + markers::STR_L2, + markers::STR_L4, + markers::STR_R1, + markers::ARR_L1, + markers::ARR_LC1, + markers::OBJ_L1, + markers::OBJ_LC1, + markers::ARR_NUM_C1, + markers::BINARY_1BYTE_LENGTH, + markers::BINARY_4BYTE_LENGTH, + markers::LOWERCASE_GUID_STRING, + markers::BASE64_STRING_LENGTH1, + markers::PACKED_7BIT_STRING_LENGTH1, + markers::INVALID, +]; + +/// A representative spread of JSON values used to produce *valid* binary buffers +/// (via the encoder) that the corruption/truncation sweeps then mutate. +fn sample_values() -> Vec { + vec![ + json!(null), + json!(true), + json!(0), + json!(31), + json!(-5_000_000_000i64), + json!(u64::MAX), + json!(1.5), + json!(""), + json!("id"), + json!("x".repeat(300)), + json!([]), + json!([1, 2, 3]), + json!({}), + json!({ "id": "1", "n": 7, "nested": { "a": [true, null] } }), + json!([[1, 2], [3, [4, 5]]]), + ] +} + +#[test] +fn decode_never_panics_on_random_bytes() { + let mut rng = SplitMix64::new(0x5eed_1234_5678_9abc); + for _ in 0..20_000 { + // Lengths up to 64 bytes; sometimes force the preamble so the decoder + // proceeds past the first-byte check into the value parser. + let len = rng.below(65) as usize; + let mut buf = Vec::with_capacity(len); + let force_preamble = rng.below(2) == 0; + for i in 0..len { + if i == 0 && force_preamble { + buf.push(PREAMBLE); + } else if rng.below(3) == 0 { + // Bias toward interesting markers to reach deeper code paths. + let idx = rng.below(INTERESTING_MARKERS.len() as u64) as usize; + buf.push(INTERESTING_MARKERS[idx]); + } else { + buf.push(rng.byte()); + } + } + // The contract: terminate with Ok or Err, never panic. + let _ = decode(&buf); + } +} + +#[test] +fn decode_never_panics_on_truncated_valid_buffers() { + // Every prefix of a valid buffer (golden corpus + encoder output for the + // sample values) must decode or error without panicking. + let mut buffers: Vec> = golden_vectors().into_iter().map(|v| v.binary).collect(); + buffers.extend(sample_values().iter().map(encode)); + + for buf in &buffers { + for cut in 0..=buf.len() { + let _ = decode(&buf[..cut]); + } + } +} + +#[test] +fn decode_never_panics_on_single_byte_corruption() { + let mut rng = SplitMix64::new(0xc0ff_ee00_d00d_1010); + for value in sample_values() { + let valid = encode(&value); + for index in 0..valid.len() { + // Try a handful of replacement bytes at each position, including + // boundary marker values that flip the parse down a different arm. + for replacement in [0x00, 0x80, 0xC0, 0xE0, 0xFF, rng.byte()] { + let mut corrupted = valid.clone(); + corrupted[index] = replacement; + let _ = decode(&corrupted); + } + } + } +} + +#[test] +fn adversarial_length_prefixes_do_not_over_allocate() { + // Buffers that declare an enormous payload but carry almost none must fail + // with a bounds error rather than panicking, hanging, or attempting a + // multi-gigabyte allocation. `read_bytes` only ever slices the existing + // buffer, so these resolve in O(1) without allocating the declared size. + let huge = u32::MAX; // ~4 GiB declared + + // StrL4 with a 4-byte length of u32::MAX but no payload. + let mut str_l4 = vec![PREAMBLE, markers::STR_L4]; + str_l4.extend_from_slice(&huge.to_le_bytes()); + assert!(decode(&str_l4).is_err()); + + // ArrL4 / ObjL4 with a giant declared body length. + for marker in [markers::ARR_L4, markers::OBJ_L4] { + let mut buf = vec![PREAMBLE, marker]; + buf.extend_from_slice(&huge.to_le_bytes()); + assert!(decode(&buf).is_err()); + } + + // Binary4ByteLength with a giant declared blob length. + let mut bin = vec![PREAMBLE, markers::BINARY_4BYTE_LENGTH]; + bin.extend_from_slice(&huge.to_le_bytes()); + assert!(decode(&bin).is_err()); + + // A uniform Int64 array claiming u16::MAX items (the max an ArrNumC2 + // count field can express): must error, not try to build a 65,535-element + // vector from a buffer that carries almost no payload. + let mut uniform = vec![PREAMBLE, markers::ARR_NUM_C2, markers::INT64]; + uniform.extend_from_slice(&(u16::MAX).to_le_bytes()); + assert!(decode(&uniform).is_err()); +} + +#[test] +fn deeply_nested_input_errors_without_stack_overflow() { + // A pathologically deep nesting of single-item arrays must hit the depth + // guard (DepthLimitExceeded) rather than overflowing the stack. 10_000 is + // far beyond MAX_DEPTH, so this exercises the guard, not a valid document. + // The recursive descent keeps a small per-level frame (leaf decoding lives + // in a separate non-inlined frame), so reaching the guard stays within an + // ordinary thread stack. + let mut buf = vec![PREAMBLE]; + buf.extend(std::iter::repeat_n(markers::ARR1, 10_000)); + buf.push(0x00); // a literal-int leaf (never reached past the guard) + assert!(decode(&buf).is_err()); +} + +#[test] +fn all_two_byte_inputs_terminate() { + // Exhaustively decode every `[0x80, b]` two-byte buffer: every single-byte + // value form (and every invalid marker) must resolve without panicking. + for b in 0u16..=255 { + let _ = decode(&[PREAMBLE, b as u8]); + } +} + +/// Runs the same buffer through both parsers of untrusted bytes and asserts +/// their agreement contract, returning the pair of results for the caller to +/// spot-check counts. +/// +/// The reference [`decode`] and the native streaming [`from_slice`] are two +/// independent parsers, and `from_slice` is the one actually wired into item +/// reads. The contract: neither may panic on any input, and whenever **both** +/// succeed they must produce the identical [`serde_json::Value`]. (One may +/// legitimately reject a buffer the other accepts only in the direction where +/// `from_slice` is stricter — e.g. exotic forms it defers to `decode` for — so +/// disagreement is asserted only when both return `Ok`.) +fn assert_decoders_agree(buf: &[u8]) { + let decoded = decode(buf); + let streamed = from_slice::(buf); + if let (Ok(a), Ok(b)) = (&decoded, &streamed) { + assert_eq!( + a, b, + "decode and from_slice disagreed on a buffer both accepted: {buf:02x?}" + ); + } +} + +#[test] +fn decode_and_from_slice_agree_on_random_bytes() { + // Mirror `decode_never_panics_on_random_bytes`, but drive both parsers so + // the native streaming path (SeqStream/MapStream termination) receives the + // same adversarial coverage as the reference decoder. + let mut rng = SplitMix64::new(0x0d1f_f00d_face_b00c); + for _ in 0..20_000 { + let len = rng.below(65) as usize; + let mut buf = Vec::with_capacity(len); + let force_preamble = rng.below(2) == 0; + for i in 0..len { + if i == 0 && force_preamble { + buf.push(PREAMBLE); + } else if rng.below(3) == 0 { + let idx = rng.below(INTERESTING_MARKERS.len() as u64) as usize; + buf.push(INTERESTING_MARKERS[idx]); + } else { + buf.push(rng.byte()); + } + } + assert_decoders_agree(&buf); + } +} + +#[test] +fn decode_and_from_slice_agree_on_truncated_and_corrupted_buffers() { + // Both parsers over every prefix and every single-byte corruption of the + // encoder's output for the sample values. + let mut rng = SplitMix64::new(0xabad_1dea_1234_5678); + for value in sample_values() { + let valid = encode(&value); + + for cut in 0..=valid.len() { + assert_decoders_agree(&valid[..cut]); + } + + for index in 0..valid.len() { + for replacement in [0x00, 0x80, 0xC0, 0xE0, 0xFF, rng.byte()] { + let mut corrupted = valid.clone(); + corrupted[index] = replacement; + assert_decoders_agree(&corrupted); + } + } + } +} + +#[test] +fn many_references_to_one_large_string_stay_bounded() { + // One large `StrL2` string followed by many `StrR2` refs must be rejected + // by the reference-expansion budget, not amplified into O(S²) `String`s. + let payload_len = 4096usize; + let reference_count = 20_000usize; + let element_count = reference_count + 1; + + // Container prefix is ArrLC4: PREAMBLE + marker + 4-byte length + 4-byte + // count, so the first element (the big string) begins at offset 10. + let string_offset: u16 = 1 + 1 + 4 + 4; + + // Body: the big StrL2 string, then `reference_count` StrR2 refs to it. + let mut body = Vec::new(); + body.push(markers::STR_L2); + body.extend_from_slice(&(payload_len as u16).to_le_bytes()); + body.extend(std::iter::repeat_n(b'a', payload_len)); + for _ in 0..reference_count { + body.push(markers::STR_R2); + body.extend_from_slice(&string_offset.to_le_bytes()); + } + + let mut buf = vec![PREAMBLE, markers::ARR_LC4]; + buf.extend_from_slice(&(body.len() as u32).to_le_bytes()); + buf.extend_from_slice(&(element_count as u32).to_le_bytes()); + buf.extend_from_slice(&body); + + // ~80 MB of expansion exceeds the budget floor, so decode must error. + assert!( + decode(&buf).is_err(), + "expected the reference-expansion budget to reject the amplified buffer" + ); + // The native path must also terminate without panicking. + let _ = from_slice::(&buf); +} + +#[test] +fn from_slice_rejects_undersized_container_count() { + // A count-framed object whose declared count (1) is smaller than the number + // of members its byte length spans (2). The reference decoder rejects this + // via its member-count validation; the native streaming path must agree + // rather than silently under-reading and reinterpreting the leftover bytes. + // + // Build the well-formed 2-member object first, then rewrite the count field + // to 1, leaving the byte length untouched. + let valid = encode(&json!({ "a": 1, "b": 2 })); + + // Locate the object marker (first byte after the preamble). It must be one + // of the count-framed `ObjLC*` forms for this rewrite to apply. + let obj_marker = valid[1]; + assert_eq!( + obj_marker, + markers::OBJ_LC1, + "encoder is expected to emit a 1-byte length+count object here" + ); + + // Layout for ObjLC1: [PREAMBLE, OBJ_LC1, length(1), count(1), ...members]. + // The count byte sits at index 3; force it to 1. + let mut undersized = valid.clone(); + undersized[3] = 1; + + // The reference decoder rejects the mismatched count... + assert!( + decode(&undersized).is_err(), + "decode should reject an object whose declared count under-counts its members" + ); + // ...and so must the native streaming deserializer. + assert!( + from_slice::(&undersized).is_err(), + "from_slice should reject an object whose declared count under-counts its members" + ); +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/markers.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/markers.rs new file mode 100644 index 00000000000..023acafac3d --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/markers.rs @@ -0,0 +1,299 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Type-marker byte constants for the Cosmos binary JSON format. +//! +//! Every value in a Cosmos binary JSON buffer is introduced by a single +//! **type-marker** byte that selects how the following bytes are interpreted. +//! The byte values here are the authoritative wire constants and **must** match +//! the service byte-for-byte; they are transcribed from the .NET reference +//! implementation `Microsoft.Azure.Cosmos/src/Json/JsonBinaryEncoding.TypeMarker.cs` +//! (see [`crate::binary_json`] module docs for the cross-reference). +//! +//! The marker space is partitioned into contiguous ranges. Range boundaries are +//! exposed as `*_MIN` (inclusive) / `*_MAX` (**exclusive**) pairs mirroring the +//! .NET `InRange(value, min, max)` convention (`value >= min && value < max`), +//! so a Rust range check is `(MIN..MAX).contains(&marker)`. +//! +//! ```text +//! 0x00..0x20 literal integer (value == marker) +//! 0x20..0x40 1-byte system string (index into the fixed dictionary) +//! 0x40..0x60 1-byte user string (index into the per-buffer dictionary) +//! 0x60..0x68 2-byte user string +//! 0x68..0x80 base64 / GUID / compressed strings +//! 0x80..0xC0 encoded-length string (length == marker & 0x7F); 0x80 is also the preamble +//! 0xC0..0xC8 variable-length / reference strings, NumberUInt64 +//! 0xC8..0xD0 fixed-width numbers +//! 0xD0..0xE0 null / bool / guid / sized ints / binary +//! 0xE0..0xE8 arrays +//! 0xE8..0xF0 objects +//! 0xF0..0xF8 uniform (typed) number arrays +//! 0xF8..=0xFF special values (0xFF == Invalid) +//! ``` + +// ───────────────────────────────────────────────────────────────────────────── +// [0x00, 0x20): Encoded literal integer (32 values) +// ───────────────────────────────────────────────────────────────────────────── + +/// First marker whose value *is* the encoded integer (`value == marker`). +pub const LITERAL_INT_MIN: u8 = 0x00; +/// Exclusive upper bound of the literal-integer range (`LITERAL_INT_MIN + 32`). +pub const LITERAL_INT_MAX: u8 = 0x20; + +// ───────────────────────────────────────────────────────────────────────────── +// [0x20, 0x40): Encoded 1-byte system string (32 values) +// ───────────────────────────────────────────────────────────────────────────── + +/// First marker for a 1-byte-encoded system string (index into the fixed +/// system-string dictionary). +pub const SYSTEM_STRING_1BYTE_MIN: u8 = 0x20; +/// Exclusive upper bound of the 1-byte system-string range. +pub const SYSTEM_STRING_1BYTE_MAX: u8 = 0x40; + +// ───────────────────────────────────────────────────────────────────────────── +// [0x40, 0x60): Encoded 1-byte user string (32 values) +// ───────────────────────────────────────────────────────────────────────────── + +/// First marker for a 1-byte-encoded user string (index into the per-buffer +/// user-string dictionary). +pub const USER_STRING_1BYTE_MIN: u8 = 0x40; +/// Exclusive upper bound of the 1-byte user-string range. +pub const USER_STRING_1BYTE_MAX: u8 = 0x60; + +// ───────────────────────────────────────────────────────────────────────────── +// [0x60, 0x68): Encoded 2-byte user string (8 values) +// ───────────────────────────────────────────────────────────────────────────── + +/// First marker for a 2-byte-encoded user string. +pub const USER_STRING_2BYTE_MIN: u8 = 0x60; +/// Exclusive upper bound of the 2-byte user-string range. +pub const USER_STRING_2BYTE_MAX: u8 = 0x68; + +// ───────────────────────────────────────────────────────────────────────────── +// [0x68, 0x80): base64 / GUID / compressed string values +// ───────────────────────────────────────────────────────────────────────────── + +/// Standard base64-encoded string, length encoded in 1 byte. +pub const BASE64_STRING_LENGTH1: u8 = 0x71; +/// Standard base64-encoded string, length encoded in 2 bytes. +pub const BASE64_STRING_LENGTH2: u8 = 0x72; +/// URL-safe base64-encoded string, length encoded in 1 byte. +pub const BASE64_URL_STRING_LENGTH1: u8 = 0x73; +/// URL-safe base64-encoded string, length encoded in 2 bytes. +pub const BASE64_URL_STRING_LENGTH2: u8 = 0x74; +/// GUID string with only lowercase characters. +pub const LOWERCASE_GUID_STRING: u8 = 0x75; +/// GUID string with only uppercase characters. +pub const UPPERCASE_GUID_STRING: u8 = 0x76; +/// Double-quoted lowercase GUID string (ETag form). +pub const DOUBLE_QUOTED_LOWERCASE_GUID_STRING: u8 = 0x77; + +/// Compressed string: lowercase hexadecimal digits packed as 4-bit characters. +pub const COMPRESSED_LOWERCASE_HEX_STRING: u8 = 0x78; +/// Compressed string: uppercase hexadecimal digits packed as 4-bit characters. +pub const COMPRESSED_UPPERCASE_HEX_STRING: u8 = 0x79; +/// Compressed string: date-time character set packed as 4-bit characters. +pub const COMPRESSED_DATE_TIME_STRING: u8 = 0x7A; +/// Compressed string: 4-bit packed characters relative to a base value. +pub const PACKED_4BIT_STRING: u8 = 0x7B; +/// Compressed string: 5-bit packed characters relative to a base value. +pub const PACKED_5BIT_STRING: u8 = 0x7C; +/// Compressed string: 6-bit packed characters relative to a base value. +pub const PACKED_6BIT_STRING: u8 = 0x7D; +/// Compressed string: 7-bit packed characters, length encoded in 1 byte. +pub const PACKED_7BIT_STRING_LENGTH1: u8 = 0x7E; +/// Compressed string: 7-bit packed characters, length encoded in 2 bytes. +pub const PACKED_7BIT_STRING_LENGTH2: u8 = 0x7F; + +// ───────────────────────────────────────────────────────────────────────────── +// [0x80, 0xC0): Encoded-length string (64 values) +// ───────────────────────────────────────────────────────────────────────────── + +/// First marker for an encoded-length string (`length == marker & 0x7F`). +/// +/// `0x80` itself is also the **buffer preamble** byte (see +/// [`crate::binary_json::PREAMBLE`]); a zero-length string at the start of a +/// buffer is therefore indistinguishable from the preamble, so the preamble is +/// always consumed first. +pub const ENCODED_STRING_LENGTH_MIN: u8 = 0x80; +/// Exclusive upper bound of the encoded-length string range +/// (`ENCODED_STRING_LENGTH_MIN + 64`). +pub const ENCODED_STRING_LENGTH_MAX: u8 = 0xC0; + +/// Mask applied to an encoded-length string marker to recover its length: +/// `length = marker & ENCODED_STRING_LENGTH_MASK`. +pub const ENCODED_STRING_LENGTH_MASK: u8 = 0x7F; + +// ───────────────────────────────────────────────────────────────────────────── +// [0xC0, 0xC8): Variable-length and reference strings, NumberUInt64 +// ───────────────────────────────────────────────────────────────────────────── + +/// Length-prefixed string with a 1-byte length. +pub const STR_L1: u8 = 0xC0; +/// Length-prefixed string with a 2-byte length. +pub const STR_L2: u8 = 0xC1; +/// Length-prefixed string with a 4-byte length. +pub const STR_L4: u8 = 0xC2; +/// Reference string addressed by a 1-byte offset to an earlier string. +pub const STR_R1: u8 = 0xC3; +/// Reference string addressed by a 2-byte offset to an earlier string. +pub const STR_R2: u8 = 0xC4; +/// Reference string addressed by a 3-byte offset to an earlier string. +pub const STR_R3: u8 = 0xC5; +/// Reference string addressed by a 4-byte offset to an earlier string. +pub const STR_R4: u8 = 0xC6; +/// 8-byte unsigned integer. +pub const NUMBER_UINT64: u8 = 0xC7; + +// ───────────────────────────────────────────────────────────────────────────── +// [0xC8, 0xD0): Fixed-width number values +// ───────────────────────────────────────────────────────────────────────────── + +/// 1-byte unsigned integer. +pub const NUMBER_UINT8: u8 = 0xC8; +/// 2-byte signed integer. +pub const NUMBER_INT16: u8 = 0xC9; +/// 4-byte signed integer. +pub const NUMBER_INT32: u8 = 0xCA; +/// 8-byte signed integer. +pub const NUMBER_INT64: u8 = 0xCB; +/// Double-precision floating-point number (the canonical JSON number form). +pub const NUMBER_DOUBLE: u8 = 0xCC; +/// Single-precision (32-bit) floating-point number. +pub const FLOAT32: u8 = 0xCD; +/// Double-precision (64-bit) floating-point number. +pub const FLOAT64: u8 = 0xCE; +/// Half-precision (16-bit) floating-point number. +pub const FLOAT16: u8 = 0xCF; + +// ───────────────────────────────────────────────────────────────────────────── +// [0xD0, 0xE0): null / bool / guid / sized ints / binary +// ───────────────────────────────────────────────────────────────────────────── + +/// JSON `null`. +pub const NULL: u8 = 0xD0; +/// JSON `false`. +pub const FALSE: u8 = 0xD1; +/// JSON `true`. +pub const TRUE: u8 = 0xD2; +/// Raw 16-byte GUID value. +pub const GUID: u8 = 0xD3; +/// 1-byte unsigned integer value. +pub const UINT8: u8 = 0xD7; +/// 1-byte signed integer value. +pub const INT8: u8 = 0xD8; +/// 2-byte signed integer value. +pub const INT16: u8 = 0xD9; +/// 4-byte signed integer value. +pub const INT32: u8 = 0xDA; +/// 8-byte signed integer value. +pub const INT64: u8 = 0xDB; +/// 4-byte unsigned integer value. +pub const UINT32: u8 = 0xDC; +/// Binary payload with a 1-byte length prefix. +pub const BINARY_1BYTE_LENGTH: u8 = 0xDD; +/// Binary payload with a 2-byte length prefix. +pub const BINARY_2BYTE_LENGTH: u8 = 0xDE; +/// Binary payload with a 4-byte length prefix. +pub const BINARY_4BYTE_LENGTH: u8 = 0xDF; + +// ───────────────────────────────────────────────────────────────────────────── +// [0xE0, 0xE8): Array type markers +// ───────────────────────────────────────────────────────────────────────────── + +/// Empty array. +pub const ARR0: u8 = 0xE0; +/// Single-item array. +pub const ARR1: u8 = 0xE1; +/// Array with a 1-byte byte-length prefix. +pub const ARR_L1: u8 = 0xE2; +/// Array with a 2-byte byte-length prefix. +pub const ARR_L2: u8 = 0xE3; +/// Array with a 4-byte byte-length prefix. +pub const ARR_L4: u8 = 0xE4; +/// Array with a 1-byte byte-length prefix followed by a 1-byte item count. +pub const ARR_LC1: u8 = 0xE5; +/// Array with a 2-byte byte-length prefix followed by a 2-byte item count. +pub const ARR_LC2: u8 = 0xE6; +/// Array with a 4-byte byte-length prefix followed by a 4-byte item count. +pub const ARR_LC4: u8 = 0xE7; + +// ───────────────────────────────────────────────────────────────────────────── +// [0xE8, 0xF0): Object type markers +// ───────────────────────────────────────────────────────────────────────────── + +/// Empty object. +pub const OBJ0: u8 = 0xE8; +/// Single-property object. +pub const OBJ1: u8 = 0xE9; +/// Object with a 1-byte byte-length prefix. +pub const OBJ_L1: u8 = 0xEA; +/// Object with a 2-byte byte-length prefix. +pub const OBJ_L2: u8 = 0xEB; +/// Object with a 4-byte byte-length prefix. +pub const OBJ_L4: u8 = 0xEC; +/// Object with a 1-byte byte-length prefix followed by a 1-byte property count. +pub const OBJ_LC1: u8 = 0xED; +/// Object with a 2-byte byte-length prefix followed by a 2-byte property count. +pub const OBJ_LC2: u8 = 0xEE; +/// Object with a 4-byte byte-length prefix followed by a 4-byte property count. +pub const OBJ_LC4: u8 = 0xEF; + +// ───────────────────────────────────────────────────────────────────────────── +// [0xF0, 0xF8): Uniform (typed) number arrays +// ───────────────────────────────────────────────────────────────────────────── + +/// Uniform number array with a 1-byte item count. +pub const ARR_NUM_C1: u8 = 0xF0; +/// Uniform number array with a 2-byte item count. +pub const ARR_NUM_C2: u8 = 0xF1; +/// Array (1-byte item count) of uniform number arrays (1-byte item count). +pub const ARR_ARR_NUM_C1C1: u8 = 0xF2; +/// Array (2-byte item count) of uniform number arrays (2-byte item count). +pub const ARR_ARR_NUM_C2C2: u8 = 0xF3; + +// ───────────────────────────────────────────────────────────────────────────── +// [0xF8, 0xFF]: Special values +// ───────────────────────────────────────────────────────────────────────────── + +/// Reserved marker used to signal an invalid type marker. +pub const INVALID: u8 = 0xFF; + +#[cfg(test)] +mod tests { + use super::*; + + /// Pin the range boundaries so an accidental edit to one constant can't + /// silently shift a whole class of markers. The `*_MAX` of one range must + /// equal the `*_MIN` of the next (the ranges are contiguous). + #[test] + fn ranges_are_contiguous() { + assert_eq!(LITERAL_INT_MAX, SYSTEM_STRING_1BYTE_MIN); + assert_eq!(SYSTEM_STRING_1BYTE_MAX, USER_STRING_1BYTE_MIN); + assert_eq!(USER_STRING_1BYTE_MAX, USER_STRING_2BYTE_MIN); + assert_eq!(ENCODED_STRING_LENGTH_MIN, 0x80); + assert_eq!(ENCODED_STRING_LENGTH_MAX, 0xC0); + assert_eq!(ENCODED_STRING_LENGTH_MAX, STR_L1); + } + + /// Each range spans the documented number of values. + #[test] + fn ranges_have_expected_widths() { + assert_eq!(LITERAL_INT_MAX - LITERAL_INT_MIN, 32); + assert_eq!(SYSTEM_STRING_1BYTE_MAX - SYSTEM_STRING_1BYTE_MIN, 32); + assert_eq!(USER_STRING_1BYTE_MAX - USER_STRING_1BYTE_MIN, 32); + assert_eq!(USER_STRING_2BYTE_MAX - USER_STRING_2BYTE_MIN, 8); + assert_eq!(ENCODED_STRING_LENGTH_MAX - ENCODED_STRING_LENGTH_MIN, 64); + } + + /// The encoded-length mask recovers the length stored in the marker. + #[test] + fn encoded_length_mask_recovers_length() { + // 0x80 | 5 == 0x85; masking off the high bit yields 5. + assert_eq!( + (ENCODED_STRING_LENGTH_MIN | 5) & ENCODED_STRING_LENGTH_MASK, + 5 + ); + assert_eq!(0x85 & ENCODED_STRING_LENGTH_MASK, 5); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs new file mode 100644 index 00000000000..7e2ffb3a72d --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/mod.rs @@ -0,0 +1,259 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Cosmos **binary JSON** codec. +//! +//! Cosmos binary JSON is a tagged byte stream that the service can persist and +//! transmit in place of UTF-8 text JSON. A buffer begins with the preamble byte +//! [`PREAMBLE`] (`0x80`); because no valid UTF-8 text JSON document starts with +//! `0x80`, the first byte unambiguously distinguishes binary from text. Each +//! value is introduced by a single **type-marker** byte (see [`markers`]) that +//! selects how the following bytes are interpreted. +//! +//! This module is schema-agnostic: it operates purely on bytes and either +//! [`serde_json::Value`] (via [`decode`] / [`encode`]) or `serde` types +//! directly (via [`from_slice`] / [`to_vec`]). +//! +//! # Reading and writing +//! +//! - Read binary bytes into a typed value with [`from_slice`], or into a +//! [`serde_json::Value`] with [`decode`]. +//! - Write a typed value to binary bytes with [`to_vec`], or a +//! [`serde_json::Value`] with [`encode`]. +//! +//! The decoder accepts every wire form the service can emit. The encoder emits +//! a valid subset of those forms rather than the most compact encoding; because +//! the service accepts the verbose form, an encode/decode round-trip preserves +//! the original value. +//! +//! The wire constants in [`markers`] match the service byte-for-byte. + +pub mod de; +pub mod error; +pub mod markers; +pub mod reader; +pub mod ser; +pub mod system_strings; +pub mod writer; + +#[cfg(test)] +mod conformance; +#[cfg(test)] +mod fuzz_tests; +#[cfg(test)] +mod vectors; + +pub use de::from_slice; +pub use error::{BinaryError, Result}; +pub use reader::decode; +pub use ser::to_vec; +pub use writer::encode; + +/// The Cosmos binary JSON preamble byte. +/// +/// Every binary JSON buffer starts with this byte. It is the basis for +/// first-byte auto-detection ([`is_binary`]): no UTF-8 text JSON document can +/// begin with `0x80` (it is a continuation byte), so its presence reliably +/// distinguishes a binary buffer from a text one. +pub const PREAMBLE: u8 = 0x80; + +/// Returns `true` if `buffer` appears to be Cosmos binary JSON. +/// +/// Detection is the single-byte test described in the spec: a buffer is binary +/// iff its first byte is the [`PREAMBLE`]. An empty buffer is not binary. +/// +/// This is intentionally independent of any HTTP content negotiation so the +/// response path can decode binary bodies even when headers are absent or +/// unexpected. +/// +/// # Examples +/// +/// ``` +/// use azure_data_cosmos_driver::binary_json::{is_binary, PREAMBLE}; +/// +/// assert!(is_binary(&[PREAMBLE, 0xD2])); // binary `true` +/// assert!(!is_binary(b"{\"id\":\"1\"}")); // text JSON +/// assert!(!is_binary(&[])); // empty +/// ``` +pub fn is_binary(buffer: &[u8]) -> bool { + buffer.first() == Some(&PREAMBLE) +} + +/// Transcodes a Cosmos binary JSON buffer to UTF-8 **text** JSON. +/// +/// This is the driver-side conversion used when an upstream SDK/app wants to +/// deal only with text JSON while still keeping the wire binary (efficient RUs +/// and network bandwidth): the request and the service response stay binary, +/// and the driver converts the binary response to text before handing it back. +/// +/// Behavior: +/// +/// - If `buffer` is Cosmos binary JSON (begins with the [`PREAMBLE`]), it is +/// decoded to a [`serde_json::Value`] and re-serialized as compact UTF-8 text +/// JSON (matching `serde_json::to_vec`). +/// - If `buffer` is already text JSON (or empty), it is returned **unchanged** +/// so the conversion is safe to apply unconditionally on a response whose +/// format was negotiated but not guaranteed. +/// +/// # Errors +/// +/// Returns a [`BinaryError`] if `buffer` is binary but malformed, or if the +/// decoded value cannot be re-serialized as JSON. +pub fn transcode_to_text(buffer: &[u8]) -> Result> { + if !is_binary(buffer) { + // Already text (or empty): nothing to convert. + return Ok(buffer.to_vec()); + } + let value = decode(buffer)?; + serde_json::to_vec(&value) + .map_err(|e| BinaryError::Custom(format!("failed to re-serialize decoded value: {e}"))) +} + +/// Transcodes a UTF-8 **text** JSON buffer to Cosmos **binary** JSON. +/// +/// This is the mirror of [`transcode_to_text`] for the **request** path: when a +/// schema-agnostic caller (for example an FFI host) deals only in text JSON but +/// the driver negotiates a binary wire, the driver converts the text request +/// body to binary before sending it. No item schema is required — the buffer is +/// parsed to a [`serde_json::Value`] and re-encoded. +/// +/// Behavior: +/// +/// - If `buffer` is already Cosmos binary JSON (begins with the [`PREAMBLE`]), +/// it is returned **unchanged** so the conversion is safe to apply +/// unconditionally (a caller that already encoded binary is not re-encoded). +/// - An empty buffer is returned unchanged (no body to encode). +/// - Otherwise `buffer` is parsed as text JSON and encoded to binary. +/// +/// # Errors +/// +/// Returns a [`BinaryError`] if `buffer` is neither binary nor valid text JSON. +pub fn transcode_to_binary(buffer: &[u8]) -> Result> { + if buffer.is_empty() || is_binary(buffer) { + // Empty, or already binary: nothing to convert. + return Ok(buffer.to_vec()); + } + let value: serde_json::Value = serde_json::from_slice(buffer) + .map_err(|e| BinaryError::Custom(format!("failed to parse text JSON request body: {e}")))?; + Ok(encode(&value)) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn preamble_is_0x80() { + assert_eq!(PREAMBLE, 0x80); + // The preamble shares its value with the start of the encoded-length + // string range; that range begins at the same byte by design. + assert_eq!(PREAMBLE, markers::ENCODED_STRING_LENGTH_MIN); + } + + #[test] + fn detects_binary_by_preamble() { + assert!(is_binary(&[PREAMBLE])); + assert!(is_binary(&[PREAMBLE, markers::TRUE])); + } + + #[test] + fn rejects_text_and_empty() { + assert!(!is_binary(b"{}")); + assert!(!is_binary(b"[1,2,3]")); + assert!(!is_binary(b"\"hello\"")); + assert!(!is_binary(&[])); + // A different leading byte is not binary even if 0x80 appears later. + assert!(!is_binary(&[0x00, PREAMBLE])); + } + + #[test] + fn transcode_binary_to_text_produces_equivalent_json() { + // A binary buffer transcodes to the same JSON serde_json would emit. + let value = serde_json::json!({ + "id": "doc-1", + "n": 42, + "flag": true, + "nested": { "arr": [1, 2, 3], "s": "café" }, + }); + let binary = encode(&value); + assert!(is_binary(&binary)); + + let text = transcode_to_text(&binary).unwrap(); + assert!(!is_binary(&text), "transcoded output must be text"); + + // Bytes match serde_json::to_vec of the same value, and re-parse equal. + assert_eq!(text, serde_json::to_vec(&value).unwrap()); + let reparsed: serde_json::Value = serde_json::from_slice(&text).unwrap(); + assert_eq!(reparsed, value); + } + + #[test] + fn transcode_passes_text_through_unchanged() { + // A text buffer is returned byte-for-byte unchanged. + let text = br#"{"id":"1","n":7}"#; + assert_eq!(transcode_to_text(text).unwrap(), text); + } + + #[test] + fn transcode_passes_empty_through_unchanged() { + assert_eq!(transcode_to_text(&[]).unwrap(), Vec::::new()); + } + + #[test] + fn transcode_errors_on_malformed_binary() { + // A lone preamble is not a complete value. + assert!(transcode_to_text(&[PREAMBLE]).is_err()); + } + + #[test] + fn transcode_text_to_binary_produces_equivalent_binary() { + // A text buffer transcodes to binary that decodes back to the same value. + let value = serde_json::json!({ + "id": "doc-1", + "n": 42, + "flag": true, + "nested": { "arr": [1, 2, 3], "s": "café" }, + }); + let text = serde_json::to_vec(&value).unwrap(); + assert!(!is_binary(&text)); + + let binary = transcode_to_binary(&text).unwrap(); + assert!(is_binary(&binary), "transcoded output must be binary"); + + // Bytes match the encoder oracle, and decode back to the same value. + assert_eq!(binary, encode(&value)); + assert_eq!(decode(&binary).unwrap(), value); + } + + #[test] + fn transcode_to_binary_passes_binary_through_unchanged() { + // A buffer that is already binary is returned byte-for-byte unchanged. + let value = serde_json::json!({ "id": "1", "n": 7 }); + let binary = encode(&value); + assert_eq!(transcode_to_binary(&binary).unwrap(), binary); + } + + #[test] + fn transcode_to_binary_passes_empty_through_unchanged() { + assert_eq!(transcode_to_binary(&[]).unwrap(), Vec::::new()); + } + + #[test] + fn transcode_to_binary_errors_on_invalid_text() { + // Not binary and not valid JSON. + assert!(transcode_to_binary(b"{not json").is_err()); + } + + #[test] + fn transcode_round_trips_text_binary_text() { + // text → binary → text is identity for a well-formed document. + let value = serde_json::json!({ "a": 1, "b": ["x", "y"], "c": null }); + let text = serde_json::to_vec(&value).unwrap(); + let binary = transcode_to_binary(&text).unwrap(); + let back = transcode_to_text(&binary).unwrap(); + assert_eq!( + serde_json::from_slice::(&back).unwrap(), + value + ); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/reader.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/reader.rs new file mode 100644 index 00000000000..48453247d3a --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/reader.rs @@ -0,0 +1,1732 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Cosmos binary JSON **decoder** (`binary` -> [`serde_json::Value`]). +//! +//! Every step is bounds-checked and returns a [`BinaryError`] rather than +//! panicking, so a malformed or truncated buffer fails gracefully. Multi-byte +//! integers and length prefixes are little-endian, matching the service. +//! +//! The decoder handles every value form the service can emit: +//! [`null`](serde_json::Value::Null), booleans, all literal, fixed-width, and +//! extended numbers, every string form (system, user, reference, encoded-length, +//! length-prefixed, GUID, base64, and compressed), the GUID value, binary blobs, +//! containers, and uniform number arrays. Two cases have no JSON representation +//! and are reported as errors: user strings (`0x40`-`0x67`) report +//! [`BinaryError::UnsupportedUserString`] (they reference an external dictionary +//! the data plane does not supply), and `Float16` (`0xCF`) plus the standalone +//! extended `UInt8` (`0xD7`) report [`BinaryError::InvalidMarker`]. + +use base64::Engine; +use serde_json::{Map, Value}; + +use std::cell::Cell; + +use super::markers::{ + ARR0, ARR1, ARR_ARR_NUM_C1C1, ARR_ARR_NUM_C2C2, ARR_L1, ARR_L2, ARR_L4, ARR_LC1, ARR_LC2, + ARR_LC4, ARR_NUM_C1, ARR_NUM_C2, BASE64_STRING_LENGTH1, BASE64_STRING_LENGTH2, + BASE64_URL_STRING_LENGTH1, BASE64_URL_STRING_LENGTH2, BINARY_1BYTE_LENGTH, BINARY_2BYTE_LENGTH, + BINARY_4BYTE_LENGTH, COMPRESSED_DATE_TIME_STRING, COMPRESSED_LOWERCASE_HEX_STRING, + COMPRESSED_UPPERCASE_HEX_STRING, DOUBLE_QUOTED_LOWERCASE_GUID_STRING, + ENCODED_STRING_LENGTH_MASK, ENCODED_STRING_LENGTH_MAX, ENCODED_STRING_LENGTH_MIN, FALSE, + FLOAT32, FLOAT64, GUID, INT16, INT32, INT64, INT8, LITERAL_INT_MAX, LITERAL_INT_MIN, + LOWERCASE_GUID_STRING, NULL, NUMBER_DOUBLE, NUMBER_INT16, NUMBER_INT32, NUMBER_INT64, + NUMBER_UINT64, NUMBER_UINT8, OBJ0, OBJ1, OBJ_L1, OBJ_L2, OBJ_L4, OBJ_LC1, OBJ_LC2, OBJ_LC4, + PACKED_4BIT_STRING, PACKED_5BIT_STRING, PACKED_6BIT_STRING, PACKED_7BIT_STRING_LENGTH1, + PACKED_7BIT_STRING_LENGTH2, STR_L1, STR_L2, STR_L4, STR_R1, STR_R2, STR_R3, STR_R4, + SYSTEM_STRING_1BYTE_MAX, SYSTEM_STRING_1BYTE_MIN, TRUE, UINT32, UINT8, UPPERCASE_GUID_STRING, + USER_STRING_1BYTE_MAX, USER_STRING_1BYTE_MIN, USER_STRING_2BYTE_MAX, USER_STRING_2BYTE_MIN, +}; +use super::system_strings::system_string_for_marker; +use super::{is_binary, BinaryError, Result}; + +/// Maximum container nesting depth the decoder will descend before returning +/// [`BinaryError::DepthLimitExceeded`]. This mirrors the .NET Cosmos JSON +/// stack's `JsonObjectState.JsonMaxNestingDepth` (256 simultaneously-open +/// containers), so the Rust decoder enforces the same nesting policy while +/// guarding against stack exhaustion from adversarial input. +const MAX_DEPTH: usize = 256; + +/// A single native scalar token read directly from the buffer, used by the +/// native serde deserializer ([`super::de`]) to feed a visitor without +/// materializing a [`serde_json::Value`]. Only the common, cheaply-decodable +/// forms are represented here; exotic string/number forms fall back to +/// [`Reader::read_value`] in the deserializer. +pub(super) enum ScalarToken<'a> { + /// `null`. + Null, + /// `true` / `false`. + Bool(bool), + /// A signed integer (literal, fixed-width, or extended). + I64(i64), + /// An unsigned integer that does not fit `i64` (`NumberUInt64`). + U64(u64), + /// A double. + F64(f64), + /// A plain UTF-8 string borrowed directly from the buffer (system, + /// encoded-length, or `StrL1/2/4` form). + Str(&'a str), +} + +/// Framing for a container being streamed by the native deserializer: either a +/// known element/member `count`, or a byte `end` offset to read until. +pub(super) struct Frame { + /// Declared element/member count, when the marker carries one. + pub(super) count: Option, + /// Absolute buffer offset at which the container's payload ends. + pub(super) end: usize, + /// Whether `end` is an authoritative byte boundary derived from a + /// length prefix (`true` for the empty `Arr0`/`Obj0` and the length-framed + /// `L*`/`LC*` markers) versus a sentinel (`false` for `Arr1`/`Obj1`, whose + /// single element has no length prefix so `end` is set to the buffer length + /// and must not be treated as the container's true end). When `true`, the + /// streaming deserializer asserts the framed byte span is fully consumed, + /// matching the reference decoder's length + count validation. + pub(super) exact_end: bool, +} + +/// The two container shapes the native deserializer streams. +pub(super) enum ContainerHeader { + /// An array; stream `visit_seq`. + Array(Frame), + /// An object; stream `visit_map`. + Object(Frame), +} + +/// Width in bytes of a little-endian length or count field. +/// +/// Length- and count-prefixed forms encode their field in 1, 2, or 4 bytes +/// depending on the marker; carrying this as an enum instead of a raw `usize` +/// keeps the width total and makes the accepted set explicit. +#[derive(Clone, Copy)] +enum FieldWidth { + /// 1-byte field. + U8, + /// 2-byte field. + U16, + /// 4-byte field. + U32, +} + +/// Decodes a complete Cosmos binary JSON buffer into a [`serde_json::Value`]. +/// +/// The buffer must begin with the [`PREAMBLE`](super::PREAMBLE) byte (`0x80`); the single +/// top-level value that follows is decoded, and any bytes left over afterwards +/// are reported as [`BinaryError::TrailingBytes`]. +/// +/// # Errors +/// +/// Returns a [`BinaryError`] if the buffer is not binary (missing preamble), +/// is truncated, contains an invalid or not-yet-supported type marker, holds a +/// malformed length, carries invalid UTF-8, or has trailing bytes. +/// +/// # Examples +/// +/// ``` +/// use azure_data_cosmos_driver::binary_json::{decode, PREAMBLE}; +/// +/// // The binary form of `true` is the preamble followed by the `true` marker. +/// let value = decode(&[PREAMBLE, 0xD2]).unwrap(); +/// assert_eq!(value, serde_json::Value::Bool(true)); +/// ``` +pub fn decode(buffer: &[u8]) -> Result { + if !is_binary(buffer) { + return Err(match buffer.first() { + Some(&found) => BinaryError::MissingPreamble { found }, + None => BinaryError::UnexpectedEof { needed: 1 }, + }); + } + + // Start reading after the one-byte preamble. The reader keeps absolute + // offsets (into `buffer`) so error positions account for the preamble. + let mut reader = Reader::new(buffer, 1); + let value = reader.read_value(0)?; + let remaining = buffer.len() - reader.pos; + if remaining != 0 { + return Err(BinaryError::TrailingBytes { remaining }); + } + Ok(value) +} + +/// A bounds-checked forward cursor over a binary JSON buffer. +/// +/// `pos` is an absolute offset into `buf`; the first value begins at `pos == 1` +/// (just past the [`PREAMBLE`](super::PREAMBLE)). Every read advances `pos` only after verifying +/// the bytes are present, so the reader never indexes out of bounds. +pub(super) struct Reader<'a> { + pub(super) buf: &'a [u8], + pub(super) pos: usize, + /// Remaining budget, in bytes, for text materialized by **reference-string** + /// resolution ([`STR_R1`](super::markers::STR_R1)-[`STR_R4`](super::markers::STR_R4)). + /// + /// Each reference decodes a *fresh owned copy* of its target string, so a + /// crafted buffer (one long string plus many short references to it) can + /// expand to O(S^2) aggregate output from a size-`S` buffer even though every + /// individual length prefix is buffer-bounded. This shared counter caps the + /// total reference-expanded bytes for one decode; exceeding it fails with + /// [`BinaryError::InvalidLength`]. Non-reference strings are backed 1:1 by + /// buffer bytes and are not charged. Shared (via [`Cell`]) across every + /// reference resolution in the value tree because they all run on the same + /// reader instance. + ref_budget: Cell, +} + +/// Reference-string expansion budget for a `buf_len`-byte buffer. +/// +/// References let a small wire payload expand into a larger logical document, +/// so the true ceiling is the Cosmos item-size limit, not the input size. The +/// floor sits above the max item size; the factor still bounds the adversarial +/// O(S^2) blow-up. +fn reference_budget(buf_len: usize) -> usize { + // Above the 16 MB absolute Cosmos item-size limit. + const FLOOR: usize = 64 * 1024 * 1024; + const FACTOR: usize = 16; + buf_len.saturating_mul(FACTOR).max(FLOOR) +} + +/// Max number of empty inner arrays a uniform array-of-arrays may declare. +/// +/// Empty inner arrays consume no body bytes, so a large `outer_count` could +/// expand a few bytes into many nodes. Bounding by buffer size keeps output +/// O(input) (the `outer_count` field is at most 2 bytes wide). +fn empty_element_budget(buf_len: usize) -> usize { + const FLOOR: usize = 64 * 1024; + const FACTOR: usize = 16; + buf_len.saturating_mul(FACTOR).max(FLOOR) +} + +impl<'a> Reader<'a> { + /// Creates a reader positioned at `pos` within `buf`. + pub(super) fn new(buf: &'a [u8], pos: usize) -> Self { + let ref_budget = Cell::new(reference_budget(buf.len())); + Self { + buf, + pos, + ref_budget, + } + } + + /// Returns the next byte without advancing the cursor. + pub(super) fn peek_u8(&self) -> Result { + self.buf + .get(self.pos) + .copied() + .ok_or(BinaryError::UnexpectedEof { needed: 1 }) + } + + /// Reads a single byte, advancing the cursor. + fn read_u8(&mut self) -> Result { + let byte = *self + .buf + .get(self.pos) + .ok_or(BinaryError::UnexpectedEof { needed: 1 })?; + self.pos += 1; + Ok(byte) + } + + /// Reads exactly `N` bytes into a fixed-size array, advancing the cursor. + fn read_array(&mut self) -> Result<[u8; N]> { + let slice = self.read_bytes(N)?; + Ok(slice + .try_into() + .expect("read_bytes returns exactly N bytes or fails")) + } + + /// Borrows the next `len` bytes, advancing the cursor. + /// + /// Returns [`BinaryError::UnexpectedEof`] if fewer than `len` bytes remain. + /// This only ever slices the existing buffer, so an attacker-controlled + /// `len` cannot trigger an allocation larger than the buffer. + fn read_bytes(&mut self, len: usize) -> Result<&'a [u8]> { + let end = self + .pos + .checked_add(len) + .ok_or(BinaryError::InvalidLength { + detail: "length prefix overflows the address space", + })?; + let slice = self + .buf + .get(self.pos..end) + .ok_or(BinaryError::UnexpectedEof { + needed: end.saturating_sub(self.buf.len()), + })?; + self.pos = end; + Ok(slice) + } + + fn read_u16_le(&mut self) -> Result { + Ok(u16::from_le_bytes(self.read_array()?)) + } + + fn read_u32_le(&mut self) -> Result { + Ok(u32::from_le_bytes(self.read_array()?)) + } + + /// Reads a 3-byte little-endian unsigned integer (the `StrR3` offset width). + fn read_u24_le(&mut self) -> Result { + let [b0, b1, b2] = self.read_bytes(3)? else { + unreachable!("read_bytes(3) returns exactly 3 bytes or fails") + }; + Ok(u32::from(*b0) | (u32::from(*b1) << 8) | (u32::from(*b2) << 16)) + } + + fn read_u64_le(&mut self) -> Result { + Ok(u64::from_le_bytes(self.read_array()?)) + } + + fn read_i8(&mut self) -> Result { + Ok(self.read_u8()? as i8) + } + + fn read_i16_le(&mut self) -> Result { + Ok(i16::from_le_bytes(self.read_array()?)) + } + + fn read_i32_le(&mut self) -> Result { + Ok(i32::from_le_bytes(self.read_array()?)) + } + + fn read_i64_le(&mut self) -> Result { + Ok(i64::from_le_bytes(self.read_array()?)) + } + + fn read_f32_le(&mut self) -> Result { + Ok(f32::from_le_bytes(self.read_array()?)) + } + + fn read_f64_le(&mut self) -> Result { + Ok(f64::from_le_bytes(self.read_array()?)) + } + + /// Reads a UTF-8 string of `len` bytes. `marker_offset` is the offset of the + /// value's type marker, used for error reporting. + fn read_string(&mut self, len: usize, marker_offset: usize) -> Result { + let bytes = self.read_bytes(len)?; + std::str::from_utf8(bytes) + .map(str::to_owned) + .map_err(|_| BinaryError::InvalidUtf8 { + offset: marker_offset, + }) + } + + /// Reads one complete value at the current position. + /// + /// `depth` is the value's nesting depth (`0` for the top-level value); + /// container children are read at `depth + 1`. Exceeding [`MAX_DEPTH`] + /// returns [`BinaryError::DepthLimitExceeded`] rather than risking stack + /// exhaustion on deeply nested adversarial input. + pub(super) fn read_value(&mut self, depth: usize) -> Result { + if depth > MAX_DEPTH { + return Err(BinaryError::DepthLimitExceeded { limit: MAX_DEPTH }); + } + // Offset of this value's type marker, captured before consuming it so + // error positions point at the marker. + let offset = self.pos; + let marker = self.read_u8()?; + + // Only the container markers recurse (into `read_value`, directly or via + // the array/object/member helpers). Every other marker is a + // non-recursive leaf, decoded in `read_leaf_value` — a separate, + // never-inlined frame. Keeping the leaf-decoding locals out of this + // function keeps `read_value`'s own frame small, so descending to + // `MAX_DEPTH` stays well within an ordinary thread stack instead of + // requiring an oversized one. + match marker { + // Arrays. + ARR0 => Ok(Value::Array(Vec::new())), + ARR1 => { + let item = self.read_value(depth + 1)?; + Ok(Value::Array(vec![item])) + } + ARR_L1 => self.read_array_value(FieldWidth::U8, false, depth), + ARR_L2 => self.read_array_value(FieldWidth::U16, false, depth), + ARR_L4 => self.read_array_value(FieldWidth::U32, false, depth), + ARR_LC1 => self.read_array_value(FieldWidth::U8, true, depth), + ARR_LC2 => self.read_array_value(FieldWidth::U16, true, depth), + ARR_LC4 => self.read_array_value(FieldWidth::U32, true, depth), + + // Objects. + OBJ0 => Ok(Value::Object(Map::new())), + OBJ1 => { + let (name, value) = self.read_member(depth + 1)?; + let mut map = Map::new(); + map.insert(name, value); + Ok(Value::Object(map)) + } + OBJ_L1 => self.read_object_value(FieldWidth::U8, false, depth), + OBJ_L2 => self.read_object_value(FieldWidth::U16, false, depth), + OBJ_L4 => self.read_object_value(FieldWidth::U32, false, depth), + OBJ_LC1 => self.read_object_value(FieldWidth::U8, true, depth), + OBJ_LC2 => self.read_object_value(FieldWidth::U16, true, depth), + OBJ_LC4 => self.read_object_value(FieldWidth::U32, true, depth), + + // Every non-container marker is a leaf value. + _ => self.read_leaf_value(marker, offset), + } + } + + /// Decodes a single non-container ("leaf") value: scalars, all string + /// forms, numbers, GUIDs, binary blobs, uniform number arrays, user-string + /// references, and back-references. + /// + /// This is deliberately **not inlined** into [`read_value`](Self::read_value) + /// and never recurses back into it, so its (comparatively large) stack frame + /// is paid only once at each leaf rather than at every level of a deeply + /// nested container. That is what lets the recursive descent reach + /// [`MAX_DEPTH`] on an ordinary thread stack. + /// + /// `offset` is the position of `marker`, used for error reporting. + #[inline(never)] + fn read_leaf_value(&mut self, marker: u8, offset: usize) -> Result { + match marker { + NULL => Ok(Value::Null), + FALSE => Ok(Value::Bool(false)), + TRUE => Ok(Value::Bool(true)), + + // Literal integer: the value is encoded in the marker itself. + LITERAL_INT_MIN..LITERAL_INT_MAX => Ok(int_value(i64::from(marker))), + + // Fixed-width numbers (little-endian payloads), both the + // self-describing `NUMBER_*` markers and the extended Cosmos + // `INT*`/`UINT*`/`FLOAT*` markers, decode through a shared helper. + // `Float16` (0xCF) and the extended `UInt8` (0xD7) have no JSON node + // type in the service and are intentionally *not* routed here, so + // they fall through to the catch-all as InvalidMarker. + NUMBER_UINT8 | NUMBER_INT16 | NUMBER_INT32 | NUMBER_INT64 | NUMBER_UINT64 + | NUMBER_DOUBLE | INT8 | INT16 | INT32 | INT64 | UINT32 | FLOAT32 | FLOAT64 => { + self.read_number_value(marker, offset) + } + + // 1-byte system string: index into the fixed dictionary. + m if (SYSTEM_STRING_1BYTE_MIN..SYSTEM_STRING_1BYTE_MAX).contains(&m) => { + let s = system_string_for_marker(m) + .ok_or(BinaryError::InvalidMarker { marker: m, offset })?; + Ok(Value::String(s.to_owned())) + } + + // Encoded-length string: the length is carried in the marker. + m if (ENCODED_STRING_LENGTH_MIN..ENCODED_STRING_LENGTH_MAX).contains(&m) => { + let len = usize::from(m & ENCODED_STRING_LENGTH_MASK); + Ok(Value::String(self.read_string(len, offset)?)) + } + + // Length-prefixed strings. + STR_L1 => { + let len = usize::from(self.read_u8()?); + Ok(Value::String(self.read_string(len, offset)?)) + } + STR_L2 => { + let len = usize::from(self.read_u16_le()?); + Ok(Value::String(self.read_string(len, offset)?)) + } + STR_L4 => { + let len = self.read_u32_le()? as usize; + Ok(Value::String(self.read_string(len, offset)?)) + } + + // GUID strings: a 16-byte encoded form expanded to the canonical + // 36-character hex text. The lowercase/uppercase variants differ + // only in hex case; the double-quoted variant additionally wraps + // the text in literal quote characters (the original JSON string + // value included the quotes). + LOWERCASE_GUID_STRING => Ok(Value::String(self.read_guid_string(false, false)?)), + UPPERCASE_GUID_STRING => Ok(Value::String(self.read_guid_string(true, false)?)), + DOUBLE_QUOTED_LOWERCASE_GUID_STRING => { + Ok(Value::String(self.read_guid_string(false, true)?)) + } + + // Base64 strings: the raw (already base64-decoded) bytes are stored + // inline; decoding re-encodes them to the JSON string. The width of + // the group-count prefix (1 vs 2 bytes) and the alphabet (standard + // vs URL-safe) depend on the marker. + BASE64_STRING_LENGTH1 => Ok(Value::String( + self.read_base64_string(FieldWidth::U8, false)?, + )), + BASE64_STRING_LENGTH2 => Ok(Value::String( + self.read_base64_string(FieldWidth::U16, false)?, + )), + BASE64_URL_STRING_LENGTH1 => Ok(Value::String( + self.read_base64_string(FieldWidth::U8, true)?, + )), + BASE64_URL_STRING_LENGTH2 => Ok(Value::String( + self.read_base64_string(FieldWidth::U16, true)?, + )), + + // Compressed strings. The 4-bit table forms map each nibble through + // a fixed character set; the packed N-bit forms unpack N-bit values + // (optionally offset by a base character). All decode to ASCII text. + COMPRESSED_LOWERCASE_HEX_STRING => Ok(Value::String( + self.read_table_string(compression::LOWERCASE_HEX)?, + )), + COMPRESSED_UPPERCASE_HEX_STRING => Ok(Value::String( + self.read_table_string(compression::UPPERCASE_HEX)?, + )), + COMPRESSED_DATE_TIME_STRING => Ok(Value::String( + self.read_table_string(compression::DATE_TIME)?, + )), + PACKED_4BIT_STRING => Ok(Value::String(self.read_packed_string( + 4, + true, + FieldWidth::U8, + )?)), + PACKED_5BIT_STRING => Ok(Value::String(self.read_packed_string( + 5, + true, + FieldWidth::U8, + )?)), + PACKED_6BIT_STRING => Ok(Value::String(self.read_packed_string( + 6, + true, + FieldWidth::U8, + )?)), + PACKED_7BIT_STRING_LENGTH1 => Ok(Value::String(self.read_packed_string( + 7, + false, + FieldWidth::U8, + )?)), + PACKED_7BIT_STRING_LENGTH2 => Ok(Value::String(self.read_packed_string( + 7, + false, + FieldWidth::U16, + )?)), + + // The GUID *value* is 16 bytes interpreted as a .NET `Guid` + // (mixed-endian) and rendered as the canonical lowercase text. This + // differs from the GUID *strings* above, which are a straight hex + // dump. JSON has no GUID type, so it maps to a string. + GUID => Ok(Value::String(self.read_guid_value()?)), + + // Binary blobs have no JSON representation; the raw bytes are mapped + // to a standard base64 string (the conventional JSON byte encoding). + BINARY_1BYTE_LENGTH => self.read_binary(FieldWidth::U8), + BINARY_2BYTE_LENGTH => self.read_binary(FieldWidth::U16), + BINARY_4BYTE_LENGTH => self.read_binary(FieldWidth::U32), + + // Uniform number arrays: a typed, marker-shared sequence of bare + // numbers (`ArrNumC*`) or a sequence of such arrays (`ArrArrNumC*`). + ARR_NUM_C1 => self.read_uniform_number_array(FieldWidth::U8), + ARR_NUM_C2 => self.read_uniform_number_array(FieldWidth::U16), + ARR_ARR_NUM_C1C1 => self.read_uniform_array_of_number_arrays(FieldWidth::U8), + ARR_ARR_NUM_C2C2 => self.read_uniform_array_of_number_arrays(FieldWidth::U16), + + // User strings reference an external string dictionary that the + // Cosmos data plane does not supply, so they cannot be resolved to + // text. We still consume the id bytes (1-byte vs 2-byte form) so the + // error reflects the correct id, then report it as unsupported. + m if (USER_STRING_1BYTE_MIN..USER_STRING_1BYTE_MAX).contains(&m) => { + let id = usize::from(m - USER_STRING_1BYTE_MIN); + Err(BinaryError::UnsupportedUserString { id }) + } + m if (USER_STRING_2BYTE_MIN..USER_STRING_2BYTE_MAX).contains(&m) => { + // Two-byte form: id = one_byte_count + low_byte + (high * 256), + // where `high` is the marker's offset from USER_STRING_2BYTE_MIN + // and `low` is the byte that follows. Mirrors .NET + // TryGetUserStringId. + let one_byte_count = usize::from(USER_STRING_1BYTE_MAX - USER_STRING_1BYTE_MIN); + let low = usize::from(self.read_u8()?); + let high = usize::from(m - USER_STRING_2BYTE_MIN); + let id = one_byte_count + low + high * 256; + Err(BinaryError::UnsupportedUserString { id }) + } + + // Reference strings point back to an earlier string's byte offset in + // the buffer. The offset width grows with the marker (1..4 bytes). + STR_R1 => { + let target = usize::from(self.read_u8()?); + self.resolve_reference(target) + } + STR_R2 => { + let target = usize::from(self.read_u16_le()?); + self.resolve_reference(target) + } + STR_R3 => { + let target = self.read_u24_le()? as usize; + self.resolve_reference(target) + } + STR_R4 => { + let target = self.read_u32_le()? as usize; + self.resolve_reference(target) + } + + // Any other byte is not a valid type marker. + other => Err(BinaryError::InvalidMarker { + marker: other, + offset, + }), + } + } + + /// Attempts to read the next value as a native scalar token, consuming it + /// only when it is one of the cheaply-decodable forms. + /// + /// Returns `Ok(Some(_))` (advancing the cursor) for `null`, booleans, every + /// literal/fixed-width/extended number, system strings, and plain + /// UTF-8 strings (encoded-length and `StrL1/2/4`). Returns `Ok(None)` + /// **without advancing** for any other marker -- containers and the exotic + /// string/number forms -- which the deserializer handles via a container + /// stream or the [`read_value`](Self::read_value) fallback respectively. + pub(super) fn try_read_native_scalar(&mut self) -> Result>> { + let offset = self.pos; + let marker = self.peek_u8()?; + let token = match marker { + NULL => ScalarToken::Null, + FALSE => ScalarToken::Bool(false), + TRUE => ScalarToken::Bool(true), + + // Literal integer: the value is encoded in the marker itself. + m if (LITERAL_INT_MIN..LITERAL_INT_MAX).contains(&m) => ScalarToken::I64(i64::from(m)), + + // Fixed-width and extended numbers all project to a JSON number. + NUMBER_UINT8 => { + self.pos += 1; + return Ok(Some(ScalarToken::I64(i64::from(self.read_u8()?)))); + } + NUMBER_INT16 | INT16 => { + self.pos += 1; + return Ok(Some(ScalarToken::I64(i64::from(self.read_i16_le()?)))); + } + NUMBER_INT32 | INT32 => { + self.pos += 1; + return Ok(Some(ScalarToken::I64(i64::from(self.read_i32_le()?)))); + } + NUMBER_INT64 | INT64 => { + self.pos += 1; + return Ok(Some(ScalarToken::I64(self.read_i64_le()?))); + } + NUMBER_UINT64 => { + self.pos += 1; + let v = self.read_u64_le()?; + // Prefer the signed projection when it fits, matching the + // `Value` decoder's number handling. + return Ok(Some(match i64::try_from(v) { + Ok(i) => ScalarToken::I64(i), + Err(_) => ScalarToken::U64(v), + })); + } + NUMBER_DOUBLE | FLOAT64 => { + self.pos += 1; + return Ok(Some(ScalarToken::F64(self.read_f64_le()?))); + } + INT8 => { + self.pos += 1; + return Ok(Some(ScalarToken::I64(i64::from(self.read_i8()?)))); + } + UINT32 => { + self.pos += 1; + return Ok(Some(ScalarToken::I64(i64::from(self.read_u32_le()?)))); + } + FLOAT32 => { + self.pos += 1; + return Ok(Some(ScalarToken::F64(f64::from(self.read_f32_le()?)))); + } + + // 1-byte system string: borrow the static dictionary entry. + m if (SYSTEM_STRING_1BYTE_MIN..SYSTEM_STRING_1BYTE_MAX).contains(&m) => { + let s = system_string_for_marker(m) + .ok_or(BinaryError::InvalidMarker { marker: m, offset })?; + self.pos += 1; + return Ok(Some(ScalarToken::Str(s))); + } + + // Encoded-length string: length carried in the marker. + m if (ENCODED_STRING_LENGTH_MIN..ENCODED_STRING_LENGTH_MAX).contains(&m) => { + self.pos += 1; + let len = usize::from(m & ENCODED_STRING_LENGTH_MASK); + return Ok(Some(ScalarToken::Str(self.read_str_slice(len, offset)?))); + } + + // Length-prefixed strings. + STR_L1 => { + self.pos += 1; + let len = usize::from(self.read_u8()?); + return Ok(Some(ScalarToken::Str(self.read_str_slice(len, offset)?))); + } + STR_L2 => { + self.pos += 1; + let len = usize::from(self.read_u16_le()?); + return Ok(Some(ScalarToken::Str(self.read_str_slice(len, offset)?))); + } + STR_L4 => { + self.pos += 1; + let len = self.read_u32_le()? as usize; + return Ok(Some(ScalarToken::Str(self.read_str_slice(len, offset)?))); + } + + // Not a native scalar: leave the cursor untouched. + _ => return Ok(None), + }; + // Single-byte tokens (null/bool/literal-int): consume just the marker. + self.pos += 1; + Ok(Some(token)) + } + + /// Borrows a `len`-byte UTF-8 slice directly from the buffer, advancing the + /// cursor. `marker_offset` positions a UTF-8 error. + fn read_str_slice(&mut self, len: usize, marker_offset: usize) -> Result<&'a str> { + let bytes = self.read_bytes(len)?; + std::str::from_utf8(bytes).map_err(|_| BinaryError::InvalidUtf8 { + offset: marker_offset, + }) + } + + /// Attempts to read a standard array/object container header, consuming the + /// marker and length/count prefix only when the marker is one of the + /// streamable container forms (`Arr0/1/L*/LC*`, `Obj0/1/L*/LC*`). + /// + /// Returns `Ok(None)` **without advancing** for anything else (including the + /// uniform number-array markers, which have no per-element framing and are + /// handled via the [`read_value`](Self::read_value) fallback). + pub(super) fn read_container_header(&mut self) -> Result> { + let marker = self.peek_u8()?; + let header = match marker { + ARR0 => ContainerHeader::Array(Frame { + count: Some(0), + end: self.pos + 1, + exact_end: true, + }), + ARR1 => { + // A one-element array with no length prefix: the element + // follows immediately, so stream by count and let the element + // read advance the cursor. + self.pos += 1; + return Ok(Some(ContainerHeader::Array(Frame { + count: Some(1), + end: self.buf.len(), + exact_end: false, + }))); + } + ARR_L1 | ARR_L2 | ARR_L4 | ARR_LC1 | ARR_LC2 | ARR_LC4 => { + let (count, end) = self.read_container_frame(marker, [ARR_L1, ARR_L2, ARR_L4])?; + return Ok(Some(ContainerHeader::Array(Frame { + count, + end, + exact_end: true, + }))); + } + OBJ0 => ContainerHeader::Object(Frame { + count: Some(0), + end: self.pos + 1, + exact_end: true, + }), + OBJ1 => { + self.pos += 1; + return Ok(Some(ContainerHeader::Object(Frame { + count: Some(1), + end: self.buf.len(), + exact_end: false, + }))); + } + OBJ_L1 | OBJ_L2 | OBJ_L4 | OBJ_LC1 | OBJ_LC2 | OBJ_LC4 => { + let (count, end) = self.read_container_frame(marker, [OBJ_L1, OBJ_L2, OBJ_L4])?; + return Ok(Some(ContainerHeader::Object(Frame { + count, + end, + exact_end: true, + }))); + } + _ => return Ok(None), + }; + // Empty-container markers (`Arr0`/`Obj0`): consume just the marker. + self.pos += 1; + Ok(Some(header)) + } + + /// Parses the length (and optional count) prefix shared by the `L*`/`LC*` + /// array and object markers, consuming the marker and prefixes. `l_markers` + /// are the three length-only markers (width 1/2/4) for this container kind; + /// the `LC*` (length + count) markers sit three positions above their `L*` + /// counterparts. + fn read_container_frame( + &mut self, + marker: u8, + l_markers: [u8; 3], + ) -> Result<(Option, usize)> { + let [l1, l2, l4] = l_markers; + let width = if marker == l1 || marker == l1 + 3 { + FieldWidth::U8 + } else if marker == l2 || marker == l2 + 3 { + FieldWidth::U16 + } else { + FieldWidth::U32 + }; + let has_count = marker == l1 + 3 || marker == l2 + 3 || marker == l4 + 3; + self.pos += 1; // consume the marker + let payload_len = self.read_len(width)?; + let count = if has_count { + Some(self.read_len(width)?) + } else { + None + }; + let end = self.bounded_end(payload_len)?; + Ok((count, end)) + } + + /// Reads a 1-, 2-, or 4-byte little-endian length or count field. + fn read_len(&mut self, width: FieldWidth) -> Result { + match width { + FieldWidth::U8 => Ok(usize::from(self.read_u8()?)), + FieldWidth::U16 => Ok(usize::from(self.read_u16_le()?)), + FieldWidth::U32 => Ok(self.read_u32_le()? as usize), + } + } + + /// Computes the absolute end offset of a `payload_len`-byte payload starting + /// at the current position, verifying it fits within the buffer. + fn bounded_end(&self, payload_len: usize) -> Result { + let end = self + .pos + .checked_add(payload_len) + .ok_or(BinaryError::InvalidLength { + detail: "container length overflows the address space", + })?; + if end > self.buf.len() { + return Err(BinaryError::UnexpectedEof { + needed: end - self.buf.len(), + }); + } + Ok(end) + } + + /// Reads a length-prefixed array body. `width` is the length/count prefix + /// width in bytes (1, 2, or 4); when `has_count` is set, a count field of + /// the same width follows the length and is validated against the number of + /// items actually decoded. + fn read_array_value( + &mut self, + width: FieldWidth, + has_count: bool, + depth: usize, + ) -> Result { + let payload_len = self.read_len(width)?; + let count = if has_count { + Some(self.read_len(width)?) + } else { + None + }; + let end = self.bounded_end(payload_len)?; + + let mut items = Vec::new(); + while self.pos < end { + let item = self.read_value(depth + 1)?; + if self.pos > end { + return Err(BinaryError::InvalidLength { + detail: "array element extends past the array's declared length", + }); + } + items.push(item); + } + + if let Some(expected) = count { + if items.len() != expected { + return Err(BinaryError::InvalidLength { + detail: "array item count does not match its declared count", + }); + } + } + Ok(Value::Array(items)) + } + + /// Reads a length-prefixed object body, mirroring [`read_array_value`] but + /// decoding name/value member pairs. The declared count (when present) is + /// the number of members, validated against the number actually decoded. + /// + /// [`read_array_value`]: Reader::read_array_value + fn read_object_value( + &mut self, + width: FieldWidth, + has_count: bool, + depth: usize, + ) -> Result { + let payload_len = self.read_len(width)?; + let count = if has_count { + Some(self.read_len(width)?) + } else { + None + }; + let end = self.bounded_end(payload_len)?; + + let mut map = Map::new(); + let mut members = 0usize; + while self.pos < end { + let (name, value) = self.read_member(depth + 1)?; + if self.pos > end { + return Err(BinaryError::InvalidLength { + detail: "object member extends past the object's declared length", + }); + } + map.insert(name, value); + members += 1; + } + + if let Some(expected) = count { + if members != expected { + return Err(BinaryError::InvalidLength { + detail: "object member count does not match its declared count", + }); + } + } + Ok(Value::Object(map)) + } + + /// Reads one object member: a string name followed by its value. The name + /// must decode to a string; any other form is reported as an + /// [`BinaryError::InvalidMarker`] at the name's marker offset, since a + /// non-string is not valid in a property-name position. + fn read_member(&mut self, depth: usize) -> Result<(String, Value)> { + let name_offset = self.pos; + // Capture the name's type marker before decoding so a non-string name + // can be reported without re-indexing the buffer. + let name_marker = self.peek_u8()?; + let name = self.read_value(depth)?; + let name = match name { + Value::String(s) => s, + _ => { + return Err(BinaryError::InvalidMarker { + marker: name_marker, + offset: name_offset, + }); + } + }; + let value = self.read_value(depth)?; + Ok((name, value)) + } + + /// Reads a GUID string: the 16-byte encoded form (following the marker) + /// expanded to the canonical `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` hex text. + /// + /// This is a straight sequential hex dump of the 16 bytes (not the .NET + /// `Guid` mixed-endian layout), mirroring .NET `DecodeGuidStringValue`. + /// `uppercase` selects the hex case. When `quoted`, the original JSON string + /// included literal quote characters, so they are re-added around the text. + fn read_guid_string(&mut self, uppercase: bool, quoted: bool) -> Result { + const DASH_POSITIONS: [usize; 4] = [4, 6, 8, 10]; + let bytes = self.read_array::<16>()?; + let digits = if uppercase { + compression::UPPERCASE_HEX + } else { + compression::LOWERCASE_HEX + }; + + // 36 hex/dash chars, plus the two optional surrounding quotes. + let mut out = String::with_capacity(if quoted { 38 } else { 36 }); + if quoted { + out.push('"'); + } + for (index, byte) in bytes.iter().enumerate() { + // A dash precedes the byte at each group boundary (after bytes 4, 6, + // 8, and 10), producing the 8-4-4-4-12 grouping. + if DASH_POSITIONS.contains(&index) { + out.push('-'); + } + out.push(char::from(digits[usize::from(byte >> 4)])); + out.push(char::from(digits[usize::from(byte & 0x0F)])); + } + if quoted { + out.push('"'); + } + Ok(out) + } + + /// Reads a base64 string. The inline payload is the **raw** (already + /// base64-decoded) bytes; this re-encodes them to the original base64 text. + /// + /// `length_width` is the width (1 or 2 bytes, little-endian) of the + /// group-count prefix that precedes a 1-byte padding field; `url_safe` + /// selects the URL-safe alphabet. The group count times four is the padded + /// base64 length, and the padding byte records how many `=` characters the + /// original text carried (or, when greater than 2, that padding was omitted, + /// in which case the encoded length shrinks accordingly). Mirrors .NET + /// `ConvertBytesToBase64String`. + fn read_base64_string(&mut self, length_width: FieldWidth, url_safe: bool) -> Result { + let groups = self.read_len(length_width)?; + let padding = self.read_u8()?; + + // Padded length is always a multiple of four; `effective_padding` is the + // literal `=` count (0..=2), or `!padding` when padding was omitted. + let padded_len = groups.checked_mul(4).ok_or(BinaryError::InvalidLength { + detail: "base64 length overflows the address space", + })?; + let omitted = padding > 2; + let effective_padding = usize::from(if omitted { !padding } else { padding }); + let final_len = padded_len + .checked_sub(if omitted { effective_padding } else { 0 }) + .ok_or(BinaryError::InvalidLength { + detail: "base64 padding exceeds the encoded length", + })?; + let raw_len = padded_len + .checked_sub(effective_padding) + .ok_or(BinaryError::InvalidLength { + detail: "base64 padding exceeds the encoded length", + })? + .checked_mul(3) + .ok_or(BinaryError::InvalidLength { + detail: "base64 length overflows the address space", + })? + / 4; + + let raw = self.read_bytes(raw_len)?; + let engine = if url_safe { + &base64::engine::general_purpose::URL_SAFE + } else { + &base64::engine::general_purpose::STANDARD + }; + let mut encoded = engine.encode(raw); + + // The padded encoding may carry trailing `=`; keep only the original + // text length (this drops padding the service chose to omit). + if final_len > encoded.len() { + return Err(BinaryError::InvalidLength { + detail: "base64 encoded length is shorter than the declared length", + }); + } + encoded.truncate(final_len); + Ok(encoded) + } + + /// Reads a 4-bit table-compressed string (lowercase hex, uppercase hex, or + /// date-time). A 1-byte prefix gives the decoded character count `len`; the + /// payload is `ceil(len / 2)` bytes, each holding two 4-bit indices into + /// `table` (low nibble first, then high nibble), mirroring .NET + /// `Decode4BitCharacterStringValue`. + fn read_table_string(&mut self, table: &[u8; 16]) -> Result { + let len = usize::from(self.read_u8()?); + let byte_count = len.div_ceil(2); + let bytes = self.read_bytes(byte_count)?; + + let mut out = String::with_capacity(len); + for (index, &byte) in bytes.iter().enumerate() { + // Low nibble is the first character of the pair. + out.push(char::from(table[usize::from(byte & 0x0F)])); + // The final byte of an odd-length string contributes only its low + // nibble; its high nibble is padding and must be zero. + let produced_low_only = index == byte_count - 1 && len % 2 == 1; + if produced_low_only { + if byte >> 4 != 0 { + return Err(BinaryError::InvalidLength { + detail: "compressed string has non-zero padding nibble", + }); + } + } else { + out.push(char::from(table[usize::from(byte >> 4)])); + } + } + Ok(out) + } + + /// Reads a packed N-bit compressed string. A length prefix (`length_width` + /// bytes, little-endian) gives the decoded character count `len`; the + /// payload is `ceil(len * bits / 8)` bytes holding `len` little-endian + /// `bits`-wide values. When `has_base`, a 1-byte base character precedes the + /// payload and is added to every unpacked value. Mirrors .NET + /// `DecodeCompressedStringValue`. + fn read_packed_string( + &mut self, + bits: u32, + has_base: bool, + length_width: FieldWidth, + ) -> Result { + let len = self.read_len(length_width)?; + let base = if has_base { self.read_u8()? } else { 0 }; + let byte_count = (len * bits as usize).div_ceil(8); + let bytes = self.read_bytes(byte_count)?; + + // Unpack `len` values of `bits` bits each, least-significant bit first, + // from a contiguous little-endian bit stream. + let mask = (1u32 << bits) - 1; + let mut out = String::with_capacity(len); + let mut bit_pos = 0usize; + for _ in 0..len { + let byte_index = bit_pos / 8; + let bit_offset = bit_pos % 8; + // A value spans at most two bytes for bits <= 8; read a little-endian + // 16-bit window so the value is always fully covered. + let lo = u32::from(bytes[byte_index]); + let hi = bytes.get(byte_index + 1).map_or(0, |&b| u32::from(b)); + let window = lo | (hi << 8); + let value = (window >> bit_offset) & mask; + // Each unpacked value is a byte; `+ base` yields the ASCII char. + let ch = (value as u8).wrapping_add(base); + out.push(char::from(ch)); + bit_pos += bits as usize; + } + Ok(out) + } + + /// Reads a GUID value: 16 bytes interpreted as a .NET `Guid` and rendered as + /// the canonical lowercase `xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx` text. + /// + /// The .NET `Guid` memory layout is mixed-endian: the first three groups + /// (4, 2, 2 bytes) are little-endian integers, while the final 8 bytes are + /// taken in order. This matches `Guid`'s in-memory representation that the + /// service writes, and differs from the GUID *string* forms (which dump the + /// 16 bytes sequentially). + fn read_guid_value(&mut self) -> Result { + let b = self.read_array::<16>()?; + Ok(format!( + "{:02x}{:02x}{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}-{:02x}{:02x}{:02x}{:02x}{:02x}{:02x}", + // Data1 (little-endian u32). + b[3], b[2], b[1], b[0], + // Data2 (little-endian u16). + b[5], b[4], + // Data3 (little-endian u16). + b[7], b[6], + // Data4 (sequential). + b[8], b[9], b[10], b[11], b[12], b[13], b[14], b[15], + )) + } + + /// Reads a binary blob: a `length_width`-byte little-endian length followed + /// by that many raw bytes, mapped to a standard base64 [`Value::String`] + /// (JSON has no native binary type). + fn read_binary(&mut self, length_width: FieldWidth) -> Result { + let len = self.read_len(length_width)?; + let bytes = self.read_bytes(len)?; + let encoded = base64::engine::general_purpose::STANDARD.encode(bytes); + Ok(Value::String(encoded)) + } + + /// Reads one **bare** number value of the given uniform-array item type. + /// + /// Inside a uniform number array the item type marker is shared, so each + /// element is just the little-endian value with no per-item marker. + /// `marker_offset` is the offset of the array's item-type marker, used to + /// report an unsupported item type. Mirrors the uniform-array branch of + /// .NET `TryGetNumberValue`. + /// + /// A uniform-array item type must be one of the **extended** + /// `INT*`/`UINT*`/`FLOAT*` markers. The self-describing `NUMBER_*` markers + /// never appear as a uniform-array item type, so they are rejected here + /// rather than accepted through the shared [`read_number_value`] decoder, + /// keeping the decoder no more permissive than the service. + /// + /// [`read_number_value`]: Self::read_number_value + fn read_bare_number(&mut self, item_marker: u8, marker_offset: usize) -> Result { + match item_marker { + INT8 | UINT8 | INT16 | INT32 | INT64 | UINT32 | FLOAT32 | FLOAT64 => { + self.read_number_value(item_marker, marker_offset) + } + other => Err(BinaryError::InvalidMarker { + marker: other, + offset: marker_offset, + }), + } + } + + /// Decodes the little-endian payload of a fixed-width number `marker` into a + /// JSON number [`Value`]. The marker has already been consumed; the payload + /// is read from the current position. `marker_offset` locates the marker for + /// error reporting. + /// + /// Shared by [`read_value`](Self::read_value) (self-describing top-level + /// numbers) and [`read_bare_number`](Self::read_bare_number) (uniform-array + /// items using the extended `INT*`/`UINT*`/`FLOAT*` markers). Each caller + /// only forwards the markers valid in its context; any other marker is + /// reported as [`BinaryError::InvalidMarker`]. + fn read_number_value(&mut self, marker: u8, marker_offset: usize) -> Result { + match marker { + NUMBER_UINT8 | UINT8 => Ok(int_value(i64::from(self.read_u8()?))), + NUMBER_INT16 | INT16 => Ok(int_value(i64::from(self.read_i16_le()?))), + NUMBER_INT32 | INT32 => Ok(int_value(i64::from(self.read_i32_le()?))), + NUMBER_INT64 | INT64 => Ok(int_value(self.read_i64_le()?)), + NUMBER_UINT64 => Ok(uint_value(self.read_u64_le()?)), + UINT32 => Ok(int_value(i64::from(self.read_u32_le()?))), + INT8 => Ok(int_value(i64::from(self.read_i8()?))), + NUMBER_DOUBLE | FLOAT64 => double_value(self.read_f64_le()?), + FLOAT32 => double_value(f64::from(self.read_f32_le()?)), + other => Err(BinaryError::InvalidMarker { + marker: other, + offset: marker_offset, + }), + } + } + + /// Reads a uniform number array (`ArrNumC1`/`ArrNumC2`). The prefix is the + /// shared item-type marker followed by a `count_width`-byte little-endian + /// item count; the body is that many bare numbers of the shared type. + fn read_uniform_number_array(&mut self, count_width: FieldWidth) -> Result { + let item_marker_offset = self.pos; + let item_marker = self.read_u8()?; + let count = self.read_len(count_width)?; + + let mut items = Vec::with_capacity(count.min(1024)); + for _ in 0..count { + items.push(self.read_bare_number(item_marker, item_marker_offset)?); + } + Ok(Value::Array(items)) + } + + /// Reads a uniform array of uniform number arrays (`ArrArrNumC1C1` / + /// `ArrArrNumC2C2`). The prefix is the inner-array type marker, the shared + /// number item-type marker, the per-inner-array number count, then the outer + /// array count (each a `count_width`-byte little-endian field). The body is + /// `outer_count` inner arrays, each holding `inner_count` bare numbers. + fn read_uniform_array_of_number_arrays(&mut self, count_width: FieldWidth) -> Result { + // Inner-array type marker: `ArrNumC1` for `ArrArrNumC1C1`, `ArrNumC2` + // for `ArrArrNumC2C2`. Validate it matches the outer count width so a + // malformed marker in this slot is rejected rather than silently + // decoded as a valid array. + let inner_array_marker_offset = self.pos; + let inner_array_marker = self.read_u8()?; + let expected_inner_array_marker = match count_width { + FieldWidth::U8 => ARR_NUM_C1, + FieldWidth::U16 => ARR_NUM_C2, + // Nested uniform number arrays only use the 1- and 2-byte count + // forms; no 4-byte (`U32`) variant exists. + FieldWidth::U32 => ARR_NUM_C2, + }; + if inner_array_marker != expected_inner_array_marker { + return Err(BinaryError::InvalidMarker { + marker: inner_array_marker, + offset: inner_array_marker_offset, + }); + } + + let item_marker_offset = self.pos; + let item_marker = self.read_u8()?; + let inner_count = self.read_len(count_width)?; + let outer_count = self.read_len(count_width)?; + + // Empty inner arrays read zero body bytes, so bound `outer_count` by a + // buffer-proportional budget to keep decode output O(input). + if inner_count == 0 && outer_count > empty_element_budget(self.buf.len()) { + return Err(BinaryError::InvalidLength { + detail: "uniform array of empty number arrays declares more elements than the input can justify", + }); + } + + let mut outer = Vec::with_capacity(outer_count.min(1024)); + for _ in 0..outer_count { + let mut inner = Vec::with_capacity(inner_count.min(1024)); + for _ in 0..inner_count { + inner.push(self.read_bare_number(item_marker, item_marker_offset)?); + } + outer.push(Value::Array(inner)); + } + Ok(Value::Array(outer)) + } + + /// Resolves a reference string ([`STR_R1`]-[`STR_R4`]) whose `target` is an + /// absolute byte offset into the buffer (the same frame as [`Reader::pos`], + /// where the [`PREAMBLE`](super::PREAMBLE) is offset `0`). + /// + /// The target must lie within the buffer and hold a string that is **not** + /// itself a reference string; this mirrors .NET's + /// `IsValidReferenceStringTarget` and makes reference chains (and therefore + /// cycles) impossible, so the lookup terminates without recursion guards. + /// The referenced string is decoded from a fresh cursor positioned at + /// `target`, leaving `self` untouched. + /// + /// [`STR_R1`]: super::markers::STR_R1 + /// [`STR_R4`]: super::markers::STR_R4 + fn resolve_reference(&self, target: usize) -> Result { + let marker = *self + .buf + .get(target) + .ok_or(BinaryError::UnresolvedReference { target })?; + + // The target must be a string, and must not itself be a reference + // string (no chains/cycles). + let is_string = (SYSTEM_STRING_1BYTE_MIN..NUMBER_UINT64).contains(&marker); + let is_reference = (STR_R1..=STR_R4).contains(&marker); + if !is_string || is_reference { + return Err(BinaryError::UnresolvedReference { target }); + } + + // Decode the referenced string from its own cursor. It is a single + // string value, so depth does not grow and a bare reader suffices. + // The target is guaranteed a non-reference string, so this sub-read + // resolves no further references and needs no shared budget. + let mut sub = Reader::new(self.buf, target); + let value = sub.read_value(0)?; + + // Charge the materialized text against the shared reference-expansion + // budget so many references to one large string cannot amplify a + // size-`S` buffer into O(S^2) aggregate output. + if let Value::String(s) = &value { + let remaining = self.ref_budget.get(); + let cost = s.len(); + if cost > remaining { + return Err(BinaryError::InvalidLength { + detail: "reference-string expansion exceeds the decode budget", + }); + } + self.ref_budget.set(remaining - cost); + } + Ok(value) + } +} + +/// Wraps a signed integer that fits in `i64` as a JSON number. +fn int_value(n: i64) -> Value { + Value::Number(n.into()) +} + +/// Wraps an unsigned 64-bit integer as a JSON number (used for `UInt64` values +/// that may exceed `i64::MAX`). +fn uint_value(n: u64) -> Value { + Value::Number(n.into()) +} + +/// Wraps a `double` as a JSON number, rejecting non-finite values that JSON +/// cannot represent. +fn double_value(n: f64) -> Result { + serde_json::Number::from_f64(n) + .map(Value::Number) + .ok_or(BinaryError::InvalidNumber { + detail: "non-finite double (NaN or infinity)", + }) +} + +/// Character lookup tables for the 4-bit table-compressed string forms. +/// +/// Each table maps a 4-bit nibble (`0x0`-`0xF`) to one ASCII byte, transcribed +/// verbatim from the .NET `StringCompressionLookupTables` `list` arrays +/// (`JsonBinaryEncoding.Chars.cs`). +mod compression { + /// Lowercase hexadecimal digits (`CompressedLowercaseHexString`). + pub(super) const LOWERCASE_HEX: &[u8; 16] = b"0123456789abcdef"; + + /// Uppercase hexadecimal digits (`CompressedUppercaseHexString`). + pub(super) const UPPERCASE_HEX: &[u8; 16] = b"0123456789ABCDEF"; + + /// Date-time character set (`CompressedDateTimeString`): space, digits, and + /// the `:`, `-`, `.`, `T`, `Z` separators used in ISO-8601 timestamps. + pub(super) const DATE_TIME: &[u8; 16] = b" 0123456789:-.TZ"; +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary_json::markers; + use crate::binary_json::vectors::golden_vectors; + use crate::binary_json::PREAMBLE; + + /// Helper: prepend the preamble to a value's marker+payload bytes. + fn buf(value_bytes: &[u8]) -> Vec { + let mut v = vec![PREAMBLE]; + v.extend_from_slice(value_bytes); + v + } + + /// The decoder reproduces every golden vector's JSON. + #[test] + fn decodes_golden_corpus() { + for vector in golden_vectors() { + let decoded = decode(&vector.binary).unwrap_or_else(|e| { + panic!("case {}: decode failed: {e}", vector.name); + }); + let expected: Value = serde_json::from_str(&vector.json).unwrap(); + assert_eq!(decoded, expected, "case {}", vector.name); + } + } + + #[test] + fn decodes_literal_integers() { + for n in 0u8..32 { + let value = decode(&buf(&[n])).unwrap(); + assert_eq!(value, serde_json::json!(n), "literal int {n}"); + } + } + + #[test] + fn rejects_non_finite_double() { + let mut nan = vec![markers::NUMBER_DOUBLE]; + nan.extend_from_slice(&f64::NAN.to_le_bytes()); + assert_eq!( + decode(&buf(&nan)), + Err(BinaryError::InvalidNumber { + detail: "non-finite double (NaN or infinity)", + }), + ); + } + + #[test] + fn rejects_missing_preamble() { + assert_eq!( + decode(b"{}"), + Err(BinaryError::MissingPreamble { found: b'{' }), + ); + } + + #[test] + fn rejects_empty_buffer() { + assert_eq!(decode(&[]), Err(BinaryError::UnexpectedEof { needed: 1 })); + } + + #[test] + fn rejects_trailing_bytes() { + // Preamble + `null` marker + one extra byte. + assert_eq!( + decode(&[PREAMBLE, markers::NULL, 0x00]), + Err(BinaryError::TrailingBytes { remaining: 1 }), + ); + } + + #[test] + fn rejects_truncated_number() { + // Int32 marker but only two payload bytes present. + assert_eq!( + decode(&[PREAMBLE, markers::NUMBER_INT32, 0x01, 0x02]), + Err(BinaryError::UnexpectedEof { needed: 2 }), + ); + } + + #[test] + fn rejects_truncated_string() { + // StrL1 claims 5 bytes but only 2 follow. + assert_eq!( + decode(&[PREAMBLE, markers::STR_L1, 5, b'h', b'i']), + Err(BinaryError::UnexpectedEof { needed: 3 }), + ); + } + + #[test] + fn rejects_invalid_utf8() { + // StrL1 of length 1 carrying a lone continuation byte (0xFF). + assert!(matches!( + decode(&[PREAMBLE, markers::STR_L1, 1, 0xFF]), + Err(BinaryError::InvalidUtf8 { .. }), + )); + } + + #[test] + fn reserved_and_invalid_markers_are_rejected() { + // 0xFF is the explicit Invalid marker; 0xD4 is a reserved/empty slot. + // Neither has a value form, so both report InvalidMarker at the marker + // offset (index 1, just past the preamble). + assert_eq!( + decode(&[PREAMBLE, markers::INVALID]), + Err(BinaryError::InvalidMarker { + marker: markers::INVALID, + offset: 1, + }), + ); + assert_eq!( + decode(&[PREAMBLE, 0xD4]), + Err(BinaryError::InvalidMarker { + marker: 0xD4, + offset: 1, + }), + ); + } + + #[test] + fn rejects_non_finite_extended_float() { + // Float32 carrying infinity has no JSON representation. + let mut inf = vec![markers::FLOAT32]; + inf.extend_from_slice(&f32::INFINITY.to_le_bytes()); + assert_eq!( + decode(&buf(&inf)), + Err(BinaryError::InvalidNumber { + detail: "non-finite double (NaN or infinity)", + }), + ); + } + + #[test] + fn float16_and_extended_uint8_have_no_json_node() { + // Float16 (0xCF) and the extended UInt8 (0xD7) map to no JSON node type + // in the service, so the decoder rejects them as invalid markers. + assert_eq!( + decode(&[PREAMBLE, markers::FLOAT16, 0x00, 0x00]), + Err(BinaryError::InvalidMarker { + marker: markers::FLOAT16, + offset: 1, + }), + ); + assert_eq!( + decode(&[PREAMBLE, markers::UINT8, 0x00]), + Err(BinaryError::InvalidMarker { + marker: markers::UINT8, + offset: 1, + }), + ); + } + + #[test] + fn rejects_truncated_extended_number() { + // Int32 marker with only two payload bytes present. + assert_eq!( + decode(&[PREAMBLE, markers::INT32, 0x01, 0x02]), + Err(BinaryError::UnexpectedEof { needed: 2 }), + ); + } + + #[test] + fn rejects_truncated_guid_string() { + // GUID string marker claims 16 encoded bytes but only 4 follow. + let mut bytes = vec![markers::LOWERCASE_GUID_STRING]; + bytes.extend_from_slice(&[0x01, 0x02, 0x03, 0x04]); + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::UnexpectedEof { needed: 12 }), + ); + } + + /// Builds a Base64StringLength1 token: marker, 1-byte group count, 1-byte + /// padding, then the raw bytes. + fn base64_len1(groups: u8, padding: u8, raw: &[u8]) -> Vec { + let mut v = vec![markers::BASE64_STRING_LENGTH1, groups, padding]; + v.extend_from_slice(raw); + v + } + + #[test] + fn rejects_truncated_base64_string() { + // Declares one group (3 raw bytes) but only one byte follows. + assert_eq!( + decode(&buf(&base64_len1(1, 0, b"f"))), + Err(BinaryError::UnexpectedEof { needed: 2 }), + ); + } + + #[test] + fn rejects_table_compressed_string_with_padding_nibble() { + // Odd length 1 but the (only) byte's high nibble is non-zero padding. + let bad = [markers::COMPRESSED_LOWERCASE_HEX_STRING, 1, 0x1F]; + assert!(matches!( + decode(&buf(&bad)), + Err(BinaryError::InvalidLength { .. }), + )); + } + + #[test] + fn rejects_truncated_compressed_string() { + // 7-bit length 4 needs ceil(4*7/8) = 4 payload bytes; only one follows. + assert_eq!( + decode(&[PREAMBLE, markers::PACKED_7BIT_STRING_LENGTH1, 4, 0x00]), + Err(BinaryError::UnexpectedEof { needed: 3 }), + ); + } + + #[test] + fn rejects_truncated_guid_value() { + // GUID value needs 16 bytes; only four follow. + let mut bytes = vec![markers::GUID]; + bytes.extend_from_slice(&[0x01, 0x02, 0x03, 0x04]); + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::UnexpectedEof { needed: 12 }), + ); + } + + #[test] + fn rejects_truncated_binary_blob() { + // Declares 10 bytes but only 2 follow. + let mut bytes = vec![markers::BINARY_1BYTE_LENGTH, 10]; + bytes.extend_from_slice(&[0x01, 0x02]); + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::UnexpectedEof { needed: 8 }), + ); + } + + #[test] + fn rejects_uniform_array_with_invalid_item_type() { + // The shared item-type marker (here NULL) is not a number type. + let bytes = [markers::ARR_NUM_C1, markers::NULL, 1, 0x00]; + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::InvalidMarker { + marker: markers::NULL, + // Marker sits at index 2: preamble (0) + ArrNumC1 (1) + item (2). + offset: 2, + }), + ); + } + + #[test] + fn rejects_uniform_array_with_self_describing_number_item_type() { + // A uniform-array item type must be an extended `INT*`/`UINT*`/`FLOAT*` + // marker. The self-describing `NUMBER_*` markers never appear in this + // position, so `NUMBER_UINT8` must be rejected rather than decoded. + let bytes = [markers::ARR_NUM_C1, markers::NUMBER_UINT8, 1, 0x00]; + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::InvalidMarker { + marker: markers::NUMBER_UINT8, + // Marker sits at index 2: preamble (0) + ArrNumC1 (1) + item (2). + offset: 2, + }), + ); + } + + #[test] + fn rejects_truncated_uniform_array() { + // Declares three Int32 items but only one value's worth of bytes follow. + let mut bytes = vec![markers::ARR_NUM_C1, markers::INT32, 3]; + bytes.extend_from_slice(&1i32.to_le_bytes()); + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::UnexpectedEof { needed: 4 }), + ); + } + + #[test] + fn decodes_uniform_array_of_empty_number_arrays() { + // `[[], []]` as ArrArrNumC1C1: inner-array marker, item-type marker, + // inner_count = 0, outer_count = 2. Empty inner arrays carry no body + // bytes; the decoder must accept this rather than reject it for having + // "more elements than remaining bytes". + let bytes = vec![ + markers::ARR_ARR_NUM_C1C1, + markers::ARR_NUM_C1, // inner-array marker + markers::INT32, // shared item-type marker + 0, // inner_count + 2, // outer_count + ]; + assert_eq!(decode(&buf(&bytes)).unwrap(), serde_json::json!([[], []]),); + } + + #[test] + fn rejects_uniform_array_with_wrong_inner_array_marker() { + // The inner-array marker slot must hold ARR_NUM_C1 for a C1C1 form; any + // other byte is a malformed buffer and must be rejected. + let bytes = vec![ + markers::ARR_ARR_NUM_C1C1, + markers::NULL, // wrong: not ARR_NUM_C1 + markers::INT32, + 0, + 1, + ]; + assert!(matches!( + decode(&buf(&bytes)), + Err(BinaryError::InvalidMarker { .. }) + )); + } + + #[test] + fn user_strings_report_unsupported() { + // 1-byte user string: id == marker - USER_STRING_1BYTE_MIN. + assert_eq!( + decode(&buf(&[markers::USER_STRING_1BYTE_MIN + 3])), + Err(BinaryError::UnsupportedUserString { id: 3 }), + ); + // The very first 1-byte user string id is 0. + assert_eq!( + decode(&buf(&[markers::USER_STRING_1BYTE_MIN])), + Err(BinaryError::UnsupportedUserString { id: 0 }), + ); + // 2-byte user string: id == one_byte_count + low + high * 256, where + // one_byte_count = USER_STRING_1BYTE_MAX - USER_STRING_1BYTE_MIN (32), + // high = marker - USER_STRING_2BYTE_MIN, low = following byte. + let one_byte_count = + usize::from(markers::USER_STRING_1BYTE_MAX - markers::USER_STRING_1BYTE_MIN); + assert_eq!( + decode(&buf(&[markers::USER_STRING_2BYTE_MIN, 5])), + Err(BinaryError::UnsupportedUserString { + id: one_byte_count + 5, + }), + ); + assert_eq!( + decode(&buf(&[markers::USER_STRING_2BYTE_MIN + 1, 5])), + Err(BinaryError::UnsupportedUserString { + id: one_byte_count + 5 + 256, + }), + ); + } + + #[test] + fn rejects_out_of_range_reference() { + // StrR1 target points past the end of the buffer. + assert_eq!( + decode(&[PREAMBLE, markers::STR_R1, 200]), + Err(BinaryError::UnresolvedReference { target: 200 }), + ); + } + + #[test] + fn rejects_reference_to_non_string() { + // StrR1 target (offset 4) lands on a literal-int marker, not a string. + // 0: PREAMBLE + // 1: ARR_L1, 2: len 4 + // 3: literal int 0 <- NOT a string + // 4: STR_R1, 5: target 3 + let payload = [0x00u8, markers::STR_R1, 3]; + let mut bytes = vec![markers::ARR_L1, payload.len() as u8]; + bytes.extend_from_slice(&payload); + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::UnresolvedReference { target: 3 }), + ); + } + + #[test] + fn rejects_reference_to_reference() { + // A StrR1 that targets another StrR1 is rejected (no chains/cycles). + // 0: PREAMBLE + // 1: ARR_L1, 2: len 4 + // 3: STR_R1, 4: target 3 (self-reference) + // 5: STR_R1, 6: target 3 + let payload = [markers::STR_R1, 3, markers::STR_R1, 3]; + let mut bytes = vec![markers::ARR_L1, payload.len() as u8]; + bytes.extend_from_slice(&payload); + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::UnresolvedReference { target: 3 }), + ); + } + + #[test] + fn rejects_count_mismatch() { + // ArrLC1 declares count 5 but only one item fits in the 1-byte payload. + let mut bytes = vec![markers::ARR_LC1, 1u8, 5u8]; + bytes.push(0x00); + assert!(matches!( + decode(&buf(&bytes)), + Err(BinaryError::InvalidLength { .. }), + )); + } + + #[test] + fn rejects_element_past_declared_length() { + // ArrL1 declares payload length 1, but its single element is an Int16 + // (3 bytes) that runs past the declared region. + let mut bytes = vec![markers::ARR_L1, 1u8, markers::NUMBER_INT16]; + bytes.extend_from_slice(&5i16.to_le_bytes()); + assert!(matches!( + decode(&buf(&bytes)), + Err(BinaryError::InvalidLength { .. }), + )); + } + + #[test] + fn rejects_non_string_object_key() { + // OBJ1 whose name slot is a literal integer (0x00) rather than a string. + assert_eq!( + decode(&buf(&[markers::OBJ1, 0x00, markers::TRUE])), + Err(BinaryError::InvalidMarker { + marker: 0x00, + offset: 2, + }), + ); + } + + #[test] + fn accepts_max_depth_nesting() { + // MAX_DEPTH nested single-item arrays around a scalar leaf is exactly at + // the limit and must decode successfully. The recursive descent keeps a + // small per-level frame (leaf decoding lives in a separate non-inlined + // frame), so this stays well within an ordinary thread stack. + let mut bytes = vec![markers::ARR1; MAX_DEPTH]; + bytes.push(0x00); // literal int 0 leaf + let mut expected = serde_json::json!(0); + for _ in 0..MAX_DEPTH { + expected = Value::Array(vec![expected]); + } + assert_eq!(decode(&buf(&bytes)).unwrap(), expected); + } + + #[test] + fn rejects_excessive_nesting() { + // One level beyond MAX_DEPTH trips the depth guard. The recursion keeps + // a small per-level frame, so reaching the guard does not stress the + // ordinary test-harness stack. + let mut bytes = vec![markers::ARR1; MAX_DEPTH + 1]; + bytes.push(0x00); + assert_eq!( + decode(&buf(&bytes)), + Err(BinaryError::DepthLimitExceeded { limit: MAX_DEPTH }), + ); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/ser.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/ser.rs new file mode 100644 index 00000000000..b04491d3306 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/ser.rs @@ -0,0 +1,921 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Native `serde` serializer for Cosmos binary JSON (`T` → `binary`). +//! +//! [`to_vec`] drives a value's own [`Serialize`] implementation straight into +//! binary bytes, without building an intermediate [`serde_json::Value`]. It +//! produces the same bytes as [`encode`](super::encode) does for the equivalent +//! [`serde_json::Value`]. +//! +//! # Enum representation +//! +//! Enums use serde's externally tagged convention (matching `serde_json`): a +//! unit variant serializes as its name string, and newtype, tuple, and struct +//! variants serialize as a single-key object `{ "Variant": }`. + +use serde::{ser, Serialize}; + +use super::writer::{ + encode_container, encode_f64, encode_i64, encode_string, encode_u64, ARRAY_LC_MARKERS, + OBJECT_LC_MARKERS, +}; +use super::{ + markers::{FALSE, NULL, TRUE}, + BinaryError, Result, PREAMBLE, +}; + +/// Magic struct name `serde_json::value::RawValue` uses for verbatim JSON; +/// rejected by [`serialize_struct`] rather than silently corrupted. +/// +/// [`serialize_struct`]: BinarySerializer::serialize_struct +const RAW_VALUE_TOKEN: &str = "$serde_json::private::RawValue"; + +/// Serializes a value into a complete Cosmos binary JSON buffer. +/// +/// The returned buffer begins with the [`PREAMBLE`] byte (`0x80`) and can be +/// round-tripped back through [`decode`](super::decode). This is the native +/// serde entry point mirroring [`serde_json::to_vec`]; it produces the same +/// bytes as `encode(&serde_json::to_value(value)?)` without building the +/// intermediate [`serde_json::Value`]. +/// +/// # Errors +/// +/// Returns [`BinaryError::Custom`] if the value's +/// [`Serialize`] implementation fails. +/// +/// # Examples +/// +/// ``` +/// use azure_data_cosmos_driver::binary_json::{decode, to_vec}; +/// +/// let value = serde_json::json!({ "id": "1", "count": 7 }); +/// let bytes = to_vec(&value).unwrap(); +/// assert_eq!(decode(&bytes).unwrap(), value); +/// ``` +pub fn to_vec(value: &T) -> Result> { + let mut out = vec![PREAMBLE]; + value.serialize(BinarySerializer { out: &mut out })?; + Ok(out) +} + +/// Serializes a single value (its type marker and payload) into `out`. +/// +/// Scalars append directly; compound types delegate to a [`ContainerBuilder`] +/// that buffers children and frames them on `end`. +struct BinarySerializer<'a> { + out: &'a mut Vec, +} + +impl<'a> ser::Serializer for BinarySerializer<'a> { + type Ok = (); + type Error = BinaryError; + + type SerializeSeq = ContainerBuilder<'a>; + type SerializeTuple = ContainerBuilder<'a>; + type SerializeTupleStruct = ContainerBuilder<'a>; + type SerializeTupleVariant = ContainerBuilder<'a>; + type SerializeMap = ContainerBuilder<'a>; + type SerializeStruct = ContainerBuilder<'a>; + type SerializeStructVariant = ContainerBuilder<'a>; + + fn serialize_bool(self, v: bool) -> Result<()> { + self.out.push(if v { TRUE } else { FALSE }); + Ok(()) + } + + fn serialize_i8(self, v: i8) -> Result<()> { + self.serialize_i64(i64::from(v)) + } + + fn serialize_i16(self, v: i16) -> Result<()> { + self.serialize_i64(i64::from(v)) + } + + fn serialize_i32(self, v: i32) -> Result<()> { + self.serialize_i64(i64::from(v)) + } + + fn serialize_i64(self, v: i64) -> Result<()> { + encode_i64(v, self.out); + Ok(()) + } + + fn serialize_i128(self, v: i128) -> Result<()> { + // Cosmos binary JSON has no 128-bit integer; encode through i64 when it + // fits (matching serde_json) and fail otherwise instead of truncating. + i64::try_from(v) + .map(|v| self.serialize_i64(v)) + .unwrap_or_else(|_| { + Err(BinaryError::Custom(format!( + "i128 value {v} out of range for Cosmos binary JSON (max i64)" + ))) + }) + } + + fn serialize_u8(self, v: u8) -> Result<()> { + self.serialize_u64(u64::from(v)) + } + + fn serialize_u16(self, v: u16) -> Result<()> { + self.serialize_u64(u64::from(v)) + } + + fn serialize_u32(self, v: u32) -> Result<()> { + self.serialize_u64(u64::from(v)) + } + + fn serialize_u64(self, v: u64) -> Result<()> { + encode_u64(v, self.out); + Ok(()) + } + + fn serialize_u128(self, v: u128) -> Result<()> { + // See `serialize_i128`. + u64::try_from(v) + .map(|v| self.serialize_u64(v)) + .unwrap_or_else(|_| { + Err(BinaryError::Custom(format!( + "u128 value {v} out of range for Cosmos binary JSON (max u64)" + ))) + }) + } + + fn serialize_f32(self, v: f32) -> Result<()> { + self.serialize_f64(f64::from(v)) + } + + fn serialize_f64(self, v: f64) -> Result<()> { + encode_f64(v, self.out); + Ok(()) + } + + fn serialize_char(self, v: char) -> Result<()> { + let mut buf = [0u8; 4]; + self.serialize_str(v.encode_utf8(&mut buf)) + } + + fn serialize_str(self, v: &str) -> Result<()> { + encode_string(v, self.out); + Ok(()) + } + + fn serialize_bytes(self, v: &[u8]) -> Result<()> { + // Mirror serde_json: a byte slice serializes as an array of byte values, + // keeping `to_vec` identical to `encode(&serde_json::to_value(v))`. + // Note this makes byte-heavy fields larger than text (each byte becomes + // a number element); the `Binary*` blob form would shrink them but break + // that parity invariant, so keep byte-heavy items on the text path. + use serde::ser::SerializeSeq; + let mut seq = self.serialize_seq(Some(v.len()))?; + for byte in v { + seq.serialize_element(byte)?; + } + seq.end() + } + + fn serialize_none(self) -> Result<()> { + self.out.push(NULL); + Ok(()) + } + + fn serialize_some(self, value: &T) -> Result<()> { + value.serialize(self) + } + + fn serialize_unit(self) -> Result<()> { + self.out.push(NULL); + Ok(()) + } + + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + self.serialize_unit() + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result<()> { + // Externally tagged: a unit variant is just its name string. + self.serialize_str(variant) + } + + fn serialize_newtype_struct( + self, + _name: &'static str, + value: &T, + ) -> Result<()> { + // Transparent: a newtype struct serializes as its inner value. + value.serialize(self) + } + + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + value: &T, + ) -> Result<()> { + // Externally tagged: `{ "Variant": }`. + let mut body = Vec::new(); + encode_string(variant, &mut body); + value.serialize(BinarySerializer { out: &mut body })?; + encode_container(OBJECT_LC_MARKERS, 1, &body, self.out); + Ok(()) + } + + fn serialize_seq(self, _len: Option) -> Result> { + Ok(ContainerBuilder::new(self.out, ContainerKind::Array)) + } + + fn serialize_tuple(self, _len: usize) -> Result> { + Ok(ContainerBuilder::new(self.out, ContainerKind::Array)) + } + + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result> { + Ok(ContainerBuilder::new(self.out, ContainerKind::Array)) + } + + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + _len: usize, + ) -> Result> { + // Externally tagged: `{ "Variant": [ ... ] }`. The outer object is + // framed when the inner array is finished, in `end`. + Ok(ContainerBuilder::new_variant( + self.out, + ContainerKind::Array, + variant, + )) + } + + fn serialize_map(self, _len: Option) -> Result> { + Ok(ContainerBuilder::new(self.out, ContainerKind::Object)) + } + + fn serialize_struct(self, name: &'static str, _len: usize) -> Result> { + // Reject `serde_json::RawValue` rather than corrupt its raw JSON into a + // stringified wrapper object; use the text path for raw JSON. + if name == RAW_VALUE_TOKEN { + return Err(BinaryError::Custom( + "serde_json RawValue is not supported by the Cosmos binary JSON serializer" + .to_owned(), + )); + } + Ok(ContainerBuilder::new(self.out, ContainerKind::Object)) + } + + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + _len: usize, + ) -> Result> { + // Externally tagged: `{ "Variant": { ... } }`. + Ok(ContainerBuilder::new_variant( + self.out, + ContainerKind::Object, + variant, + )) + } +} + +/// Whether a [`ContainerBuilder`] frames its contents as an array or an object. +enum ContainerKind { + Array, + Object, +} + +/// Serializes map keys, which must project to a string (a property name). +/// +/// Mirrors `serde_json`'s map-key handling: strings, chars, and integers are +/// accepted (integers are stringified); every other type is rejected so a +/// non-string key surfaces at serialization time rather than producing a buffer +/// that only fails to decode later. +struct MapKeySerializer<'a> { + out: &'a mut Vec, +} + +fn key_must_be_a_string() -> BinaryError { + BinaryError::Custom("map key must serialize to a string".to_owned()) +} + +impl ser::Serializer for MapKeySerializer<'_> { + type Ok = (); + type Error = BinaryError; + + type SerializeSeq = ser::Impossible<(), BinaryError>; + type SerializeTuple = ser::Impossible<(), BinaryError>; + type SerializeTupleStruct = ser::Impossible<(), BinaryError>; + type SerializeTupleVariant = ser::Impossible<(), BinaryError>; + type SerializeMap = ser::Impossible<(), BinaryError>; + type SerializeStruct = ser::Impossible<(), BinaryError>; + type SerializeStructVariant = ser::Impossible<(), BinaryError>; + + fn serialize_str(self, v: &str) -> Result<()> { + encode_string(v, self.out); + Ok(()) + } + + fn serialize_char(self, v: char) -> Result<()> { + encode_string(v.encode_utf8(&mut [0u8; 4]), self.out); + Ok(()) + } + + fn serialize_i8(self, v: i8) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_i16(self, v: i16) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_i32(self, v: i32) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_i64(self, v: i64) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_i128(self, v: i128) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_u8(self, v: u8) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_u16(self, v: u16) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_u32(self, v: u32) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_u64(self, v: u64) -> Result<()> { + self.serialize_str(&v.to_string()) + } + fn serialize_u128(self, v: u128) -> Result<()> { + self.serialize_str(&v.to_string()) + } + + fn serialize_unit_variant( + self, + _name: &'static str, + _variant_index: u32, + variant: &'static str, + ) -> Result<()> { + self.serialize_str(variant) + } + + fn serialize_newtype_struct( + self, + _name: &'static str, + value: &T, + ) -> Result<()> { + value.serialize(self) + } + + fn serialize_bool(self, _v: bool) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_f32(self, _v: f32) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_f64(self, _v: f64) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_bytes(self, _v: &[u8]) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_none(self) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_some(self, _value: &T) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_unit(self) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_unit_struct(self, _name: &'static str) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_newtype_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _value: &T, + ) -> Result<()> { + Err(key_must_be_a_string()) + } + fn serialize_seq(self, _len: Option) -> Result { + Err(key_must_be_a_string()) + } + fn serialize_tuple(self, _len: usize) -> Result { + Err(key_must_be_a_string()) + } + fn serialize_tuple_struct( + self, + _name: &'static str, + _len: usize, + ) -> Result { + Err(key_must_be_a_string()) + } + fn serialize_tuple_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(key_must_be_a_string()) + } + fn serialize_map(self, _len: Option) -> Result { + Err(key_must_be_a_string()) + } + fn serialize_struct(self, _name: &'static str, _len: usize) -> Result { + Err(key_must_be_a_string()) + } + fn serialize_struct_variant( + self, + _name: &'static str, + _variant_index: u32, + _variant: &'static str, + _len: usize, + ) -> Result { + Err(key_must_be_a_string()) + } +} + +/// Buffers a compound value's children, then frames them on `end`. +/// +/// Children are serialized into `buffer` (a scratch buffer); `end` emits the +/// appropriate `LC*` marker, byte length, and element count into the parent +/// `out`, then appends the buffer. When `variant` is set, the framed container +/// is itself wrapped in a single-key `{ "Variant": }` object to +/// realize serde's externally-tagged enum representation. +struct ContainerBuilder<'a> { + out: &'a mut Vec, + buffer: Vec, + count: usize, + kind: ContainerKind, + variant: Option<&'static str>, +} + +impl<'a> ContainerBuilder<'a> { + fn new(out: &'a mut Vec, kind: ContainerKind) -> Self { + Self { + out, + buffer: Vec::new(), + count: 0, + kind, + variant: None, + } + } + + fn new_variant(out: &'a mut Vec, kind: ContainerKind, variant: &'static str) -> Self { + Self { + out, + buffer: Vec::new(), + count: 0, + kind, + variant: Some(variant), + } + } + + /// Serializes `value` into the scratch buffer and bumps the element count. + fn push_element(&mut self, value: &T) -> Result<()> { + value.serialize(BinarySerializer { + out: &mut self.buffer, + })?; + self.count += 1; + Ok(()) + } + + /// Serializes `key` into the scratch buffer **without** bumping the count. + /// Object element count tracks key/value *pairs*, so only the value bumps. + /// + /// Map keys must project to a string; non-string keys are rejected via + /// [`MapKeySerializer`] (matching `serde_json`). + fn push_key(&mut self, key: &T) -> Result<()> { + key.serialize(MapKeySerializer { + out: &mut self.buffer, + }) + } + + /// Serializes a struct field name (a static string) into the scratch buffer. + fn push_field_name(&mut self, key: &'static str) { + encode_string(key, &mut self.buffer); + } + + /// Frames the buffered container into the parent buffer, wrapping it in a + /// single-key object first when this builder represents an enum variant. + fn finish(self) -> Result<()> { + let markers = match self.kind { + ContainerKind::Array => ARRAY_LC_MARKERS, + ContainerKind::Object => OBJECT_LC_MARKERS, + }; + match self.variant { + None => encode_container(markers, self.count, &self.buffer, self.out), + Some(variant) => { + // Build the inner container, then wrap it in `{ variant: inner }`. + let mut inner = Vec::new(); + encode_container(markers, self.count, &self.buffer, &mut inner); + let mut wrapper = Vec::new(); + encode_string(variant, &mut wrapper); + wrapper.extend_from_slice(&inner); + encode_container(OBJECT_LC_MARKERS, 1, &wrapper, self.out); + } + } + Ok(()) + } +} + +impl ser::SerializeSeq for ContainerBuilder<'_> { + type Ok = (); + type Error = BinaryError; + + fn serialize_element(&mut self, value: &T) -> Result<()> { + self.push_element(value) + } + + fn end(self) -> Result<()> { + self.finish() + } +} + +impl ser::SerializeTuple for ContainerBuilder<'_> { + type Ok = (); + type Error = BinaryError; + + fn serialize_element(&mut self, value: &T) -> Result<()> { + self.push_element(value) + } + + fn end(self) -> Result<()> { + self.finish() + } +} + +impl ser::SerializeTupleStruct for ContainerBuilder<'_> { + type Ok = (); + type Error = BinaryError; + + fn serialize_field(&mut self, value: &T) -> Result<()> { + self.push_element(value) + } + + fn end(self) -> Result<()> { + self.finish() + } +} + +impl ser::SerializeTupleVariant for ContainerBuilder<'_> { + type Ok = (); + type Error = BinaryError; + + fn serialize_field(&mut self, value: &T) -> Result<()> { + self.push_element(value) + } + + fn end(self) -> Result<()> { + self.finish() + } +} + +impl ser::SerializeMap for ContainerBuilder<'_> { + type Ok = (); + type Error = BinaryError; + + fn serialize_key(&mut self, key: &T) -> Result<()> { + self.push_key(key) + } + + fn serialize_value(&mut self, value: &T) -> Result<()> { + self.push_element(value) + } + + fn end(self) -> Result<()> { + self.finish() + } +} + +impl ser::SerializeStruct for ContainerBuilder<'_> { + type Ok = (); + type Error = BinaryError; + + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<()> { + self.push_field_name(key); + self.push_element(value) + } + + fn end(self) -> Result<()> { + self.finish() + } +} + +impl ser::SerializeStructVariant for ContainerBuilder<'_> { + type Ok = (); + type Error = BinaryError; + + fn serialize_field( + &mut self, + key: &'static str, + value: &T, + ) -> Result<()> { + self.push_field_name(key); + self.push_element(value) + } + + fn end(self) -> Result<()> { + self.finish() + } +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary_json::{decode, encode}; + use serde::{Deserialize, Serialize}; + use serde_json::json; + use std::collections::BTreeMap; + + /// Asserts that the native serializer produces exactly the same bytes as + /// the `Value`-based encoder for the given JSON value. + fn assert_parity(value: serde_json::Value) { + let native = to_vec(&value).unwrap(); + let via_value = encode(&value); + assert_eq!( + native, via_value, + "native to_vec must match encode(&Value) for {value:?}" + ); + // And the bytes must round-trip back to the original value. + assert_eq!(decode(&native).unwrap(), value); + } + + #[test] + fn map_with_integer_keys_stringifies_them() { + // serde_json stringifies integer map keys; the native serializer does + // the same so the container's property names stay strings on the wire. + let mut map = BTreeMap::new(); + map.insert(1u32, "a"); + map.insert(2u32, "b"); + let bytes = to_vec(&map).unwrap(); + assert_eq!(decode(&bytes).unwrap(), json!({ "1": "a", "2": "b" })); + } + + #[test] + fn map_with_non_string_keys_is_rejected() { + // A bool key has no string projection and must be rejected at + // serialization time (matching serde_json), not silently encoded. + let mut map = BTreeMap::new(); + map.insert(true, 1); + map.insert(false, 2); + let err = to_vec(&map).unwrap_err(); + assert!( + matches!(err, BinaryError::Custom(ref m) if m.contains("map key")), + "expected a map-key error, got {err:?}" + ); + } + + #[test] + fn parity_scalars() { + assert_parity(json!(null)); + assert_parity(json!(true)); + assert_parity(json!(false)); + assert_parity(json!(0)); + assert_parity(json!(31)); + assert_parity(json!(32)); + assert_parity(json!(-1)); + assert_parity(json!(i64::MAX)); + assert_parity(json!(u64::MAX)); + assert_parity(json!(1.5)); + assert_parity(json!("hello")); + assert_parity(json!("")); + } + + #[test] + fn parity_long_string() { + // Exercise the length-prefixed string forms beyond the encoded-length + // range (< 64 bytes). + assert_parity(json!("x".repeat(100))); + assert_parity(json!("y".repeat(70_000))); + } + + #[test] + fn parity_arrays_and_objects() { + assert_parity(json!([])); + assert_parity(json!([1, 2, 3])); + assert_parity(json!({})); + assert_parity(json!({ "id": "doc-1", "count": 42, "nested": { "ok": true } })); + assert_parity(json!({ "a": [1, { "b": [true, null, "x"] }], "c": 3.5 })); + } + + #[test] + fn to_vec_begins_with_preamble() { + let bytes = to_vec(&json!({ "id": "1" })).unwrap(); + assert_eq!(bytes.first(), Some(&PREAMBLE)); + } + + #[test] + fn raw_value_is_rejected_rather_than_corrupted() { + // Mirrors how `RawValue`'s `Serialize` impl drives a serializer (the + // driver doesn't enable serde_json's `raw_value` feature). + struct RawValueLike; + impl Serialize for RawValueLike { + fn serialize( + &self, + serializer: S, + ) -> std::result::Result { + use serde::ser::SerializeStruct; + let mut s = serializer.serialize_struct("$serde_json::private::RawValue", 1)?; + s.serialize_field("$serde_json::private::RawValue", r#"{"a":1}"#)?; + s.end() + } + } + let err = to_vec(&RawValueLike).unwrap_err(); + assert!( + matches!(err, BinaryError::Custom(msg) if msg.contains("RawValue")), + "expected a RawValue rejection error" + ); + } + + #[derive(Serialize, Deserialize, PartialEq, Debug)] + struct Product { + id: String, + count: u64, + tags: Vec, + } + + #[test] + fn typed_struct_round_trips() { + // The native serializer preserves struct *declaration* order (like + // `serde_json::to_vec`), whereas `serde_json::to_value` sorts object + // keys (its `Map` is a `BTreeMap` without `preserve_order`). So the + // bytes intentionally differ from the `Value` encoder for named + // structs; the correctness bar is a faithful round-trip. + let product = Product { + id: "p1".to_owned(), + count: 7, + tags: vec!["a".to_owned(), "b".to_owned()], + }; + let native = to_vec(&product).unwrap(); + let decoded: Product = serde_json::from_value(decode(&native).unwrap()).unwrap(); + assert_eq!(decoded, product); + } + + #[test] + fn typed_struct_preserves_field_declaration_order() { + // Field order on the wire must be id, count, tags (declaration order), + // matching `serde_json::to_vec`, not the alphabetized `to_value` order. + // The decoder normalizes keys into a sorted map, so assert on the raw + // wire bytes: each field-name string is embedded verbatim, so their + // byte offsets reflect emission order. + let product = Product { + id: "p1".to_owned(), + count: 7, + tags: vec![], + }; + let bytes = to_vec(&product).unwrap(); + let offset = |needle: &str| { + bytes + .windows(needle.len()) + .position(|w| w == needle.as_bytes()) + .unwrap_or_else(|| panic!("field name {needle:?} not found in wire bytes")) + }; + let (id_at, count_at, tags_at) = (offset("id"), offset("count"), offset("tags")); + assert!( + id_at < count_at && count_at < tags_at, + "expected declaration order id < count < tags, got offsets \ + id={id_at}, count={count_at}, tags={tags_at}" + ); + } + + #[derive(Serialize, Deserialize, PartialEq, Debug)] + enum Shape { + Unit, + Newtype(u32), + Tuple(u8, u8), + Struct { width: u32, height: u32 }, + } + + #[test] + fn enum_variants_round_trip_externally_tagged() { + // Externally-tagged variants round-trip through decode + + // serde_json::from_value. (Byte-parity with the `Value` encoder is not + // asserted for the struct variant, whose named fields keep declaration + // order rather than the alphabetized `to_value` order.) + for shape in [ + Shape::Unit, + Shape::Newtype(5), + Shape::Tuple(1, 2), + Shape::Struct { + width: 3, + height: 4, + }, + ] { + let native = to_vec(&shape).unwrap(); + let decoded: Shape = serde_json::from_value(decode(&native).unwrap()).unwrap(); + assert_eq!(decoded, shape); + } + } + + #[test] + fn unit_variant_serializes_as_name_string() { + // A unit variant is externally tagged as its bare name string. + assert_eq!( + decode(&to_vec(&Shape::Unit).unwrap()).unwrap(), + json!("Unit") + ); + } + + #[test] + fn newtype_variant_serializes_as_tagged_object() { + // `{ "Newtype": 5 }`. + assert_eq!( + decode(&to_vec(&Shape::Newtype(5)).unwrap()).unwrap(), + json!({ "Newtype": 5 }) + ); + } + + /// A tiny deterministic LCG so the generative parity test needs no external + /// RNG dependency and reproduces the same values on every run. + struct Lcg(u64); + + impl Lcg { + fn next_u64(&mut self) -> u64 { + // Numerical Recipes LCG constants. + self.0 = self.0.wrapping_mul(6364136223846793005).wrapping_add(1); + self.0 + } + + fn below(&mut self, n: u64) -> u64 { + self.next_u64() % n + } + } + + /// Builds a random JSON value up to `depth` levels deep. Object keys are + /// generated so that both encode paths see identical `serde_json::Value` + /// input (same key ordering), keeping the byte-parity assertion valid. + fn random_value(rng: &mut Lcg, depth: u32) -> serde_json::Value { + // At depth 0 only scalars are produced to bound recursion. + let arms = if depth == 0 { 6 } else { 8 }; + match rng.below(arms) { + 0 => serde_json::Value::Null, + 1 => json!(rng.next_u64().is_multiple_of(2)), + 2 => json!(rng.below(64) as i64), // hits the literal-int range + 3 => json!((rng.next_u64() as i64).wrapping_sub(i64::MAX / 2)), // wide i64 + 4 => json!((rng.next_u64() as f64) / 7.0), // double + 5 => { + let len = rng.below(80) as usize; // spans encoded-length + StrL1 + json!("s".repeat(len)) + } + 6 => { + let n = rng.below(5) as usize; + let items: Vec<_> = (0..n).map(|_| random_value(rng, depth - 1)).collect(); + serde_json::Value::Array(items) + } + _ => { + let n = rng.below(5) as usize; + let mut map = serde_json::Map::new(); + for i in 0..n { + // Distinct, deterministic keys; Map orders them for us. + map.insert(format!("k{i}"), random_value(rng, depth - 1)); + } + serde_json::Value::Object(map) + } + } + } + + #[test] + fn generative_parity_native_matches_value_encoder() { + // Property: for any `serde_json::Value`, the native serializer emits the + // exact same bytes as the `Value` encoder, and those bytes round-trip. + // (Parity holds because both paths observe the same `Value` — identical + // key ordering — unlike typed structs, which preserve declaration order.) + let mut rng = Lcg(0x1234_5678_9abc_def0); + for _ in 0..2_000 { + let value = random_value(&mut rng, 4); + let native = to_vec(&value).unwrap(); + assert_eq!( + native, + encode(&value), + "native to_vec diverged from encode(&Value) for {value:?}" + ); + assert_eq!( + decode(&native).unwrap(), + value, + "round-trip mismatch for {value:?}" + ); + } + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/system_strings.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/system_strings.rs new file mode 100644 index 00000000000..6ee53c3ce71 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/system_strings.rs @@ -0,0 +1,189 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! The Cosmos binary JSON **system-string dictionary**. +//! +//! System strings are a fixed dictionary of common Cosmos property names and +//! GeoJSON keywords. A 1-byte system-string type marker (in the range +//! [`SYSTEM_STRING_1BYTE_MIN`]..[`SYSTEM_STRING_1BYTE_MAX`]) encodes a string by +//! index into this table: `index = marker - SYSTEM_STRING_1BYTE_MIN`. +//! +//! The table matches the service byte-for-byte and in the same order. + +use super::markers::{SYSTEM_STRING_1BYTE_MAX, SYSTEM_STRING_1BYTE_MIN}; + +/// The number of entries in the system-string dictionary. +pub const SYSTEM_STRING_COUNT: usize = 32; + +/// The system-string dictionary, indexed by system-string id. +/// +/// Entry `i` is encoded by the 1-byte type marker +/// `SYSTEM_STRING_1BYTE_MIN + i`. The order is significant and matches the +/// service; do not sort or reorder. +pub const SYSTEM_STRINGS: [&str; SYSTEM_STRING_COUNT] = [ + "$s", // 0 + "$t", // 1 + "$v", // 2 + "_attachments", // 3 + "_etag", // 4 + "_rid", // 5 + "_self", // 6 + "_ts", // 7 + "attachments/", // 8 + "coordinates", // 9 + "geometry", // 10 + "GeometryCollection", // 11 + "id", // 12 + "url", // 13 + "Value", // 14 + "label", // 15 + "LineString", // 16 + "link", // 17 + "MultiLineString", // 18 + "MultiPoint", // 19 + "MultiPolygon", // 20 + "name", // 21 + "Name", // 22 + "Type", // 23 + "Point", // 24 + "Polygon", // 25 + "properties", // 26 + "type", // 27 + "value", // 28 + "Feature", // 29 + "FeatureCollection", // 30 + "_id", // 31 +]; + +/// Returns the system string for a dictionary `index`, or `None` if the index +/// is out of range. +/// +/// This is the lookup the **decoder** performs when it reads a 1-byte +/// system-string marker. +/// +/// # Examples +/// +/// ``` +/// use azure_data_cosmos_driver::binary_json::system_strings::system_string; +/// +/// assert_eq!(system_string(12), Some("id")); +/// assert_eq!(system_string(5), Some("_rid")); +/// assert_eq!(system_string(32), None); +/// ``` +pub fn system_string(index: usize) -> Option<&'static str> { + SYSTEM_STRINGS.get(index).copied() +} + +/// Returns the system string addressed by a 1-byte system-string `marker` +/// (in `SYSTEM_STRING_1BYTE_MIN..SYSTEM_STRING_1BYTE_MAX`), or `None` if the +/// marker is outside that range or addresses an index past the table. +/// +/// Convenience wrapper that performs the `marker - SYSTEM_STRING_1BYTE_MIN` +/// index arithmetic for the decoder. +pub fn system_string_for_marker(marker: u8) -> Option<&'static str> { + if (SYSTEM_STRING_1BYTE_MIN..SYSTEM_STRING_1BYTE_MAX).contains(&marker) { + system_string(usize::from(marker - SYSTEM_STRING_1BYTE_MIN)) + } else { + None + } +} + +#[cfg(test)] +mod tests { + use super::*; + + /// Cross-check every entry's byte length against the buckets the .NET + /// `GetSystemStringIdLength{N}` reverse-lookup functions sort them into. + /// This guards the transcription independently of the forward order. + #[test] + fn entry_lengths_match_dotnet_buckets() { + // (index, expected utf-8 byte length) for every entry, taken from the + // .NET length-bucketed lookup functions. + let expected: [(usize, usize); SYSTEM_STRING_COUNT] = [ + (0, 2), // $s + (1, 2), // $t + (2, 2), // $v + (3, 12), // _attachments + (4, 5), // _etag + (5, 4), // _rid + (6, 5), // _self + (7, 3), // _ts + (8, 12), // attachments/ + (9, 11), // coordinates + (10, 8), // geometry + (11, 18), // GeometryCollection + (12, 2), // id + (13, 3), // url + (14, 5), // Value + (15, 5), // label + (16, 10), // LineString + (17, 4), // link + (18, 15), // MultiLineString + (19, 10), // MultiPoint + (20, 12), // MultiPolygon + (21, 4), // name + (22, 4), // Name + (23, 4), // Type + (24, 5), // Point + (25, 7), // Polygon + (26, 10), // properties + (27, 4), // type + (28, 5), // value + (29, 7), // Feature + (30, 17), // FeatureCollection + (31, 3), // _id + ]; + for (index, len) in expected { + assert_eq!( + SYSTEM_STRINGS[index].len(), + len, + "entry {index} ({:?}) has unexpected length", + SYSTEM_STRINGS[index], + ); + } + } + + /// No two entries are identical (the dictionary is a set). + #[test] + fn entries_are_unique() { + for i in 0..SYSTEM_STRINGS.len() { + for j in (i + 1)..SYSTEM_STRINGS.len() { + assert_ne!( + SYSTEM_STRINGS[i], SYSTEM_STRINGS[j], + "duplicate system string at indices {i} and {j}", + ); + } + } + } + + #[test] + fn system_string_lookup_bounds() { + assert_eq!(system_string(0), Some("$s")); + assert_eq!(system_string(31), Some("_id")); + assert_eq!(system_string(32), None); + assert_eq!(system_string(usize::MAX), None); + } + + /// The marker convenience wrapper maps the 1-byte system-string range onto + /// the table and rejects markers outside it. + #[test] + fn marker_lookup_maps_range() { + assert_eq!( + system_string_for_marker(SYSTEM_STRING_1BYTE_MIN), + Some("$s") + ); + // Last valid system-string marker addresses the last entry. + assert_eq!( + system_string_for_marker(SYSTEM_STRING_1BYTE_MAX - 1), + Some("_id"), + ); + // The id marker: SYSTEM_STRING_1BYTE_MIN + 12. + assert_eq!( + system_string_for_marker(SYSTEM_STRING_1BYTE_MIN + 12), + Some("id"), + ); + // Out of range: a user-string marker is not a system string. + assert_eq!(system_string_for_marker(SYSTEM_STRING_1BYTE_MAX), None); + assert_eq!(system_string_for_marker(0x00), None); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/vectors.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/vectors.rs new file mode 100644 index 00000000000..2e142c52b92 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/vectors.rs @@ -0,0 +1,88 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Golden binary-JSON decode vectors (test-support). +//! +//! Each [`BinaryVector`] pairs a complete Cosmos binary JSON buffer with the +//! text JSON it decodes to. The corpus lives in a shared, human-reviewable JSON +//! data file (`testdata/binary_json_vectors.json`) embedded via [`include_str!`] +//! so it can be reviewed and shared across language SDKs. +//! +//! The module is compiled only under `cfg(test)`. + +use serde::Deserialize; + +/// The embedded corpus data file (see the module docs for the format). +const CORPUS: &str = include_str!("../../testdata/binary_json_vectors.json"); + +/// A single golden vector: a complete binary JSON buffer and the canonical text +/// JSON it decodes to. +pub(crate) struct BinaryVector { + /// Human-readable case name (used in assertion messages). + pub name: String, + /// The complete binary JSON buffer (including the `0x80` preamble). + pub binary: Vec, + /// The text JSON the buffer decodes to. + pub json: String, +} + +/// The on-disk shape of a corpus record: a case name, the binary buffer as a +/// space-separated hex string, and the expected JSON value written inline. +#[derive(Deserialize)] +struct RawVector { + name: String, + binary: String, + json: serde_json::Value, +} + +/// Parses the embedded JSON corpus into golden vectors. +/// +/// The file is a JSON array of objects with `name`, `binary` (space-separated +/// hex bytes), and `json` (the expected value inline). Panics on a malformed +/// corpus, since it is a compile-time-embedded test fixture. +pub(crate) fn golden_vectors() -> Vec { + let raw: Vec = + serde_json::from_str(CORPUS).expect("corpus must be a valid JSON array of vectors"); + + let vectors: Vec = raw + .into_iter() + .map(|r| BinaryVector { + name: r.name, + binary: parse_hex(&r.binary), + json: r.json.to_string(), + }) + .collect(); + + assert!(!vectors.is_empty(), "corpus must not be empty"); + vectors +} + +/// Parses space-separated 2-digit hex bytes (e.g. `"80 D0"`) into bytes. +fn parse_hex(text: &str) -> Vec { + text.split_whitespace() + .map(|byte| u8::from_str_radix(byte, 16).expect("corpus binary is valid hex")) + .collect() +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary_json::{is_binary, PREAMBLE}; + + /// Every golden buffer is detected as binary, starts with the preamble, and + /// carries non-empty name/json fields, with unique names. + #[test] + fn vectors_are_well_formed() { + let vectors = golden_vectors(); + for v in &vectors { + assert!(is_binary(&v.binary), "{}: not detected as binary", v.name); + assert_eq!(v.binary[0], PREAMBLE, "{}: missing preamble", v.name); + assert!(!v.json.is_empty(), "{}: empty expected json", v.name); + } + let mut names: Vec<&str> = vectors.iter().map(|v| v.name.as_str()).collect(); + names.sort_unstable(); + let count = names.len(); + names.dedup(); + assert_eq!(names.len(), count, "corpus contains duplicate vector names"); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/writer.rs b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/writer.rs new file mode 100644 index 00000000000..6fb82c1e710 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/binary_json/writer.rs @@ -0,0 +1,423 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Cosmos binary JSON encoder ([`serde_json::Value`] → `binary`). +//! +//! [`encode`] produces a valid binary buffer for any [`serde_json::Value`], +//! using a subset of the wire forms: +//! +//! - `null` / `false` / `true` singletons, +//! - numbers as a literal int (`0`–`31`), `Int64`, `UInt64`, or `Double`, +//! - strings as an encoded-length string (≤ 63 bytes) or `StrL1`/`StrL2`/`StrL4`, +//! - arrays and objects as the length+count `ArrLC*` / `ObjLC*` forms. +//! +//! It does not emit the more compact forms (system/user strings, +//! reference-string dedup, compressed strings, the `Arr0`/`Arr1`/`Obj0`/`Obj1` +//! container forms, or uniform number arrays). The decoder accepts all of +//! those, so an encode/decode round-trip reproduces the original value. + +use serde_json::Value; + +use super::markers::{ + ARR_LC1, ARR_LC2, ARR_LC4, ENCODED_STRING_LENGTH_MAX, ENCODED_STRING_LENGTH_MIN, FALSE, + LITERAL_INT_MAX, NULL, NUMBER_DOUBLE, NUMBER_INT64, NUMBER_UINT64, OBJ_LC1, OBJ_LC2, OBJ_LC4, + STR_L1, STR_L2, STR_L4, TRUE, +}; +use super::PREAMBLE; + +/// The number of distinct encoded-length string markers, i.e. the maximum +/// string length (in bytes) that fits the encoded-length form (`0`–`63`). +const ENCODED_STRING_LENGTH_SPAN: usize = + (ENCODED_STRING_LENGTH_MAX - ENCODED_STRING_LENGTH_MIN) as usize; + +/// The `LC1`/`LC2`/`LC4` length+count markers for **arrays**, passed to +/// [`encode_container`]. Shared with the native serde serializer. +pub(super) const ARRAY_LC_MARKERS: [u8; 3] = [ARR_LC1, ARR_LC2, ARR_LC4]; + +/// The `LC1`/`LC2`/`LC4` length+count markers for **objects**, passed to +/// [`encode_container`]. Shared with the native serde serializer. +pub(super) const OBJECT_LC_MARKERS: [u8; 3] = [OBJ_LC1, OBJ_LC2, OBJ_LC4]; + +/// Encodes a [`serde_json::Value`] into a complete Cosmos binary JSON buffer. +/// +/// The returned buffer begins with the [`PREAMBLE`] byte +/// (`0x80`) and can be round-tripped back through [`decode`](super::decode). +/// +/// # Examples +/// +/// ``` +/// use azure_data_cosmos_driver::binary_json::{decode, encode}; +/// +/// let value = serde_json::json!({ "id": "1", "count": 7 }); +/// let bytes = encode(&value); +/// assert_eq!(decode(&bytes).unwrap(), value); +/// ``` +pub fn encode(value: &Value) -> Vec { + let mut out = vec![PREAMBLE]; + encode_value(value, &mut out); + out +} + +/// Appends the encoding of `value` (its type marker and payload) to `out`. +fn encode_value(value: &Value, out: &mut Vec) { + match value { + Value::Null => out.push(NULL), + Value::Bool(false) => out.push(FALSE), + Value::Bool(true) => out.push(TRUE), + Value::Number(n) => encode_number(n, out), + Value::String(s) => encode_string(s, out), + Value::Array(items) => { + let mut body = Vec::new(); + for item in items { + encode_value(item, &mut body); + } + encode_container(ARRAY_LC_MARKERS, items.len(), &body, out); + } + Value::Object(map) => { + let mut body = Vec::new(); + for (key, val) in map { + encode_string(key, &mut body); + encode_value(val, &mut body); + } + encode_container(OBJECT_LC_MARKERS, map.len(), &body, out); + } + } +} + +/// Encodes a JSON number as a literal int (`0`–`31`), `Int64`, `UInt64`, or +/// `Double` — the minimal set that covers every [`serde_json::Number`]. +fn encode_number(n: &serde_json::Number, out: &mut Vec) { + if let Some(i) = n.as_i64() { + encode_i64(i, out); + } else if let Some(u) = n.as_u64() { + // Only reached when the value exceeds `i64::MAX` (so `as_i64` is `None`). + encode_u64(u, out); + } else { + // A `serde_json::Number` that is neither `i64` nor `u64` is an `f64`, + // and JSON numbers are always finite, so `as_f64` yields a value here. + let f = n + .as_f64() + .expect("serde_json::Number is i64, u64, or finite f64"); + encode_f64(f, out); + } +} + +/// Encodes a signed integer as a literal int (`0`–`31`) or `Int64`. +/// +/// Shared by the [`Value`]-based [`encode`] path and the native serde +/// serializer so both emit identical bytes for the same integer. +pub(super) fn encode_i64(i: i64, out: &mut Vec) { + if (0..i64::from(LITERAL_INT_MAX)).contains(&i) { + // Literal int: the value is the marker. + out.push(i as u8); + } else { + out.push(NUMBER_INT64); + out.extend_from_slice(&i.to_le_bytes()); + } +} + +/// Encodes an unsigned integer as a literal int (`0`–`31`), `Int64` (when it +/// still fits `i64`), or `UInt64`. +/// +/// Shared by the [`Value`]-based [`encode`] path and the native serde +/// serializer. +pub(super) fn encode_u64(u: u64, out: &mut Vec) { + if let Ok(i) = i64::try_from(u) { + // Fits `i64`, so route through the signed path to keep literal-int and + // `Int64` selection identical to the `Value` encoder. + encode_i64(i, out); + } else { + out.push(NUMBER_UINT64); + out.extend_from_slice(&u.to_le_bytes()); + } +} + +/// Encodes a floating-point number as an IEEE-754 `Double`. +/// +/// Non-finite values (`NaN`, `±∞`) have no JSON representation, so they are +/// emitted as `null` — mirroring [`serde_json`], which serializes a non-finite +/// float as `null`. This keeps `to_vec(item)` byte-identical to +/// `encode(&serde_json::to_value(item))` and matches the decoder, which rejects +/// a non-finite `Double`. +/// +/// Shared by the [`Value`]-based [`encode`] path and the native serde +/// serializer. +pub(super) fn encode_f64(f: f64, out: &mut Vec) { + if !f.is_finite() { + out.push(NULL); + return; + } + out.push(NUMBER_DOUBLE); + out.extend_from_slice(&f.to_le_bytes()); +} + +/// Encodes a string as an encoded-length string (≤ 63 bytes, length baked into +/// the marker) or a length-prefixed `StrL1`/`StrL2`/`StrL4`. +/// +/// Shared by the [`Value`]-based [`encode`] path and the native serde +/// serializer. +pub(super) fn encode_string(s: &str, out: &mut Vec) { + let bytes = s.as_bytes(); + let len = bytes.len(); + if len < ENCODED_STRING_LENGTH_SPAN { + out.push(ENCODED_STRING_LENGTH_MIN | (len as u8)); + } else if len <= u8::MAX as usize { + out.push(STR_L1); + out.push(len as u8); + } else if len <= u16::MAX as usize { + out.push(STR_L2); + out.extend_from_slice(&(len as u16).to_le_bytes()); + } else { + // Cosmos caps request bodies far below `u32::MAX`, so a `u32` length is + // always sufficient on the data plane. + out.push(STR_L4); + out.extend_from_slice(&(len as u32).to_le_bytes()); + } + out.extend_from_slice(bytes); +} + +/// Writes a length+count container: the marker, the payload byte length, the +/// item/member count, then the pre-encoded `body`. The narrowest of the three +/// `LC1`/`LC2`/`LC4` markers whose length and count fields both fit is used. +/// +/// Shared by the [`Value`]-based [`encode`] path and the native serde +/// serializer, which buffers each container's body in a scratch `Vec` and then +/// calls this to frame it. +pub(super) fn encode_container(lc_markers: [u8; 3], count: usize, body: &[u8], out: &mut Vec) { + let [lc1, lc2, lc4] = lc_markers; + let len = body.len(); + if len <= u8::MAX as usize && count <= u8::MAX as usize { + out.push(lc1); + out.push(len as u8); + out.push(count as u8); + } else if len <= u16::MAX as usize && count <= u16::MAX as usize { + out.push(lc2); + out.extend_from_slice(&(len as u16).to_le_bytes()); + out.extend_from_slice(&(count as u16).to_le_bytes()); + } else { + // Cosmos caps request bodies far below u32::MAX, so the widest LC4 + // markers always suffice; guard against silently truncating a larger + // in-memory value into an invalid buffer. + debug_assert!( + len <= u32::MAX as usize && count <= u32::MAX as usize, + "container length/count exceeds u32::MAX" + ); + out.push(lc4); + out.extend_from_slice(&(len as u32).to_le_bytes()); + out.extend_from_slice(&(count as u32).to_le_bytes()); + } + out.extend_from_slice(body); +} + +#[cfg(test)] +mod tests { + use super::*; + use crate::binary_json::markers; + use serde_json::json; + + /// Builds the expected buffer: preamble, then `head`, then `payload`. + fn buf(head: &[u8], payload: &[u8]) -> Vec { + let mut v = vec![PREAMBLE]; + v.extend_from_slice(head); + v.extend_from_slice(payload); + v + } + + #[test] + fn encodes_null_and_booleans() { + assert_eq!(encode(&Value::Null), vec![PREAMBLE, markers::NULL]); + assert_eq!(encode(&json!(true)), vec![PREAMBLE, markers::TRUE]); + assert_eq!(encode(&json!(false)), vec![PREAMBLE, markers::FALSE]); + } + + #[test] + fn encodes_literal_int_in_marker() { + // Small non-negative integers are encoded as just the literal-int marker. + assert_eq!(encode(&json!(0)), vec![PREAMBLE, 0]); + assert_eq!(encode(&json!(7)), vec![PREAMBLE, 7]); + assert_eq!(encode(&json!(31)), vec![PREAMBLE, 31]); + } + + #[test] + fn encodes_int64_and_uint64() { + // 32 no longer fits the literal-int range, so it becomes an Int64. + assert_eq!( + encode(&json!(32)), + vec![PREAMBLE, markers::NUMBER_INT64, 32, 0, 0, 0, 0, 0, 0, 0], + ); + assert_eq!( + encode(&json!(i64::MAX)), + vec![ + PREAMBLE, + markers::NUMBER_INT64, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0x7F + ], + ); + // Values beyond i64::MAX use the UInt64 form. + assert_eq!( + encode(&json!(u64::MAX)), + vec![ + PREAMBLE, + markers::NUMBER_UINT64, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF, + 0xFF + ], + ); + } + + #[test] + fn encodes_double() { + // 1.5 as an IEEE-754 little-endian Double. + assert_eq!( + encode(&json!(1.5)), + vec![ + PREAMBLE, + markers::NUMBER_DOUBLE, + 0, + 0, + 0, + 0, + 0, + 0, + 0xF8, + 0x3F + ], + ); + } + + #[test] + fn encodes_non_finite_double_as_null() { + // Non-finite floats have no JSON representation. `serde_json` serializes + // them as `null`, so the binary encoder must do the same to keep + // `encode` byte-identical to `encode(&serde_json::to_value(..))` and to + // match the decoder (which rejects a non-finite `Double`). + // + // Note: a `serde_json::Value` cannot itself hold a non-finite number + // (`json!(f64::NAN)` is `Null`), so exercise the low-level helper + // directly to prove the encoder branch. + let mut nan = vec![PREAMBLE]; + encode_f64(f64::NAN, &mut nan); + assert_eq!(nan, vec![PREAMBLE, markers::NULL]); + + let mut inf = vec![PREAMBLE]; + encode_f64(f64::INFINITY, &mut inf); + assert_eq!(inf, vec![PREAMBLE, markers::NULL]); + + let mut neg_inf = vec![PREAMBLE]; + encode_f64(f64::NEG_INFINITY, &mut neg_inf); + assert_eq!(neg_inf, vec![PREAMBLE, markers::NULL]); + } + + #[test] + fn encodes_empty_string_as_encoded_length_marker() { + // Empty string -> a single encoded-length marker (0x80 | 0). + assert_eq!( + encode(&json!("")), + vec![PREAMBLE, markers::ENCODED_STRING_LENGTH_MIN], + ); + } + + #[test] + fn encodes_strings_at_width_boundaries() { + // 63 bytes: the maximum encoded-length form (0x80 | 63 == 0xBF). + assert_eq!(encode(&json!("x".repeat(63))), buf(&[0xBF], &[b'x'; 63]),); + // 64 bytes: first StrL1 (marker 0xC0, 1-byte length 64). + assert_eq!( + encode(&json!("y".repeat(64))), + buf(&[markers::STR_L1, 64], &[b'y'; 64]), + ); + // 256 bytes: first StrL2 (marker 0xC1, 2-byte little-endian length). + assert_eq!( + encode(&json!("p".repeat(256))), + buf(&[markers::STR_L2, 0x00, 0x01], &[b'p'; 256]), + ); + } + + #[test] + fn encodes_empty_array_as_length_count_form() { + // Empty array -> ArrLC1 with length 0 and count 0. + assert_eq!(encode(&json!([])), vec![PREAMBLE, markers::ARR_LC1, 0, 0]); + } + + #[test] + fn encodes_mixed_array() { + // [null, true, "x", 3.5] -> ArrLC1, len 13, count 4, then the elements. + assert_eq!( + encode(&json!([null, true, "x", 3.5])), + vec![ + PREAMBLE, + markers::ARR_LC1, + 0x0D, // payload length + 0x04, // element count + markers::NULL, + markers::TRUE, + 0x81, + b'x', // encoded-length "x" + markers::NUMBER_DOUBLE, + 0, + 0, + 0, + 0, + 0, + 0, + 0x0C, + 0x40, // 3.5 + ], + ); + } + + #[test] + fn encodes_empty_object_as_length_count_form() { + assert_eq!(encode(&json!({})), vec![PREAMBLE, markers::OBJ_LC1, 0, 0]); + } + + #[test] + fn encodes_object_with_nested_array() { + // { "a": 1, "b": [2, 3] } -> ObjLC1, len 10, count 2, then the members. + assert_eq!( + encode(&json!({ "a": 1, "b": [2, 3] })), + vec![ + PREAMBLE, + markers::OBJ_LC1, + 0x0A, // payload length + 0x02, // member count + 0x81, + b'a', // key "a" + 0x01, // value 1 (literal int) + 0x81, + b'b', // key "b" + markers::ARR_LC1, + 0x02, + 0x02, + 0x02, + 0x03, // [2, 3] + ], + ); + } + + #[test] + fn encodes_large_array_with_two_byte_container() { + // 300 single-byte elements push the container past the 1-byte length and + // count fields, so ArrLC2 (2-byte length + 2-byte count) is used. + let value = Value::Array((0..300).map(|_| json!(0)).collect()); + let encoded = encode(&value); + assert_eq!(encoded[0], PREAMBLE); + assert_eq!(encoded[1], markers::ARR_LC2); + // 2-byte little-endian length (300) then count (300). + assert_eq!(&encoded[2..6], &[0x2C, 0x01, 0x2C, 0x01]); + assert_eq!(encoded.len(), 6 + 300); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/cosmos_driver.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/cosmos_driver.rs index 3414323aceb..6dfa4f4df08 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/cosmos_driver.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/cosmos_driver.rs @@ -70,6 +70,19 @@ const DTX_OUTER_JITTER_RATIO: f64 = 0.25; const ACCOUNT_PROPERTIES_CONNECTIVITY_MAX_RETRIES: u32 = 2; const ACCOUNT_PROPERTIES_CONNECTIVITY_BASE_DELAY: Duration = Duration::from_millis(100); +/// Serialization formats advertised (`x-ms-cosmos-supported-serialization-formats`) +/// on point operations when binary encoding is enabled. Point operations +/// advertise `CosmosBinary` only — matching the .NET SDK's point-op default +/// (`RequestInvokerHandler` sets `BinarySerializationFormat = +/// SupportedSerializationFormats.CosmosBinary`) — so the service is required to +/// reply in binary, preserving the read-side RU/COGS benefit the caller opted +/// into. When the caller also asked for a text payload +/// (`request_text_response`), the driver transcodes the guaranteed-binary +/// response back to text after receiving it, keeping the wire binary in both +/// directions. (The broader `JsonText,CosmosBinary` negotiation applies to +/// query/feed, which is not yet wired.) +const BINARY_NEGOTIATION_FORMATS: &str = "CosmosBinary"; + fn should_retry_account_properties_connectivity_error( error: &crate::error::CosmosError, request_sent: RequestSentStatus, @@ -2284,10 +2297,9 @@ impl CosmosDriver { options: OperationOptions, ) -> crate::error::Result> { // PATCH is a virtual operation type: dispatch it to the dedicated - // Read-Modify-Write handler before any of the standard pipeline steps - // run, because the handler issues its own Read/Replace operations - // through this same entry point. `Box::pin` is required so the - // resulting async future has a fixed size even though it can recurse. + // Read-Modify-Write handler, which issues its own Read/Replace + // operations through this same entry point. `Box::pin` gives the + // recursive future a fixed size. if operation.operation_type() == crate::models::OperationType::Patch { let max_attempts = operation.patch_max_attempts(); return Box::pin(async { @@ -2303,14 +2315,95 @@ impl CosmosDriver { .await; } + // Resolve binary encoding through the same layered view as every other + // option, and only honor it for point **item** operations (the resource + // must be a `Document`; query/feed/batch and every control-plane + // resource are deferred per the binary-encoding spec). + let binary = + if Self::binary_encoding_applies(operation.resource_type(), operation.operation_type()) + { + self.operation_options_view(&options) + .binary_encoding() + .cloned() + .unwrap_or_default() + } else { + crate::options::BinaryEncodingOptions::default() + }; + let operation = if binary.enabled { + Self::apply_request_binary_encoding(operation)? + } else { + operation + }; + + let transcode_response_to_text = binary.enabled && binary.request_text_response; + // TODO: This boxing is a temporary fix to avoid a large future. // We need to do some refactoring here to shrink the future size and avoid this heap allocation if possible. - Box::pin(async { + let response = Box::pin(async { let container = operation.container().cloned(); let mut plan = Box::pin(self.plan_operation(operation, &options, None)).await?; self.execute_plan(&mut plan, container, options).await }) - .await + .await?; + + // Driver-side transcoding: convert the binary response body to text + // when the caller asked for a text payload over a binary wire. + if transcode_response_to_text { + if let Some(mut response) = response { + response.transcode_body_to_text()?; + return Ok(Some(response)); + } + } + Ok(response) + } + + /// Whether binary encoding applies to an operation. + /// + /// Honored only for point item operations: the resource must be a + /// [`ResourceType::Document`] and the operation one of create/read/replace/ + /// upsert. Control-plane resources share those operation types but must + /// never be binary encoded (some carry JSON bodies). + fn binary_encoding_applies( + resource_type: crate::models::ResourceType, + operation_type: crate::models::OperationType, + ) -> bool { + resource_type == crate::models::ResourceType::Document + && operation_type.supports_binary_encoding() + } + + /// Applies request-side binary encoding to an operation: transcodes a text + /// request body to Cosmos binary JSON (an already-binary or empty body is + /// passed through) and advertises binary responses via the + /// `x-ms-cosmos-supported-serialization-formats` header. + /// + /// This is schema-agnostic — it operates on the raw body bytes — so a + /// caller that deals only in text JSON gets a binary wire without encoding + /// anything itself. + fn apply_request_binary_encoding( + operation: CosmosOperation, + ) -> crate::error::Result { + // Transcode a non-empty *text* body to binary. A body that is already + // binary (the SDK's typed fast path) or empty is left in place — no + // clone — so only genuinely text bodies pay the conversion. + let transcoded = match operation.body() { + Some(body) if !body.is_empty() && !crate::binary_json::is_binary(body) => { + Some(crate::binary_json::transcode_to_binary(body).map_err(|e| { + crate::error::CosmosError::builder() + .with_status(crate::error::CosmosStatus::SERIALIZATION_REQUEST_BODY_INVALID) + .with_message(format!( + "failed to transcode text request body to Cosmos binary JSON: {e}" + )) + .with_source(e) + .build() + })?) + } + _ => None, + }; + let operation = match transcoded { + Some(bytes) => operation.with_body(bytes), + None => operation, + }; + Ok(operation.with_supported_serialization_formats(BINARY_NEGOTIATION_FORMATS)) } /// Executes a singleton operation (operations which return only a single result). @@ -5658,4 +5751,127 @@ mod tests { "logical PK must resolve to a single owning range, got {id}", ); } + + // ── apply_request_binary_encoding (schema-agnostic request-side encode) ── + + #[test] + fn binary_encoding_applies_only_to_document_item_ops() { + use crate::models::{OperationType, ResourceType}; + + // Point item ops on `Document` are the only combinations that qualify. + for op in [ + OperationType::Create, + OperationType::Read, + OperationType::Replace, + OperationType::Upsert, + ] { + assert!( + CosmosDriver::binary_encoding_applies(ResourceType::Document, op), + "Document + {op:?} should be binary-encodable", + ); + } + + // Non-item operation types on `Document` are excluded (query/feed/delete/patch). + for op in [ + OperationType::Delete, + OperationType::Query, + OperationType::ReadFeed, + OperationType::Patch, + ] { + assert!( + !CosmosDriver::binary_encoding_applies(ResourceType::Document, op), + "Document + {op:?} must not be binary-encoded", + ); + } + + // Control-plane resources share the create/read/replace/upsert operation + // types but must NEVER be binary encoded — some carry JSON bodies. + for rt in [ + ResourceType::Database, + ResourceType::DocumentCollection, + ResourceType::Offer, + ResourceType::StoredProcedure, + ResourceType::Trigger, + ResourceType::UserDefinedFunction, + ] { + for op in [ + OperationType::Create, + OperationType::Read, + OperationType::Replace, + OperationType::Upsert, + ] { + assert!( + !CosmosDriver::binary_encoding_applies(rt, op), + "{rt:?} + {op:?} must not be binary-encoded (control plane)", + ); + } + } + } + + fn binary_encoding_test_operation(body: Vec) -> CosmosOperation { + let container = epk_test_container(r#"{"paths":["/pk"],"version":2}"#); + let item = + crate::models::ItemReference::from_name(&container, PartitionKey::from("pk1"), "doc1"); + CosmosOperation::create_item(item).with_body(body) + } + + #[test] + fn apply_request_binary_encoding_transcodes_text_body_to_binary() { + // A caller (e.g. FFI) hands a TEXT JSON body; the driver transcodes it + // to Cosmos binary JSON and advertises binary responses. The caller + // never encoded binary itself. + let text = serde_json::to_vec(&serde_json::json!({ "id": "doc1", "n": 7 })).unwrap(); + assert!(!crate::binary_json::is_binary(&text)); + + let op = binary_encoding_test_operation(text); + let op = CosmosDriver::apply_request_binary_encoding(op).unwrap(); + + let body = op.body().expect("body present"); + assert!( + crate::binary_json::is_binary(body), + "text body must be transcoded to binary on the wire", + ); + // Decodes back to the same value. + assert_eq!( + crate::binary_json::decode(body).unwrap(), + serde_json::json!({ "id": "doc1", "n": 7 }), + ); + // Advertises binary responses. + assert_eq!( + op.request_headers() + .supported_serialization_formats + .as_deref(), + Some("CosmosBinary"), + ); + } + + #[test] + fn apply_request_binary_encoding_passes_binary_body_through() { + // A typed consumer (Rust SDK) may pre-encode to binary; the driver's + // request-side transcode sees an already-binary body and passes it + // through unchanged. + let binary = crate::binary_json::encode(&serde_json::json!({ "id": "doc1", "n": 7 })); + let op = binary_encoding_test_operation(binary.clone()); + let op = CosmosDriver::apply_request_binary_encoding(op).unwrap(); + + assert_eq!(op.body().unwrap(), binary.as_slice()); + assert_eq!( + op.request_headers() + .supported_serialization_formats + .as_deref(), + Some("CosmosBinary"), + ); + } + + #[test] + fn apply_request_binary_encoding_errors_on_invalid_text_body() { + // A body that is neither binary nor valid JSON surfaces as a + // request-body serialization error. + let op = binary_encoding_test_operation(b"{not json".to_vec()); + let err = CosmosDriver::apply_request_binary_encoding(op).unwrap_err(); + assert_eq!( + err.status().sub_status(), + Some(crate::error::SubStatusCode::SERIALIZATION_REQUEST_BODY_INVALID), + ); + } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/patch_handler.rs b/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/patch_handler.rs index df331af2dff..59adcce8a6c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/patch_handler.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/driver/pipeline/patch_handler.rs @@ -44,7 +44,7 @@ use crate::models::{ CosmosOperation, CosmosResponse, PartitionKeyKind, PatchInstructions, PatchOperation, Precondition, SessionToken, }; -use crate::options::OperationOptions; +use crate::options::{BinaryEncodingOptions, OperationOptions}; use async_trait::async_trait; use azure_core::http::{Etag, StatusCode}; use std::num::NonZeroU8; @@ -108,9 +108,14 @@ pub(crate) async fn execute( pub(crate) async fn execute_with_dispatcher( dispatcher: &D, operation: CosmosOperation, - options: OperationOptions, + mut options: OperationOptions, max_attempts: Option, ) -> crate::error::Result { + // PATCH is excluded from binary encoding. Force it off *explicitly*: + // `None` would inherit a lower layer (e.g. an account/client that enabled + // binary), which would then flow into the internal Read/Replace sub-ops. + options.binary_encoding = Some(BinaryEncodingOptions::new().with_enabled(false)); + // -- 1. Reject caller-set preconditions -- // // PATCH manages its own `If-Match` precondition internally — the handler @@ -1076,7 +1081,7 @@ mod tests { use crate::diagnostics::DiagnosticsContextBuilder; use crate::models::{ActivityId, CosmosResponseHeaders, CosmosStatus, RequestCharge}; - use crate::options::DiagnosticsOptions; + use crate::options::{BinaryEncodingOptions, DiagnosticsOptions}; use std::sync::{Arc, Mutex}; /// A pre-baked response a [`ScriptedDispatcher`] returns for a single @@ -1901,4 +1906,73 @@ mod tests { // source (the Replace), per `aggregate_sub_operations`'s contract. assert_eq!(returned.activity_id(), handed_out[1].activity_id()); } + + #[tokio::test] + async fn rmw_forces_binary_encoding_off_on_forwarded_sub_ops() { + // A caller may set `binary_encoding` on a patch; the handler must force + // it OFF explicitly (not `None`, which would inherit an account/client + // default) so the forwarded Read/Replace sub-ops stay text. + struct OptionsCapturingDispatcher { + binary_encodings: Mutex>>, + } + + #[async_trait] + impl SubOperationDispatcher for OptionsCapturingDispatcher { + async fn execute_operation( + &self, + operation: CosmosOperation, + options: OperationOptions, + ) -> crate::error::Result { + self.binary_encodings + .lock() + .unwrap() + .push(options.binary_encoding.clone()); + let body = match operation.operation_type() { + OperationType::Read => br#"{"id":"doc1","pk":"pk1","visits":0}"#.to_vec(), + OperationType::Replace => br#"{"id":"doc1","pk":"pk1","visits":1}"#.to_vec(), + other => panic!("unexpected sub-op {other:?}"), + }; + let mut headers = CosmosResponseHeaders::new(); + headers.etag = Some(Etag::from("\"v1\"")); + let diagnostics = Arc::new( + DiagnosticsContextBuilder::new( + ActivityId::new_uuid(), + Arc::new(DiagnosticsOptions::default()), + ) + .complete(), + ); + Ok(from_local_body_and_driver_headers( + body, + headers, + CosmosStatus::from_parts(StatusCode::Ok, None), + diagnostics, + )) + } + } + + let dispatcher = OptionsCapturingDispatcher { + binary_encodings: Mutex::new(Vec::new()), + }; + + // Caller opts a patch into binary encoding + text response. + let mut options = OperationOptions::default(); + options.binary_encoding = Some( + BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true), + ); + + execute_with_dispatcher(&dispatcher, canonical_patch_op(), options, None) + .await + .expect("PATCH should succeed"); + + let captured = dispatcher.binary_encodings.lock().unwrap().clone(); + assert_eq!(captured.len(), 2, "expected one Read + one Replace sub-op"); + let disabled = Some(BinaryEncodingOptions::new().with_enabled(false)); + assert!( + captured.iter().all(|be| *be == disabled), + "patch must force binary_encoding OFF (explicit disabled, not inherit) \ + on every forwarded sub-op, got {captured:?}", + ); + } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/error/cosmos_status.rs b/sdk/cosmos/azure_data_cosmos_driver/src/error/cosmos_status.rs index f5cf6d3aebb..09a43cbc963 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/error/cosmos_status.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/error/cosmos_status.rs @@ -1117,6 +1117,9 @@ impl SubStatusCode { /// `crate::error::Error::serialization`. pub const SERIALIZATION_RESPONSE_BODY_INVALID: SubStatusCode = SubStatusCode(20020); + /// Request body failed to serialize (20021). + pub const SERIALIZATION_REQUEST_BODY_INVALID: SubStatusCode = SubStatusCode(20021); + // ----- Authentication boundary mapping code (20402) ----- /// Credential / AAD token acquisition failed before the request was @@ -1917,6 +1920,13 @@ impl CosmosStatus { sub_status: Some(SubStatusCode::SERIALIZATION_RESPONSE_BODY_INVALID), }; + /// Request body failed to serialize (HTTP 400, sub-status 20021). The + /// caller supplied an item that could not be encoded. + pub const SERIALIZATION_REQUEST_BODY_INVALID: CosmosStatus = CosmosStatus { + status_code: StatusCode::BadRequest, + sub_status: Some(SubStatusCode::SERIALIZATION_REQUEST_BODY_INVALID), + }; + /// AAD / credential provider token acquisition failed /// (HTTP 401, sub-status 20402). pub const AUTHENTICATION_TOKEN_ACQUISITION_FAILED: CosmosStatus = CosmosStatus { diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/dispatch.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/dispatch.rs index 6e98df40b1a..3d374d46a6a 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/dispatch.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/dispatch.rs @@ -69,6 +69,11 @@ pub(crate) struct ParsedRequest { /// throughput instead of silently falling back to `ContainerConfig::default()` /// (which has no provisioned RU/s and disables throttling for the container). pub offer_throughput: Option, + /// Whether the client advertised that it accepts Cosmos binary JSON in the + /// response, via `x-ms-cosmos-supported-serialization-formats` containing + /// `CosmosBinary`. When set, item read/write responses encode their body as + /// binary so the full encode → store → decode loop can be exercised locally. + pub binary_response: bool, #[allow(dead_code)] pub offer_autopilot_settings: Option, #[allow(dead_code)] @@ -112,6 +117,8 @@ static END_EPK: HeaderName = HeaderName::from_static("x-ms-end-epk"); static READ_FEED_KEY_TYPE: HeaderName = HeaderName::from_static("x-ms-read-key-type"); static IS_BATCH_REQUEST: HeaderName = HeaderName::from_static("x-ms-cosmos-is-batch-request"); static OFFER_THROUGHPUT: HeaderName = HeaderName::from_static("x-ms-offer-throughput"); +static SUPPORTED_SERIALIZATION_FORMATS: HeaderName = + HeaderName::from_static("x-ms-cosmos-supported-serialization-formats"); static OFFER_AUTOPILOT_SETTINGS: HeaderName = HeaderName::from_static("x-ms-cosmos-offer-autopilot-settings"); @@ -179,6 +186,19 @@ pub(crate) fn parse_request(request: &Request) -> ParsedRequest { .get_optional_str(&OFFER_AUTOPILOT_SETTINGS) .map(|s| s.to_string()); + // The client advertises binary-response support via + // `x-ms-cosmos-supported-serialization-formats: JsonText,CosmosBinary`. + // Matching the .NET flag enum, the presence of a `CosmosBinary` token (case- + // insensitive, comma-separated) means the client can decode a binary + // response. + let binary_response = headers + .get_optional_str(&SUPPORTED_SERIALIZATION_FORMATS) + .map(|v| { + v.split(',') + .any(|fmt| fmt.trim().eq_ignore_ascii_case("CosmosBinary")) + }) + .unwrap_or(false); + let path = url.path(); // Reject trailing slashes after the leading `/`. `/dbs/mydb/colls/mycoll/docs/` // would otherwise parse to depth=5 and misroute to Create. Only the root @@ -246,6 +266,7 @@ pub(crate) fn parse_request(request: &Request) -> ParsedRequest { activity_id, content_response_on_write, offer_throughput, + binary_response, offer_autopilot_settings, max_item_count, continuation, @@ -702,4 +723,58 @@ mod tests { let parsed = parse_request(&req); assert_eq!(parsed.operation, OperationType::ReadAccount); } + + #[test] + fn binary_response_true_when_cosmosbinary_advertised() { + // The default binary-encoding negotiation header advertises both text + // and binary; the emulator must reply with binary. + let mut req = make_request("GET", "/dbs/mydb/colls/mycoll/docs/doc1"); + insert_header( + &mut req, + SUPPORTED_SERIALIZATION_FORMATS.clone(), + "JsonText,CosmosBinary", + ); + assert!(parse_request(&req).binary_response); + } + + #[test] + fn binary_response_false_when_only_jsontext_advertised() { + // `request_text_response` makes the SDK advertise only `JsonText`, so + // the emulator must reply with text even though the request body may be + // binary. This is the response-side of the `request_text_response` + // option. + let mut req = make_request("GET", "/dbs/mydb/colls/mycoll/docs/doc1"); + insert_header( + &mut req, + SUPPORTED_SERIALIZATION_FORMATS.clone(), + "JsonText", + ); + assert!(!parse_request(&req).binary_response); + } + + #[test] + fn binary_response_false_when_header_absent() { + // No negotiation header (binary encoding disabled) ⇒ text response. + let req = make_request("GET", "/dbs/mydb/colls/mycoll/docs/doc1"); + assert!(!parse_request(&req).binary_response); + } + + #[test] + fn binary_response_detects_cosmosbinary_regardless_of_order_or_case() { + // The token match is case-insensitive and order-independent, matching + // the .NET flag-enum parse. + for value in [ + "CosmosBinary", + "CosmosBinary,JsonText", + "cosmosbinary", + "JsonText, CosmosBinary", + ] { + let mut req = make_request("GET", "/dbs/mydb/colls/mycoll/docs/doc1"); + insert_header(&mut req, SUPPORTED_SERIALIZATION_FORMATS.clone(), value); + assert!( + parse_request(&req).binary_response, + "value {value:?} should negotiate a binary response", + ); + } + } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs index cd4f5f19b35..457bd9df6ab 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/operations.rs @@ -24,7 +24,9 @@ use super::response::headers::{ }; #[cfg(feature = "preview_dtx")] use super::response::headers::{ETAG, REQUEST_CHARGE, SESSION_TOKEN, SUBSTATUS}; -use super::response::{error_response, success_response, ResponseBuilder}; +use super::response::{ + error_response, success_response, success_response_with_format, ResponseBuilder, +}; use super::ru_model::RuChargingModel; use super::session::SessionToken; use super::store::{ @@ -657,6 +659,7 @@ pub(crate) async fn handle_operation( end_epk: None, is_query_plan: false, is_batch: false, + binary_response: false, is_upsert: matches!(operation_type, OperationType::Upsert), }; @@ -1461,6 +1464,7 @@ pub(crate) async fn handle_operation( end_epk: None, is_query_plan: false, is_batch: false, + binary_response: false, is_upsert: false, } } @@ -4178,6 +4182,21 @@ fn check_throttle( None } +/// Parses a request body as either Cosmos binary JSON (when it begins with the +/// `0x80` preamble) or UTF-8 text JSON. +/// +/// This mirrors the SDK's response-side auto-detection so the emulator accepts +/// binary-encoded item writes when the client negotiated binary, letting the +/// full encode → store → decode loop be validated locally. Returns `Err(())` on +/// a malformed body; callers turn that into a `400 BadRequest`. +fn decode_request_body(request_body: &[u8]) -> Result { + if crate::binary_json::is_binary(request_body) { + crate::binary_json::decode(request_body).map_err(|_| ()) + } else { + serde_json::from_slice(request_body).map_err(|_| ()) + } +} + async fn handle_create( store: &Arc, region_name: &str, @@ -4207,7 +4226,7 @@ async fn handle_create_locked( return resp; } - let mut body: serde_json::Value = match serde_json::from_slice(request_body) { + let mut body: serde_json::Value = match decode_request_body(request_body) { Ok(v) => v, Err(_) => { return error_response( @@ -4384,9 +4403,16 @@ async fn handle_create_locked( store.replicate(region_name, db_id, coll_id, &doc, false); let builder = if parsed.content_response_on_write { - success_response(StatusCode::Created, &response_body, charge, &token, start) - .with_etag(&doc.etag) - .with_lsn(doc.lsn) + success_response_with_format( + StatusCode::Created, + &response_body, + parsed.binary_response, + charge, + &token, + start, + ) + .with_etag(&doc.etag) + .with_lsn(doc.lsn) } else { ResponseBuilder::new(StatusCode::Created, start) .with_request_charge(charge) @@ -4617,9 +4643,16 @@ fn handle_read( match result { Some(Ok((body, etag, token, charge, lsn, headers))) => { - let builder = success_response(StatusCode::Ok, &body, charge, &token, start) - .with_etag(&etag) - .with_lsn(lsn); + let builder = success_response_with_format( + StatusCode::Ok, + &body, + parsed.binary_response, + charge, + &token, + start, + ) + .with_etag(&etag) + .with_lsn(lsn); decorate_point_response(builder, headers, Some(lsn)).build() } Some(Err(response)) => response, @@ -4657,7 +4690,7 @@ async fn handle_replace_locked( return resp; } - let mut body: serde_json::Value = match serde_json::from_slice(request_body) { + let mut body: serde_json::Value = match decode_request_body(request_body) { Ok(v) => v, Err(_) => { return error_response( @@ -4947,9 +4980,16 @@ async fn handle_replace_locked( store.replicate(region_name, db_id, coll_id, &doc, false); let builder = if parsed.content_response_on_write { - success_response(StatusCode::Ok, &response_body, charge, &token, start) - .with_etag(&doc.etag) - .with_lsn(doc.lsn) + success_response_with_format( + StatusCode::Ok, + &response_body, + parsed.binary_response, + charge, + &token, + start, + ) + .with_etag(&doc.etag) + .with_lsn(doc.lsn) } else { ResponseBuilder::new(StatusCode::Ok, start) .with_request_charge(charge) @@ -4994,7 +5034,7 @@ async fn handle_upsert_locked( return resp; } - let mut body: serde_json::Value = match serde_json::from_slice(request_body) { + let mut body: serde_json::Value = match decode_request_body(request_body) { Ok(v) => v, Err(_) => { return error_response( @@ -5156,9 +5196,16 @@ async fn handle_upsert_locked( store.replicate(region_name, db_id, coll_id, &doc, false); let builder = if parsed.content_response_on_write { - success_response(status, &response_body, charge, &token, start) - .with_etag(&doc.etag) - .with_lsn(doc.lsn) + success_response_with_format( + status, + &response_body, + parsed.binary_response, + charge, + &token, + start, + ) + .with_etag(&doc.etag) + .with_lsn(doc.lsn) } else { ResponseBuilder::new(status, start) .with_request_charge(charge) diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs index 92055ca909f..b72d07ba2df 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/in_memory_emulator/response.rs @@ -215,8 +215,25 @@ impl ResponseBuilder { self } - pub fn with_json_body(mut self, body: &serde_json::Value) -> Self { - self.body = serde_json::to_vec(body).unwrap_or_default(); + pub fn with_json_body(self, body: &serde_json::Value) -> Self { + self.with_value_body(body, false) + } + + /// Sets the body from a JSON value, encoded as Cosmos binary JSON when + /// `binary` is set (the client negotiated it) or UTF-8 text JSON otherwise. + /// + /// The binary form begins with the `0x80` preamble, which the SDK + /// auto-detects from the first byte, so the `Content-Type` stays + /// `application/json` either way (mirroring the real service). + pub fn with_value_body(mut self, body: &serde_json::Value, binary: bool) -> Self { + self.body = if binary { + crate::binary_json::encode(body) + } else { + // The emulator owns these `Value`s, so a serialization failure is a + // bug in the emulator — fail loudly rather than emit an empty body + // that would mask the defect downstream. + serde_json::to_vec(body).expect("emulator response body must serialize to JSON") + }; self } @@ -238,11 +255,25 @@ pub(crate) fn success_response( charge: f64, session_token: &str, start: Instant, +) -> ResponseBuilder { + success_response_with_format(status, body, false, charge, session_token, start) +} + +/// Like [`success_response`], but encodes the body as Cosmos binary JSON when +/// `binary` is set. Used by the item read/write handlers to honor a client that +/// negotiated binary responses via `x-ms-cosmos-supported-serialization-formats`. +pub(crate) fn success_response_with_format( + status: StatusCode, + body: &serde_json::Value, + binary: bool, + charge: f64, + session_token: &str, + start: Instant, ) -> ResponseBuilder { ResponseBuilder::new(status, start) .with_request_charge(charge) .with_session_token(session_token) - .with_json_body(body) + .with_value_body(body, binary) } /// Creates an error response. diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/lib.rs b/sdk/cosmos/azure_data_cosmos_driver/src/lib.rs index cd94c0ef497..48758d805a5 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/lib.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/lib.rs @@ -20,6 +20,7 @@ //! raw bytes (`&[u8]`) and return buffered responses (`Vec`). Serialization is handled by //! the consuming SDK in its native language. +pub mod binary_json; pub mod diagnostics; pub mod driver; pub mod error; diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_headers.rs b/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_headers.rs index 34e5235e18a..67612e9f629 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_headers.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_headers.rs @@ -43,6 +43,10 @@ pub(crate) mod request_header_names { pub const IS_QUERY: &str = "x-ms-documentdb-isquery"; pub const IS_QUERY_PLAN_REQUEST: &str = "x-ms-cosmos-is-query-plan-request"; pub const SUPPORTED_QUERY_FEATURES: &str = "x-ms-cosmos-supported-query-features"; + /// Advertises which serialization formats the client accepts in responses + /// (e.g. `JsonText,CosmosBinary`). The service uses it to decide whether to + /// reply with Cosmos binary JSON instead of text. + pub const SUPPORTED_SERIALIZATION_FORMATS: &str = "x-ms-cosmos-supported-serialization-formats"; pub const QUERY_VERSION: &str = "x-ms-cosmos-query-version"; pub const IS_UPSERT: &str = "x-ms-documentdb-is-upsert"; pub const MAX_ITEM_COUNT: &str = "x-ms-max-item-count"; @@ -281,6 +285,15 @@ pub struct CosmosRequestHeaders { /// Sent on query plan requests to indicate which query capabilities the /// client supports. The backend uses this to shape its response. pub supported_query_features: Option>, + + /// Serialization formats the client accepts in responses + /// (`x-ms-cosmos-supported-serialization-formats`). + /// + /// When set (e.g. `JsonText,CosmosBinary`), the service may reply with + /// Cosmos binary JSON, which the SDK auto-detects and decodes. `None` omits + /// the header, so the service replies with text JSON as before. The driver + /// is a passthrough here — the SDK decides the value per its enablement. + pub supported_serialization_formats: Option>, } impl CosmosRequestHeaders { @@ -380,15 +393,30 @@ impl CosmosRequestHeaders { if let Some(features) = self.supported_query_features.as_ref() { headers.insert( request_header_names::SUPPORTED_QUERY_FEATURES, - match features { - Cow::Borrowed(s) => HeaderValue::from(*s), - Cow::Owned(s) => HeaderValue::from(s.clone()), - }, + header_value_from_cow(features), + ); + } + if let Some(formats) = self.supported_serialization_formats.as_ref() { + headers.insert( + request_header_names::SUPPORTED_SERIALIZATION_FORMATS, + header_value_from_cow(formats), ); } } } +/// Converts a `Cow<'static, str>` header value into a [`HeaderValue`]. +/// +/// A `Cow::Borrowed` holds a `&'static str`, so it wraps into a `HeaderValue` +/// with no allocation; a `Cow::Owned` must clone once because `HeaderValue` +/// owns a `Cow<'static, str>` and this borrows `&self`. +fn header_value_from_cow(value: &Cow<'static, str>) -> HeaderValue { + match value { + Cow::Borrowed(s) => HeaderValue::from(*s), + Cow::Owned(s) => HeaderValue::from(s), + } +} + /// Autoscale throughput settings for the `x-ms-cosmos-offer-autopilot-settings` header. #[derive(Clone, Debug, Default, Serialize)] #[serde(rename_all = "camelCase")] @@ -1357,6 +1385,35 @@ mod tests { ); } + #[test] + fn write_to_headers_emits_supported_serialization_formats() { + let cosmos_headers = CosmosRequestHeaders { + supported_serialization_formats: Some("JsonText,CosmosBinary".into()), + ..Default::default() + }; + let mut headers = Headers::new(); + cosmos_headers.write_to_headers(&mut headers); + assert_eq!( + headers.get_optional_str(&HeaderName::from_static( + "x-ms-cosmos-supported-serialization-formats" + )), + Some("JsonText,CosmosBinary") + ); + } + + #[test] + fn write_to_headers_omits_supported_serialization_formats_when_none() { + let cosmos_headers = CosmosRequestHeaders::default(); + let mut headers = Headers::new(); + cosmos_headers.write_to_headers(&mut headers); + assert_eq!( + headers.get_optional_str(&HeaderName::from_static( + "x-ms-cosmos-supported-serialization-formats" + )), + None + ); + } + /// Round-trips a fully-populated [`CosmosResponseHeaders`] through /// [`to_raw_headers`](CosmosResponseHeaders::to_raw_headers) followed /// by [`from_headers`](CosmosResponseHeaders::from_headers) and diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_operation.rs b/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_operation.rs index c3905213be9..92d4b6f3557 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_operation.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_operation.rs @@ -242,6 +242,20 @@ impl CosmosOperation { self } + /// Advertises which serialization formats the client accepts in the response + /// (the `x-ms-cosmos-supported-serialization-formats` request header), e.g. + /// `JsonText,CosmosBinary`. + /// + /// When set, the service may reply with Cosmos binary JSON, which the SDK + /// auto-detects and decodes; when unset, the response stays text JSON. + pub fn with_supported_serialization_formats( + mut self, + formats: impl Into>, + ) -> Self { + self.request_headers.supported_serialization_formats = Some(formats.into()); + self + } + /// Sets the maximum number of items the server should return per page /// (the `x-ms-max-item-count` request header). /// @@ -1068,4 +1082,18 @@ mod tests { let resource_ref: CosmosResourceReference = item_ref.into(); let _op = CosmosOperation::new(OperationType::Create, resource_ref, None); } + + #[test] + fn with_supported_serialization_formats_sets_header_field() { + let item_ref = + ItemReference::from_name(&test_container(), PartitionKey::from("pk1"), "doc1"); + let op = CosmosOperation::create_item(item_ref) + .with_supported_serialization_formats("JsonText,CosmosBinary"); + assert_eq!( + op.request_headers() + .supported_serialization_formats + .as_deref(), + Some("JsonText,CosmosBinary"), + ); + } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_response.rs b/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_response.rs index 3b7c246be35..ab1c24de43c 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_response.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/models/cosmos_response.rs @@ -45,8 +45,14 @@ impl CosmosResponsePayload { pub(crate) fn headers(&self) -> &CosmosResponseHeaders { &self.headers } -} + /// Transcodes any binary JSON body to text JSON in place. + fn transcode_body_to_text(&mut self) -> crate::error::Result<()> { + let body = std::mem::take(&mut self.body); + self.body = body.transcode_to_text()?; + Ok(()) + } +} /// Result of a Cosmos DB operation. /// /// Contains the response body (as a [`ResponseBody`] of one or more @@ -130,6 +136,22 @@ impl CosmosResponse { self.payload.into_body() } + /// Transcodes a binary JSON response body to text JSON in place. + /// + /// Applied by the driver when the operation negotiated binary on the wire + /// and the caller requested a text response + /// ([`BinaryEncodingOptions::request_text_response`](crate::options::BinaryEncodingOptions)): + /// the wire stays binary in both directions, and the driver converts the + /// binary response payload to text before returning it. A text (or empty) + /// body is left unchanged. + /// + /// # Errors + /// + /// Returns an error if a binary payload is malformed. + pub(crate) fn transcode_body_to_text(&mut self) -> crate::error::Result<()> { + self.payload.transcode_body_to_text() + } + /// Returns a reference to the extracted headers. pub fn headers(&self) -> &CosmosResponseHeaders { self.payload.headers() diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/models/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/src/models/mod.rs index 9e0addc336e..260f920811e 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/models/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/models/mod.rs @@ -636,6 +636,20 @@ impl OperationType { ) } + /// True for the point item ops (create/read/replace/upsert) eligible for + /// binary encoding. Necessary but not sufficient: the full gate also + /// requires [`ResourceType::Document`] (see + /// `CosmosDriver::binary_encoding_applies`). + pub(crate) fn supports_binary_encoding(self) -> bool { + matches!( + self, + OperationType::Create + | OperationType::Read + | OperationType::Replace + | OperationType::Upsert + ) + } + /// Returns the HTTP method for this operation type. pub fn http_method(self) -> azure_core::http::Method { use azure_core::http::Method; @@ -877,6 +891,35 @@ mod tests { use super::*; use serde::{Deserialize, Serialize}; + #[test] + fn supports_binary_encoding_covers_only_bodied_point_ops() { + // Matches the binary-encoding spec §2 scope table: create/read/replace/ + // upsert. `delete` is excluded (no request or response body); query, + // feed, batch, and stored-procedure paths are deferred. + for op in [ + OperationType::Create, + OperationType::Read, + OperationType::Replace, + OperationType::Upsert, + ] { + assert!(op.supports_binary_encoding(), "{op:?} should be supported"); + } + for op in [ + OperationType::Delete, + OperationType::Query, + OperationType::SqlQuery, + OperationType::ReadFeed, + OperationType::Batch, + OperationType::Execute, + OperationType::Patch, + ] { + assert!( + !op.supports_binary_encoding(), + "{op:?} should not be supported" + ); + } + } + #[test] fn partition_key_version_numeric_mapping() { assert_eq!( diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/models/response_body.rs b/sdk/cosmos/azure_data_cosmos_driver/src/models/response_body.rs index a49c82e1ac0..b75648d9a95 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/models/response_body.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/models/response_body.rs @@ -120,55 +120,92 @@ impl ResponseBody { } } - /// Deserializes a single-payload body as JSON of type `T`. + /// Deserializes a single-payload body as type `T`, transparently accepting + /// either Cosmos binary JSON or UTF-8 text JSON (auto-detected by the + /// `0x80` preamble). /// /// Returns an error if the body is a feed [`Items`](Self::Items) response /// or if the body is [`NoPayload`](Self::NoPayload) (nothing to parse). pub fn into_single(self) -> crate::error::Result { let bytes = self.single()?; - serde_json::from_slice(&bytes).map_err(|e| { - crate::error::CosmosError::builder() - .with_status(crate::error::CosmosStatus::SERIALIZATION_RESPONSE_BODY_INVALID) - .with_message("failed to deserialize response body") - .with_source(e) - .build() - }) + deserialize_response(&bytes, "failed to deserialize response body") } /// Deserializes every item in a feed response, or the single payload, as - /// JSON of type `T`. A [`NoPayload`](Self::NoPayload) body yields an empty - /// `Vec`. + /// type `T`. A [`NoPayload`](Self::NoPayload) body yields an empty `Vec`. + /// + /// Each buffer is decoded transparently as either Cosmos binary JSON or + /// UTF-8 text JSON (auto-detected by the `0x80` preamble). pub fn into_items(self) -> crate::error::Result> { match self { Self::NoPayload => Ok(Vec::new()), Self::Bytes(b) => { - let item = serde_json::from_slice(&b).map_err(|e| { - crate::error::CosmosError::builder() - .with_status( - crate::error::CosmosStatus::SERIALIZATION_RESPONSE_BODY_INVALID, - ) - .with_message("failed to deserialize response body") - .with_source(e) - .build() - })?; + let item = deserialize_response(&b, "failed to deserialize response body")?; Ok(vec![item]) } Self::Items(items) => items .into_iter() - .map(|b| { - serde_json::from_slice(&b).map_err(|e| { - crate::error::CosmosError::builder() - .with_status( - crate::error::CosmosStatus::SERIALIZATION_RESPONSE_BODY_INVALID, - ) - .with_message("failed to deserialize feed item") - .with_source(e) - .build() - }) - }) + // NOTE: `deserialize_response` auto-detects binary per slice via + // the `0x80` preamble, but the feed pipeline that produces + // `Self::Items` splits the `Documents` array by scanning **text** + // JSON — it is not binary-aware. A single-preamble binary feed + // envelope would therefore be sliced into sub-documents *without* + // preambles, which `is_binary` would then route to the text path. + // This is inert today because query/feed binary negotiation is + // deferred (the service does not emit binary feeds without the + // negotiation header), so the binary `Items` branch is only + // exercised by hand-prefixed synthetic tests. When feed/query + // binary negotiation is added, the feed splitter must be made + // binary-aware (or each slice re-prefixed) before this path can + // decode real binary feeds. + .map(|b| deserialize_response(&b, "failed to deserialize feed item")) .collect(), } } + + /// Transcodes any Cosmos **binary** JSON payload(s) to UTF-8 **text** JSON + /// in place, leaving text payloads unchanged. + /// + /// Used by the driver when the operation requested a text response while + /// keeping the wire binary (see + /// [`BinaryEncodingOptions::request_text_response`](crate::options::BinaryEncodingOptions)). + /// The conversion is schema-agnostic: each buffer is decoded with + /// [`binary_json::decode`](crate::binary_json::decode) and re-serialized as + /// compact text JSON. A [`NoPayload`](Self::NoPayload) body is a no-op. + /// + /// # Errors + /// + /// Returns an error if a binary payload is malformed. + pub(crate) fn transcode_to_text(self) -> crate::error::Result { + fn convert(bytes: &Bytes) -> crate::error::Result { + // Already-text payloads are left unchanged: return a cheap refcount + // clone instead of round-tripping through `transcode_to_text` (which + // would copy the buffer into a fresh `Vec`). + if !crate::binary_json::is_binary(bytes) { + return Ok(bytes.clone()); + } + let text = crate::binary_json::transcode_to_text(bytes).map_err(|e| { + crate::error::CosmosError::builder() + .with_status(crate::error::CosmosStatus::SERIALIZATION_RESPONSE_BODY_INVALID) + .with_message(format!("failed to transcode binary response to text: {e}")) + .with_source(e) + .build() + })?; + Ok(Bytes::from(text)) + } + + match self { + Self::NoPayload => Ok(Self::NoPayload), + Self::Bytes(b) => Ok(Self::Bytes(convert(&b)?)), + Self::Items(items) => { + let converted = items + .into_iter() + .map(|b| convert(&b)) + .collect::>>()?; + Ok(Self::Items(converted)) + } + } + } } impl From for ResponseBody { @@ -183,6 +220,42 @@ impl From> for ResponseBody { } } +/// Builds a `SERIALIZATION_RESPONSE_BODY_INVALID` error carrying `message` and +/// the underlying `source`. +fn invalid_body_error(message: &'static str, source: E) -> crate::error::CosmosError +where + E: std::error::Error + Send + Sync + 'static, +{ + crate::error::CosmosError::builder() + .with_status(crate::error::CosmosStatus::SERIALIZATION_RESPONSE_BODY_INVALID) + .with_message(message) + .with_source(source) + .build() +} + +/// Deserializes a response buffer as `T`, transparently accepting either Cosmos +/// binary JSON or UTF-8 text JSON. +/// +/// A buffer that begins with the binary preamble (`0x80`, detected by +/// [`is_binary`](crate::binary_json::is_binary)) is deserialized directly by the +/// binary-JSON codec's native serde deserializer +/// ([`from_slice`](crate::binary_json::from_slice)) — driving `T::deserialize` +/// straight off the bytes with no intermediate [`serde_json::Value`]; any other +/// buffer is parsed directly as text JSON. Because no UTF-8 text JSON document +/// can begin with `0x80` (it is a UTF-8 continuation byte), the detection is +/// unambiguous and the text path is byte-for-byte unchanged. `message` is the +/// error context attached on failure. +fn deserialize_response( + bytes: &[u8], + message: &'static str, +) -> crate::error::Result { + if crate::binary_json::is_binary(bytes) { + crate::binary_json::from_slice(bytes).map_err(|e| invalid_body_error(message, e)) + } else { + serde_json::from_slice(bytes).map_err(|e| invalid_body_error(message, e)) + } +} + #[cfg(test)] mod tests { use super::*; @@ -350,4 +423,176 @@ mod tests { fn is_empty_false_for_non_empty_bytes() { assert!(!ResponseBody::Bytes(Bytes::from_static(b"x")).is_empty()); } + + // ── Binary JSON auto-detection (P1e) ──────────────────────────────────── + // + // The encoder is a later phase, so these vectors are hand-encoded. Each + // buffer starts with the `0x80` preamble; `into_single` / `into_items` + // detect it and route through the binary-JSON decoder. + + use crate::binary_json::{markers, PREAMBLE}; + + /// Wraps already-encoded value bytes in a complete binary buffer (preamble + /// prefix), returning shared [`Bytes`]. + fn binary(value_bytes: &[u8]) -> Bytes { + let mut buf = vec![PREAMBLE]; + buf.extend_from_slice(value_bytes); + Bytes::from(buf) + } + + /// Encodes a `{"id": n}` object: `OBJ1` (single property), the 1-byte + /// system string for `id` (index 12), then the literal-int value `n` + /// (`n` must be < 32 to use the literal-int form). + fn id_object(n: u8) -> Vec { + assert!(n < 32, "literal-int form requires n < 32"); + vec![markers::OBJ1, markers::SYSTEM_STRING_1BYTE_MIN + 12, n] + } + + /// Encodes an encoded-length string value (the marker carries the length). + fn enc_str(s: &str) -> Vec { + let mut v = vec![markers::ENCODED_STRING_LENGTH_MIN | (s.len() as u8)]; + v.extend_from_slice(s.as_bytes()); + v + } + + #[derive(serde::Deserialize, PartialEq, Debug)] + struct Item { + id: u32, + } + + #[test] + fn into_single_decodes_binary_object() { + // Binary `{"id": 7}` decodes through the typed point-read path. + let body = ResponseBody::Bytes(binary(&id_object(7))); + let item: Item = body.into_single().unwrap(); + assert_eq!(item, Item { id: 7 }); + } + + #[test] + fn into_items_decodes_binary_bytes_variant() { + let body = ResponseBody::Bytes(binary(&id_object(9))); + let items: Vec = body.into_items().unwrap(); + assert_eq!(items, vec![Item { id: 9 }]); + } + + #[test] + fn into_items_decodes_binary_items_variant() { + let body = ResponseBody::from_items(vec![binary(&id_object(1)), binary(&id_object(2))]); + let items: Vec = body.into_items().unwrap(); + assert_eq!(items, vec![Item { id: 1 }, Item { id: 2 }]); + } + + #[test] + fn into_single_decodes_binary_feed_envelope() { + // Proves the query path: the whole `{"Documents":[…],"_count":N}` + // envelope is decoded from binary and deserialized into a typed feed + // body in one pass (as the SDK does via `into_single::>`). + #[derive(serde::Deserialize, PartialEq, Debug)] + struct Feed { + #[serde(rename = "Documents")] + documents: Vec, + #[serde(rename = "_count")] + count: u32, + } + + // Documents: ARR1 wrapping a single `{"id": 1}`. + let mut documents = vec![markers::ARR1]; + documents.extend_from_slice(&id_object(1)); + + // OBJ_L1 envelope with two members. + let mut payload = Vec::new(); + payload.extend_from_slice(&enc_str("Documents")); + payload.extend_from_slice(&documents); + payload.extend_from_slice(&enc_str("_count")); + payload.push(0x01); // literal int 1 + let mut envelope = vec![markers::OBJ_L1, payload.len() as u8]; + envelope.extend_from_slice(&payload); + + let body = ResponseBody::Bytes(binary(&envelope)); + let feed: Feed = body.into_single().unwrap(); + assert_eq!( + feed, + Feed { + documents: vec![Item { id: 1 }], + count: 1, + } + ); + } + + #[test] + fn text_json_still_deserializes_unchanged() { + // A text buffer never begins with `0x80`, so it takes the unchanged + // text path even with auto-detection in place. + let body = ResponseBody::Bytes(Bytes::from_static(br#"{"id":5}"#)); + let item: Item = body.into_single().unwrap(); + assert_eq!(item, Item { id: 5 }); + } + + #[test] + fn malformed_binary_body_errors() { + // A lone preamble is a truncated binary buffer; decoding fails and the + // error surfaces as a response-body deserialization error. + let body = ResponseBody::Bytes(Bytes::from_static(&[PREAMBLE])); + let result: crate::error::Result = body.into_single(); + assert!(result.is_err()); + } + + // ── Binary → text transcoding ─────────────────────────────────────────── + + #[test] + fn transcode_bytes_binary_to_text() { + // A binary `{"id": 7}` body transcodes to text bytes (no `0x80`) that + // deserialize to the same value. + let body = ResponseBody::Bytes(binary(&id_object(7))); + let text = body.transcode_to_text().unwrap(); + match &text { + ResponseBody::Bytes(b) => { + assert!(!crate::binary_json::is_binary(b), "must be text now"); + let item: Item = serde_json::from_slice(b).unwrap(); + assert_eq!(item, Item { id: 7 }); + } + _ => panic!("expected Bytes variant"), + } + } + + #[test] + fn transcode_items_binary_to_text() { + let body = ResponseBody::from_items(vec![binary(&id_object(1)), binary(&id_object(2))]); + let text = body.transcode_to_text().unwrap(); + match &text { + ResponseBody::Items(items) => { + assert_eq!(items.len(), 2); + for b in items { + assert!(!crate::binary_json::is_binary(b)); + } + } + _ => panic!("expected Items variant"), + } + } + + #[test] + fn transcode_text_body_unchanged() { + // Text passes through byte-for-byte. + let body = ResponseBody::Bytes(Bytes::from_static(br#"{"id":5}"#)); + let text = body.transcode_to_text().unwrap(); + match &text { + ResponseBody::Bytes(b) => assert_eq!(&b[..], br#"{"id":5}"#), + _ => panic!("expected Bytes variant"), + } + } + + #[test] + fn transcode_no_payload_is_noop() { + let body = ResponseBody::NoPayload; + assert!(matches!( + body.transcode_to_text().unwrap(), + ResponseBody::NoPayload + )); + } + + #[test] + fn transcode_malformed_binary_errors() { + let body = ResponseBody::Bytes(Bytes::from_static(&[PREAMBLE])); + assert!(body.transcode_to_text().is_err()); + } } diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/options/binary_encoding.rs b/sdk/cosmos/azure_data_cosmos_driver/src/options/binary_encoding.rs new file mode 100644 index 00000000000..ad76a41b3f8 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/src/options/binary_encoding.rs @@ -0,0 +1,82 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! [`BinaryEncodingOptions`] — driver-level Cosmos binary JSON encoding options. + +/// Options controlling Cosmos **binary JSON** on the wire for an operation. +/// +/// These options are **schema-agnostic**, so they live in the driver. They are +/// set on +/// [`OperationOptions::binary_encoding`](crate::options::OperationOptions::binary_encoding) +/// and participate in the standard runtime → account → operation layered +/// resolution. +/// +/// The driver performs the byte-level transcoding both ways when needed, so a +/// caller can deal purely in text JSON and still get an efficient binary wire: +/// +/// - [`enabled`](Self::enabled) — put **binary on the wire**. On the request +/// path, a text-JSON body is transcoded to binary before it is sent (an +/// already-binary body is passed through). The client also advertises +/// `CosmosBinary`, so the service response comes back binary. +/// - [`request_text_response`](Self::request_text_response) — hand the caller +/// **text JSON back**. The wire still stays binary; the driver transcodes the +/// binary response to text before returning it. Has no effect unless +/// [`enabled`](Self::enabled) is `true`. +/// +/// A typed consumer may pre-encode its request body straight from +/// `T: Serialize` as an optimization; the driver's request-side transcoding then +/// sees an already-binary body and passes it through unchanged. +/// +/// # Examples +/// +/// ```rust +/// use azure_data_cosmos_driver::options::BinaryEncodingOptions; +/// +/// // Binary on the wire, but transcode the response back to text. +/// let options = BinaryEncodingOptions::new() +/// .with_enabled(true) +/// .with_request_text_response(true); +/// assert!(options.enabled); +/// assert!(options.request_text_response); +/// ``` +#[derive(Clone, Debug, Default, PartialEq, Eq)] +#[non_exhaustive] +pub struct BinaryEncodingOptions { + /// Whether Cosmos binary JSON is used on the wire for the operation. + /// + /// When `true`, the request body is sent as binary (the driver transcodes a + /// text body to binary first, or passes an already-binary body through) and + /// the client advertises that it accepts binary responses. + pub enabled: bool, + + /// Whether the driver hands the caller **text** JSON even when binary + /// encoding is [`enabled`](Self::enabled). + /// + /// When `false` (the default), the binary response is returned as-is. When + /// `true`, the wire stays binary in both directions and the driver + /// transcodes the binary response to text JSON before returning it. Has no + /// effect when [`enabled`](Self::enabled) is `false`. + pub request_text_response: bool, +} + +impl BinaryEncodingOptions { + /// Creates binary-encoding options with defaults (disabled). + pub fn new() -> Self { + Self::default() + } + + /// Sets whether Cosmos binary JSON is used on the wire. + pub fn with_enabled(mut self, enabled: bool) -> Self { + self.enabled = enabled; + self + } + + /// Sets whether the driver transcodes the binary response back to text JSON. + /// + /// See [`request_text_response`](Self::request_text_response) for the + /// behavior. + pub fn with_request_text_response(mut self, request_text_response: bool) -> Self { + self.request_text_response = request_text_response; + self + } +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/options/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/src/options/mod.rs index 4c12ea43b45..7aa193ce775 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/options/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/options/mod.rs @@ -16,6 +16,7 @@ //! initialization time and do not participate in per-operation layered resolution. mod availability_strategy; +mod binary_encoding; mod connection_pool; mod diagnostics_options; mod driver_options; @@ -30,6 +31,7 @@ mod region; mod throughput_control; pub use availability_strategy::{AvailabilityStrategy, HedgeThreshold, HedgingStrategy}; +pub use binary_encoding::BinaryEncodingOptions; pub use connection_pool::{ConnectionPoolOptions, ConnectionPoolOptionsBuilder}; pub use diagnostics_options::{ DiagnosticsOptions, DiagnosticsOptionsBuilder, DiagnosticsVerbosity, diff --git a/sdk/cosmos/azure_data_cosmos_driver/src/options/operation_options.rs b/sdk/cosmos/azure_data_cosmos_driver/src/options/operation_options.rs index 51e4ccb5081..61b5dafb060 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/src/options/operation_options.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/src/options/operation_options.rs @@ -12,8 +12,8 @@ use azure_data_cosmos_macros::CosmosOptions; use crate::{ models::ThroughputControlGroupName, options::{ - AvailabilityStrategy, ContentResponseOnWrite, EndToEndOperationLatencyPolicy, - ExcludedRegions, PriorityLevel, ReadConsistencyStrategy, + AvailabilityStrategy, BinaryEncodingOptions, ContentResponseOnWrite, + EndToEndOperationLatencyPolicy, ExcludedRegions, PriorityLevel, ReadConsistencyStrategy, }, }; @@ -142,6 +142,15 @@ pub struct OperationOptions { // Additional headers beyond those natively supported by the driver. // May be removed in the future as we analyze exactly what options are needed. pub custom_headers: Option>, + + /// Cosmos binary JSON encoding for this operation. + /// + /// Controls whether the operation uses binary on the wire and whether the + /// driver transcodes the response back to text. Schema-agnostic, so it is + /// honored uniformly for the Rust SDK and FFI callers. `None` inherits from + /// a lower level (default: text JSON, no binary). See + /// [`BinaryEncodingOptions`]. + pub binary_encoding: Option, } /// Retry behavior for requests throttled by the service (HTTP 429, @@ -505,6 +514,56 @@ mod tests { ); } + #[test] + fn builder_round_trips_binary_encoding() { + let options = OperationOptionsBuilder::new() + .with_binary_encoding( + BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true), + ) + .build(); + + assert_eq!( + options.binary_encoding, + Some( + BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true) + ) + ); + } + + #[test] + fn binary_encoding_resolves_via_view() { + use std::sync::Arc; + + let account_be = BinaryEncodingOptions::new().with_enabled(true); + let operation_be = BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true); + + let account = Arc::new(OperationOptions { + binary_encoding: Some(account_be.clone()), + ..Default::default() + }); + + // Operation layer wins over account. + let operation = OperationOptions { + binary_encoding: Some(operation_be.clone()), + ..Default::default() + }; + let view_op_overrides = + OperationOptionsView::new(None, None, Some(account.clone()), Some(&operation)); + assert_eq!(view_op_overrides.binary_encoding(), Some(&operation_be)); + + // When the operation layer leaves it unset, the account value applies. + let empty_operation = OperationOptions::default(); + let view_account_wins = + OperationOptionsView::new(None, None, Some(account), Some(&empty_operation)); + assert_eq!(view_account_wins.binary_encoding(), Some(&account_be)); + } + /// The nested [`ThrottlingRetryOptions`] group must participate in the /// standard runtime → account → operation → environment layered /// resolution on a *per-inner-field* basis. A finer-grained per-field diff --git a/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json b/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json new file mode 100644 index 00000000000..61943444e85 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/testdata/binary_json_vectors.json @@ -0,0 +1,63 @@ +[ + { "name": "null", "binary": "80 D0", "json": null }, + { "name": "false", "binary": "80 D1", "json": false }, + { "name": "true", "binary": "80 D2", "json": true }, + { "name": "literal_int_zero", "binary": "80 00", "json": 0 }, + { "name": "literal_int_max", "binary": "80 1F", "json": 31 }, + { "name": "uint8", "binary": "80 C8 C8", "json": 200 }, + { "name": "int16", "binary": "80 C9 18 FC", "json": -1000 }, + { "name": "int32", "binary": "80 CA 70 11 01 00", "json": 70000 }, + { "name": "int64", "binary": "80 CB 00 0E FA D5 FE FF FF FF", "json": -5000000000 }, + { "name": "uint64", "binary": "80 C7 FE FF FF FF FF FF FF FF", "json": 18446744073709551614 }, + { "name": "double", "binary": "80 CC 00 00 00 00 00 00 0C 40", "json": 3.5 }, + { "name": "ext_int8", "binary": "80 D8 FB", "json": -5 }, + { "name": "ext_int16", "binary": "80 D9 18 FC", "json": -1000 }, + { "name": "ext_int32", "binary": "80 DA 90 EE FE FF", "json": -70000 }, + { "name": "ext_int64", "binary": "80 DB 00 0E FA D5 FE FF FF FF", "json": -5000000000 }, + { "name": "ext_uint32", "binary": "80 DC FE FF FF FF", "json": 4294967294 }, + { "name": "float32", "binary": "80 CD 00 00 C0 3F", "json": 1.5 }, + { "name": "float64", "binary": "80 CE 00 00 00 00 00 00 02 C0", "json": -2.25 }, + { "name": "system_string_id", "binary": "80 2C", "json": "id" }, + { "name": "encoded_length_string_empty", "binary": "80 80", "json": "" }, + { "name": "encoded_length_string_hi", "binary": "80 82 68 69", "json": "hi" }, + { "name": "str_l1_hello", "binary": "80 C0 05 68 65 6C 6C 6F", "json": "hello" }, + { "name": "str_l2_300a", "binary": "80 C1 2C 01 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61 61", "json": "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" }, + { "name": "guid_string_lower", "binary": "80 75 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F", "json": "00010203-0405-0607-0809-0a0b0c0d0e0f" }, + { "name": "guid_string_upper", "binary": "80 76 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F", "json": "00010203-0405-0607-0809-0A0B0C0D0E0F" }, + { "name": "guid_string_quoted", "binary": "80 77 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F", "json": "\"00010203-0405-0607-0809-0a0b0c0d0e0f\"" }, + { "name": "base64_foo", "binary": "80 71 01 00 66 6F 6F", "json": "Zm9v" }, + { "name": "base64_fo", "binary": "80 71 01 01 66 6F", "json": "Zm8=" }, + { "name": "base64_a", "binary": "80 71 01 02 41", "json": "QQ==" }, + { "name": "base64_a_omitted_padding", "binary": "80 71 01 FD 41", "json": "QQ" }, + { "name": "base64_len2_foobar", "binary": "80 72 02 00 00 66 6F 6F 62 61 72", "json": "Zm9vYmFy" }, + { "name": "base64_std_special", "binary": "80 71 01 00 FB FF FE", "json": "+//+" }, + { "name": "base64_url_special", "binary": "80 73 01 00 FB FF FE", "json": "-__-" }, + { "name": "compressed_lower_hex", "binary": "80 78 04 A1 B2", "json": "1a2b" }, + { "name": "compressed_upper_hex", "binary": "80 79 04 A1 B2", "json": "1A2B" }, + { "name": "compressed_lower_hex_odd", "binary": "80 78 01 0F", "json": "f" }, + { "name": "compressed_datetime", "binary": "80 7A 07 13 53 1C 02", "json": "2024-01" }, + { "name": "packed_7bit", "binary": "80 7E 02 C8 34", "json": "Hi" }, + { "name": "packed_4bit", "binary": "80 7B 04 30 10 32", "json": "0123" }, + { "name": "packed_5bit", "binary": "80 7C 03 61 20 08", "json": "abc" }, + { "name": "packed_6bit", "binary": "80 7D 04 61 40 20 0C", "json": "abcd" }, + { "name": "packed_7bit_len2", "binary": "80 7F 02 00 C8 34", "json": "Hi" }, + { "name": "guid_value", "binary": "80 D3 00 01 02 03 04 05 06 07 08 09 0A 0B 0C 0D 0E 0F", "json": "03020100-0504-0706-0809-0a0b0c0d0e0f" }, + { "name": "binary_deadbeef", "binary": "80 DD 04 DE AD BE EF", "json": "3q2+7w==" }, + { "name": "binary_empty", "binary": "80 DD 00", "json": "" }, + { "name": "binary_2byte", "binary": "80 DE 03 00 01 02 03", "json": "AQID" }, + { "name": "uniform_int32", "binary": "80 F0 DA 03 01 00 00 00 02 00 00 00 03 00 00 00", "json": [1,2,3] }, + { "name": "uniform_uint8", "binary": "80 F0 D7 03 0A 14 1E", "json": [10,20,30] }, + { "name": "uniform_empty", "binary": "80 F0 DA 00", "json": [] }, + { "name": "uniform_int16_c2", "binary": "80 F1 D9 03 00 FF FF 00 00 E8 03", "json": [-1,0,1000] }, + { "name": "uniform_float32", "binary": "80 F0 CD 02 00 00 C0 3F 00 00 80 BE", "json": [1.5,-0.25] }, + { "name": "uniform_arr_of_arr", "binary": "80 F2 F0 DA 02 02 01 00 00 00 02 00 00 00 03 00 00 00 04 00 00 00", "json": [[1,2],[3,4]] }, + { "name": "empty_array", "binary": "80 E0", "json": [] }, + { "name": "empty_object", "binary": "80 E8", "json": {} }, + { "name": "single_item_array", "binary": "80 E1 D2", "json": [true] }, + { "name": "single_item_object", "binary": "80 E9 2C D2", "json": {"id":true} }, + { "name": "array_l1", "binary": "80 E2 03 00 01 D0", "json": [0,1,null] }, + { "name": "array_lc1", "binary": "80 E5 03 03 00 01 D0", "json": [0,1,null] }, + { "name": "object_l1", "binary": "80 EA 04 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "object_lc1", "binary": "80 ED 04 02 2C 00 3B 01", "json": {"id":0,"type":1} }, + { "name": "nested_containers", "binary": "80 E2 05 E1 00 E9 2C 01", "json": [[0],{"id":1}] } +] \ No newline at end of file diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/binary_response_format.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/binary_response_format.rs new file mode 100644 index 00000000000..2fb78ad8d55 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/binary_response_format.rs @@ -0,0 +1,220 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Response-format negotiation tests. +//! +//! These tests exercise the **response side** of Cosmos binary JSON +//! negotiation at the emulator: given a binary request body, what serialization +//! format does the service send *back*? The answer is driven entirely by the +//! `x-ms-cosmos-supported-serialization-formats` request header: +//! +//! - `JsonText,CosmosBinary` (or `CosmosBinary` alone) → the service replies +//! with **binary** (body begins with the `0x80` preamble). +//! - `JsonText` alone → the service replies with **text**, even though the +//! request body was binary. +//! - no header (binary encoding disabled) → **text**. +//! +//! Note the SDK-level `BinaryEncodingOptions::request_text_response` does **not** +//! send `JsonText` alone: point operations keep advertising `CosmosBinary` (so +//! the wire stays binary) and the *driver* transcodes the binary response to +//! text. These tests cover the underlying emulator format decision directly. +//! +//! The tests send a Cosmos-binary request body directly through the in-memory +//! emulator and inspect the **raw** response bytes, so they assert on the actual +//! wire format rather than a decoded value. + +use super::*; +use azure_core::http::headers::HeaderValue; +use azure_data_cosmos_driver::binary_json::{self, PREAMBLE}; + +/// Builds a create-item POST whose body is Cosmos **binary** JSON, optionally +/// advertising a response-format via `x-ms-cosmos-supported-serialization-formats`. +fn create_binary_item_request( + gateway_url: &str, + db: &str, + coll: &str, + body: &serde_json::Value, + pk: &str, + serialization_formats: Option<&str>, +) -> Request { + let url = format!("{}/dbs/{}/colls/{}/docs", gateway_url, db, coll); + let mut req = Request::new(Url::parse(&url).unwrap(), Method::Post); + + // The request body is binary — begins with the 0x80 preamble. + let binary_body = binary_json::encode(body); + assert_eq!( + binary_body.first(), + Some(&PREAMBLE), + "test setup: request body must be binary", + ); + req.set_body(binary_body); + + req.headers_mut() + .insert(PARTITION_KEY.clone(), HeaderValue::from(pk.to_string())); + req.headers_mut() + .insert(CONTENT_RESPONSE.clone(), HeaderValue::from_static("True")); + if let Some(formats) = serialization_formats { + req.headers_mut().insert( + SUPPORTED_SERIALIZATION_FORMATS.clone(), + HeaderValue::from(formats.to_string()), + ); + } + req +} + +/// Default negotiation (`JsonText,CosmosBinary`): the response body is binary. +/// +/// This documents the *current* behavior — when binary encoding is enabled with +/// its default options, the service (emulator) sends the response back as +/// **binary**. +#[tokio::test] +async fn enabled_default_negotiation_yields_binary_response() { + let ctx = setup_single_region().await; + let body = serde_json::json!({ "id": "bin-1", "pk": "pk1", "value": 42 }); + + let req = create_binary_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &body, + r#"["pk1"]"#, + Some("JsonText,CosmosBinary"), + ); + let response = ctx.emulator.execute_request(&req).await.unwrap(); + let (status, _headers, raw) = collect_raw_response(response).await; + + assert_eq!(status, StatusCode::Created); + assert_eq!( + raw.first(), + Some(&PREAMBLE), + "default negotiation must return a binary (0x80) response body", + ); + assert!( + binary_json::is_binary(&raw), + "response body must be detected as binary", + ); + + // And it decodes back to the stored document. + let decoded: serde_json::Value = binary_json::decode(&raw).unwrap(); + assert_eq!(decoded["id"], "bin-1"); + assert_eq!(decoded["value"], 42); +} + +/// Explicit `JsonText`-only negotiation: the response body is **text**, even +/// though the request body was binary. +/// +/// This asserts the emulator's underlying format decision when a request +/// advertises `JsonText` alone. Note this is **not** the SDK-level +/// `request_text_response` behavior (which keeps advertising `CosmosBinary` and +/// has the driver transcode the binary response); it is the lower-level +/// negotiation primitive that mode does not use. +#[tokio::test] +async fn jsontext_only_negotiation_yields_text_response_despite_binary_request() { + let ctx = setup_single_region().await; + let body = serde_json::json!({ "id": "text-1", "pk": "pk1", "value": 7 }); + + let req = create_binary_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &body, + r#"["pk1"]"#, + Some("JsonText"), + ); + let response = ctx.emulator.execute_request(&req).await.unwrap(); + let (status, _headers, raw) = collect_raw_response(response).await; + + assert_eq!(status, StatusCode::Created); + assert_ne!( + raw.first(), + Some(&PREAMBLE), + "text negotiation must NOT return a binary (0x80) response body", + ); + assert!( + !binary_json::is_binary(&raw), + "response body must NOT be detected as binary", + ); + + // The raw bytes are valid UTF-8 text JSON and decode directly. + let text = std::str::from_utf8(&raw).expect("text response must be valid UTF-8"); + let decoded: serde_json::Value = serde_json::from_str(text).unwrap(); + assert_eq!(decoded["id"], "text-1"); + assert_eq!(decoded["value"], 7); +} + +/// No negotiation header (binary encoding disabled): the response is text. +#[tokio::test] +async fn no_negotiation_header_yields_text_response() { + let ctx = setup_single_region().await; + let body = serde_json::json!({ "id": "notneg-1", "pk": "pk1", "value": 1 }); + + let req = create_binary_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &body, + r#"["pk1"]"#, + None, + ); + let response = ctx.emulator.execute_request(&req).await.unwrap(); + let (status, _headers, raw) = collect_raw_response(response).await; + + assert_eq!(status, StatusCode::Created); + assert!( + !binary_json::is_binary(&raw), + "absent negotiation header must yield a text response", + ); + let decoded: serde_json::Value = serde_json::from_slice(&raw).unwrap(); + assert_eq!(decoded["id"], "notneg-1"); +} + +/// A read of a binary-written document, requested with `JsonText`, comes back as +/// text — proving the stored value is format-agnostic and the response format is +/// governed purely by the read's negotiation header. +#[tokio::test] +async fn text_read_of_binary_written_item_yields_text_response() { + let ctx = setup_single_region().await; + let body = serde_json::json!({ "id": "mixed-1", "pk": "pk1", "value": 314 }); + + // Write with default (binary) negotiation → stored via the binary request + // body, and (per the first test) echoed back as binary. + let write = create_binary_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + &body, + r#"["pk1"]"#, + Some("JsonText,CosmosBinary"), + ); + let write_resp = ctx.emulator.execute_request(&write).await.unwrap(); + let (write_status, _h, write_raw) = collect_raw_response(write_resp).await; + assert_eq!(write_status, StatusCode::Created); + assert!( + binary_json::is_binary(&write_raw), + "write response should be binary under default negotiation", + ); + + // Read the same item, but advertise only JsonText → text response. + let mut read = read_item_request( + &ctx.gateway_url, + "testdb", + "testcoll", + "mixed-1", + r#"["pk1"]"#, + ); + read.headers_mut().insert( + SUPPORTED_SERIALIZATION_FORMATS.clone(), + HeaderValue::from_static("JsonText"), + ); + let read_resp = ctx.emulator.execute_request(&read).await.unwrap(); + let (read_status, _h, read_raw) = collect_raw_response(read_resp).await; + + assert_eq!(read_status, StatusCode::Ok); + assert!( + !binary_json::is_binary(&read_raw), + "text-negotiated read must return a text response even though the item was written binary", + ); + let decoded: serde_json::Value = serde_json::from_slice(&read_raw).unwrap(); + assert_eq!(decoded["id"], "mixed-1"); + assert_eq!(decoded["value"], 314); +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs index 2951e653697..41ad83a5761 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/mod.rs @@ -5,6 +5,7 @@ pub mod account_metadata_refresh; pub mod batch; +pub mod binary_response_format; pub mod control_plane; #[cfg(feature = "preview_dtx")] pub mod distributed_transaction; @@ -133,6 +134,11 @@ pub static IS_UPSERT: HeaderName = HeaderName::from_static("x-ms-documentdb-is-u pub static CONTENT_RESPONSE: HeaderName = HeaderName::from_static("x-ms-cosmos-populate-content-response-on-write"); pub static IF_MATCH: HeaderName = HeaderName::from_static("if-match"); +/// The client's response-format negotiation header. When it contains a +/// `CosmosBinary` token the emulator replies with a binary body; otherwise it +/// replies with text. +pub static SUPPORTED_SERIALIZATION_FORMATS: HeaderName = + HeaderName::from_static("x-ms-cosmos-supported-serialization-formats"); /// Helper to create a POST request to create a document. pub fn create_item_request( @@ -279,3 +285,14 @@ pub async fn collect_response( }; (status, headers, body) } + +/// Fully buffers a response and returns status, headers, and the **raw** body +/// bytes (undecoded), so tests can inspect the on-the-wire serialization format +/// (e.g. the `0x80` binary preamble vs text JSON). +pub async fn collect_raw_response(response: AsyncRawResponse) -> (StatusCode, Headers, Vec) { + let raw = response.try_into_raw_response().await.unwrap(); + let status = raw.status(); + let headers = raw.headers().clone(); + let body_bytes = raw.body().as_ref().to_vec(); + (status, headers, body_bytes) +} diff --git a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/topology_refresh_on_substatus.rs b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/topology_refresh_on_substatus.rs index b57af347e7b..6bfbd1ca391 100644 --- a/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/topology_refresh_on_substatus.rs +++ b/sdk/cosmos/azure_data_cosmos_driver/tests/in_memory_emulator_tests/topology_refresh_on_substatus.rs @@ -335,7 +335,12 @@ async fn upsert_item_403_1008_triggers_refresh_and_cross_region_retry() { } /// **403/1008 — bounded bubble-up.** All-region 1008 retries must terminate. -#[tokio::test] +// Runs on tokio's paused clock: this test exhausts the full 120-attempt +// backend-failover budget, and each attempt sleeps `BACKEND_FAILOVER_RETRY_INTERVAL` +// (1s) via `azure_core::sleep` (tokio timer). Paused time auto-advances those +// sleeps instantly, so the test asserts the same outcome without ~120s of real +// wall-clock sleeping. +#[tokio::test(start_paused = true)] async fn all_regions_403_1008_bounded_retries_then_bubble_up() { let recorder = HostRecorder::new(); let east_rule = region_fault_rule( @@ -490,7 +495,9 @@ async fn write_403_3_triggers_topology_refresh() { } /// **403/3 — bounded bubble-up.** All-region 403/3 retries must terminate. -#[tokio::test] +// Paused clock: exhausts the 120-attempt backend-failover budget (1s backoff +// each) with both regions faulted; paused time makes the retry sleeps instant. +#[tokio::test(start_paused = true)] async fn all_regions_403_3_bounded_retries_then_bubble_up() { let recorder = HostRecorder::new(); let east = region_fault_rule( @@ -596,7 +603,10 @@ async fn write_403_3_second_attempt_targets_different_region() { } /// **403/3 — persistent one-region failures must not pin retries there.** -#[tokio::test] +// Paused clock: the persistent-fault retry loop backs off 1s per attempt via a +// tokio timer; paused time collapses those sleeps so the runaway guard never +// costs real wall-clock time. +#[tokio::test(start_paused = true)] async fn write_403_3_persistent_fault_pins_retries_to_same_region() { let recorder = HostRecorder::new(); let rule = region_fault_rule( @@ -681,7 +691,10 @@ async fn write_403_3_persistent_fault_pins_retries_to_same_region() { } /// **403/3 — backend-driven failover honors caller `excluded_regions`.** -#[tokio::test] +// Paused clock: West is excluded so East's persistent 403/3 exhausts the +// 120-attempt budget (1s backoff each) before bubbling up; paused time makes it +// instant. +#[tokio::test(start_paused = true)] async fn write_403_3_retry_honors_excluded_region() { let recorder = HostRecorder::new(); let rule = region_fault_rule( @@ -739,7 +752,9 @@ async fn write_403_3_retry_honors_excluded_region() { } /// **403/1008 — backend-driven failover honors caller `excluded_regions`.** -#[tokio::test] +// Paused clock: West is excluded so East's persistent 403/1008 exhausts the +// 120-attempt budget (1s backoff each); paused time makes it instant. +#[tokio::test(start_paused = true)] async fn create_item_403_1008_retry_honors_excluded_region() { let recorder = HostRecorder::new(); let rule = region_fault_rule( @@ -797,7 +812,9 @@ async fn create_item_403_1008_retry_honors_excluded_region() { } /// **GetDatabaseAccount metadata refresh is independent of `excluded_regions`.** -#[tokio::test] +// Paused clock: the excluded-region retry path exhausts the backend-failover +// budget with 1s backoffs; paused time collapses those sleeps. +#[tokio::test(start_paused = true)] async fn metadata_refresh_ignores_excluded_regions() { let recorder = HostRecorder::new(); let rule = region_fault_rule( @@ -869,7 +886,9 @@ async fn metadata_refresh_ignores_excluded_regions() { } /// **Regional-endpoint metadata-refresh fallback ignores `excluded_regions`.** -#[tokio::test] +// Paused clock: the excluded-region regional-fallback retry path exhausts the +// backend-failover budget with 1s backoffs; paused time collapses those sleeps. +#[tokio::test(start_paused = true)] async fn metadata_refresh_regional_fallback_ignores_excluded_regions() { let recorder = HostRecorder::new(); let data_plane_rule = region_fault_rule( diff --git a/sdk/cosmos/azure_data_cosmos_driver_native/include/azurecosmosdriver.h b/sdk/cosmos/azure_data_cosmos_driver_native/include/azurecosmosdriver.h index a7c155adefe..453b6eb7772 100644 --- a/sdk/cosmos/azure_data_cosmos_driver_native/include/azurecosmosdriver.h +++ b/sdk/cosmos/azure_data_cosmos_driver_native/include/azurecosmosdriver.h @@ -1037,6 +1037,27 @@ typedef struct cosmos_operation_options_t { * Number of entries in `custom_headers`. */ uintptr_t custom_headers_len; + /** + * Whether Cosmos binary JSON is used on the wire. Tri-state bool + * (`0` unset / `1` false / `2` true). + * + * When true, the driver transcodes a **text** request body to binary + * before sending it (an already-binary body is passed through) and + * advertises `CosmosBinary`, so the caller never encodes binary itself. + * An explicit `false` forces binary **off** for this operation regardless + * of any account/runtime default; `unset` inherits a lower layer (text by + * default). + */ + int8_t binary_encoding_enabled; + /** + * Whether the driver transcodes the binary response back to **text** JSON. + * Tri-state bool (`0` unset / `1` false / `2` true). + * + * Only meaningful when [`binary_encoding_enabled`](Self::binary_encoding_enabled) + * is true: the wire stays binary in both directions and the driver hands + * back text. `unset` / `false` returns the binary response as-is. + */ + int8_t binary_encoding_request_text_response; } cosmos_operation_options_t; /** diff --git a/sdk/cosmos/azure_data_cosmos_driver_native/src/op_request.rs b/sdk/cosmos/azure_data_cosmos_driver_native/src/op_request.rs index 41e57fafef6..f03a021fbba 100644 --- a/sdk/cosmos/azure_data_cosmos_driver_native/src/op_request.rs +++ b/sdk/cosmos/azure_data_cosmos_driver_native/src/op_request.rs @@ -45,8 +45,8 @@ use azure_data_cosmos_driver::models::{ MaxItemCountHint, PartitionKey, Precondition, SessionToken, ThroughputControlGroupName, }; use azure_data_cosmos_driver::options::{ - ContentResponseOnWrite, EndToEndOperationLatencyPolicy, ExcludedRegions, OperationOptions, - ReadConsistencyStrategy, Region, ThroughputControlOptions, + BinaryEncodingOptions, ContentResponseOnWrite, EndToEndOperationLatencyPolicy, ExcludedRegions, + OperationOptions, ReadConsistencyStrategy, Region, ThroughputControlOptions, }; use crate::account_ref::AccountRefHandle; @@ -285,6 +285,24 @@ pub struct CosmosOperationOptions { pub custom_headers: *const CosmosHeaderKv, /// Number of entries in `custom_headers`. pub custom_headers_len: usize, + + /// Whether Cosmos binary JSON is used on the wire. Tri-state bool + /// (`0` unset / `1` false / `2` true). + /// + /// When true, the driver transcodes a **text** request body to binary + /// before sending it (an already-binary body is passed through) and + /// advertises `CosmosBinary`, so the caller never encodes binary itself. + /// An explicit `false` forces binary **off** for this operation regardless + /// of any account/runtime default; `unset` inherits a lower layer (text by + /// default). + pub binary_encoding_enabled: i8, + /// Whether the driver transcodes the binary response back to **text** JSON. + /// Tri-state bool (`0` unset / `1` false / `2` true). + /// + /// Only meaningful when [`binary_encoding_enabled`](Self::binary_encoding_enabled) + /// is true: the wire stays binary in both directions and the driver hands + /// back text. `unset` / `false` returns the binary response as-is. + pub binary_encoding_request_text_response: i8, } impl CosmosOperationOptions { @@ -339,6 +357,21 @@ impl CosmosOperationOptions { opts.custom_headers = Some(headers); } + // Binary encoding is a whole-value option. It is tri-state: `unset` + // leaves `binary_encoding` as `None` (inherit a lower layer), while an + // explicit `true`/`false` is honored as `Some(..)` so a host can force + // binary off regardless of any account/runtime default. The + // `request_text_response` flag is only meaningful when binary is on. + if let Some(enabled) = decode_tristate_bool(self.binary_encoding_enabled)? { + let request_text_response = + decode_tristate_bool(self.binary_encoding_request_text_response)?.unwrap_or(false); + opts.binary_encoding = Some( + BinaryEncodingOptions::new() + .with_enabled(enabled) + .with_request_text_response(request_text_response), + ); + } + Ok(opts) } } @@ -393,6 +426,8 @@ pub extern "C" fn cosmos_operation_options_default() -> CosmosOperationOptions { excluded_regions_len: 0, custom_headers: std::ptr::null(), custom_headers_len: 0, + binary_encoding_enabled: TRISTATE_UNSET, + binary_encoding_request_text_response: TRISTATE_UNSET, } } @@ -1098,6 +1133,8 @@ mod tests { assert_eq!(o.excluded_regions_len, 0); assert!(o.custom_headers.is_null()); assert_eq!(o.custom_headers_len, 0); + assert_eq!(o.binary_encoding_enabled, TRISTATE_UNSET); + assert_eq!(o.binary_encoding_request_text_response, TRISTATE_UNSET); } #[test] @@ -1115,6 +1152,62 @@ mod tests { assert_eq!(driver.end_to_end_latency_policy, None); assert_eq!(driver.excluded_regions, None); assert!(driver.throughput_control.is_none()); + assert!(driver.binary_encoding.is_none()); + } + + #[test] + fn binary_encoding_flags_convert_to_driver_option() { + // enabled = true, request_text = true → the driver option is built with + // both flags set. An FFI host that deals only in text can thus get a + // binary wire and a text response without encoding anything itself. + let mut o = cosmos_operation_options_default(); + o.binary_encoding_enabled = TRISTATE_TRUE; + o.binary_encoding_request_text_response = TRISTATE_TRUE; + // SAFETY: all pointer fields are NULL / len 0. + let driver = unsafe { o.to_driver() }.expect("options convert"); + let be = driver.binary_encoding.expect("binary encoding set"); + assert!(be.enabled); + assert!(be.request_text_response); + } + + #[test] + fn binary_encoding_enabled_without_text_response() { + let mut o = cosmos_operation_options_default(); + o.binary_encoding_enabled = TRISTATE_TRUE; + // request_text_response left unset → defaults to false. + // SAFETY: all pointer fields are NULL / len 0. + let driver = unsafe { o.to_driver() }.expect("options convert"); + let be = driver.binary_encoding.expect("binary encoding set"); + assert!(be.enabled); + assert!(!be.request_text_response); + } + + #[test] + fn binary_encoding_unset_yields_no_option() { + // enabled unset → no binary-encoding option at all (inherit a lower + // layer), even if the text-response flag is set (a no-op when unset). + let mut o = cosmos_operation_options_default(); + o.binary_encoding_request_text_response = TRISTATE_TRUE; + // SAFETY: all pointer fields are NULL / len 0. + let driver = unsafe { o.to_driver() }.expect("options convert"); + assert!(driver.binary_encoding.is_none()); + } + + #[test] + fn binary_encoding_explicit_false_is_honored() { + // enabled = false is an *explicit* opt-out: it must produce + // `Some(disabled)` so the driver forces binary off for this operation + // rather than inheriting an account/runtime default (which `None` does). + let mut o = cosmos_operation_options_default(); + o.binary_encoding_enabled = TRISTATE_FALSE; + // request_text_response set but irrelevant when binary is off. + o.binary_encoding_request_text_response = TRISTATE_TRUE; + // SAFETY: all pointer fields are NULL / len 0. + let driver = unsafe { o.to_driver() }.expect("options convert"); + let be = driver + .binary_encoding + .expect("explicit false must be honored as Some(disabled)"); + assert!(!be.enabled); } #[test] diff --git a/sdk/cosmos/azure_data_cosmos_perf/Cargo.toml b/sdk/cosmos/azure_data_cosmos_perf/Cargo.toml index bb33a8e79ff..e277a66377b 100644 --- a/sdk/cosmos/azure_data_cosmos_perf/Cargo.toml +++ b/sdk/cosmos/azure_data_cosmos_perf/Cargo.toml @@ -36,6 +36,14 @@ uuid.workspace = true # Optional: tokio runtime metrics (scheduling delay, poll times, worker utilization) tokio-metrics = { workspace = true, optional = true } +[dev-dependencies] +# Test-only: the binary round-trip fuzzer generates arbitrary JSON, canonicalizes +# it (RFC 8785), and hashes the canonical form for differential comparison. +arbitrary = { workspace = true } +arbitrary-json = { workspace = true } +json-canon = { workspace = true } +sha2 = { workspace = true } + [features] default = [] tokio-console = ["dep:console-subscriber", "tokio/tracing"] diff --git a/sdk/cosmos/azure_data_cosmos_perf/README.md b/sdk/cosmos/azure_data_cosmos_perf/README.md index 88d9d1bd021..5df858fb986 100644 --- a/sdk/cosmos/azure_data_cosmos_perf/README.md +++ b/sdk/cosmos/azure_data_cosmos_perf/README.md @@ -20,6 +20,19 @@ From the repository root: cargo build -p azure_data_cosmos_perf ``` +## Sample JSON corpus (`testdata/`, not tracked) + +The opt-in test `tests/binary_sampled_testdata.rs` samples a large corpus of +representative JSON payloads under `testdata/*.json` (~500 MB). This corpus is +kept as a **local copy** and is intentionally **not tracked in source control** +to keep the repository small. + +Nothing in the build or the CI gates depends on it: the test is gated behind +`test_category = "binary_encoding"` (ignored otherwise) and requires a live +account, and the benchmarks generate their own synthetic data. To run that test, +restore the `testdata/*.json` files locally first; if they are missing, +`load_sample_pool` fails with a message explaining how to restore them. + ## Usage ### Key Authentication diff --git a/sdk/cosmos/azure_data_cosmos_perf/build.rs b/sdk/cosmos/azure_data_cosmos_perf/build.rs new file mode 100644 index 00000000000..9a3b22c46bb --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_perf/build.rs @@ -0,0 +1,11 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +// Registers custom cfgs used by this crate's integration tests. +// +// Some CI/build setups enable `-W unexpected-cfgs`, and in newer Rust toolchains +// unknown cfg names are warned/denied unless explicitly declared via check-cfg. +fn main() { + // Allow `#[cfg_attr(not(test_category = "..."), ignore)]` in `tests/*.rs`. + println!("cargo:rustc-check-cfg=cfg(test_category, values(\"binary_encoding\"))"); +} diff --git a/sdk/cosmos/azure_data_cosmos_perf/tests/binary_roundtrip_fuzzer.rs b/sdk/cosmos/azure_data_cosmos_perf/tests/binary_roundtrip_fuzzer.rs new file mode 100644 index 00000000000..c59ce291df9 --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_perf/tests/binary_roundtrip_fuzzer.rs @@ -0,0 +1,1507 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! End-to-end **round-trip fuzzer** for Cosmos binary JSON encoding. +//! +//! Generates random JSON documents with a seeded PRNG, stores and reads each one +//! back through a live Cosmos account under several binary-encoding +//! configurations, and asserts the value survives unchanged by comparing a +//! **Cosmos-compatible canonical form** of what was sent against what came back. +//! +//! See the design doc: +//! `azure_data_cosmos_driver/docs/BINARY_ENCODING_ROUNDTRIP_FUZZER.md`. +//! +//! # Running +//! +//! ```bash +//! # Smoke run against a local emulator: +//! AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=...;AccountKey=...;' \ +//! AZURE_COSMOS_ALLOW_INVALID_CERT=true \ +//! RUSTFLAGS='--cfg test_category="binary_encoding"' \ +//! cargo test -p azure_data_cosmos_perf --test binary_roundtrip_fuzzer -- --nocapture +//! +//! # Multi-day soak (millions of docs), release build: +//! AZURE_COSMOS_CONNECTION_STRING='...' AZURE_COSMOS_FUZZ_ITERATIONS=5000000 \ +//! RUSTFLAGS='--cfg test_category="binary_encoding"' \ +//! cargo test -p azure_data_cosmos_perf --test binary_roundtrip_fuzzer --release -- --nocapture +//! +//! # Reproduce a failure exactly: +//! AZURE_COSMOS_FUZZ_SEED= ... cargo test ... +//! ``` +//! +//! Every run prints its seed; a failing document is reproduced deterministically +//! by re-running with `AZURE_COSMOS_FUZZ_SEED=`. + +#![allow(clippy::large_futures)] + +use std::error::Error; + +use arbitrary::{Arbitrary, Unstructured}; +use arbitrary_json::ArbitraryValue; +use azure_core::http::StatusCode; +use azure_data_cosmos::models::ContainerProperties; +use azure_data_cosmos::options::{ + BinaryEncodingOptions, ConnectionPoolOptions, ContentResponseOnWrite, ItemWriteOptions, + OperationOptions, Region, ServerCertificateValidation, +}; +use azure_data_cosmos::{ + AccountEndpoint, AccountReference, CosmosClient, CosmosRuntime, RoutingStrategy, +}; +use azure_data_cosmos_driver::models::ConnectionString; +use serde_json::{Map, Number, Value}; +use sha2::{Digest, Sha256}; +use uuid::Uuid; + +const CONNECTION_STRING_ENV_VAR: &str = "AZURE_COSMOS_CONNECTION_STRING"; +const ALLOW_INVALID_CERT_ENV_VAR: &str = "AZURE_COSMOS_ALLOW_INVALID_CERT"; +const DATABASE_NAME_ENV_VAR: &str = "AZURE_COSMOS_BINARY_TEST_DATABASE"; +const CONTAINER_NAME_ENV_VAR: &str = "AZURE_COSMOS_BINARY_TEST_CONTAINER"; +const ITERATIONS_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_ITERATIONS"; +const SEED_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_SEED"; +const MAX_DEPTH_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_MAX_DEPTH"; +const WIDE_NUMBERS_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_WIDE_NUMBERS"; +const UNICODE_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_UNICODE"; +const BREADTH_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_BREADTH"; +const CALIBRATE_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_CALIBRATE"; +const PRINT_ENV_VAR: &str = "AZURE_COSMOS_FUZZ_PRINT"; + +const DEFAULT_DATABASE_NAME: &str = "binary-fuzz-db"; +const DEFAULT_CONTAINER_NAME: &str = "binary-fuzz-ct"; +const PARTITION_KEY_PATH: &str = "/pk"; + +const DEFAULT_ITERATIONS: u64 = 200; +const DEFAULT_MAX_DEPTH: u32 = 6; +/// Default maximum number of child fields/elements generated at each container +/// level (the branching factor). Higher values produce larger, wider documents. +const DEFAULT_BREADTH: u32 = 6; + +// ───────────────────────────────────────────────────────────────────────────── +// Seeded PRNG (SplitMix64) — deterministic and dependency-feature-free, matching +// the codebase's in-tree fuzz convention so a failure reproduces from its seed. +// ───────────────────────────────────────────────────────────────────────────── + +struct SplitMix64 { + state: u64, +} + +impl SplitMix64 { + fn new(seed: u64) -> Self { + Self { state: seed } + } + + fn next_u64(&mut self) -> u64 { + self.state = self.state.wrapping_add(0x9E37_79B9_7F4A_7C15); + let mut z = self.state; + z = (z ^ (z >> 30)).wrapping_mul(0xBF58_476D_1CE4_E5B9); + z = (z ^ (z >> 27)).wrapping_mul(0x94D0_49BB_1331_11EB); + z ^ (z >> 31) + } + + /// Uniform integer in `[0, n)`. `n` must be non-zero. + fn below(&mut self, n: u64) -> u64 { + self.next_u64() % n + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Configuration +// ───────────────────────────────────────────────────────────────────────────── + +struct FuzzConfig { + iterations: u64, + seed: u64, + max_depth: u32, + wide_numbers: bool, + unicode: bool, + breadth: u32, + calibrate: bool, + print_docs: bool, +} + +impl FuzzConfig { + fn from_env() -> Self { + let seed = std::env::var(SEED_ENV_VAR) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or_else(|| { + // Non-deterministic default seed derived from the wall clock. + std::time::SystemTime::now() + .duration_since(std::time::UNIX_EPOCH) + .map(|d| d.as_nanos() as u64) + .unwrap_or(0x1234_5678_9ABC_DEF0) + }); + Self { + iterations: env_u64(ITERATIONS_ENV_VAR, DEFAULT_ITERATIONS), + seed, + max_depth: env_u64(MAX_DEPTH_ENV_VAR, DEFAULT_MAX_DEPTH as u64) as u32, + wide_numbers: env_bool(WIDE_NUMBERS_ENV_VAR, false), + unicode: env_bool(UNICODE_ENV_VAR, true), + breadth: env_u64(BREADTH_ENV_VAR, DEFAULT_BREADTH as u64).max(1) as u32, + calibrate: env_bool(CALIBRATE_ENV_VAR, false), + print_docs: env_bool(PRINT_ENV_VAR, false), + } + } +} + +fn env_u64(name: &str, default: u64) -> u64 { + std::env::var(name) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(default) +} + +fn env_bool(name: &str, default: bool) -> bool { + std::env::var(name) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(default) +} + +// ───────────────────────────────────────────────────────────────────────────── +// JSON generator (arbitrary-json, seeded from the PRNG) +// ───────────────────────────────────────────────────────────────────────────── + +/// Generates a random JSON **object** suitable as a Cosmos item body. +/// +/// Uses a **hybrid** strategy: a depth-controlled *skeleton* guarantees the +/// document actually reaches a target nesting depth (drawn from `[1, max_depth]`), +/// while every leaf and filler branch is irregular JSON — a mix of hand-rolled +/// typed scalars (integers, floats, alphabetic / alphanumeric / free-text / +/// non-ASCII strings, booleans, nulls, number arrays) and [`arbitrary_json`] +/// subtrees. This fixes the `arbitrary_iter` shallowness (it stops nesting +/// almost immediately regardless of byte budget), so `max_depth` now +/// meaningfully scales structure and `breadth` scales width. Everything is +/// driven by the [`SplitMix64`] seed stream, so the same `AZURE_COSMOS_FUZZ_SEED` +/// reproduces the same document. +/// +/// Every document also carries a **sampler** subtree ([`gen_sampler`]) that +/// guarantees at least one value of each category appears, so a single run +/// exercises numeric, alphabetic, alphanumeric, free-text, and non-ASCII data +/// under multi-level nesting. +/// +/// A [`bound_value`] pass applies the `wide_numbers` / `unicode` knobs: by +/// default numbers are clamped into the calibrated-safe envelope (design doc +/// §3.2) and strings to ASCII, unless explicitly widened. The hand-rolled +/// scalars already respect those knobs directly. +fn gen_object(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Map { + let max_depth = cfg.max_depth.max(1); + // Target nesting depth for this document's spine, in [1, max_depth]. + let target_depth = 1 + rng.below(max_depth as u64) as u32; + + let mut map = Map::new(); + // A guaranteed sampler covering every value category (numeric, alphabetic, + // alphanumeric, free text, non-ASCII, boolean, null, number array, nested). + // Keyed distinctly from the caller-reserved `id`/`pk` so it is never + // overwritten. + map.insert("_sampler".to_string(), gen_sampler(rng, cfg)); + // A spread of irregular root fields (typed scalars + arbitrary-json subtrees). + for _ in 0..rng.below(cfg.breadth as u64 + 1) { + map.insert(gen_key(rng), gen_filler_value(rng, cfg)); + } + // The spine field guarantees the target depth is reached. Its key avoids the + // caller-reserved `id`/`pk`/`_sampler` (and empty) so nothing overwrites the + // deep subtree. + let mut spine_key = gen_key(rng); + while spine_key.is_empty() || spine_key == "id" || spine_key == "pk" || spine_key == "_sampler" + { + spine_key.push('_'); + } + map.insert(spine_key, gen_spine(rng, cfg, target_depth)); + + map +} + +/// Number of PRNG bytes fed to `arbitrary-json` for one filler subtree. A larger +/// budget lets `arbitrary-json` build bigger, deeper irregular subtrees. +const FILLER_BUDGET: usize = 256; + +/// Refills `n` bytes deterministically from the PRNG. +fn fill_bytes(rng: &mut SplitMix64, n: usize) -> Vec { + let mut bytes = Vec::with_capacity(n); + while bytes.len() < n { + bytes.extend_from_slice(&rng.next_u64().to_le_bytes()); + } + bytes +} + +/// A random object key from `arbitrary-json`'s string generator. +fn gen_key(rng: &mut SplitMix64) -> String { + let bytes = fill_bytes(rng, 16); + let mut u = Unstructured::new(&bytes); + String::arbitrary(&mut u).unwrap_or_default() +} + +// ───────────────────────────────────────────────────────────────────────────── +// Typed scalar generators (character classes + numbers), seeded from the PRNG. +// ───────────────────────────────────────────────────────────────────────────── + +/// ASCII letters, for the alphabetic string class. +const ALPHA_CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz"; +/// ASCII letters + digits, for the alphanumeric string class. +const ALPHANUMERIC_CHARS: &[u8] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789"; +/// A spread of non-ASCII scalars across scripts, symbols, and astral-plane +/// emoji — exercises multi-byte UTF-8 and surrogate-pair paths in the codec. +const NON_ASCII_CHARS: &[char] = &[ + 'é', 'ñ', 'ü', 'ß', 'ç', 'å', 'ø', 'Ω', 'λ', 'π', 'µ', 'я', 'ж', 'д', 'α', 'β', '中', '文', + '日', '本', '語', '한', '국', 'ع', 'ب', '€', '£', '¥', '©', '™', '—', '…', '→', '∑', '≈', '♠', + '☃', '😀', '🚀', '🌍', '🎉', '𝄞', '𐍈', +]; + +/// A uniform integer in `[lo, hi]` (inclusive). `lo <= hi` required. +fn gen_int_in(rng: &mut SplitMix64, lo: i64, hi: i64) -> i64 { + let span = (hi - lo) as u64 + 1; + lo + rng.below(span) as i64 +} + +/// A random ASCII string drawn from `pool`, length in `[1, max_len]`. +fn gen_string_from(rng: &mut SplitMix64, pool: &[u8], max_len: usize) -> String { + let len = 1 + rng.below(max_len as u64) as usize; + (0..len) + .map(|_| pool[rng.below(pool.len() as u64) as usize] as char) + .collect() +} + +/// A string mixing alphanumeric and non-ASCII scalars, length in `[1, max_len]`. +/// Falls back to alphanumeric-only when `unicode` generation is disabled so the +/// ASCII envelope contract (design doc §3.2) still holds. +fn gen_unicode_string(rng: &mut SplitMix64, cfg: &FuzzConfig, max_len: usize) -> String { + if !cfg.unicode { + return gen_string_from(rng, ALPHANUMERIC_CHARS, max_len); + } + let len = 1 + rng.below(max_len as u64) as usize; + (0..len) + .map(|_| { + if rng.below(2) == 0 { + ALPHANUMERIC_CHARS[rng.below(ALPHANUMERIC_CHARS.len() as u64) as usize] as char + } else { + NON_ASCII_CHARS[rng.below(NON_ASCII_CHARS.len() as u64) as usize] + } + }) + .collect() +} + +/// An envelope-safe integer (`±1_000_000`). +fn gen_envelope_int(rng: &mut SplitMix64) -> Value { + Value::Number(Number::from(gen_int_in(rng, -1_000_000, 1_000_000))) +} + +/// An envelope-safe two-decimal float (`±100_000.00`). +fn gen_envelope_float(rng: &mut SplitMix64) -> Value { + let cents = gen_int_in(rng, -10_000_000, 10_000_000); + Number::from_f64(cents as f64 / 100.0) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(0))) +} + +/// A single number: envelope-safe by default; when `wide_numbers` is set, +/// occasionally a wide value beyond `2^53` that drives the calibrated +/// string-token comparison path (design doc §3.1). +fn gen_number(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + if cfg.wide_numbers && rng.below(4) == 0 { + return Value::Number(Number::from(rng.next_u64() as i64)); + } + if rng.below(2) == 0 { + gen_envelope_int(rng) + } else { + gen_envelope_float(rng) + } +} + +/// One rich scalar spanning the value taxonomy: integer, float, alphabetic, +/// alphanumeric, free text, non-ASCII, boolean, or null. +fn gen_scalar(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + match rng.below(8) { + 0 | 1 => gen_number(rng, cfg), + 2 => Value::String(gen_string_from(rng, ALPHA_CHARS, 24)), + 3 => Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 24)), + 4 => Value::String(gen_unicode_string(rng, cfg, 24)), + 5 => Value::String(gen_unicode_string(rng, cfg, 80)), // longer free text + 6 => Value::Bool(rng.below(2) == 0), + _ => Value::Null, + } +} + +/// A sampler object guaranteeing every value category appears in the document at +/// least once: integer, float, alphabetic, alphanumeric, free text, non-ASCII, +/// boolean, null, a homogeneous number array, and a small nested object. +fn gen_sampler(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + let mut map = Map::new(); + map.insert("int".into(), gen_envelope_int(rng)); + map.insert("float".into(), gen_envelope_float(rng)); + map.insert( + "alpha".into(), + Value::String(gen_string_from(rng, ALPHA_CHARS, 24)), + ); + map.insert( + "alphanumeric".into(), + Value::String(gen_string_from(rng, ALPHANUMERIC_CHARS, 24)), + ); + map.insert( + "text".into(), + Value::String(gen_unicode_string(rng, cfg, 64)), + ); + map.insert( + "unicode".into(), + Value::String(gen_unicode_string(rng, cfg, 24)), + ); + map.insert("flag".into(), Value::Bool(rng.below(2) == 0)); + map.insert("empty".into(), Value::Null); + let count = 1 + rng.below(8); + let numbers = (0..count).map(|_| gen_envelope_int(rng)).collect(); + map.insert("numbers".into(), Value::Array(numbers)); + // A small nested object so the sampler itself has a second level. + let mut nested = Map::new(); + nested.insert("mixed".into(), gen_scalar(rng, cfg)); + nested.insert( + "list".into(), + Value::Array( + (0..1 + rng.below(4)) + .map(|_| gen_scalar(rng, cfg)) + .collect(), + ), + ); + map.insert("nested".into(), Value::Object(nested)); + Value::Object(map) +} + +/// A small, irregular filler value. Draws from typed scalars, mixed +/// arrays/objects of typed scalars, homogeneous number arrays (to exercise the +/// uniform-number wire forms), and `arbitrary-json` subtrees — so filler is both +/// varied and non-trivial in size. Already respects the `wide_numbers`/`unicode` +/// knobs. +fn gen_filler_value(rng: &mut SplitMix64, cfg: &FuzzConfig) -> Value { + match rng.below(10) { + // Homogeneous number array (uniform-number wire forms). + 0 => { + let len = rng.below(8); + let arr = (0..len).map(|_| gen_envelope_int(rng)).collect(); + Value::Array(arr) + } + // Typed scalars across the character/number classes. + 1..=3 => gen_scalar(rng, cfg), + // A short mixed-type array. + 4 => { + let len = 1 + rng.below(cfg.breadth as u64 + 1); + Value::Array((0..len).map(|_| gen_scalar(rng, cfg)).collect()) + } + // A small object of typed scalars. + 5 => { + let mut map = Map::new(); + for _ in 0..1 + rng.below(cfg.breadth as u64) { + map.insert(gen_key(rng), gen_scalar(rng, cfg)); + } + Value::Object(map) + } + // An `arbitrary-json` subtree (bigger byte budget), envelope-bounded. + _ => { + let bytes = fill_bytes(rng, FILLER_BUDGET); + let mut u = Unstructured::new(&bytes); + let mut v: Value = ArbitraryValue::arbitrary(&mut u) + .map(Into::into) + .unwrap_or(Value::Null); + bound_value(&mut v, cfg); + v + } + } +} + +/// Builds a nested container chain `depth` levels deep, with several irregular +/// filler siblings at each level, guaranteeing the document reaches `depth`. +/// Each level is randomly an object or an array; exactly one child continues the +/// spine deeper. The sibling count scales with `breadth`, so deeper documents +/// are also wider. +fn gen_spine(rng: &mut SplitMix64, cfg: &FuzzConfig, depth: u32) -> Value { + if depth == 0 { + return gen_filler_value(rng, cfg); + } + if rng.below(2) == 0 { + // Object: filler fields + one spine field going deeper. + let mut map = Map::new(); + for _ in 0..rng.below(cfg.breadth as u64 + 1) { + map.insert(gen_key(rng), gen_filler_value(rng, cfg)); + } + let mut key = gen_key(rng); + while key.is_empty() { + key.push('_'); + } + map.insert(key, gen_spine(rng, cfg, depth - 1)); + Value::Object(map) + } else { + // Array: filler elements + one spine element going deeper. + let mut arr = Vec::new(); + for _ in 0..rng.below(cfg.breadth as u64 + 1) { + arr.push(gen_filler_value(rng, cfg)); + } + arr.push(gen_spine(rng, cfg, depth - 1)); + Value::Array(arr) + } +} + +/// Recursively applies the generation bounds to a value: clamps numbers into the +/// calibrated-safe envelope unless `wide_numbers`, and drops non-ASCII from +/// strings unless `unicode`. Leaves structure otherwise untouched. +fn bound_value(value: &mut Value, cfg: &FuzzConfig) { + match value { + Value::Number(n) => { + if !cfg.wide_numbers { + *value = clamp_number_to_envelope(n); + } + } + Value::String(s) => { + if !cfg.unicode && !s.is_ascii() { + *s = s.chars().filter(char::is_ascii).collect(); + } + } + Value::Array(items) => { + for item in items.iter_mut() { + bound_value(item, cfg); + } + } + Value::Object(map) => { + for v in map.values_mut() { + bound_value(v, cfg); + } + } + _ => {} + } +} + +/// Clamps a number into the **backend-safe** envelope (design doc §3.2): bounded +/// integers and two-decimal floats, matching what the calibrated +/// [`normalize_number`] models without `--wide-numbers`. +fn clamp_number_to_envelope(n: &Number) -> Value { + if let Some(i) = n.as_i64() { + Value::Number(Number::from(i.rem_euclid(2_000_001) - 1_000_000)) + } else if let Some(u) = n.as_u64() { + Value::Number(Number::from((u % 2_000_001) as i64 - 1_000_000)) + } else if let Some(f) = n.as_f64() { + // Two decimal places within ±100_000 keeps it inside the calibrated + // envelope; a non-finite arbitrary float collapses to 0. + let bounded = if f.is_finite() { + ((f % 100_000.0) * 100.0).round() / 100.0 + } else { + 0.0 + }; + Number::from_f64(bounded) + .map(Value::Number) + .unwrap_or_else(|| Value::Number(Number::from(0))) + } else { + Value::Number(Number::from(0)) + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Cosmos-compatible canonicalization (design doc §3) +// ───────────────────────────────────────────────────────────────────────────── + +/// Produces the canonical string for a JSON value: the calibrated Cosmos number +/// rewrite ([`normalize_numbers`]) followed by RFC 8785 (JCS) structural +/// canonicalization via [`json_canon`] — whitespace removed, object keys sorted, +/// strings minimally escaped. Two values with the same canonical string are +/// considered equal after a round-trip. +/// +/// Numbers are normalized **first** so the JCS serializer's own number +/// formatting no longer affects the comparison; the only Cosmos-specific step is +/// [`normalize_numbers`] (design doc §3.1). +fn canonicalize(value: &Value) -> String { + let normalized = normalize_numbers(value); + json_canon::to_string(&normalized).expect("normalized value always canonicalizes") +} + +/// Rewrites a single JSON number to its **Cosmos-calibrated** canonical +/// [`Value`]. **This is the tuning surface** — see the design doc §3.1. It is +/// the one number-specific step that must stay under our control (RFC 8785 / JCS +/// number formatting is *not* the backend's store-time rewrite); the structural +/// canonicalization around it can be delegated to a standard JCS serializer. +/// +/// Rules (calibrated against a live account, design doc §3.1): +/// - integers with magnitude `< 2^53` → exact integer (JCS-safe); +/// - integers with magnitude `>= 2^53` that fit `i64` → exact **string token** +/// (Cosmos preserves them exactly, but RFC 8785 / JCS refuses to emit integers +/// beyond the safe range, so they are compared as a stable decimal token); +/// - integers above `i64::MAX` → **string token** of the `f64` form (the backend +/// stores them as IEEE-754 doubles), so a sent `u64` and its returned double +/// map to the same token; +/// - integral-valued floats below `2^53` (e.g. `1.0`) → integer form (the +/// backend drops the trailing `.0`); +/// - integral-valued floats `>= 2^53` → `f64` string token (matches the lossy +/// double case above); +/// - other finite floats → kept as `f64` (JCS-safe); +/// - non-finite (`NaN` / `±∞`) → `null`. +/// +/// The string tokens are only ever compared for equality (never parsed back), so +/// representing an out-of-JCS-range number as a token is sound: any two values +/// Cosmos would round-trip to each other produce the identical token. +fn normalize_number(n: &Number) -> Value { + if let Some(i) = n.as_i64() { + if (i.unsigned_abs() as f64) < JCS_SAFE_INT_LIMIT { + Value::Number(Number::from(i)) + } else { + Value::String(i.to_string()) + } + } else if let Some(u) = n.as_u64() { + // u > i64::MAX: Cosmos stores it as a lossy double; token from the double. + Value::String(cosmos_double_token(u as f64)) + } else if let Some(f) = n.as_f64() { + if !f.is_finite() { + Value::Null + } else if f.fract() == 0.0 && f.abs() < JCS_SAFE_INT_LIMIT { + Value::Number(Number::from(f as i64)) + } else if f.fract() == 0.0 { + // Integral but out of the JCS-safe range → double token. + Value::String(cosmos_double_token(f)) + } else { + // Non-integral finite float is JCS-safe as a number. + Number::from_f64(f) + .map(Value::Number) + .unwrap_or(Value::Null) + } + } else { + Value::Null + } +} + +/// `2^53`: the largest magnitude RFC 8785 (JCS) will emit as an integer. At or +/// beyond this, `json-canon` refuses integer output and Cosmos stores `u64` +/// above `i64::MAX` lossily as doubles, so such numbers are canonicalized as +/// string tokens (see [`normalize_number`]). +const JCS_SAFE_INT_LIMIT: f64 = 9_007_199_254_740_992.0; + +/// A stable decimal token for a Cosmos-stored double, from the `f64` value. +/// Both a sent `u64` and its returned scientific-notation double parse to the +/// same `f64`, so they produce the same token. +fn cosmos_double_token(f: f64) -> String { + format!("{f}") +} + +/// Recursively rewrites every number in `value` to its Cosmos-calibrated form +/// (see [`normalize_number`]), leaving all other value kinds unchanged. The +/// result is a `Value` ready for a standard (JCS) structural canonicalization +/// pass — the number rewrite has already been applied, so the structural +/// serializer's own number formatting no longer changes the comparison. +fn normalize_numbers(value: &Value) -> Value { + match value { + Value::Number(n) => normalize_number(n), + Value::Array(items) => Value::Array(items.iter().map(normalize_numbers).collect()), + Value::Object(map) => Value::Object( + map.iter() + .map(|(k, v)| (k.clone(), normalize_numbers(v))) + .collect(), + ), + other => other.clone(), + } +} + +/// Canonicalizes and returns `(canonical_string, SHA-256 digest)`. +/// +/// The digest is a cryptographic hash of the canonical string, so it is stable +/// across runs and platforms — suitable for a durable corpus of expected `H0` +/// values ("store the hash once, compare later"). +fn canonical_hash(value: &Value) -> (String, [u8; 32]) { + let s = canonicalize(value); + let digest: [u8; 32] = Sha256::digest(s.as_bytes()).into(); + (s, digest) +} + +/// Formats a 32-byte digest as lowercase hex, for mismatch reporting. +fn hex(digest: &[u8; 32]) -> String { + let mut out = String::with_capacity(64); + for b in digest { + out.push_str(&format!("{b:02x}")); + } + out +} + +/// Normalizes a value by one JSON serialize→parse pass. Any Cosmos round-trip +/// (through text or binary, plus the backend's own store rewrite) puts the value +/// through at least one serialize→parse, which can shift a `from_f64` float to a +/// neighboring value with a shorter shortest-form. Computing the **sent** +/// canonical form from the normalized value puts sent and round-tripped +/// documents on equal footing. Normalization is idempotent (a parsed value's +/// shortest serialization round-trips back to itself). +fn normalize(value: &Value) -> Value { + let text = serde_json::to_string(value).expect("value always serializes"); + serde_json::from_str(&text).expect("serialized value always parses") +} + +/// Projects a returned document to only the keys present in `sent`, so +/// service-added system fields (`_rid`, `_etag`, `_ts`, ...) don't affect the +/// comparison. +fn project_to_sent_keys(sent: &Map, got: &Value) -> Value { + let got_obj = match got.as_object() { + Some(o) => o, + None => return got.clone(), + }; + let mut out = Map::new(); + for key in sent.keys() { + if let Some(v) = got_obj.get(key) { + out.insert(key.clone(), v.clone()); + } + } + Value::Object(out) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Client / account setup +// ───────────────────────────────────────────────────────────────────────────── + +/// One binary-encoding configuration exercised per generated document. +struct RunConfig { + label: &'static str, + binary: Option, +} + +fn run_configs() -> Vec { + vec![ + RunConfig { + label: "text-control", + binary: None, + }, + RunConfig { + label: "binary", + binary: Some(BinaryEncodingOptions::new().with_enabled(true)), + }, + RunConfig { + label: "binary+text-response", + binary: Some( + BinaryEncodingOptions::new() + .with_enabled(true) + .with_request_text_response(true), + ), + }, + ] +} + +/// Builds a Cosmos client for a given binary-encoding configuration. +async fn build_client( + binary: &Option, +) -> Result> { + let connection_string = std::env::var(CONNECTION_STRING_ENV_VAR).map_err(|_| { + format!("{CONNECTION_STRING_ENV_VAR} must be set to a Cosmos DB connection string") + })?; + let connection_string: ConnectionString = connection_string.parse()?; + + let endpoint: AccountEndpoint = connection_string.account_endpoint().parse()?; + let account = AccountReference::with_authentication_key( + endpoint, + connection_string.account_key().clone(), + ); + + let mut builder = CosmosClient::builder(); + if let Some(options) = binary { + builder = builder.with_binary_encoding_options(options.clone()); + } + + let allow_invalid_cert = env_bool(ALLOW_INVALID_CERT_ENV_VAR, false); + if allow_invalid_cert { + let runtime = CosmosRuntime::builder() + .with_connection_pool( + ConnectionPoolOptions::builder() + .with_server_certificate_validation( + ServerCertificateValidation::RequiredUnlessEmulator, + ) + .build()?, + ) + .build() + .await?; + builder = builder.with_runtime(runtime); + } + + let client = builder + .build(account, RoutingStrategy::ProximityTo(Region::EAST_US)) + .await?; + Ok(client) +} + +fn ignore_conflict(result: azure_data_cosmos::Result) -> Result<(), Box> { + match result { + Ok(_) => Ok(()), + Err(e) if e.status().status_code() == StatusCode::Conflict => Ok(()), + Err(e) => Err(e.into()), + } +} + +fn write_options_with_content() -> ItemWriteOptions { + let mut operation = OperationOptions::default(); + operation.content_response_on_write = Some(ContentResponseOnWrite::Enabled); + ItemWriteOptions::default().with_operation_options(operation) +} + +// ───────────────────────────────────────────────────────────────────────────── +// The fuzzer test +// ───────────────────────────────────────────────────────────────────────────── + +#[tokio::test] +#[cfg_attr( + not(test_category = "binary_encoding"), + ignore = "requires test_category 'binary_encoding' and a live account connection string" +)] +async fn binary_encoding_roundtrip_fuzz() -> Result<(), Box> { + let cfg = FuzzConfig::from_env(); + + if cfg.calibrate { + return run_calibration().await; + } + + println!( + "binary_roundtrip_fuzzer: seed={} iterations={} max_depth={} breadth={} wide_numbers={} unicode={}", + cfg.seed, cfg.iterations, cfg.max_depth, cfg.breadth, cfg.wide_numbers, cfg.unicode + ); + println!("Reproduce this run with {SEED_ENV_VAR}={}", cfg.seed); + + let configs = run_configs(); + + // One client per config (binary encoding is resolved at build time). + let mut clients = Vec::new(); + for rc in &configs { + clients.push((rc.label, build_client(&rc.binary).await?)); + } + + // Ensure the target database + container exist on each client's account + // (they share the same account, so the first client suffices). + let database_name = + std::env::var(DATABASE_NAME_ENV_VAR).unwrap_or_else(|_| DEFAULT_DATABASE_NAME.to_string()); + let container_name = std::env::var(CONTAINER_NAME_ENV_VAR) + .unwrap_or_else(|_| DEFAULT_CONTAINER_NAME.to_string()); + + let setup_client = &clients[0].1; + ignore_conflict(setup_client.create_database(&database_name, None).await)?; + let setup_db = setup_client.database_client(&database_name); + ignore_conflict( + setup_db + .create_container( + ContainerProperties::new(container_name.clone(), PARTITION_KEY_PATH.into()), + None, + ) + .await, + )?; + + let mut rng = SplitMix64::new(cfg.seed); + let mut checked: u64 = 0; + + for iter in 0..cfg.iterations { + // Generate the document body once per iteration so all three configs + // test the *same value* three ways. Each config gets a distinct `id` + // below — the same document stored under multiple configs would + // otherwise collide on the `(pk, id)` key and fail with 409 Conflict. + let base_doc = gen_object(&mut rng, &cfg); + let pk = format!("pk-{}", rng.below(16)); + + // Optionally print the generated document (pretty JSON) so a run can be + // eyeballed. Enable with `AZURE_COSMOS_FUZZ_PRINT=true`. + if cfg.print_docs { + println!( + "--- iter {iter} (seed={}) ---\n{}", + cfg.seed, + serde_json::to_string_pretty(&Value::Object(base_doc.clone())) + .unwrap_or_else(|_| "".to_string()) + ); + } + + for (label, client) in &clients { + let container = client + .database_client(&database_name) + .container_client(&container_name) + .await?; + + // Distinct id per config so the three stores don't conflict. + let id = Uuid::new_v4().to_string(); + let mut doc = base_doc.clone(); + doc.insert("id".to_string(), Value::String(id.clone())); + doc.insert("pk".to_string(), Value::String(pk.clone())); + + // Compute the sent canonical form from a normalized copy so it + // matches documents that have been through the backend's + // serialize→parse. + let (sent_canon, sent_hash) = canonical_hash(&normalize(&Value::Object(doc.clone()))); + + let context = format!("iter={iter} config={label} id={id} seed={}", cfg.seed); + + // CREATE with content response (exercises the response decode path). + let created = container + .create_item(&pk, &id, &doc, Some(write_options_with_content())) + .await + .map_err(|e| format!("{context}: create failed: {e}"))?; + let created_doc: Value = created + .into_model() + .map_err(|e| format!("{context}: create response decode failed: {e}"))?; + assert_roundtrip( + &doc, + &created_doc, + &sent_canon, + &sent_hash, + &context, + "create", + ); + + // READ back. + let read = container + .read_item(&pk, &id, None) + .await + .map_err(|e| format!("{context}: read failed: {e}"))?; + let read_doc: Value = read + .into_model() + .map_err(|e| format!("{context}: read response decode failed: {e}"))?; + assert_roundtrip(&doc, &read_doc, &sent_canon, &sent_hash, &context, "read"); + + // REPLACE the item with the same value (exercises the replace point + // op's request encode + response decode). Binary encoding is honored + // for replace, so this drives the encoder/decoder just like create. + let replaced = container + .replace_item(&pk, &id, &doc, Some(write_options_with_content())) + .await + .map_err(|e| format!("{context}: replace failed: {e}"))?; + let replaced_doc: Value = replaced + .into_model() + .map_err(|e| format!("{context}: replace response decode failed: {e}"))?; + assert_roundtrip( + &doc, + &replaced_doc, + &sent_canon, + &sent_hash, + &context, + "replace", + ); + + // UPSERT the same value (upsert is a point op that also carries a + // body; here it updates the existing item). Covers the upsert + // request-encode + response-decode path. + let upserted = container + .upsert_item(&pk, &id, &doc, Some(write_options_with_content())) + .await + .map_err(|e| format!("{context}: upsert failed: {e}"))?; + let upserted_doc: Value = upserted + .into_model() + .map_err(|e| format!("{context}: upsert response decode failed: {e}"))?; + assert_roundtrip( + &doc, + &upserted_doc, + &sent_canon, + &sent_hash, + &context, + "upsert", + ); + + // Four point-op round-trips this config: create, read, replace, upsert. + checked += 4; + } + + if (iter + 1) % 100 == 0 { + println!("... {} iterations, {checked} round-trips OK", iter + 1); + } + } + + println!( + "binary_roundtrip_fuzzer: DONE — {} documents × {} configs × 4 point ops = {checked} round-trips, all canonical-equal (seed={})", + cfg.iterations, + configs.len(), + cfg.seed + ); + Ok(()) +} + +// ───────────────────────────────────────────────────────────────────────────── +// Calibration mode +// ───────────────────────────────────────────────────────────────────────────── + +/// A numeric edge case: a human label and the **exact JSON literal** to store. +/// The literal is parsed with `serde_json` so its precise form is preserved. +struct NumberProbe { + label: &'static str, + literal: &'static str, +} + +/// The spread of numeric forms whose backend rewrite we want to learn. These are +/// the cases the design doc (§3.1) flags as `[CALIBRATE]`. +const NUMBER_PROBES: &[NumberProbe] = &[ + NumberProbe { + label: "integer_zero", + literal: "0", + }, + NumberProbe { + label: "negative_zero", + literal: "-0", + }, + NumberProbe { + label: "integral_float_1.0", + literal: "1.0", + }, + NumberProbe { + label: "integral_float_20.0", + literal: "20.0", + }, + NumberProbe { + label: "integral_float_exp_2e1", + literal: "2e1", + }, + NumberProbe { + label: "small_fraction_0.5", + literal: "0.5", + }, + NumberProbe { + label: "repeating_0.1", + literal: "0.1", + }, + NumberProbe { + label: "sum_0.1_plus_0.2", + literal: "0.30000000000000004", + }, + NumberProbe { + label: "high_precision_pi", + literal: "3.141592653589793", + }, + NumberProbe { + label: "large_exponent", + literal: "1e20", + }, + NumberProbe { + label: "small_exponent", + literal: "1e-20", + }, + NumberProbe { + label: "negative_large_exp", + literal: "-1.5e18", + }, + NumberProbe { + label: "i64_max", + literal: "9223372036854775807", + }, + NumberProbe { + label: "i64_min", + literal: "-9223372036854775808", + }, + NumberProbe { + label: "u64_max_minus_1", + literal: "18446744073709551614", + }, + NumberProbe { + label: "just_above_i64", + literal: "9223372036854775808", + }, + NumberProbe { + label: "trailing_zeros_1.2300", + literal: "1.2300", + }, + NumberProbe { + label: "leading_int_0e0", + literal: "0e0", + }, +]; + +/// **Calibration mode** (design doc §3.1): stores each numeric probe through the +/// binary path, reads it back, and prints how the backend rewrote it alongside +/// how `canonicalize_number` currently renders it. Any `DIFF` row is a number +/// form the canonicalizer does not yet model — tune `canonicalize_number` (or +/// narrow the generator) until the calibration table is all `MATCH`. +/// +/// This is a **diagnostic** that prints a table; it does not assert (a `DIFF` is +/// expected the first time and is the signal to tune, not a test failure). Run +/// it with `AZURE_COSMOS_FUZZ_CALIBRATE=true` against a live account. +async fn run_calibration() -> Result<(), Box> { + println!("binary_roundtrip_fuzzer: CALIBRATION MODE — learning the backend's number rewrite"); + println!("(store each probe via binary encoding, read back, compare canonical forms)\n"); + + // Use the binary config so the full encode→store→decode path is exercised. + let client = build_client(&Some(BinaryEncodingOptions::new().with_enabled(true))).await?; + + let database_name = + std::env::var(DATABASE_NAME_ENV_VAR).unwrap_or_else(|_| DEFAULT_DATABASE_NAME.to_string()); + let container_name = std::env::var(CONTAINER_NAME_ENV_VAR) + .unwrap_or_else(|_| DEFAULT_CONTAINER_NAME.to_string()); + + ignore_conflict(client.create_database(&database_name, None).await)?; + let db = client.database_client(&database_name); + ignore_conflict( + db.create_container( + ContainerProperties::new(container_name.clone(), PARTITION_KEY_PATH.into()), + None, + ) + .await, + )?; + let container = db.container_client(&container_name).await?; + + println!( + "{:<26} {:<24} {:<24} {:<24} {}", + "probe", "sent-literal", "our-canonical", "backend-returned", "status" + ); + println!("{}", "-".repeat(120)); + + let mut diffs = 0u32; + for probe in NUMBER_PROBES { + // Parse the exact literal (skip probes serde_json cannot represent). + let Ok(number_value) = serde_json::from_str::(probe.literal) else { + println!( + "{:<26} {:<24} (serde_json cannot parse this literal)", + probe.label, probe.literal + ); + continue; + }; + + let id = Uuid::new_v4().to_string(); + let pk = "calibration".to_string(); + let doc = serde_json::json!({ "id": id, "pk": pk, "n": number_value }); + + container + .create_item(&pk, &id, &doc, Some(write_options_with_content())) + .await + .map_err(|e| format!("{}: create failed: {e}", probe.label))?; + let read = container + .read_item(&pk, &id, None) + .await + .map_err(|e| format!("{}: read failed: {e}", probe.label))?; + let read_doc: Value = read + .into_model() + .map_err(|e| format!("{}: read decode failed: {e}", probe.label))?; + + let returned_n = read_doc.get("n").cloned().unwrap_or(Value::Null); + // The backend's raw JSON text rendering of the number. + let backend_returned = serde_json::to_string(&returned_n).unwrap_or_default(); + // How our canonicalizer renders the sent value vs the returned value. + let (our_canonical, _) = canonical_hash(&number_value); + let (returned_canonical, _) = canonical_hash(&returned_n); + + let status = if our_canonical == returned_canonical { + "MATCH" + } else { + diffs += 1; + "DIFF <-- tune normalize_number" + }; + + println!( + "{:<26} {:<24} {:<24} {:<24} {}", + probe.label, probe.literal, our_canonical, backend_returned, status + ); + } + + println!("{}", "-".repeat(120)); + if diffs == 0 { + println!("CALIBRATION: all probes MATCH — normalize_number models the backend rewrite."); + } else { + println!( + "CALIBRATION: {diffs} probe(s) DIFF — update `normalize_number` to match the backend-returned column above." + ); + } + Ok(()) +} + +/// Asserts the returned document, projected to the sent keys, canonicalizes to +/// the same form (and hash) as what was sent. On mismatch, prints both canonical +/// forms and the reproduction seed. +fn assert_roundtrip( + sent: &Map, + got: &Value, + sent_canon: &str, + sent_hash: &[u8; 32], + context: &str, + phase: &str, +) { + let projected = project_to_sent_keys(sent, got); + let (got_canon, got_hash) = canonical_hash(&projected); + if &got_hash != sent_hash || got_canon != sent_canon { + panic!( + "{context}: {phase} round-trip MISMATCH\n sent (sha256 {}): {sent_canon}\n got (sha256 {}): {got_canon}\n reproduce with {SEED_ENV_VAR} from the context above", + hex(sent_hash), + hex(&got_hash), + ); + } +} + +// ───────────────────────────────────────────────────────────────────────────── +// Offline unit tests — validate the canonicalizer and generator without a live +// account. These run under a normal `cargo test -p azure_data_cosmos_perf`. +// ───────────────────────────────────────────────────────────────────────────── + +#[cfg(test)] +mod tests { + use super::*; + + fn canon(value: &Value) -> String { + canonicalize(value) + } + + #[test] + fn canonicalize_sorts_object_keys() { + let a = serde_json::json!({ "b": 1, "a": 2, "c": 3 }); + let b = serde_json::json!({ "c": 3, "a": 2, "b": 1 }); + assert_eq!(canon(&a), canon(&b)); + assert_eq!(canon(&a), r#"{"a":2,"b":1,"c":3}"#); + } + + #[test] + fn canonicalize_drops_whitespace_and_preserves_array_order() { + let v: Value = serde_json::from_str(" [ 1 ,2, 3 ] ").unwrap(); + assert_eq!(canon(&v), "[1,2,3]"); + } + + #[test] + fn canonicalize_normalizes_integral_floats_to_integers() { + // 1.0 and 1 must canonicalize identically (mirrors the backend rewrite). + assert_eq!(canon(&serde_json::json!(1.0)), "1"); + assert_eq!(canon(&serde_json::json!(1)), "1"); + assert_eq!(canon(&serde_json::json!(20.0)), "20"); + assert_eq!(canon(&serde_json::json!(-0.0)), "0"); + } + + #[test] + fn canonicalize_keeps_non_integral_floats() { + assert_eq!(canon(&serde_json::json!(3.5)), "3.5"); + assert_eq!(canon(&serde_json::json!(-2.25)), "-2.25"); + } + + #[test] + fn canonicalize_large_unsigned_integer_matches_backend_double() { + // Calibrated (§3.1): the backend stores integers above i64::MAX as + // doubles and returns them in scientific notation, so the canonicalizer + // models that — a large u64 canonicalizes identically to the double form + // the backend returns. Because RFC 8785 (JCS) refuses to emit integers + // beyond 2^53, these are compared as stable string tokens (of the f64), + // which keeps sent and round-tripped values comparable. + let sent_u64: Value = serde_json::from_str("18446744073709551614").unwrap(); + let backend_double: Value = serde_json::from_str("1.8446744073709552e+19").unwrap(); + assert_eq!(canon(&sent_u64), canon(&backend_double)); + + // 2^63 (just above i64::MAX) behaves the same way. + let sent_2p63: Value = serde_json::from_str("9223372036854775808").unwrap(); + let backend_2p63: Value = serde_json::from_str("9.223372036854776e+18").unwrap(); + assert_eq!(canon(&sent_2p63), canon(&backend_2p63)); + + // i64::MAX exceeds the JCS-safe integer range, so it canonicalizes to + // an exact decimal string token (quoted by JCS), not a bare number. + let i64_max: Value = serde_json::from_str("9223372036854775807").unwrap(); + assert_eq!(canon(&i64_max), r#""9223372036854775807""#); + + // A JCS-safe integer stays a bare number. + assert_eq!(canon(&serde_json::json!(1_000_000)), "1000000"); + } + + #[test] + fn normalize_numbers_rewrites_every_number_in_the_tree() { + // The Cosmos-calibrated number rewrite applies recursively through + // arrays and nested objects, leaving non-number values untouched. + let input = serde_json::json!({ + "int": 5, + "integral_float": 1.0, + "fraction": 2.5, + "arr": [1.0, 2.0, 3.5], + "nested": { "big": 18446744073709551614u64, "s": "x", "b": true, "n": null } + }); + let out = normalize_numbers(&input); + + // Integral floats collapse to integers; fractions stay; big u64 → f64. + assert_eq!(out["int"], serde_json::json!(5)); + assert_eq!(out["integral_float"], serde_json::json!(1)); + assert_eq!(out["fraction"], serde_json::json!(2.5)); + assert_eq!(out["arr"], serde_json::json!([1, 2, 3.5])); + assert_eq!( + out["nested"]["big"], + normalize_number(&serde_json::from_str::("18446744073709551614").unwrap()) + ); + // Non-number leaves pass through unchanged. + assert_eq!(out["nested"]["s"], serde_json::json!("x")); + assert_eq!(out["nested"]["b"], serde_json::json!(true)); + assert_eq!(out["nested"]["n"], Value::Null); + } + + #[test] + fn normalize_numbers_is_idempotent() { + // Applying the rewrite twice yields the same tree — a prerequisite for + // comparing a normalized sent doc against a normalized returned doc. + let input = serde_json::json!({ + "a": 1.0, "b": [2.0, 3.5, 18446744073709551614u64], "c": { "d": 9223372036854775807i64 } + }); + let once = normalize_numbers(&input); + let twice = normalize_numbers(&once); + assert_eq!(once, twice); + } + + #[test] + fn canonical_hash_is_stable_and_matches_json_canon() { + // The digest is a deterministic function of the canonical string, and + // the canonical string is the JCS form of the number-normalized value. + let v = serde_json::json!({ "b": 1.0, "a": [2.0, 3.5], "c": "x" }); + let (s1, h1) = canonical_hash(&v); + let (s2, h2) = canonical_hash(&v); + assert_eq!(s1, s2); + assert_eq!(h1, h2); + + // Structural equivalence (key order / whitespace / integral floats) maps + // to the same digest. + let equiv = serde_json::from_str::(r#" { "c":"x", "a":[2,3.5], "b":1 } "#).unwrap(); + let (_, h_equiv) = canonical_hash(&equiv); + assert_eq!(h1, h_equiv); + } + + #[test] + fn project_strips_service_fields() { + let sent: Map = serde_json::from_value(serde_json::json!({ + "id": "x", "pk": "p", "value": 1 + })) + .unwrap(); + let got = serde_json::json!({ + "id": "x", "pk": "p", "value": 1, + "_rid": "abc", "_etag": "\"y\"", "_ts": 123 + }); + let projected = project_to_sent_keys(&sent, &got); + assert_eq!(canon(&projected), canon(&Value::Object(sent))); + } + + #[test] + fn generator_is_deterministic_for_a_seed() { + let cfg = FuzzConfig { + iterations: 0, + seed: 42, + max_depth: 4, + wide_numbers: false, + unicode: true, + breadth: DEFAULT_BREADTH, + calibrate: false, + print_docs: false, + }; + let mut a = SplitMix64::new(cfg.seed); + let mut b = SplitMix64::new(cfg.seed); + let doc_a = Value::Object(gen_object(&mut a, &cfg)); + let doc_b = Value::Object(gen_object(&mut b, &cfg)); + assert_eq!(canon(&doc_a), canon(&doc_b)); + } + + /// Nesting depth of a JSON value (scalars are depth 0). + fn depth_of(v: &Value) -> u32 { + match v { + Value::Array(items) => 1 + items.iter().map(depth_of).max().unwrap_or(0), + Value::Object(map) => 1 + map.values().map(depth_of).max().unwrap_or(0), + _ => 0, + } + } + + #[test] + fn generator_depth_scales_with_max_depth() { + // Guards the hybrid-skeleton generator: the average nesting depth must + // grow with `max_depth` (the old arbitrary-json-only generator was flat + // at ~1.3 regardless of the knob). We assert a conservative lower bound + // on the average and that the deepest doc reaches near the target. + fn avg_and_max_depth(max_depth: u32) -> (f64, u32) { + let cfg = FuzzConfig { + iterations: 0, + seed: 1784944014111583800, + max_depth, + wide_numbers: false, + unicode: true, + breadth: DEFAULT_BREADTH, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + let n = 1000u32; + let mut sum = 0u64; + let mut max_seen = 0u32; + for _ in 0..n { + let d = depth_of(&Value::Object(gen_object(&mut rng, &cfg))); + sum += d as u64; + max_seen = max_seen.max(d); + } + (sum as f64 / n as f64, max_seen) + } + + let (avg3, max3) = avg_and_max_depth(3); + let (avg8, max8) = avg_and_max_depth(8); + + // Depth clearly scales with the knob (not flat like the old generator). + assert!( + avg8 > avg3 + 1.0, + "avg depth should grow with max_depth: avg@3={avg3:.2}, avg@8={avg8:.2}" + ); + // The deepest documents actually approach the requested depth. + assert!( + max3 >= 3, + "max depth @3 should reach the target, got {max3}" + ); + assert!( + max8 >= 8, + "max depth @8 should reach the target, got {max8}" + ); + } + + #[test] + fn generated_documents_normalize_idempotently() { + // Sanity: after one serialize→parse normalization, a generated doc is + // stable — a second round-trip does not change its canonical form. This + // is the invariant the fuzzer relies on to compare the (normalized) sent + // doc against a backend-round-tripped one. + let cfg = FuzzConfig { + iterations: 0, + seed: 7, + max_depth: 5, + wide_numbers: true, + unicode: true, + breadth: DEFAULT_BREADTH, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + for _ in 0..500 { + let doc = Value::Object(gen_object(&mut rng, &cfg)); + let once = normalize(&doc); + let twice = normalize(&once); + assert_eq!( + canon(&once), + canon(&twice), + "normalization not idempotent for doc: {}", + serde_json::to_string(&doc).unwrap() + ); + } + } + + #[test] + fn generated_documents_cover_all_value_categories() { + // The sampler subtree guarantees every value category appears in each + // document: integer, float, alphabetic, alphanumeric, non-ASCII string, + // boolean, null, a number array, and multi-level nesting. This asserts + // the "really complex JSON" contract holds for a spread of seeds. + fn walk(v: &Value, seen: &mut Categories, max_depth: &mut u32, depth: u32) { + *max_depth = (*max_depth).max(depth); + match v { + Value::Null => seen.null = true, + Value::Bool(_) => seen.boolean = true, + Value::Number(n) => { + if n.is_f64() { + seen.float = true; + } else { + seen.integer = true; + } + } + Value::String(s) => { + if !s.is_empty() && s.chars().all(|c| c.is_ascii_alphabetic()) { + seen.alphabetic = true; + } + if !s.is_empty() && s.chars().all(|c| c.is_ascii_alphanumeric()) { + seen.alphanumeric = true; + } + if !s.is_ascii() { + seen.non_ascii = true; + } + } + Value::Array(items) => { + seen.array = true; + for item in items { + walk(item, seen, max_depth, depth + 1); + } + } + Value::Object(map) => { + seen.object = true; + for child in map.values() { + walk(child, seen, max_depth, depth + 1); + } + } + } + } + + #[derive(Default)] + struct Categories { + integer: bool, + float: bool, + alphabetic: bool, + alphanumeric: bool, + non_ascii: bool, + boolean: bool, + null: bool, + array: bool, + object: bool, + } + + let cfg = FuzzConfig { + iterations: 0, + seed: 0xC0FFEE, + max_depth: 6, + wide_numbers: false, + unicode: true, + breadth: DEFAULT_BREADTH, + calibrate: false, + print_docs: false, + }; + let mut rng = SplitMix64::new(cfg.seed); + + let mut all = Categories::default(); + let mut deepest = 0u32; + for _ in 0..50 { + let doc = Value::Object(gen_object(&mut rng, &cfg)); + walk(&doc, &mut all, &mut deepest, 0); + } + + assert!(all.integer, "no integer produced"); + assert!(all.float, "no float produced"); + assert!(all.alphabetic, "no alphabetic string produced"); + assert!(all.alphanumeric, "no alphanumeric string produced"); + assert!(all.non_ascii, "no non-ASCII string produced"); + assert!(all.boolean, "no boolean produced"); + assert!(all.null, "no null produced"); + assert!(all.array, "no array produced"); + assert!(all.object, "no nested object produced"); + // Multi-level nesting: the guaranteed sampler alone reaches depth ≥ 3, + // and the spine pushes documents deeper. + assert!( + deepest >= 4, + "documents should reach multi-level nesting, deepest={deepest}" + ); + } + + /// Prints a few sample generated documents as pretty JSON so the generator + /// output can be eyeballed **offline** (no Cosmos account). Ignored by + /// default; run explicitly with `--ignored --nocapture`: + /// + /// ```bash + /// cargo test -p azure_data_cosmos_perf --test binary_roundtrip_fuzzer \ + /// print_sample_documents -- --ignored --nocapture + /// ``` + /// + /// Control shape/size via the same env vars as a live run, e.g. + /// `AZURE_COSMOS_FUZZ_SEED`, `AZURE_COSMOS_FUZZ_MAX_DEPTH`, + /// `AZURE_COSMOS_FUZZ_BREADTH`, `AZURE_COSMOS_FUZZ_WIDE_NUMBERS`; the count + /// defaults to 3 (override with `AZURE_COSMOS_FUZZ_PRINT_COUNT`). + #[test] + #[ignore = "prints sample JSON on demand; run with --ignored --nocapture"] + fn print_sample_documents() { + let cfg = FuzzConfig::from_env(); + let count = std::env::var("AZURE_COSMOS_FUZZ_PRINT_COUNT") + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(3); + println!( + "print_sample_documents: seed={} max_depth={} breadth={} wide_numbers={} unicode={}", + cfg.seed, cfg.max_depth, cfg.breadth, cfg.wide_numbers, cfg.unicode + ); + let mut rng = SplitMix64::new(cfg.seed); + for i in 0..count { + let doc = Value::Object(gen_object(&mut rng, &cfg)); + println!( + "--- sample {i} ---\n{}", + serde_json::to_string_pretty(&doc).unwrap() + ); + } + } + + #[test] + fn calibration_probes_are_valid_and_unique() { + // Every calibration probe literal must parse as a JSON number, and the + // labels must be unique (they key the printed calibration table). + let mut labels: Vec<&str> = Vec::new(); + for probe in NUMBER_PROBES { + let value: Value = serde_json::from_str(probe.literal).unwrap_or_else(|e| { + panic!( + "probe {} literal {:?} invalid: {e}", + probe.label, probe.literal + ) + }); + assert!( + value.is_number(), + "probe {} literal {:?} is not a JSON number", + probe.label, + probe.literal + ); + labels.push(probe.label); + } + labels.sort_unstable(); + let count = labels.len(); + labels.dedup(); + assert_eq!(labels.len(), count, "duplicate calibration probe label"); + } +} diff --git a/sdk/cosmos/azure_data_cosmos_perf/tests/binary_sampled_testdata.rs b/sdk/cosmos/azure_data_cosmos_perf/tests/binary_sampled_testdata.rs new file mode 100644 index 00000000000..2ce032819fd --- /dev/null +++ b/sdk/cosmos/azure_data_cosmos_perf/tests/binary_sampled_testdata.rs @@ -0,0 +1,318 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +//! Live integration test that samples the bundled `testdata/*.json` corpus and +//! round-trips the sampled documents through Cosmos DB using **binary JSON** +//! encoding. +//! +//! The perf crate ships a large collection of representative JSON payloads in +//! `testdata/`. This test picks random documents out of that corpus, injects an +//! `id` and a `pk` (the container is partitioned on `/pk`), then creates each +//! document and reads it back — with the SDK's binary-encoding preview enabled — +//! asserting the fields we wrote survive the binary request/response round-trip. +//! +//! # Test data dependency +//! +//! This test reads the `testdata/*.json` corpus **at runtime** (via +//! [`load_sample_pool`], `std::fs::read_dir`), so the files must be present on +//! disk under `azure_data_cosmos_perf/testdata/` when the test runs. That corpus +//! (~500 MB) is intentionally **not tracked in the source repo** to keep the +//! repository small; it is kept as a local copy. If the directory is missing or +//! empty, [`load_sample_pool`] returns an error telling you to restore it. +//! +//! Nothing in the build or the CI gates depends on this corpus: the test is +//! gated behind `test_category = "binary_encoding"` (ignored otherwise) and +//! requires a live account, and the benchmarks generate their own synthetic +//! data — so removing the files from source control does not affect compilation +//! or any gate. +//! +//! Binary encoding is enabled explicitly on the client via +//! `CosmosClientBuilder::with_binary_encoding_options`, which the SDK resolves +//! **once at client-build time**. This test therefore sets it when building the +//! client. +//! +//! # Running +//! +//! Provide a live account connection string (which carries both the endpoint +//! and the account key) and select the `binary_encoding` test category: +//! +//! ```bash +//! AZURE_COSMOS_CONNECTION_STRING='AccountEndpoint=...;AccountKey=...;' \ +//! RUSTFLAGS='--cfg test_category="binary_encoding"' \ +//! cargo test -p azure_data_cosmos_perf --test binary_sampled_testdata +//! ``` +//! +//! The test targets the `binary-encoding-perf-db` database and +//! `binary-encoding-perf-ct` container (partition key `/pk`) by default, +//! creating them if they do not already exist. Override the names with +//! `AZURE_COSMOS_BINARY_TEST_DATABASE` / `AZURE_COSMOS_BINARY_TEST_CONTAINER`. +//! To run against a local emulator, also set `AZURE_COSMOS_ALLOW_INVALID_CERT=true`. + +#![allow(clippy::large_futures)] + +use std::error::Error; +use std::path::{Path, PathBuf}; + +use azure_core::http::StatusCode; +use azure_data_cosmos::models::ContainerProperties; +use azure_data_cosmos::options::{ + BinaryEncodingOptions, ConnectionPoolOptions, ContentResponseOnWrite, ItemWriteOptions, + OperationOptions, Region, ServerCertificateValidation, +}; +use azure_data_cosmos::{ + AccountEndpoint, AccountReference, CosmosClient, CosmosRuntime, RoutingStrategy, +}; +use azure_data_cosmos_driver::models::ConnectionString; +use rand::RngExt; +use serde_json::{Map, Value}; +use uuid::Uuid; + +const CONNECTION_STRING_ENV_VAR: &str = "AZURE_COSMOS_CONNECTION_STRING"; +const ALLOW_INVALID_CERT_ENV_VAR: &str = "AZURE_COSMOS_ALLOW_INVALID_CERT"; +const DATABASE_NAME_ENV_VAR: &str = "AZURE_COSMOS_BINARY_TEST_DATABASE"; +const CONTAINER_NAME_ENV_VAR: &str = "AZURE_COSMOS_BINARY_TEST_CONTAINER"; + +const DEFAULT_DATABASE_NAME: &str = "binary-encoding-perf-db"; +const DEFAULT_CONTAINER_NAME: &str = "binary-encoding-perf-ct"; +const PARTITION_KEY_PATH: &str = "/pk"; + +/// Number of documents sampled from the corpus per test run. +const SAMPLE_COUNT: usize = 25; + +/// Reads all bundled `testdata/*.json` files and flattens them into a pool of +/// candidate JSON **objects** (non-object top-level values and array elements +/// are skipped, since only objects can carry the injected `id`/`pk` fields). +/// +/// The `testdata/` corpus is a local copy that is intentionally not tracked in +/// the source repo (see the module docs). If the directory is missing or holds +/// no usable objects, this returns an error explaining how to restore it. +fn load_sample_pool() -> Result>, Box> { + let testdata_dir = PathBuf::from(env!("CARGO_MANIFEST_DIR")).join("testdata"); + + let entries = std::fs::read_dir(&testdata_dir).map_err(|e| { + format!( + "failed to read the test-data corpus under {} ({e}). This corpus is a \ + local copy (not tracked in source control); restore the \ + `testdata/*.json` files before running this test.", + testdata_dir.display() + ) + })?; + + let mut pool = Vec::new(); + for entry in entries { + let path = entry?.path(); + if path + .extension() + .and_then(|e| e.to_str()) + .map(str::to_ascii_lowercase) + != Some("json".to_string()) + { + continue; + } + collect_objects_from_file(&path, &mut pool); + } + + if pool.is_empty() { + return Err(format!( + "no candidate JSON objects found under {} — the local test-data corpus \ + is missing or empty; restore the `testdata/*.json` files.", + testdata_dir.display() + ) + .into()); + } + Ok(pool) +} + +/// Parses a single testdata file and appends any JSON objects it contains to +/// `pool`. A file may be a top-level object, an array of objects, or an object +/// whose values contain arrays of objects (e.g. GeoJSON `features`). Unparsable +/// files are skipped rather than failing the whole run. +fn collect_objects_from_file(path: &Path, pool: &mut Vec>) { + let Ok(bytes) = std::fs::read(path) else { + return; + }; + let Ok(value) = serde_json::from_slice::(&bytes) else { + return; + }; + collect_objects_from_value(value, pool); +} + +/// Recursively harvests JSON objects from a value, descending into arrays and +/// into the array-valued fields of objects. +fn collect_objects_from_value(value: Value, pool: &mut Vec>) { + match value { + Value::Array(items) => { + for item in items { + collect_objects_from_value(item, pool); + } + } + Value::Object(map) => { + // Descend into array-valued fields (e.g. GeoJSON `features`) so + // container documents like `{ "features": [ {...}, {...} ] }` + // contribute their inner objects too. + for nested in map.values() { + if let Value::Array(items) = nested { + for item in items { + if let Value::Object(obj) = item { + pool.push(obj.clone()); + } + } + } + } + pool.push(map); + } + _ => {} + } +} + +/// Builds a Cosmos client from the connection string, enabling binary encoding. +async fn build_client() -> Result> { + let connection_string = std::env::var(CONNECTION_STRING_ENV_VAR).map_err(|_| { + format!("{CONNECTION_STRING_ENV_VAR} must be set to a Cosmos DB connection string") + })?; + let connection_string: ConnectionString = connection_string.parse()?; + + let endpoint: AccountEndpoint = connection_string.account_endpoint().parse()?; + let account = AccountReference::with_authentication_key( + endpoint, + connection_string.account_key().clone(), + ); + + // Enable binary encoding explicitly via the client option rather than the + // process environment, so the setting is scoped to this client. + let mut builder = CosmosClient::builder() + .with_binary_encoding_options(BinaryEncodingOptions::new().with_enabled(true)); + + let allow_invalid_cert = std::env::var(ALLOW_INVALID_CERT_ENV_VAR) + .ok() + .and_then(|v| v.parse::().ok()) + .unwrap_or(false); + if allow_invalid_cert { + let runtime = CosmosRuntime::builder() + .with_connection_pool( + ConnectionPoolOptions::builder() + .with_server_certificate_validation( + ServerCertificateValidation::RequiredUnlessEmulator, + ) + .build()?, + ) + .build() + .await?; + builder = builder.with_runtime(runtime); + } + + let client = builder + .build(account, RoutingStrategy::ProximityTo(Region::EAST_US)) + .await?; + Ok(client) +} + +/// Maps a 409 Conflict result to `Ok(())` so "resource already exists" is not +/// treated as a failure when ensuring the database/container exist. +fn ignore_conflict(result: azure_data_cosmos::Result) -> Result<(), Box> { + match result { + Ok(_) => Ok(()), + Err(e) if e.status().status_code() == StatusCode::Conflict => Ok(()), + Err(e) => Err(e.into()), + } +} + +/// Write options that request the service echo the stored document back, so the +/// binary **response** decode path is exercised on every write. +fn write_options_with_content() -> ItemWriteOptions { + let mut operation = OperationOptions::default(); + operation.content_response_on_write = Some(ContentResponseOnWrite::Enabled); + ItemWriteOptions::default().with_operation_options(operation) +} + +/// Asserts that every field we sent in `sent` is present and equal in the +/// service-returned `got`. The service adds system fields (`_rid`, `_etag`, +/// `_ts`, ...) to stored documents, so a full-object equality would spuriously +/// fail — we only verify the fields we control round-tripped intact. +fn assert_sent_fields_round_tripped(sent: &Map, got: &Value, context: &str) { + let got = got + .as_object() + .unwrap_or_else(|| panic!("{context}: response body was not a JSON object: {got}")); + for (key, expected) in sent { + let actual = got + .get(key) + .unwrap_or_else(|| panic!("{context}: response missing field {key:?}")); + assert_eq!( + actual, expected, + "{context}: field {key:?} did not round-trip", + ); + } +} + +/// Samples documents from the bundled corpus and round-trips each one through +/// create + read using binary encoding, asserting the sent fields survive. +#[tokio::test] +#[cfg_attr( + not(test_category = "binary_encoding"), + ignore = "requires test_category 'binary_encoding' and a live account connection string" +)] +async fn binary_round_trips_sampled_testdata() -> Result<(), Box> { + let pool = load_sample_pool()?; + println!("Loaded {} candidate documents from testdata/", pool.len()); + + let client = build_client().await?; + + // Target database/container names default to generic values but can be + // overridden via env vars so the test can run against caller-owned + // resources. + let database_name = + std::env::var(DATABASE_NAME_ENV_VAR).unwrap_or_else(|_| DEFAULT_DATABASE_NAME.to_string()); + let container_name = std::env::var(CONTAINER_NAME_ENV_VAR) + .unwrap_or_else(|_| DEFAULT_CONTAINER_NAME.to_string()); + + // Ensure the target database and container (partitioned on /pk) exist, + // treating a 409 Conflict as "already created". + ignore_conflict(client.create_database(&database_name, None).await)?; + let db_client = client.database_client(&database_name); + ignore_conflict( + db_client + .create_container( + ContainerProperties::new(container_name.clone(), PARTITION_KEY_PATH.into()), + None, + ) + .await, + )?; + let container = db_client.container_client(&container_name).await?; + + let mut rng = rand::rng(); + let sampled: Vec> = (0..SAMPLE_COUNT) + .map(|_| pool[rng.random_range(0..pool.len())].clone()) + .collect(); + + for (i, base) in sampled.into_iter().enumerate() { + // Inject the id and partition key. The corpus documents may already + // carry an `id`/`pk`, which we overwrite to guarantee uniqueness and a + // valid single-value partition key. + let id = Uuid::new_v4().to_string(); + let pk = format!("pk-{}", rng.random_range(0..16)); + + let mut doc = base; + doc.insert("id".to_string(), Value::String(id.clone())); + doc.insert("pk".to_string(), Value::String(pk.clone())); + + let context = format!("sample #{i} (id={id})"); + + // CREATE with content response: the service echoes the stored document + // back through the binary path. + let created = container + .create_item(&pk, &id, &doc, Some(write_options_with_content())) + .await?; + assert_eq!(created.status(), StatusCode::Created, "{context}: create"); + let created_doc: Value = created.into_model()?; + assert_sent_fields_round_tripped(&doc, &created_doc, &format!("{context}: create echo")); + + // READ back through the binary path and verify the stored values. + let read = container.read_item(&pk, &id, None).await?; + assert_eq!(read.status(), StatusCode::Ok, "{context}: read"); + let read_doc: Value = read.into_model()?; + assert_sent_fields_round_tripped(&doc, &read_doc, &format!("{context}: read")); + } + + println!("Round-tripped {SAMPLE_COUNT} sampled documents through binary encoding."); + Ok(()) +}