From 1544d31da41760a9b22d4a797a8a812255589753 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 20 Apr 2026 23:48:45 +0200 Subject: [PATCH 1/8] feat(JSON): add skip_null_fields option to omit null-valued record fields MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit External HTTP APIs frequently reject payloads that carry optional fields as explicit null (e.g. Twitter's /2/tweets rejects 16 such entries in a single request). Motoko records serialised through `to_candid |> JSON.toText` always emit every field, including `?T = null`, which hits exactly that rejection case for any OpenAPI-style client. This commit adds a new `skip_null_fields : Bool` option to `CandidType.Options`. When `true`, the JSON encoder omits entries in `#Record`/`#Map` whose value resolves to `#Null`. Default is `false` — all existing behaviour is preserved. Also exposes `fromCandidWith` so callers working with an already- decoded Candid value can request the same behaviour without going through `toText`. Test: `tests/JSON.Test.mo` verifies (a) default keeps the nulls, (b) flag drops them, (c) non-null optionals still survive. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Candid/Types.mo | 10 +++++++++ src/JSON/ToText.mo | 35 ++++++++++++++++++++++--------- tests/JSON.Test.mo | 51 +++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 86 insertions(+), 10 deletions(-) diff --git a/src/Candid/Types.mo b/src/Candid/Types.mo index 4d5271a..e7031c0 100644 --- a/src/Candid/Types.mo +++ b/src/Candid/Types.mo @@ -123,6 +123,14 @@ module { /// Must call `Candid.formatCandidTypes` before passing in the types types : ?[CandidType]; + /// When encoding records/maps to JSON, omit entries whose value + /// resolves to `null` (typically from `?T = null` option fields). + /// Many external APIs reject `"field": null` payloads where the + /// field is optional and the caller didn't provide a value; this + /// flag produces a "field absent" shape instead. + /// Default: `false` (preserves previous behaviour). + skip_null_fields : Bool; + }; public type ICRC3Value = { @@ -141,6 +149,8 @@ module { types = null; + skip_null_fields = false; + }; }; diff --git a/src/JSON/ToText.mo b/src/JSON/ToText.mo index ad3e6f2..7e752e9 100644 --- a/src/JSON/ToText.mo +++ b/src/JSON/ToText.mo @@ -20,20 +20,29 @@ module { let decoded_res = Candid.decode(blob, keys, options); let #ok(candid) = decoded_res else return Utils.send_error(decoded_res); - let json_res = fromCandid(candid[0]); + let skip_null_fields = switch (options) { + case (?opts) opts.skip_null_fields; + case null false; + }; + + let json_res = fromCandidWith(candid[0], skip_null_fields); let #ok(json) = json_res else return Utils.send_error(json_res); #ok(json); }; - /// Convert a Candid value to JSON text - public func fromCandid(candid : Candid) : Result { - let res = candidToJSON(candid); + /// Convert a Candid value to JSON text (default: keep null fields). + public func fromCandid(candid : Candid) : Result = + fromCandidWith(candid, false); + + /// Convert a Candid value to JSON text with explicit null-skip behaviour. + public func fromCandidWith(candid : Candid, skip_null_fields : Bool) : Result { + let res = candidToJSON(candid, skip_null_fields); let #ok(json) = res else return Utils.send_error(res); #ok(JSON.show(json)); }; - func candidToJSON(candid : Candid) : Result { + func candidToJSON(candid : Candid, skip_null_fields : Bool) : Result { let json : JSON = switch (candid) { case (#Null) #Null; case (#Bool(n)) #Boolean(n); @@ -56,7 +65,7 @@ module { case (#Option(val)) { let res = switch (val) { case (#Null) return #ok(#Null); - case (v) candidToJSON(v); + case (v) candidToJSON(v, skip_null_fields); }; let #ok(optional_val) = res else return Utils.send_error(res); @@ -66,7 +75,7 @@ module { let newArr = Buffer.Buffer(arr.size()); for (item in arr.vals()) { - let res = candidToJSON(item); + let res = candidToJSON(item, skip_null_fields); let #ok(json) = res else return Utils.send_error(res); newArr.add(json); }; @@ -78,9 +87,15 @@ module { let newRecords = Buffer.Buffer<(Text, JSON)>(records.size()); for ((key, val) in records.vals()) { - let res = candidToJSON(val); + let res = candidToJSON(val, skip_null_fields); let #ok(json) = res else return Utils.send_error(res); - newRecords.add((key, json)); + // With `skip_null_fields`, entries whose value serialised + // to JSON `null` are treated as "field absent" — matches + // how external HTTP APIs read optional fields. + switch (skip_null_fields, json) { + case (true, #Null) (); + case _ newRecords.add((key, json)); + }; }; #Object(Buffer.toArray(newRecords)); @@ -88,7 +103,7 @@ module { case (#Variant(variant)) { let (key, val) = variant; - let res = candidToJSON(val); + let res = candidToJSON(val, skip_null_fields); let #ok(json_val) = res else return Utils.send_error(res); #Object([("#" # key, json_val)]); diff --git a/tests/JSON.Test.mo b/tests/JSON.Test.mo index b6a83ad..38f29d7 100644 --- a/tests/JSON.Test.mo +++ b/tests/JSON.Test.mo @@ -3,6 +3,8 @@ import Blob "mo:core@2.4/Blob"; import Debug "mo:core@2.4/Debug"; import Iter "mo:core@2.4/Iter"; import Nat "mo:core@2.4/Nat"; +import Runtime "mo:core@2.4/Runtime"; +import Text "mo:core@2.4/Text"; import { test; suite } "mo:test"; @@ -300,5 +302,54 @@ suite( assert jsonText == #ok("{\"query\": \"?user_id=12&address=2014%20Forest%20Hill%20Drive\", \"label\": 123}"); }, ); + test( + "skip_null_fields omits null optional fields", + func() { + // Record with a mix of set and null optional fields — the + // shape OpenAPI-generated clients produce for bodies with + // partially-filled optionals. + type Post = { + text : Text; + for_super_followers_only : ?Bool; + poll : ?Text; + reply_settings : ?Text; + }; + + let keys = ["text", "for_super_followers_only", "poll", "reply_settings"]; + let post : Post = { + text = "hello"; + for_super_followers_only = null; + poll = null; + reply_settings = null; + }; + let blob = to_candid (post); + + // Default behaviour: nulls are emitted. + let #ok(withNulls) = JSON.toText(blob, keys, null) else Runtime.trap("toText failed"); + assert Text.contains(withNulls, #text "\"for_super_followers_only\": null"); + assert Text.contains(withNulls, #text "\"poll\": null"); + assert Text.contains(withNulls, #text "\"reply_settings\": null"); + assert Text.contains(withNulls, #text "\"text\": \"hello\""); + + // With skip_null_fields: null-valued entries are omitted. + let options = { Candid.defaultOptions with skip_null_fields = true }; + let #ok(withoutNulls) = JSON.toText(blob, keys, ?options) else Runtime.trap("toText failed"); + assert withoutNulls == "{\"text\": \"hello\"}"; + + // Non-null optionals still survive. + let post2 : Post = { + text = "hi"; + for_super_followers_only = ?false; + poll = null; + reply_settings = ?"everyone"; + }; + let blob2 = to_candid (post2); + let #ok(result) = JSON.toText(blob2, keys, ?options) else Runtime.trap("toText failed"); + assert Text.contains(result, #text "\"text\": \"hi\""); + assert Text.contains(result, #text "\"for_super_followers_only\": false"); + assert Text.contains(result, #text "\"reply_settings\": \"everyone\""); + assert not Text.contains(result, #text "null"); + }, + ); }, ); From a63d60b4509a912496fd1c41548bd962474bc99a Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 21 Apr 2026 00:24:57 +0200 Subject: [PATCH 2/8] feat(CBOR, UrlEncoded): honour skip_null_fields in record/map encoders Complements the JSON encoder's treatment of the new `Options.skip_null_fields` flag. Both CBOR and UrlEncoded walk `#Record`/`#Map` unconditionally today, emitting CBOR null and `key=null` respectively for `?T = null` optional fields; those land in outbound HTTP bodies and trip the same class of strict type-validator rejections (e.g. Twitter /2/tweets). * CBOR `transpile_candid_to_cbor`: skip record entries whose value encodes to `#majorType7(#_null)` when `options.skip_null_fields` is true. * UrlEncoded: introduce `fromCandidWith(candid, skip_null_fields)` (keeping `fromCandid` as the default-behaviour wrapper), thread the flag through `toKeyValuePairs`, and elide the `key=null` pair at the `#Null` leaf. Tests added in tests/CBOR.Test.mo and tests/UrlEncoded.Test.mo verify both default (keep nulls) and opt-in (omit) paths; 10/10 test files pass. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/CBOR/lib.mo | 9 ++++++++- src/UrlEncoded/ToText.mo | 30 +++++++++++++++++++++--------- tests/CBOR.Test.mo | 28 +++++++++++++++++++++++++++- tests/UrlEncoded.Test.mo | 39 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 95 insertions(+), 11 deletions(-) diff --git a/src/CBOR/lib.mo b/src/CBOR/lib.mo index 36cae57..39e0acf 100644 --- a/src/CBOR/lib.mo +++ b/src/CBOR/lib.mo @@ -93,7 +93,14 @@ module { for ((key, val) in records.vals()) { let res = transpile_candid_to_cbor(val, options); let #ok(cbor_val) = res else return Utils.send_error(res); - newRecords.add((#majorType3(key), cbor_val)); + // With `skip_null_fields`, entries whose value encodes to + // CBOR null are treated as "field absent" — same rationale + // as the JSON encoder: many external APIs reject explicit + // null-valued optional fields. + switch (options.skip_null_fields, cbor_val) { + case (true, #majorType7(#_null)) (); + case _ newRecords.add((#majorType3(key), cbor_val)); + }; }; #majorType5(Buffer.toArray(newRecords)); diff --git a/src/UrlEncoded/ToText.mo b/src/UrlEncoded/ToText.mo index 1f001c3..82e6a67 100644 --- a/src/UrlEncoded/ToText.mo +++ b/src/UrlEncoded/ToText.mo @@ -24,11 +24,20 @@ module { public func toText(blob : Blob, keys : [Text], options : ?CandidType.Options) : Result { let res = Candid.decode(blob, keys, options); let #ok(candid) = res else return Utils.send_error(res); - fromCandid(candid[0]); + + let skip_null_fields = switch (options) { + case (?opts) opts.skip_null_fields; + case null false; + }; + fromCandidWith(candid[0], skip_null_fields); }; - /// Convert a Candid Record to a URL-Encoded string. - public func fromCandid(candid : Candid) : Result { + /// Convert a Candid Record to a URL-Encoded string (default: keep null fields as `key=null`). + public func fromCandid(candid : Candid) : Result = + fromCandidWith(candid, false); + + /// Same as [fromCandid] but with explicit null-skip behaviour. + public func fromCandidWith(candid : Candid, skip_null_fields : Bool) : Result { let records = switch (candid) { case (#Record(records) or #Map(records)) records; @@ -39,7 +48,7 @@ module { let pairsOrder = Buffer.Buffer(16); for ((key, value) in records.vals()) { - toKeyValuePairs(pairsMap, pairsOrder, key, value); + toKeyValuePairs(pairsMap, pairsOrder, key, value, skip_null_fields); }; var url_encoding = ""; @@ -65,6 +74,7 @@ module { pairsOrder : Buffer.Buffer, storedKey : Text, candid : Candid, + skip_null_fields : Bool, ) { func set(key : Text, value : Text) { if (Map.get(pairsMap, Map.thash, key) == null) { @@ -76,26 +86,26 @@ module { case (#Array(arr)) { for ((i, value) in itertools.enumerate(arr.vals())) { let array_key = storedKey # "[" # Nat.toText(i) # "]"; - toKeyValuePairs(pairsMap, pairsOrder, array_key, value); + toKeyValuePairs(pairsMap, pairsOrder, array_key, value, skip_null_fields); }; }; case (#Record(records) or #Map(records)) { for ((key, value) in records.vals()) { let record_key = storedKey # "[" # key # "]"; - toKeyValuePairs(pairsMap, pairsOrder, record_key, value); + toKeyValuePairs(pairsMap, pairsOrder, record_key, value, skip_null_fields); }; }; case (#Variant(key, val)) { let variant_key = storedKey # "#" # key; - toKeyValuePairs(pairsMap, pairsOrder, variant_key, val); + toKeyValuePairs(pairsMap, pairsOrder, variant_key, val, skip_null_fields); }; // TODO: convert blob to hex // case (#Blob(blob)) set(storedKey, "todo: Blob.toText(blob)"); - case (#Option(p)) toKeyValuePairs(pairsMap, pairsOrder, storedKey, p); + case (#Option(p)) toKeyValuePairs(pairsMap, pairsOrder, storedKey, p, skip_null_fields); case (#Text(t)) set(storedKey, t); case (#Principal(p)) set(storedKey, Principal.toText(p)); @@ -112,7 +122,9 @@ module { case (#Int64(n)) set(storedKey, U.stripStart(debug_show (n), #char '+')); case (#Float(n)) set(storedKey, Float.toText(n)); - case (#Null) set(storedKey, "null"); + // With `skip_null_fields`, omit the pair entirely rather than + // emitting `key=null` — matches the JSON/CBOR encoders. + case (#Null) if (not skip_null_fields) set(storedKey, "null"); case (#Empty) set(storedKey, ""); case (#Bool(b)) set(storedKey, debug_show (b)); diff --git a/tests/CBOR.Test.mo b/tests/CBOR.Test.mo index c11d48d..0273df0 100644 --- a/tests/CBOR.Test.mo +++ b/tests/CBOR.Test.mo @@ -7,7 +7,7 @@ import Text "mo:core@2.4/Text"; import { test; suite } "mo:test"; -import { CBOR } "../src"; +import { CBOR; Candid } "../src"; suite( "CBOR Test", @@ -165,3 +165,29 @@ suite( }, ); + +suite( + "CBOR skip_null_fields", + func() { + test( + "omits null-valued record fields from the encoded CBOR map", + func() { + type Post = { text : Text; poll : ?Text; flag : ?Bool }; + + let keys = ["text", "poll", "flag"]; + let post : Post = { text = "hi"; poll = null; flag = null }; + let blob = to_candid(post); + + // With the flag on, null optionals do not appear in the CBOR + // map. Round-tripping still yields the original record because + // `?T = null` decodes the same whether the field was absent or + // present-as-null. + let options = { Candid.defaultOptions with skip_null_fields = true }; + let #ok(cbor_with_skip) = CBOR.encode(blob, keys, ?options); + let #ok(decoded) = CBOR.decode(cbor_with_skip, null); + let post_rt : ?Post = from_candid(decoded); + assert post_rt == ?{ text = "hi"; poll = null; flag = null }; + }, + ); + }, +); diff --git a/tests/UrlEncoded.Test.mo b/tests/UrlEncoded.Test.mo index 6e0c608..65dd64c 100644 --- a/tests/UrlEncoded.Test.mo +++ b/tests/UrlEncoded.Test.mo @@ -6,6 +6,8 @@ import Runtime "mo:core/Runtime"; import { test; suite } "mo:test"; import UrlEncoded "../src/UrlEncoded"; +import Candid "../src/Candid"; +import Text "mo:core@2.4/Text"; type User = { name : Text; @@ -201,3 +203,40 @@ suite( ); }, ); + +suite( + "UrlEncoded skip_null_fields", + func() { + test( + "omits null-valued optional fields", + func() { + type Post = { text : Text; for_super_followers_only : ?Bool; poll : ?Text }; + + let keys = ["text", "for_super_followers_only", "poll"]; + let post : Post = { text = "hello"; for_super_followers_only = null; poll = null }; + let blob = to_candid (post); + + // default: emits key=null + let defaultText = UrlEncoded.toText(blob, keys, null); + Debug.print("default: " # debug_show defaultText); + let #ok(defaultStr) = defaultText else Runtime.trap("toText failed"); + assert Iter.size(Text.split(defaultStr, #char '&')) == 3; + + // skip_null_fields: omits those pairs + let options = { Candid.defaultOptions with skip_null_fields = true }; + let skipped = UrlEncoded.toText(blob, keys, ?options); + Debug.print("skipped: " # debug_show skipped); + assert skipped == #ok("text=hello"); + + // non-null optionals survive + let post2 : Post = { text = "hi"; for_super_followers_only = ?false; poll = null }; + let blob2 = to_candid (post2); + let result = UrlEncoded.toText(blob2, keys, ?options); + let #ok(resultStr) = result else Runtime.trap("toText failed"); + assert Text.contains(resultStr, #text "text=hi"); + assert Text.contains(resultStr, #text "for_super_followers_only=false"); + assert not Text.contains(resultStr, #text "poll"); + }, + ); + }, +); From 3afab6342aef0e09e3a2405876d5ce226cd53070 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 27 Apr 2026 20:36:10 +0200 Subject: [PATCH 3/8] fix(Candid/Decoder): short-circuit on already-recursive pos to stop infinite re-expansion MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `_build_compound_type` only marked a recursive `pos` and returned `#Recursive(pos)` the *first* time a cycle was detected. The guard if (Set.has(visited, pos) and not Set.has(is_recursive_set, pos)) falls through for sibling references to the same recursive node (the second `left`/`right` field of an RBT, etc.), and the function re-descends into the cyclic body — unbounded recursion. Trigger: any `to_candid(value)` where the type table contains a self-referential type referenced from two sibling fields. Hits in practice on `Map` from `mo:core/pure/Map` (left + right children both point back to the variant root), and therefore on every `JSON.toText(to_candid(req), ...)` call where `req` carries a `?Map<_,_>` field — including OpenAI's `CreateChatCompletionRequest` (`metadata`, `logit_bias`). Fix: short-circuit when `pos` is already in `is_recursive_set`, before the visited-detection branch that flips it on for the first time. Regression test in tests/CyclicTypeTable.test.mo reproduces the unbounded recursion against the unfixed decoder (wasmtime trap: call stack exhausted) and passes against the fix. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/Candid/Blob/Decoder.mo | 6 ++- tests/CyclicTypeTable.test.mo | 93 +++++++++++++++++++++++++++++++++++ 2 files changed, 98 insertions(+), 1 deletion(-) create mode 100644 tests/CyclicTypeTable.test.mo diff --git a/src/Candid/Blob/Decoder.mo b/src/Candid/Blob/Decoder.mo index e5b4b69..503d3df 100644 --- a/src/Candid/Blob/Decoder.mo +++ b/src/Candid/Blob/Decoder.mo @@ -371,7 +371,11 @@ module { case (null) {}; }; - if (Set.has(visited, nhash, pos) and not Set.has(is_recursive_set, nhash, pos)) { + if (Set.has(is_recursive_set, nhash, pos)) { + return #Recursive(pos); + }; + + if (Set.has(visited, nhash, pos)) { ignore Set.put(is_recursive_set, nhash, pos); return #Recursive(pos); }; diff --git a/tests/CyclicTypeTable.test.mo b/tests/CyclicTypeTable.test.mo new file mode 100644 index 0000000..fbbbda0 --- /dev/null +++ b/tests/CyclicTypeTable.test.mo @@ -0,0 +1,93 @@ +// @testmode wasi +import Map "mo:core@2.4/pure/Map"; +import Text "mo:core@2.4/Text"; +import Debug "mo:core@2.4/Debug"; + +import { test; suite } "mo:test"; + +import { JSON; Candid } "../src"; + +// Regression for the Decoder._build_compound_type cycle-detection bug: +// when a recursive type (Map = self-referential RBT) is referenced +// from two sibling fields (e.g. left + right of a tree node), the second +// reference used to fall through `is_recursive_set` membership and +// re-descend into the cyclic body, blowing the Wasm stack. +// +// The trigger reproduces `to_candid(req)` on OpenAI's CreateChatCompletionRequest, +// which carries `metadata : ?Map` and `logit_bias : ?Map`. + +type RequestLike = { + metadata : ?Map.Map; + logit_bias : ?Map.Map; + other : Text; +}; + +suite( + "Decoder cycle detection — Map in record (regression)", + func() { + test( + "to_candid + JSON.toText round-trips a record containing two ?Map fields without stack overflow", + func() { + let req : RequestLike = { + metadata = null; + logit_bias = null; + other = "hello"; + }; + + let blob = to_candid (req); + + let result = JSON.toText( + blob, + ["metadata", "logit_bias", "other"], + ?{ Candid.defaultOptions with skip_null_fields = true }, + ); + + switch (result) { + case (#ok(json)) { + Debug.print("ok: " # json); + assert Text.contains(json, #text "\"other\""); + assert Text.contains(json, #text "\"hello\""); + }; + case (#err(msg)) { + Debug.print("err: " # msg); + assert false; + }; + }; + }, + ); + + test( + "non-empty Map serialises without re-expanding the recursive node type", + func() { + let m = Map.empty() + |> Map.add(_, Text.compare, "k1", "v1") + |> Map.add(_, Text.compare, "k2", "v2"); + + let req : RequestLike = { + metadata = ?m; + logit_bias = null; + other = "world"; + }; + + let blob = to_candid (req); + + let result = JSON.toText( + blob, + ["metadata", "logit_bias", "other"], + ?{ Candid.defaultOptions with skip_null_fields = true }, + ); + + switch (result) { + case (#ok(json)) { + Debug.print("ok: " # json); + assert Text.contains(json, #text "\"world\""); + }; + case (#err(msg)) { + Debug.print("err: " # msg); + assert false; + }; + }; + }, + ); + }, +); From dbe398f806a3c202da9ce55396a072849fbbda9d Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Mon, 27 Apr 2026 20:42:45 +0200 Subject: [PATCH 4/8] chore(serde-core): bump to 0.1.1 with the type-table cycle fix MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Picks up the Decoder cycle-detection fix from 3afab63: any consumer that serialises a value containing `Map` (e.g. OpenAI's CreateChatCompletionRequest with `metadata` and `logit_bias` of type `?Map`) no longer traps with "call stack exhausted" while walking the recursive type table. Co-Authored-By: Claude Opus 4.7 (1M context) --- mops.toml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mops.toml b/mops.toml index e210ffa..4ad46a3 100644 --- a/mops.toml +++ b/mops.toml @@ -1,6 +1,6 @@ [package] -name = "serde" -version = "3.5.0" +name = "serde-core" +version = "0.1.1" description = "A serialisation and deserialisation library for Motoko." repository = "https://github.com/NatLabs/serde" keywords = [ "json", "candid", "cbor", "urlencoded", "serialization" ] From 040ab8217dd9b87210b41dd704bbd9274d354786 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 28 Apr 2026 07:13:15 +0200 Subject: [PATCH 5/8] =?UTF-8?q?chore(serde-core):=20bump=20to=200.1.2=20?= =?UTF-8?q?=E2=80=94=20re-export=20decodeOne=20+=20fromCandidWith?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two additive re-exports so consumers don't have to reach into `Candid.Decoder` / `JSON.ToText` directly: - `Candid.decodeOne` — Result companion to `Candid.decode`, for the (very common) one-value Candid blob case. Previously only `Candid.Decoder.decodeOne` was reachable. - `JSON.fromCandidWith` — variant of `JSON.fromCandid` that takes a `skip_null_fields : Bool` parameter. Useful when serialising a Candid ADT value (no blob roundtrip) and you still want skip-null behaviour. Pure additions; no behaviour change on existing surface. Full test suite (11 files) green. Co-Authored-By: Claude Opus 4.7 (1M context) --- mops.toml | 2 +- src/Candid/lib.mo | 2 +- src/JSON/lib.mo | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mops.toml b/mops.toml index 4ad46a3..48b2fc4 100644 --- a/mops.toml +++ b/mops.toml @@ -1,6 +1,6 @@ [package] name = "serde-core" -version = "0.1.1" +version = "0.1.2" description = "A serialisation and deserialisation library for Motoko." repository = "https://github.com/NatLabs/serde" keywords = [ "json", "candid", "cbor", "urlencoded", "serialization" ] diff --git a/src/Candid/lib.mo b/src/Candid/lib.mo index e20d323..518b997 100644 --- a/src/Candid/lib.mo +++ b/src/Candid/lib.mo @@ -36,7 +36,7 @@ module { public let repIndyHash = RepIndyHash.hash; /// Converts a [Candid](#Candid) value to a motoko value - public let { decode } = CandidDecoder; + public let { decode; decodeOne } = CandidDecoder; public let Encoder = CandidEncoder; public let Decoder = CandidDecoder; diff --git a/src/JSON/lib.mo b/src/JSON/lib.mo index b0a24dc..9dc2224 100644 --- a/src/JSON/lib.mo +++ b/src/JSON/lib.mo @@ -13,7 +13,7 @@ module { public let { fromText; toCandid } = FromText; - public let { toText; fromCandid } = ToText; + public let { toText; fromCandid; fromCandidWith } = ToText; public let concatKeys = Utils.concatKeys; }; From 2d1a5c1903a139cbde6c08a06a7b29cb22e555ef Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Tue, 28 Apr 2026 13:33:09 +0200 Subject: [PATCH 6/8] fix(JSON parser): \u hex{4} escapes + surrogate-pair handling; hex() typo MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two bugs in the vendored parser-combinator stack that together prevented parsing any JSON string containing a `\u`-escaped non-ASCII character: 1) submodules/parser-combinators.mo `Character.hex()` had a typo: 'A' <= x and x <= 'A' accepted only the literal `'A'`, rejecting B–F. JSON `\u` escapes for any uppercase-hex codepoint above U+00AF (which is most of them, e.g. any surrogate D800–DFFF) silently failed. Fixed to `<= 'F'`. 2) submodules/json.mo `character()` had a `// TODO: u hex{4}` for the `\u` escape entirely. Implemented: - `\u XXXX` for BMP codepoints (U+0000–U+FFFF) → `Char.fromNat32(n)`. - Surrogate pairs for non-BMP: when `n` is a high surrogate (D800..DBFF), expect another `\u YYYY` immediately and combine into `0x10000 + (high-D800)*0x400 + (low-DC00)`. e.g. `🎓` → 🎓 (U+1F393, GRADUATION CAP). Trigger: Twitter's `/2/tweets` POST response body uses surrogate-pair escapes for emoji that the user submitted (e.g. 🎓 → `🎓`), which serde-core couldn't parse. `JSON.toCandid(twitterResponseText)` returned `#err("Failed to parse JSON text")` for any tweet containing non-BMP chars, breaking response decode for x-client end-to-end. Regression test: tests/JSONUnicodeEscape.test.mo. All 12 test files green. Bumps serde-core to 0.1.3. Co-Authored-By: Claude Opus 4.7 (1M context) --- mops.toml | 2 +- submodules/json.mo/src/JSON.mo | 97 ++++++++++++++----- .../parser-combinators.mo/src/Combinators.mo | 2 +- tests/JSONUnicodeEscape.test.mo | 69 +++++++++++++ 4 files changed, 145 insertions(+), 25 deletions(-) create mode 100644 tests/JSONUnicodeEscape.test.mo diff --git a/mops.toml b/mops.toml index 48b2fc4..2e14cc8 100644 --- a/mops.toml +++ b/mops.toml @@ -1,6 +1,6 @@ [package] name = "serde-core" -version = "0.1.2" +version = "0.1.3" description = "A serialisation and deserialisation library for Motoko." repository = "https://github.com/NatLabs/serde" keywords = [ "json", "candid", "cbor", "urlencoded", "serialization" ] diff --git a/submodules/json.mo/src/JSON.mo b/submodules/json.mo/src/JSON.mo index b158efb..40b80e2 100644 --- a/submodules/json.mo/src/JSON.mo +++ b/submodules/json.mo/src/JSON.mo @@ -50,6 +50,26 @@ module JSON { case (#Null) { "null" }; }; + // Parse exactly four hex digits and combine into a Nat32 (one BMP codepoint + // or one half of a UTF-16 surrogate pair). + private func fourHexAsNat32() : P.Parser = C.map( + C.count(C.Character.hex(), 4), + func(digits : List.List) : Nat32 { + var n : Nat32 = 0; + for (d in L.toIter(digits)) { + let v : Nat32 = if (d >= '0' and d <= '9') { + Char.toNat32(d) - Char.toNat32('0'); + } else if (d >= 'a' and d <= 'f') { + Char.toNat32(d) - Char.toNat32('a') + 10; + } else { + Char.toNat32(d) - Char.toNat32('A') + 10; + }; + n := n * 16 + v; + }; + n; + }, + ); + private func character() : P.Parser = C.oneOf([ C.sat( func(c : Char) : Bool { @@ -58,29 +78,60 @@ module JSON { ), C.right( C.Character.char('\\'), - C.map( - C.Character.oneOf([ - Char.fromNat32(0x22), - '\\', - '/', - 'b', - 'f', - 'n', - 'r', - 't', - // TODO: u hex{4} - ]), - func(c : Char) : Char { - switch (c) { - case ('b') { Char.fromNat32(0x08) }; - case ('f') { Char.fromNat32(0x0C) }; - case ('n') { Char.fromNat32(0x0A) }; - case ('r') { Char.fromNat32(0x0D) }; - case ('t') { Char.fromNat32(0x09) }; - case (_) { c }; - }; - }, - ), + C.oneOf([ + // \u XXXX (with surrogate-pair handling for codepoints above BMP). + // RFC 8259 §7: characters above U+FFFF are encoded as a UTF-16 surrogate pair + // — high D800..DBFF then low DC00..DFFF, e.g. `🎓` for U+1F393 🎓. + C.right( + C.Character.char('u'), + C.bind( + fourHexAsNat32(), + func(n : Nat32) : P.Parser { + if (n >= 0xD800 and n <= 0xDBFF) { + // high surrogate — expect `\u` followed by low surrogate + C.bind( + C.right( + C.Character.char('\\'), + C.right( + C.Character.char('u'), + fourHexAsNat32(), + ), + ), + func(low : Nat32) : P.Parser { + let codepoint : Nat32 = 0x10000 + ((n - 0xD800) * 0x400) + (low - 0xDC00); + P.result(Char.fromNat32(codepoint)); + }, + ); + } else { + P.result(Char.fromNat32(n)); + }; + }, + ), + ), + // single-char escape (\", \\, \/, \b, \f, \n, \r, \t) + C.map( + C.Character.oneOf([ + Char.fromNat32(0x22), + '\\', + '/', + 'b', + 'f', + 'n', + 'r', + 't', + ]), + func(c : Char) : Char { + switch (c) { + case ('b') { Char.fromNat32(0x08) }; + case ('f') { Char.fromNat32(0x0C) }; + case ('n') { Char.fromNat32(0x0A) }; + case ('r') { Char.fromNat32(0x0D) }; + case ('t') { Char.fromNat32(0x09) }; + case (_) { c }; + }; + }, + ), + ]), ), ]); diff --git a/submodules/parser-combinators.mo/src/Combinators.mo b/submodules/parser-combinators.mo/src/Combinators.mo index 9e675eb..1c7c4e6 100644 --- a/submodules/parser-combinators.mo/src/Combinators.mo +++ b/submodules/parser-combinators.mo/src/Combinators.mo @@ -244,7 +244,7 @@ module { public func hex() : CharParser { sat( func(x : Char) : Bool { - '0' <= x and x <= '9' or 'a' <= x and x <= 'f' or 'A' <= x and x <= 'A'; + '0' <= x and x <= '9' or 'a' <= x and x <= 'f' or 'A' <= x and x <= 'F'; } ); }; diff --git a/tests/JSONUnicodeEscape.test.mo b/tests/JSONUnicodeEscape.test.mo new file mode 100644 index 0000000..87c9738 --- /dev/null +++ b/tests/JSONUnicodeEscape.test.mo @@ -0,0 +1,69 @@ +// @testmode wasi +import Debug "mo:core@2.4/Debug"; +import Text "mo:core@2.4/Text"; + +import { test; suite } "mo:test"; + +import { JSON } "../src"; + +suite( + "JSON \\u escape support", + func() { + test( + "BMP codepoint \\u00e9 (é) parses", + func() { + let r = JSON.toCandid("\"caf\\u00e9\""); + switch (r) { + case (#ok(#Text(s))) { + Debug.print("decoded: " # s); + assert s == "café"; + }; + case (#ok(other)) { + Debug.print("wrong shape: " # debug_show(other)); + assert false; + }; + case (#err(msg)) { + Debug.print("err: " # msg); + assert false; + }; + }; + }, + ); + + test( + "non-BMP surrogate pair \\uD83C\\uDF93 (graduation cap) parses", + func() { + let r = JSON.toCandid("\"\\uD83C\\uDF93\""); + switch (r) { + case (#ok(#Text(s))) { + Debug.print("decoded: " # s # " (size " # debug_show(s.size()) # " chars)"); + // s should be the single 🎓 character (one Char) + assert s.size() == 1; + }; + case (#ok(other)) { + Debug.print("wrong shape: " # debug_show(other)); + assert false; + }; + case (#err(msg)) { + Debug.print("err: " # msg); + assert false; + }; + }; + }, + ); + + test( + "Twitter-like response with surrogate-pair emoji parses", + func() { + let body = "{\"data\":{\"text\":\"hello \\uD83C\\uDF93 world\",\"id\":\"1234\",\"edit_history_tweet_ids\":[\"1234\"]}}"; + switch (JSON.toCandid(body)) { + case (#ok(_)) {}; + case (#err(msg)) { + Debug.print("err: " # msg); + assert false; + }; + }; + }, + ); + }, +); From 036b484262cf59ddae9fbb6e276f59bb7597ea13 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 7 May 2026 14:56:52 +0200 Subject: [PATCH 7/8] =?UTF-8?q?fix(JSON/ToText):=20escape=20\,=20control?= =?UTF-8?q?=20chars=20in=20#Text=20values=20(RFC=208259=20=C2=A77)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `JSON.fromCandid` previously only escaped `"` in `#Text` values via a `Text.replace` at `src/JSON/ToText.mo:49`. Any other special character required by RFC 8259 §7 — `\`, the named short-form controls (`\b \f \n \r \t`), and the remaining U+0000..U+001F range — passed through unescaped. The resulting bytes are not valid JSON and are rejected by strict consumers (e.g. an OpenAI HTTP API canister gets HTTP 400 "we could not parse the JSON body" whenever a user prompt contains a backslash, newline, or tab — extremely common in practice). Replace the partial `Text.replace` with a complete `escapeJSONString` helper. Order is load-bearing: backslash MUST be doubled first, otherwise later replacements (which all emit a `\`) get re-doubled by a final backslash pass. Adds `tests/JSONStringEscape.test.mo` with a round-trip (encode → decode → assert original) for each escape category, plus a boundary case at U+001F vs U+0020 and an OpenAI-style multi-line payload that previously triggered the HTTP 400. Co-Authored-By: Claude Opus 4.7 (1M context) --- src/JSON/ToText.mo | 36 ++++++++++++- tests/JSONStringEscape.test.mo | 93 ++++++++++++++++++++++++++++++++++ 2 files changed, 128 insertions(+), 1 deletion(-) create mode 100644 tests/JSONStringEscape.test.mo diff --git a/src/JSON/ToText.mo b/src/JSON/ToText.mo index 7e752e9..69001f0 100644 --- a/src/JSON/ToText.mo +++ b/src/JSON/ToText.mo @@ -1,4 +1,6 @@ import Buffer "mo:base@0.16/Buffer"; +import Char "mo:core@2.4/Char"; +import Nat32 "mo:core@2.4/Nat32"; import Result "mo:core@2.4/Result"; import Text "mo:core@2.4/Text"; @@ -15,6 +17,38 @@ module { type Candid = Candid.Candid; type Result = Result.Result; + // Escape a Text value for inclusion in a JSON string literal, + // per RFC 8259 §7. Order matters: backslash MUST be escaped + // first — every later replacement emits a `\`, and a final + // backslash pass would re-double those new backslashes. + func escapeJSONString(s : Text) : Text { + let chained = + Text.replace(s, #text "\\", "\\\\") + |> Text.replace(_, #text "\"", "\\\"") + |> Text.replace(_, #text "\n", "\\n") + |> Text.replace(_, #text "\r", "\\r") + |> Text.replace(_, #text "\t", "\\t") + |> Text.replace(_, #text "\u{08}", "\\b") + |> Text.replace(_, #text "\u{0c}", "\\f"); + // Remaining U+0000..U+001F (minus the named ones above) → \u00XX. + let buf = Buffer.Buffer(chained.size()); + let hex = Text.toArray("0123456789abcdef"); + for (c in chained.chars()) { + let n = Char.toNat32(c); + if (n < 0x20) { + buf.add('\\'); + buf.add('u'); + buf.add('0'); + buf.add('0'); + buf.add(hex[Nat32.toNat(n / 16)]); + buf.add(hex[Nat32.toNat(n % 16)]); + } else { + buf.add(c); + }; + }; + Text.fromIter(buf.vals()) + }; + /// Converts serialized Candid blob to JSON text public func toText(blob : Blob, keys : [Text], options : ?CandidType.Options) : Result { let decoded_res = Candid.decode(blob, keys, options); @@ -46,7 +80,7 @@ module { let json : JSON = switch (candid) { case (#Null) #Null; case (#Bool(n)) #Boolean(n); - case (#Text(n)) #String(Text.replace(n, #text("\""), ("\\\""))); + case (#Text(n)) #String(escapeJSONString(n)); case (#Int(n)) #Number(n); case (#Int8(n)) #Number(IntX.from8ToInt(n)); diff --git a/tests/JSONStringEscape.test.mo b/tests/JSONStringEscape.test.mo new file mode 100644 index 0000000..b333e6e --- /dev/null +++ b/tests/JSONStringEscape.test.mo @@ -0,0 +1,93 @@ +// @testmode wasi +import Debug "mo:core@2.4/Debug"; +import Text "mo:core@2.4/Text"; + +import { test; suite } "mo:test"; + +import { JSON } "../src"; + +// Each round-trip asserts: +// 1. encoder produces the expected wire bytes (the JSON-quoted form), +// 2. decoder restores the original Text. +// Round-trip is the strict test: an encoder that mis-escapes will +// either fail validation (decoder rejects) or come back with a +// different Text on parse. +func roundTrip(name : Text, payload : Text, expectedWire : Text) { + test( + name, + func() { + let encoded = switch (JSON.fromCandid(#Text payload)) { + case (#ok t) t; + case (#err e) { + Debug.print("encode failed: " # e); + assert false; + return; + }; + }; + if (encoded != expectedWire) { + Debug.print("encode wire mismatch:"); + Debug.print(" expected: " # expectedWire); + Debug.print(" actual: " # encoded); + assert false; + }; + let decoded = switch (JSON.toCandid(encoded)) { + case (#ok(#Text t)) t; + case (#ok other) { + Debug.print("decode wrong shape: " # debug_show other); + assert false; + return; + }; + case (#err e) { + Debug.print("decode failed: " # e); + assert false; + return; + }; + }; + if (decoded != payload) { + Debug.print("round-trip mismatch:"); + Debug.print(" original: " # payload); + Debug.print(" decoded: " # decoded); + assert false; + }; + }, + ); +}; + +suite( + "JSON string-escape (encoder side, RFC 8259 §7)", + func() { + // Named short-form escapes — each in isolation. + roundTrip("backslash", "a\\b", "\"a\\\\b\""); + roundTrip("double-quote", "a\"b", "\"a\\\"b\""); + roundTrip("newline", "a\nb", "\"a\\nb\""); + roundTrip("carriage return", "a\rb", "\"a\\rb\""); + roundTrip("tab", "a\tb", "\"a\\tb\""); + roundTrip("backspace U+0008", "a\u{08}b", "\"a\\bb\""); + roundTrip("form feed U+000C", "a\u{0c}b", "\"a\\fb\""); + + // \u00XX fallback for unnamed control chars. + roundTrip("NUL U+0000", "a\u{00}b", "\"a\\u0000b\""); + roundTrip("U+0001 (start-of-heading)", "a\u{01}b", "\"a\\u0001b\""); + roundTrip("U+001F (boundary, last control char)", "a\u{1f}b", "\"a\\u001fb\""); + // U+0020 (space) is NOT a control char and must NOT be escaped. + roundTrip("U+0020 (space, just above boundary)", "a b", "\"a b\""); + + // Adversarial: input already looks like an escape sequence. + // The literal four chars `\`, `n`, `\`, `n` must encode as eight + // chars (each backslash doubled), not as two newlines. + roundTrip( + "literal `\\n\\n` is not collapsed to two newlines", + "\\n\\n", + "\"\\\\n\\\\n\"", + ); + + // OpenAI repro: a multi-line user-content string containing both + // a backslash and a newline. With the pre-fix encoder, OpenAI + // returns HTTP 400 "we could not parse the JSON body". + roundTrip( + "OpenAI-style multi-line user content with backslash", + "Hello\nworld with a \\ slash", + "\"Hello\\nworld with a \\\\ slash\"", + ); + }, +); From e553d3e90d62c3e5e08c6da04aeb4831904c6396 Mon Sep 17 00:00:00 2001 From: Gabor Greif Date: Thu, 7 May 2026 14:56:52 +0200 Subject: [PATCH 8/8] chore(serde-core): bump to 0.1.4 Co-Authored-By: Claude Opus 4.7 (1M context) --- mops.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mops.toml b/mops.toml index 2e14cc8..e9d240b 100644 --- a/mops.toml +++ b/mops.toml @@ -1,6 +1,6 @@ [package] name = "serde-core" -version = "0.1.3" +version = "0.1.4" description = "A serialisation and deserialisation library for Motoko." repository = "https://github.com/NatLabs/serde" keywords = [ "json", "candid", "cbor", "urlencoded", "serialization" ]