From d2ebf73047fb75398b218981116529be181e7a94 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:10:04 +0000 Subject: [PATCH 1/5] fix(json): throw instead of dropping data for overlapping JSON paths MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ClickHouse accepts a JSON column which declares a path both as a value and as the parent of another path, for example `JSON(a Int64, a.b Int64)`. Both paths are typed, so both are present in every row, and the server renders the row with a duplicate key: `{"a":0,"a":{"b":7}}`. A JsonObject cannot hold two values for one key. The wire puts `a.b` before `a`, so the leaf overwrote the subtree, and reading `{"a":{"b":7}}` back out of that column returned `{"a":0}`. Well-formed input, silent wrong answer. With non-Nullable paths the driver cannot tell a stored `0` from an absent path — the server collapses both — so no rule recovers the real value. The only choice is between a wrong answer and a clear failure. ReadAsJsonObject now throws a SerializationException naming the two paths when both sides of an overlap hold a value: a parent which holds a scalar or an array, and a leaf value landing on a subtree which is not all-null. The representable cases are unchanged — a null leaf still keeps the subtree, and a leaf value still replaces an all-null subtree — so `Nullable` overlaps, which fill in one side per row, keep reading as before. The parent walk previously cast the occupant to JsonObject, which surfaced the same collision as an InvalidCastException for two overlapping dynamic paths. That cast is now the explicit throw. The all-null scan runs only on a collision, so a non-overlapping schema never pays for it. --- CHANGELOG.md | 3 + .../Types/JsonTypeTests.cs | 94 ++++++++++++++++++ ClickHouse.Driver/Types/JsonType.cs | 98 ++++++++++++++++--- docs/overview.mdx | 14 +++ 4 files changed, 194 insertions(+), 15 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index a8cdecd67..9477b2a3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,9 @@ Unreleased --- +**Bug Fixes:** +* **Breaking Change**: Fixed a `JSON` column with overlapping typed paths silently losing data. ClickHouse accepts a column which declares a path both as a value and as the parent of another path — for example `JSON(a Int64, a.b Int64)` — and sends both paths in every row, so the row renders as a document with a duplicate key (`{"a":0,"a":{"b":7}}`). A `JsonObject` cannot hold two values for one key, so the driver silently kept whichever path the server sent last: reading `{"a":{"b":7}}` back out of that column returned `{"a":0}` and the `7` was gone. Such a row now throws a `SerializationException` naming the two paths. With non-`Nullable` paths the driver cannot tell a stored `0` from an absent path, so no rule can recover the real value — the choice is between a wrong answer and a clear failure. Where the overlap is declared `Nullable`, the row normally fills in one side only and continues to read as before. Use `JsonReadMode.String` to read such a column as the server's JSON text, duplicate key included. + v1.4.0 --- diff --git a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs index 1513c39cc..f3db7efce 100644 --- a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs +++ b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs @@ -1180,6 +1180,14 @@ public static IEnumerable TypedPathNullValueTestCases() "{\"a\":{\"b\":null}}") .SetName("TypedLeafPathOverlappingTypedSubtreeWithNullValues"); + // Mirror of the case above: the value belongs to the leaf, and the overlapping subtree + // (`{"b":null}`) carries none, so the leaf's value replaces it. + yield return new TestCaseData( + "a Nullable(Int64), a.b Nullable(Int64)", + "{\"a\": 5}", + "{\"a\":5}") + .SetName("TypedLeafPathOverlappingTypedSubtreeWithLeafValue"); + // Both `a` and `a.b` are typed and the value belongs to the subtree. Descending into an // object at a path that overlaps a typed leaf was added after 25.8: 26.3 and later route // the value to `a.b`, while 25.8 parses `{"b":7}` against `a Nullable(Int64)` and rejects @@ -1237,6 +1245,92 @@ await connection.ExecuteStatementAsync( Assert.That(result.ToJsonString(), Is.EqualTo("{}")); } + public static IEnumerable OverlappingPathsHoldingValuesTestCases() + { + // A non-Nullable typed path is never absent — an absent path materializes as 0 — so `a` + // always holds a value and always collides with the `a.b` subtree, whichever of the two + // the document fills in. + yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": {\"b\": 7}}") + .SetName("OverlappingNonNullableTypedPathsWithValueInSubtree"); + + yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": 5}") + .SetName("OverlappingNonNullableTypedPathsWithValueInLeaf"); + + // Nullable typed paths only collide where the document itself carries both, which needs a + // duplicate key. ClickHouse accepts the document and keeps both values. + yield return new TestCaseData("JSON(a Nullable(Int64), a.b Nullable(Int64))", "{\"a\": 5, \"a\": {\"b\": 7}}") + .SetName("OverlappingNullableTypedPathsWithDuplicateKeyDocument"); + + // The same collision with no hints at all: `a` and `a.b` are both dynamic paths. + yield return new TestCaseData("JSON", "{\"a\": 5, \"a\": {\"b\": 7}}") + .SetName("OverlappingDynamicPathsWithDuplicateKeyDocument"); + } + + [Test] + [RequiredFeature(Feature.Json)] + [TestCaseSource(nameof(OverlappingPathsHoldingValuesTestCases))] + public async Task Read_WithOverlappingPathsHoldingValues_ShouldThrow(string columnType, string jsonData) + { + var targetTable = CreateTableName(); + + await connection.ExecuteStatementAsync( + $"CREATE OR REPLACE TABLE {targetTable} (data {columnType}) ENGINE = Memory;"); + await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{jsonData}')"); + + using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + + // The column is decoded by Read, so the throw lands there rather than on GetValue. + var exception = Assert.Throws(() => reader.Read()); + + Assert.That(exception.Message, Does.Contain("'a'"), exception.Message); + Assert.That(exception.Message, Does.Contain("'a.b'"), exception.Message); + Assert.That(exception.Message, Does.Contain("JsonReadMode.String"), exception.Message); + } + + [Test] + [RequiredFeature(Feature.Json)] + public async Task Read_WithOverlappingPathsHoldingValues_ShouldBeReadableAsString() + { + var targetTable = CreateTableName(); + + await connection.ExecuteStatementAsync( + $"CREATE OR REPLACE TABLE {targetTable} (data JSON(a Int64, a.b Int64)) ENGINE = Memory;"); + await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{{\"a\": {{\"b\": 7}}}}')"); + + using var stringClient = TestUtilities.GetTestClickHouseClient(jsonReadMode: JsonReadMode.String); + + using var reader = await stringClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + ClassicAssert.IsTrue(reader.Read()); + + Assert.That(reader.GetValue(0), Is.EqualTo("{\"a\":0,\"a\":{\"b\":7}}")); + } + + [Test] + [RequiredFeature(Feature.Json)] + public async Task Read_WithNestedPathsThatDoNotOverlap_ShouldReadEveryPath() + { + var targetTable = CreateTableName(); + + await connection.ExecuteStatementAsync( + $"CREATE OR REPLACE TABLE {targetTable} " + + "(data JSON(a.b Int64, a.c Nullable(Int64), a.d.e String, f Int64)) ENGINE = Memory;"); + await connection.ExecuteStatementAsync( + $"INSERT INTO {targetTable} VALUES ('{{\"a\": {{\"b\": 1, \"d\": {{\"e\": \"x\"}}}}, \"f\": 2}}')"); + + using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + ClassicAssert.IsTrue(reader.Read()); + + var result = (JsonObject)reader.GetValue(0); + + // The server sends the paths in its own order, so assert per path rather than on the + // rendered document. + var a = (JsonObject)result["a"]; + Assert.That((long)a["b"], Is.EqualTo(1L)); + Assert.That(a["c"], Is.Null); + Assert.That((string)((JsonObject)a["d"])["e"], Is.EqualTo("x")); + Assert.That((long)result["f"], Is.EqualTo(2L)); + } + private class NullableHintedData { public int? Value { get; set; } diff --git a/ClickHouse.Driver/Types/JsonType.cs b/ClickHouse.Driver/Types/JsonType.cs index f70b28bca..7e8cb5f35 100644 --- a/ClickHouse.Driver/Types/JsonType.cs +++ b/ClickHouse.Driver/Types/JsonType.cs @@ -82,29 +82,48 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) } var pathParts = name.Split('.'); + var depth = 0; foreach (var part in pathParts.SkipLast1(1)) { - if (current[part] is { } existing) + depth++; + switch (current[part]) { - current = (JsonObject)existing; - } - else - { - // Either the parent is not there yet, or it holds the null of an overlapping - // typed leaf path (ClickHouse allows both `a` and `a.b` to be typed); the - // subtree replaces that null so the deeper path stays readable. - var newCurrent = new JsonObject(); - current[part] = newCurrent; - current = newCurrent; + case JsonObject existing: + current = existing; + break; + + case null: + // Either the parent is not there yet, or it holds the null of an overlapping + // leaf path (ClickHouse allows both `a` and `a.b` to be declared); the + // subtree replaces that null so the deeper path stays readable. + var newCurrent = new JsonObject(); + current[part] = newCurrent; + current = newCurrent; + break; + + default: + // The parent holds a value of its own, so this row needs both a value and a + // subtree under one key. Nothing can be dropped without losing data. + throw OverlappingPathsException(string.Join(".", pathParts.Take(depth)), name); } } var leaf = pathParts.Last(); - if (jsonNode is null && current[leaf] is JsonObject) + if (current[leaf] is JsonObject occupant) { - // Mirror of the walk above for the opposite path order: a typed leaf path's null - // must not erase the subtree an overlapping deeper typed path already filled in. - continue; + // A deeper overlapping path was read first and put its subtree here. + if (jsonNode is null) + { + // The null of the leaf path must not erase that subtree. + continue; + } + + if (!IsAllNull(occupant, out var occupiedPath)) + { + throw OverlappingPathsException(name, $"{name}.{occupiedPath}"); + } + + // The subtree carries no value, so the leaf's value replaces it. } current[leaf] = jsonNode; @@ -113,6 +132,55 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) return root; } + /// + /// Reports whether every value the object holds, at any depth, is a JSON null. An empty object + /// holds no value, so it counts as all-null. An array counts as a value. + /// + /// + /// Dotted path of the first value which is not null, relative to ; + /// null when the object is all-null. + /// + private static bool IsAllNull(JsonObject value, out string valuePath) + { + foreach (var property in value) + { + if (property.Value is null) + { + continue; + } + + if (property.Value is JsonObject nested) + { + if (IsAllNull(nested, out var nestedPath)) + { + continue; + } + + valuePath = $"{property.Key}.{nestedPath}"; + return false; + } + + valuePath = property.Key; + return false; + } + + valuePath = null; + return true; + } + + /// + /// Builds the error for a row where one path holds a value and is also the parent of another + /// path which holds a value. ClickHouse accepts such a column (for example + /// JSON(a Int64, a.b Int64)) and renders the row with a duplicate key, which a + /// cannot hold. + /// + private static SerializationException OverlappingPathsException(string path, string nestedPath) => + new SerializationException( + $"JSON paths '{path}' and '{nestedPath}' overlap and both hold a value in this row. " + + $"'{path}' is a value and is also the parent of '{nestedPath}', so the server sends the row with a " + + $"duplicate '{path}' key, which a JsonObject cannot hold. " + + $"Read this column with JsonReadMode.String to get the server's JSON text unchanged."); + public override ParameterizedType Parse( SyntaxTreeNode node, Func parseClickHouseType, diff --git a/docs/overview.mdx b/docs/overview.mdx index c98d4e286..320c7b909 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -1505,6 +1505,20 @@ var settings = new ClickHouseClientSettings("Host=localhost") // "Host=localhost;JsonReadMode=String" ``` +##### Overlapping typed paths + +ClickHouse accepts a column which declares a path both as a value and as the parent of another +path, for example `JSON(a Int64, a.b Int64)`. Both paths are present in every row, so the server +renders the row with a duplicate key: `{"a":0,"a":{"b":7}}`. A `JsonObject` cannot hold two values +for one key, so `JsonReadMode.Binary` throws a `SerializationException` naming the two paths. + +This applies only where both paths hold a value in that row. Where the overlap is declared with +`Nullable` types, the row usually fills in one side only — `JSON(a Nullable(Int64), a.b +Nullable(Int64))` reads `{"a":5}` and `{"a":{"b":7}}` normally — so nothing throws. + +Read such a column with `JsonReadMode.String` to get the server's JSON text unchanged, duplicate +key included. + --- #### Map type {#type-map-reading-map} From 313c98a1475c3adbb617f3df44e2194956a54dd6 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:26:20 +0000 Subject: [PATCH 2/5] test(json): cover a two-level overlapping path The all-null scan recurses, but every overlap case stopped one level down, so neither recursive branch ran. `JSON(a Int64, a.b.c Int64)` reaches the branch which finds a value and has to report `a.b.c` rather than the first key under `a`; the Nullable counterpart reaches the branch which finds the whole subtree null and lets the leaf's value replace it. --- .../Types/JsonTypeTests.cs | 25 ++++++++++++++----- 1 file changed, 19 insertions(+), 6 deletions(-) diff --git a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs index f3db7efce..189a086d2 100644 --- a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs +++ b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs @@ -1188,6 +1188,14 @@ public static IEnumerable TypedPathNullValueTestCases() "{\"a\":5}") .SetName("TypedLeafPathOverlappingTypedSubtreeWithLeafValue"); + // Same, with the overlapping subtree two levels deep: it is all-null throughout, so the + // leaf's value still replaces it. + yield return new TestCaseData( + "a Nullable(Int64), a.b.c Nullable(Int64)", + "{\"a\": 5}", + "{\"a\":5}") + .SetName("TypedLeafPathOverlappingDeeperTypedSubtreeWithLeafValue"); + // Both `a` and `a.b` are typed and the value belongs to the subtree. Descending into an // object at a path that overlaps a typed leaf was added after 25.8: 26.3 and later route // the value to `a.b`, while 25.8 parses `{"b":7}` against `a Nullable(Int64)` and rejects @@ -1250,26 +1258,31 @@ public static IEnumerable OverlappingPathsHoldingValuesTestCases() // A non-Nullable typed path is never absent — an absent path materializes as 0 — so `a` // always holds a value and always collides with the `a.b` subtree, whichever of the two // the document fills in. - yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": {\"b\": 7}}") + yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": {\"b\": 7}}", "a.b") .SetName("OverlappingNonNullableTypedPathsWithValueInSubtree"); - yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": 5}") + yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": 5}", "a.b") .SetName("OverlappingNonNullableTypedPathsWithValueInLeaf"); + // The value is deeper than the path it collides with, so the error has to walk down to it + // rather than name the first key under `a`. + yield return new TestCaseData("JSON(a Int64, a.b.c Int64)", "{\"a\": {\"b\": {\"c\": 7}}}", "a.b.c") + .SetName("OverlappingNonNullableTypedPathsWithValueInDeeperSubtree"); + // Nullable typed paths only collide where the document itself carries both, which needs a // duplicate key. ClickHouse accepts the document and keeps both values. - yield return new TestCaseData("JSON(a Nullable(Int64), a.b Nullable(Int64))", "{\"a\": 5, \"a\": {\"b\": 7}}") + yield return new TestCaseData("JSON(a Nullable(Int64), a.b Nullable(Int64))", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") .SetName("OverlappingNullableTypedPathsWithDuplicateKeyDocument"); // The same collision with no hints at all: `a` and `a.b` are both dynamic paths. - yield return new TestCaseData("JSON", "{\"a\": 5, \"a\": {\"b\": 7}}") + yield return new TestCaseData("JSON", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") .SetName("OverlappingDynamicPathsWithDuplicateKeyDocument"); } [Test] [RequiredFeature(Feature.Json)] [TestCaseSource(nameof(OverlappingPathsHoldingValuesTestCases))] - public async Task Read_WithOverlappingPathsHoldingValues_ShouldThrow(string columnType, string jsonData) + public async Task Read_WithOverlappingPathsHoldingValues_ShouldThrow(string columnType, string jsonData, string expectedNestedPath) { var targetTable = CreateTableName(); @@ -1283,7 +1296,7 @@ await connection.ExecuteStatementAsync( var exception = Assert.Throws(() => reader.Read()); Assert.That(exception.Message, Does.Contain("'a'"), exception.Message); - Assert.That(exception.Message, Does.Contain("'a.b'"), exception.Message); + Assert.That(exception.Message, Does.Contain($"'{expectedNestedPath}'"), exception.Message); Assert.That(exception.Message, Does.Contain("JsonReadMode.String"), exception.Message); } From ef063d4fab47aeae25299832016903d67eb0d740 Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:46:26 +0000 Subject: [PATCH 3/5] fix(json): treat an object-valued path as a value, not as a subtree MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A Map path, or a Dynamic one holding an object, decodes to a JsonObject — the same type as the container the path walk builds for a deeper path. The walk could not tell them apart, so it descended into the value and merged the deeper path into it. Two values the server sent under one key came back as one merged object, and where they shared a key the map's own entry was overwritten and gone: `JSON(a Map(String, Int64))` holding `{"a":{"b":1},"a.b":7}` read back as `{"a":{"b":7}}`. Object values are now tracked by reference for the row being read, so the walk throws on one instead of descending. The set is allocated only where such a path exists, which a column of plain scalar paths never has. The leaf now also mirrors the null rule onto the incoming value: an empty or all-null object holds nothing, so it gives way to the subtree rather than colliding with it. Without that, `JSON(a Map(String, Int64), `a.b` Int64)` threw on every row — every key of the document is routed to a path of its own, so the map is always empty, and no data was at risk. Both sides of the rule are now order-independent: whichever of the two paths the wire puts first, a row where both hold a value throws, and a row where one holds nothing reads the other. --- CHANGELOG.md | 2 +- .../Types/JsonTypeTests.cs | 37 ++++++++++ ClickHouse.Driver/Types/JsonType.cs | 70 +++++++++++++------ docs/overview.mdx | 16 +++-- 4 files changed, 95 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9477b2a3a..e72d50e7e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Unreleased --- **Bug Fixes:** -* **Breaking Change**: Fixed a `JSON` column with overlapping typed paths silently losing data. ClickHouse accepts a column which declares a path both as a value and as the parent of another path — for example `JSON(a Int64, a.b Int64)` — and sends both paths in every row, so the row renders as a document with a duplicate key (`{"a":0,"a":{"b":7}}`). A `JsonObject` cannot hold two values for one key, so the driver silently kept whichever path the server sent last: reading `{"a":{"b":7}}` back out of that column returned `{"a":0}` and the `7` was gone. Such a row now throws a `SerializationException` naming the two paths. With non-`Nullable` paths the driver cannot tell a stored `0` from an absent path, so no rule can recover the real value — the choice is between a wrong answer and a clear failure. Where the overlap is declared `Nullable`, the row normally fills in one side only and continues to read as before. Use `JsonReadMode.String` to read such a column as the server's JSON text, duplicate key included. +* **Breaking Change**: Fixed a `JSON` column with overlapping paths silently losing data. ClickHouse accepts a column which declares a path both as a value and as the parent of another path — for example `JSON(a Int64, a.b Int64)` — and sends both paths in every row, so the row renders as a document with a duplicate key (`{"a":0,"a":{"b":7}}`). A `JsonObject` cannot hold two values for one key, so the driver silently kept whichever path the server sent last: reading `{"a":{"b":7}}` back out of that column returned `{"a":0}` and the `7` was gone. The same loss hit a `Map` path overlapped by a deeper path (`JSON(a Map(String, Int64))` holding `{"a":{"b":1},"a.b":7}` returned `{"a":{"b":7}}`, dropping the map's own entry), except that there the two values were merged into one object rather than one replacing the other. A row where both sides of such an overlap hold a value now throws a `SerializationException` naming the two paths. With non-`Nullable` paths the driver cannot tell a stored `0` from an absent path, so no rule can recover the real value — the choice is between a wrong answer and a clear failure. A side which holds nothing is not a collision: a null, an empty object, and an all-null subtree all give way to the side which has the data, so `Nullable` overlaps continue to read as before. Use `JsonReadMode.String` to read such a column as the server's JSON text, duplicate key included. v1.4.0 --- diff --git a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs index 189a086d2..e6e751893 100644 --- a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs +++ b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs @@ -1277,6 +1277,20 @@ public static IEnumerable OverlappingPathsHoldingValuesTestCases() // The same collision with no hints at all: `a` and `a.b` are both dynamic paths. yield return new TestCaseData("JSON", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") .SetName("OverlappingDynamicPathsWithDuplicateKeyDocument"); + + // A Map path decodes to a JsonObject, which looks exactly like a subtree built for a + // deeper path. The map's entries are a value of `a` all the same, and the server sends + // them under a key of their own, so merging `a.b` into them would drop one of the two. + yield return new TestCaseData("JSON(a Map(String, Int64))", "{\"a\": {\"x\": 1}, \"a.b\": 7}", "a.b") + .SetName("OverlappingMapPathAndDeeperPathWithDisjointKeys"); + + // The same, where the map's own key is the one the deeper path would take. + yield return new TestCaseData("JSON(a Map(String, Int64))", "{\"a\": {\"b\": 1}, \"a.b\": 7}", "a.b") + .SetName("OverlappingMapPathAndDeeperPathWithTheSameKey"); + + // An array value cannot hold a subtree at all. + yield return new TestCaseData("JSON(a Array(Int64))", "{\"a\": [1, 2], \"a.b\": 7}", "a.b") + .SetName("OverlappingArrayPathAndDeeperPath"); } [Test] @@ -1300,6 +1314,29 @@ await connection.ExecuteStatementAsync( Assert.That(exception.Message, Does.Contain("JsonReadMode.String"), exception.Message); } + [Test] + [RequiredFeature(Feature.Json)] + public async Task Read_WithOverlappingEmptyObjectValue_ShouldKeepTheSubtree() + { + var targetTable = CreateTableName(); + + // Every key of the document is routed to a path of its own — `a.b` typed, `a.x` dynamic — + // so the map at `a` is empty in every row. + await connection.ExecuteStatementAsync( + $"CREATE OR REPLACE TABLE {targetTable} (data JSON(a Map(String, Int64), `a.b` Int64)) ENGINE = Memory;"); + await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{{\"a\": {{\"b\": 7, \"x\": 1}}}}')"); + + using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + ClassicAssert.IsTrue(reader.Read()); + + var result = (JsonObject)reader.GetValue(0); + + // An empty map holds no value, so it neither collides with the subtree nor erases it. + var a = (JsonObject)result["a"]; + Assert.That((long)a["b"], Is.EqualTo(7L)); + Assert.That((long)a["x"], Is.EqualTo(1L)); + } + [Test] [RequiredFeature(Feature.Json)] public async Task Read_WithOverlappingPathsHoldingValues_ShouldBeReadableAsString() diff --git a/ClickHouse.Driver/Types/JsonType.cs b/ClickHouse.Driver/Types/JsonType.cs index 7e8cb5f35..5de588332 100644 --- a/ClickHouse.Driver/Types/JsonType.cs +++ b/ClickHouse.Driver/Types/JsonType.cs @@ -64,6 +64,12 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) { JsonObject root = new(); + // A Map path, or a Dynamic one holding an object, decodes to a JsonObject which is that + // path's value — not a container built here to hold deeper paths. The two are the same + // type, so the values are tracked by reference. Allocated only where such a path exists, + // which a column of plain scalar paths never has. + HashSet objectValues = null; + var nfields = reader.Read7BitEncodedInt(); for (int i = 0; i < nfields; i++) { @@ -86,44 +92,55 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) foreach (var part in pathParts.SkipLast1(1)) { depth++; - switch (current[part]) + var occupant = current[part]; + + if (occupant is JsonObject subtree && !IsObjectValue(objectValues, subtree)) + { + // A container built earlier in this row for an overlapping deeper path. + current = subtree; + } + else if (occupant is null || (occupant is JsonObject empty && IsAllNull(empty, out _))) + { + // Nothing is there yet, or what is there holds no value — the null of an + // overlapping leaf path, or an empty object value. ClickHouse allows both `a` + // and `a.b` to be declared; the subtree replaces it so the deeper path stays + // readable. + var newCurrent = new JsonObject(); + current[part] = newCurrent; + current = newCurrent; + } + else { - case JsonObject existing: - current = existing; - break; - - case null: - // Either the parent is not there yet, or it holds the null of an overlapping - // leaf path (ClickHouse allows both `a` and `a.b` to be declared); the - // subtree replaces that null so the deeper path stays readable. - var newCurrent = new JsonObject(); - current[part] = newCurrent; - current = newCurrent; - break; - - default: - // The parent holds a value of its own, so this row needs both a value and a - // subtree under one key. Nothing can be dropped without losing data. - throw OverlappingPathsException(string.Join(".", pathParts.Take(depth)), name); + // The parent holds a value of its own, so this row needs both a value and a + // subtree under one key. Nothing can be dropped without losing data. + throw OverlappingPathsException(string.Join(".", pathParts.Take(depth)), name); } } var leaf = pathParts.Last(); - if (current[leaf] is JsonObject occupant) + if (current[leaf] is JsonObject occupied) { // A deeper overlapping path was read first and put its subtree here. - if (jsonNode is null) + if (jsonNode is null || (jsonNode is JsonObject incoming && IsAllNull(incoming, out _))) { - // The null of the leaf path must not erase that subtree. + // This path holds no value — a null, or an empty object value — so it must not + // erase the subtree. continue; } - if (!IsAllNull(occupant, out var occupiedPath)) + if (!IsAllNull(occupied, out var occupiedPath)) { throw OverlappingPathsException(name, $"{name}.{occupiedPath}"); } - // The subtree carries no value, so the leaf's value replaces it. + // The subtree carries no value either, so this path's value replaces it. + } + + if (jsonNode is JsonObject objectValue) + { + // Mark it as a value so that an overlapping deeper path read later throws instead + // of descending into it and merging the two. + (objectValues ??= new HashSet(ObjectReferenceEqualityComparer.Instance)).Add(objectValue); } current[leaf] = jsonNode; @@ -132,6 +149,13 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) return root; } + /// + /// Reports whether the object is a path's own value rather than a container built to hold + /// deeper paths. + /// + private static bool IsObjectValue(HashSet objectValues, JsonObject value) => + objectValues is not null && objectValues.Contains(value); + /// /// Reports whether every value the object holds, at any depth, is a JSON null. An empty object /// holds no value, so it counts as all-null. An array counts as a value. diff --git a/docs/overview.mdx b/docs/overview.mdx index 320c7b909..2e2b924ad 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -1505,16 +1505,20 @@ var settings = new ClickHouseClientSettings("Host=localhost") // "Host=localhost;JsonReadMode=String" ``` -##### Overlapping typed paths +##### Overlapping paths ClickHouse accepts a column which declares a path both as a value and as the parent of another path, for example `JSON(a Int64, a.b Int64)`. Both paths are present in every row, so the server renders the row with a duplicate key: `{"a":0,"a":{"b":7}}`. A `JsonObject` cannot hold two values -for one key, so `JsonReadMode.Binary` throws a `SerializationException` naming the two paths. - -This applies only where both paths hold a value in that row. Where the overlap is declared with -`Nullable` types, the row usually fills in one side only — `JSON(a Nullable(Int64), a.b -Nullable(Int64))` reads `{"a":5}` and `{"a":{"b":7}}` normally — so nothing throws. +for one key, so `JsonReadMode.Binary` throws a `SerializationException` naming the two paths. The +same holds where the value is a `Map`, as in `JSON(a Map(String, Int64))` read from a row which +also has a dynamic `a.b`. + +This applies only where both sides hold a value in that row. A side which holds nothing — a null, +an empty object, or a subtree whose values are all null — gives way to the side which has the data. +An overlap declared with `Nullable` types therefore fills in one side per row and reads without +error: `JSON(a Nullable(Int64), a.b Nullable(Int64))` gives `{"a":5}` and `{"a":{"b":7}}` as +expected. Read such a column with `JsonReadMode.String` to get the server's JSON text unchanged, duplicate key included. From 98b8be2841bbb657e3af65517631b7a84f24698f Mon Sep 17 00:00:00 2001 From: Polyglot AI <293096396+polyglotAI-bot@users.noreply.github.com> Date: Sun, 23 Aug 2026 18:57:11 +0000 Subject: [PATCH 4/5] test(json): keep the overlap cases readable on 25.8 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Three of the new documents only insert on 26.3 and later. A nested `{"a": {"b": 7}}` is parsed against the overlapping `a Int64` by 25.8 and rejected with INCORRECT_DATA. The flattened `{"a.b": 7}` key stores identically on every server in the matrix — both render `{"a":0,"a":{"b":7}}` — so the cases now use it, as TypedPathNullValueTestCases already does for the same reason. A duplicate-key document is rejected outright by 25.8 unless duplicated paths are skipped, which would drop one of the two values and remove the thing under test. Those two cases are gated at 26.3, the oldest server which keeps both. The empty-object case relied on 25.8 and 26.7 routing a nested document to the same paths, which they do not: 26.7 empties the map and 25.8 fills it. Leaving the map out of the document empties it on both. --- .../Types/JsonTypeTests.cs | 39 ++++++++++++------- 1 file changed, 24 insertions(+), 15 deletions(-) diff --git a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs index e6e751893..d2e63342f 100644 --- a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs +++ b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs @@ -1255,10 +1255,16 @@ await connection.ExecuteStatementAsync( public static IEnumerable OverlappingPathsHoldingValuesTestCases() { + // The documents below reach a path under `a` through a flattened key rather than a nested + // object, because 25.8 parses `{"b": 7}` against the overlapping `a Int64` and rejects the + // insert with INCORRECT_DATA. The two forms are indistinguishable once stored — both give + // `{"a":0,"a":{"b":7}}` — so the read path under test is the same either way. See + // TypedPathNullValueTestCases for the same limitation. + // A non-Nullable typed path is never absent — an absent path materializes as 0 — so `a` // always holds a value and always collides with the `a.b` subtree, whichever of the two // the document fills in. - yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": {\"b\": 7}}", "a.b") + yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a.b\": 7}", "a.b") .SetName("OverlappingNonNullableTypedPathsWithValueInSubtree"); yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": 5}", "a.b") @@ -1266,17 +1272,22 @@ public static IEnumerable OverlappingPathsHoldingValuesTestCases() // The value is deeper than the path it collides with, so the error has to walk down to it // rather than name the first key under `a`. - yield return new TestCaseData("JSON(a Int64, a.b.c Int64)", "{\"a\": {\"b\": {\"c\": 7}}}", "a.b.c") + yield return new TestCaseData("JSON(a Int64, a.b.c Int64)", "{\"a.b.c\": 7}", "a.b.c") .SetName("OverlappingNonNullableTypedPathsWithValueInDeeperSubtree"); // Nullable typed paths only collide where the document itself carries both, which needs a - // duplicate key. ClickHouse accepts the document and keeps both values. - yield return new TestCaseData("JSON(a Nullable(Int64), a.b Nullable(Int64))", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") - .SetName("OverlappingNullableTypedPathsWithDuplicateKeyDocument"); + // duplicate key. 26.3 and later accept the document and keep both values; 25.8 rejects it + // with INCORRECT_DATA unless duplicated paths are skipped, which would drop one of the two + // values and so remove the very thing under test. + if (TestUtilities.ServerVersion >= Version.Parse("26.3")) + { + yield return new TestCaseData("JSON(a Nullable(Int64), a.b Nullable(Int64))", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") + .SetName("OverlappingNullableTypedPathsWithDuplicateKeyDocument"); - // The same collision with no hints at all: `a` and `a.b` are both dynamic paths. - yield return new TestCaseData("JSON", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") - .SetName("OverlappingDynamicPathsWithDuplicateKeyDocument"); + // The same collision with no hints at all: `a` and `a.b` are both dynamic paths. + yield return new TestCaseData("JSON", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") + .SetName("OverlappingDynamicPathsWithDuplicateKeyDocument"); + } // A Map path decodes to a JsonObject, which looks exactly like a subtree built for a // deeper path. The map's entries are a value of `a` all the same, and the server sends @@ -1320,11 +1331,11 @@ public async Task Read_WithOverlappingEmptyObjectValue_ShouldKeepTheSubtree() { var targetTable = CreateTableName(); - // Every key of the document is routed to a path of its own — `a.b` typed, `a.x` dynamic — - // so the map at `a` is empty in every row. + // The row fills in the typed `a.b` and leaves the map at `a` empty, so the server renders + // it as `{"a":{},"a":{"b":7}}` — an overlap where only one side holds a value. await connection.ExecuteStatementAsync( $"CREATE OR REPLACE TABLE {targetTable} (data JSON(a Map(String, Int64), `a.b` Int64)) ENGINE = Memory;"); - await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{{\"a\": {{\"b\": 7, \"x\": 1}}}}')"); + await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{{\"a.b\": 7}}')"); using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); ClassicAssert.IsTrue(reader.Read()); @@ -1332,9 +1343,7 @@ await connection.ExecuteStatementAsync( var result = (JsonObject)reader.GetValue(0); // An empty map holds no value, so it neither collides with the subtree nor erases it. - var a = (JsonObject)result["a"]; - Assert.That((long)a["b"], Is.EqualTo(7L)); - Assert.That((long)a["x"], Is.EqualTo(1L)); + Assert.That(result.ToJsonString(), Is.EqualTo("{\"a\":{\"b\":7}}")); } [Test] @@ -1345,7 +1354,7 @@ public async Task Read_WithOverlappingPathsHoldingValues_ShouldBeReadableAsStrin await connection.ExecuteStatementAsync( $"CREATE OR REPLACE TABLE {targetTable} (data JSON(a Int64, a.b Int64)) ENGINE = Memory;"); - await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{{\"a\": {{\"b\": 7}}}}')"); + await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{{\"a.b\": 7}}')"); using var stringClient = TestUtilities.GetTestClickHouseClient(jsonReadMode: JsonReadMode.String); From 6316a6af73d94cbcb02b8869fe24b67ff236ca06 Mon Sep 17 00:00:00 2001 From: Alex Soffronow-Pagonidis Date: Mon, 24 Aug 2026 11:42:14 +0200 Subject: [PATCH 5/5] feat(json): add AllowDuplicateJsonKeys to read an overlap lossily An overlap where both paths hold a value has no faithful JsonObject representation, so reading such a row throws. AllowDuplicateJsonKeys keeps the column readable instead: the driver keeps whichever of the two values the row carries last and drops the other. A path whose parent holds a scalar or an array still throws, because a subtree cannot be placed under either. Co-Authored-By: Claude --- CHANGELOG.md | 2 +- .../Types/JsonTypeTests.cs | 75 +++++++++++++------ .../ADO/ClickHouseClientSettings.cs | 16 ++++ .../ADO/ClickHouseConnectionStringBuilder.cs | 7 ++ ClickHouse.Driver/ADO/ClickHouseDefaults.cs | 5 ++ ClickHouse.Driver/ClickHouseClient.cs | 2 +- .../PublicAPI/PublicAPI.Unshipped.txt | 5 ++ ClickHouse.Driver/TypeSettings.cs | 4 +- ClickHouse.Driver/Types/JsonType.cs | 16 ++-- docs/overview.mdx | 16 ++++ 10 files changed, 118 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index e72d50e7e..2a8130cfa 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ Unreleased --- **Bug Fixes:** -* **Breaking Change**: Fixed a `JSON` column with overlapping paths silently losing data. ClickHouse accepts a column which declares a path both as a value and as the parent of another path — for example `JSON(a Int64, a.b Int64)` — and sends both paths in every row, so the row renders as a document with a duplicate key (`{"a":0,"a":{"b":7}}`). A `JsonObject` cannot hold two values for one key, so the driver silently kept whichever path the server sent last: reading `{"a":{"b":7}}` back out of that column returned `{"a":0}` and the `7` was gone. The same loss hit a `Map` path overlapped by a deeper path (`JSON(a Map(String, Int64))` holding `{"a":{"b":1},"a.b":7}` returned `{"a":{"b":7}}`, dropping the map's own entry), except that there the two values were merged into one object rather than one replacing the other. A row where both sides of such an overlap hold a value now throws a `SerializationException` naming the two paths. With non-`Nullable` paths the driver cannot tell a stored `0` from an absent path, so no rule can recover the real value — the choice is between a wrong answer and a clear failure. A side which holds nothing is not a collision: a null, an empty object, and an all-null subtree all give way to the side which has the data, so `Nullable` overlaps continue to read as before. Use `JsonReadMode.String` to read such a column as the server's JSON text, duplicate key included. +* **Breaking Change**: Fixed a `JSON` column with overlapping paths silently losing data. ClickHouse accepts a column which declares a path both as a value and as the parent of another path — for example `JSON(a Int64, a.b Int64)` — and sends both paths in every row, so the row renders as a document with a duplicate key (`{"a":0,"a":{"b":7}}`). A `JsonObject` cannot hold two values for one key, so the driver silently kept whichever path the server sent last: reading `{"a":{"b":7}}` back out of that column returned `{"a":0}` and the `7` was gone. The same loss hit a `Map` path overlapped by a deeper path (`JSON(a Map(String, Int64))` holding `{"a":{"b":1},"a.b":7}` returned `{"a":{"b":7}}`, dropping the map's own entry), except that there the two values were merged into one object rather than one replacing the other. A row where both sides of such an overlap hold a value now throws a `SerializationException` naming the two paths. With non-`Nullable` paths the driver cannot tell a stored `0` from an absent path, so no rule can recover the real value — the choice is between a wrong answer and a clear failure. A side which holds nothing is not a collision: a null, an empty object, and an all-null subtree all give way to the side which has the data, so `Nullable` overlaps continue to read as before. Use `JsonReadMode.String` to read such a column as the server's JSON text, duplicate key included. To restore the previous behavior and keep reading the column as a `JsonObject`, set the new `AllowDuplicateJsonKeys` setting (connection string key `AllowDuplicateJsonKeys=true`, or `ClickHouseClientSettings.AllowDuplicateJsonKeys`); the driver then keeps whichever value the row carries last, with the data loss described above. An overlap whose parent holds a scalar or an array still throws — that shape raised `InvalidCastException` before and is now a `SerializationException`. v1.4.0 --- diff --git a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs index d2e63342f..42ad0ea26 100644 --- a/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs +++ b/ClickHouse.Driver.Tests/Types/JsonTypeTests.cs @@ -1253,7 +1253,12 @@ await connection.ExecuteStatementAsync( Assert.That(result.ToJsonString(), Is.EqualTo("{}")); } - public static IEnumerable OverlappingPathsHoldingValuesTestCases() + /// + /// Document read when AllowDuplicateJsonKeys is set, which keeps whichever of the two + /// values the row carries last; null where the overlap stays unreadable because the + /// parent holds a scalar or an array, which no subtree can be placed under. + /// + private static IEnumerable<(string Name, string ColumnType, string JsonData, string NestedPath, string WhenAllowed)> OverlappingPathShapes() { // The documents below reach a path under `a` through a flattened key rather than a nested // object, because 25.8 parses `{"b": 7}` against the overlapping `a Int64` and rejects the @@ -1264,16 +1269,16 @@ public static IEnumerable OverlappingPathsHoldingValuesTestCases() // A non-Nullable typed path is never absent — an absent path materializes as 0 — so `a` // always holds a value and always collides with the `a.b` subtree, whichever of the two // the document fills in. - yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a.b\": 7}", "a.b") - .SetName("OverlappingNonNullableTypedPathsWithValueInSubtree"); + yield return ("OverlappingNonNullableTypedPathsWithValueInSubtree", + "JSON(a Int64, a.b Int64)", "{\"a.b\": 7}", "a.b", "{\"a\":0}"); - yield return new TestCaseData("JSON(a Int64, a.b Int64)", "{\"a\": 5}", "a.b") - .SetName("OverlappingNonNullableTypedPathsWithValueInLeaf"); + yield return ("OverlappingNonNullableTypedPathsWithValueInLeaf", + "JSON(a Int64, a.b Int64)", "{\"a\": 5}", "a.b", "{\"a\":5}"); // The value is deeper than the path it collides with, so the error has to walk down to it // rather than name the first key under `a`. - yield return new TestCaseData("JSON(a Int64, a.b.c Int64)", "{\"a.b.c\": 7}", "a.b.c") - .SetName("OverlappingNonNullableTypedPathsWithValueInDeeperSubtree"); + yield return ("OverlappingNonNullableTypedPathsWithValueInDeeperSubtree", + "JSON(a Int64, a.b.c Int64)", "{\"a.b.c\": 7}", "a.b.c", "{\"a\":0}"); // Nullable typed paths only collide where the document itself carries both, which needs a // duplicate key. 26.3 and later accept the document and keep both values; 25.8 rejects it @@ -1281,33 +1286,48 @@ public static IEnumerable OverlappingPathsHoldingValuesTestCases() // values and so remove the very thing under test. if (TestUtilities.ServerVersion >= Version.Parse("26.3")) { - yield return new TestCaseData("JSON(a Nullable(Int64), a.b Nullable(Int64))", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") - .SetName("OverlappingNullableTypedPathsWithDuplicateKeyDocument"); - - // The same collision with no hints at all: `a` and `a.b` are both dynamic paths. - yield return new TestCaseData("JSON", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b") - .SetName("OverlappingDynamicPathsWithDuplicateKeyDocument"); + yield return ("OverlappingNullableTypedPathsWithDuplicateKeyDocument", + "JSON(a Nullable(Int64), a.b Nullable(Int64))", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b", "{\"a\":5}"); + + // The same collision with no hints at all: `a` and `a.b` are both dynamic paths. The + // scalar `a` is sent first here, so the subtree has nowhere to go even when duplicate + // keys are allowed. + yield return ("OverlappingDynamicPathsWithDuplicateKeyDocument", + "JSON", "{\"a\": 5, \"a\": {\"b\": 7}}", "a.b", null); } // A Map path decodes to a JsonObject, which looks exactly like a subtree built for a // deeper path. The map's entries are a value of `a` all the same, and the server sends // them under a key of their own, so merging `a.b` into them would drop one of the two. - yield return new TestCaseData("JSON(a Map(String, Int64))", "{\"a\": {\"x\": 1}, \"a.b\": 7}", "a.b") - .SetName("OverlappingMapPathAndDeeperPathWithDisjointKeys"); + yield return ("OverlappingMapPathAndDeeperPathWithDisjointKeys", + "JSON(a Map(String, Int64))", "{\"a\": {\"x\": 1}, \"a.b\": 7}", "a.b", "{\"a\":{\"x\":1,\"b\":7}}"); // The same, where the map's own key is the one the deeper path would take. - yield return new TestCaseData("JSON(a Map(String, Int64))", "{\"a\": {\"b\": 1}, \"a.b\": 7}", "a.b") - .SetName("OverlappingMapPathAndDeeperPathWithTheSameKey"); + yield return ("OverlappingMapPathAndDeeperPathWithTheSameKey", + "JSON(a Map(String, Int64))", "{\"a\": {\"b\": 1}, \"a.b\": 7}", "a.b", "{\"a\":{\"b\":7}}"); // An array value cannot hold a subtree at all. - yield return new TestCaseData("JSON(a Array(Int64))", "{\"a\": [1, 2], \"a.b\": 7}", "a.b") - .SetName("OverlappingArrayPathAndDeeperPath"); + yield return ("OverlappingArrayPathAndDeeperPath", + "JSON(a Array(Int64))", "{\"a\": [1, 2], \"a.b\": 7}", "a.b", null); + } + + public static IEnumerable OverlappingPathsHoldingValuesTestCases() + { + foreach (var (name, columnType, jsonData, nestedPath, whenAllowed) in OverlappingPathShapes()) + { + yield return new TestCaseData(columnType, jsonData, false, nestedPath, null) + .SetName(name); + + yield return new TestCaseData(columnType, jsonData, true, nestedPath, whenAllowed) + .SetName($"{name}_DuplicateKeysAllowed"); + } } [Test] [RequiredFeature(Feature.Json)] [TestCaseSource(nameof(OverlappingPathsHoldingValuesTestCases))] - public async Task Read_WithOverlappingPathsHoldingValues_ShouldThrow(string columnType, string jsonData, string expectedNestedPath) + public async Task Read_WithOverlappingPathsHoldingValues_ShouldThrowUnlessDuplicateKeysAreAllowed( + string columnType, string jsonData, bool allowDuplicateJsonKeys, string expectedNestedPath, string expectedJson) { var targetTable = CreateTableName(); @@ -1315,7 +1335,20 @@ await connection.ExecuteStatementAsync( $"CREATE OR REPLACE TABLE {targetTable} (data {columnType}) ENGINE = Memory;"); await connection.ExecuteStatementAsync($"INSERT INTO {targetTable} VALUES ('{jsonData}')"); - using var reader = await connection.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + var settings = new ClickHouseClientSettings(TestUtilities.GetTestClickHouseClientSettings()) + { + AllowDuplicateJsonKeys = allowDuplicateJsonKeys, + }; + using var overlapClient = new ClickHouseClient(settings); + + using var reader = await overlapClient.ExecuteReaderAsync($"SELECT data FROM {targetTable}"); + + if (expectedJson is not null) + { + ClassicAssert.IsTrue(reader.Read()); + Assert.That(((JsonObject)reader.GetValue(0)).ToJsonString(), Is.EqualTo(expectedJson)); + return; + } // The column is decoded by Read, so the throw lands there rather than on GetValue. var exception = Assert.Throws(() => reader.Read()); diff --git a/ClickHouse.Driver/ADO/ClickHouseClientSettings.cs b/ClickHouse.Driver/ADO/ClickHouseClientSettings.cs index a4b537b14..d78e01601 100644 --- a/ClickHouse.Driver/ADO/ClickHouseClientSettings.cs +++ b/ClickHouse.Driver/ADO/ClickHouseClientSettings.cs @@ -92,6 +92,7 @@ public ClickHouseClientSettings(ClickHouseClientSettings other) JsonWriteMode = other.JsonWriteMode; MapReadMode = other.MapReadMode; + AllowDuplicateJsonKeys = other.AllowDuplicateJsonKeys; // Copy parameter type resolver ParameterTypeResolver = other.ParameterTypeResolver; @@ -365,6 +366,17 @@ public string SessionId /// public MapReadMode MapReadMode { get; init; } = ClickHouseDefaults.MapReadMode; + /// + /// Gets or sets how to read a JSON row where one path holds a value and is also the parent of + /// another path which holds a value. The server sends such a row with a duplicate key, which a + /// JsonObject cannot hold. + /// false (default): throw, because either value can only be kept by dropping the other + /// true: keep whichever value the row carries last and drop the other + /// Applies to JsonReadMode.Binary and JsonReadMode.None. JsonReadMode.String returns the + /// server's JSON text unchanged and is unaffected. + /// + public bool AllowDuplicateJsonKeys { get; init; } = ClickHouseDefaults.AllowDuplicateJsonKeys; + /// /// Gets or sets a custom resolver for mapping .NET types to ClickHouse types /// during @-style parameter substitution. When set, this resolver is consulted @@ -461,6 +473,7 @@ internal static ClickHouseClientSettings FromConnectionStringBuilder(ClickHouseC JsonReadMode = builder.JsonReadMode, JsonWriteMode = builder.JsonWriteMode, MapReadMode = builder.MapReadMode, + AllowDuplicateJsonKeys = builder.AllowDuplicateJsonKeys, AcceptEncoding = builder.AcceptEncoding, }; @@ -511,6 +524,7 @@ public bool Equals(ClickHouseClientSettings other) JsonReadMode == other.JsonReadMode && JsonWriteMode == other.JsonWriteMode && MapReadMode == other.MapReadMode && + AllowDuplicateJsonKeys == other.AllowDuplicateJsonKeys && ParameterTypeResolver == other.ParameterTypeResolver && ParameterFormatter == other.ParameterFormatter && ReadValueConverter == other.ReadValueConverter && @@ -555,6 +569,7 @@ public override int GetHashCode() hash.Add(JsonReadMode); hash.Add(JsonWriteMode); hash.Add(MapReadMode); + hash.Add(AllowDuplicateJsonKeys); hash.Add(ParameterTypeResolver); hash.Add(ParameterFormatter); hash.Add(ReadValueConverter); @@ -608,6 +623,7 @@ public override string ToString() $"ReadBufferSize={ReadBufferSize};" + $"JsonReadMode={JsonReadMode};JsonWriteMode={JsonWriteMode};" + $"MapReadMode={MapReadMode};" + + $"AllowDuplicateJsonKeys={AllowDuplicateJsonKeys};" + $"UseFormDataParameters={UseFormDataParameters}"; if (!string.IsNullOrEmpty(AcceptEncoding)) diff --git a/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs b/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs index f7f91eff9..e6675d356 100644 --- a/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs +++ b/ClickHouse.Driver/ADO/ClickHouseConnectionStringBuilder.cs @@ -97,6 +97,12 @@ public bool ReadStringsAsByteArrays set => this["ReadStringsAsByteArrays"] = value; } + public bool AllowDuplicateJsonKeys + { + get => GetBooleanOrDefault("AllowDuplicateJsonKeys", ClickHouseDefaults.AllowDuplicateJsonKeys); + set => this["AllowDuplicateJsonKeys"] = value; + } + /// /// Gets or sets the size, in bytes, of the buffer used when reading HTTP query responses. /// Default: 8 KiB @@ -260,6 +266,7 @@ public static ClickHouseConnectionStringBuilder FromSettings(ClickHouseClientSet JsonReadMode = settings.JsonReadMode, JsonWriteMode = settings.JsonWriteMode, MapReadMode = settings.MapReadMode, + AllowDuplicateJsonKeys = settings.AllowDuplicateJsonKeys, AcceptEncoding = settings.AcceptEncoding, }; diff --git a/ClickHouse.Driver/ADO/ClickHouseDefaults.cs b/ClickHouse.Driver/ADO/ClickHouseDefaults.cs index bf053b4ed..6c1039fa1 100644 --- a/ClickHouse.Driver/ADO/ClickHouseDefaults.cs +++ b/ClickHouse.Driver/ADO/ClickHouseDefaults.cs @@ -72,6 +72,11 @@ public static class ClickHouseDefaults /// public const MapReadMode MapReadMode = ClickHouse.Driver.MapReadMode.Dictionary; + /// + /// Default handling of a JSON row where two overlapping paths both hold a value (false = throw) + /// + public const bool AllowDuplicateJsonKeys = false; + /// /// Default setting for server certificate validation (false = validate certificates) /// diff --git a/ClickHouse.Driver/ClickHouseClient.cs b/ClickHouse.Driver/ClickHouseClient.cs index 180f9fa4e..7c67d2665 100644 --- a/ClickHouse.Driver/ClickHouseClient.cs +++ b/ClickHouse.Driver/ClickHouseClient.cs @@ -171,7 +171,7 @@ internal string RedactedConnectionString /// /// Gets the type settings for serialization. /// - internal TypeSettings TypeSettings => new(Settings.UseCustomDecimals, Settings.ReadStringsAsByteArrays, jsonTypeRegistry, Settings.JsonReadMode, Settings.JsonWriteMode, Settings.MapReadMode); + internal TypeSettings TypeSettings => new(Settings.UseCustomDecimals, Settings.ReadStringsAsByteArrays, jsonTypeRegistry, Settings.JsonReadMode, Settings.JsonWriteMode, Settings.MapReadMode, Settings.AllowDuplicateJsonKeys); /// /// Gets the per-client POCO type registry shared by binary insert and read materialization. diff --git a/ClickHouse.Driver/PublicAPI/PublicAPI.Unshipped.txt b/ClickHouse.Driver/PublicAPI/PublicAPI.Unshipped.txt index bedf44477..0f7d54dd8 100644 --- a/ClickHouse.Driver/PublicAPI/PublicAPI.Unshipped.txt +++ b/ClickHouse.Driver/PublicAPI/PublicAPI.Unshipped.txt @@ -258,3 +258,8 @@ ClickHouse.Driver.ADO.ClickHouseClientSettings.MapReadMode.init -> void ClickHouse.Driver.ADO.ClickHouseConnectionStringBuilder.MapReadMode.get -> ClickHouse.Driver.MapReadMode ClickHouse.Driver.ADO.ClickHouseConnectionStringBuilder.MapReadMode.set -> void const ClickHouse.Driver.ADO.ClickHouseDefaults.MapReadMode = ClickHouse.Driver.MapReadMode.Dictionary -> ClickHouse.Driver.MapReadMode +ClickHouse.Driver.ADO.ClickHouseClientSettings.AllowDuplicateJsonKeys.get -> bool +ClickHouse.Driver.ADO.ClickHouseClientSettings.AllowDuplicateJsonKeys.init -> void +ClickHouse.Driver.ADO.ClickHouseConnectionStringBuilder.AllowDuplicateJsonKeys.get -> bool +ClickHouse.Driver.ADO.ClickHouseConnectionStringBuilder.AllowDuplicateJsonKeys.set -> void +const ClickHouse.Driver.ADO.ClickHouseDefaults.AllowDuplicateJsonKeys = false -> bool diff --git a/ClickHouse.Driver/TypeSettings.cs b/ClickHouse.Driver/TypeSettings.cs index 397256f85..e2d8914a6 100644 --- a/ClickHouse.Driver/TypeSettings.cs +++ b/ClickHouse.Driver/TypeSettings.cs @@ -2,7 +2,7 @@ namespace ClickHouse.Driver; -internal record struct TypeSettings(bool useBigDecimal, bool readStringsAsByteArrays, JsonTypeRegistry jsonTypeRegistry, JsonReadMode jsonReadMode, JsonWriteMode jsonWriteMode, MapReadMode mapReadMode = MapReadMode.Dictionary) +internal record struct TypeSettings(bool useBigDecimal, bool readStringsAsByteArrays, JsonTypeRegistry jsonTypeRegistry, JsonReadMode jsonReadMode, JsonWriteMode jsonWriteMode, MapReadMode mapReadMode = MapReadMode.Dictionary, bool allowDuplicateJsonKeys = false) { - public static TypeSettings Default => new TypeSettings(true, false, null, JsonReadMode.Binary, JsonWriteMode.String, MapReadMode.Dictionary); + public static TypeSettings Default => new TypeSettings(true, false, null, JsonReadMode.Binary, JsonWriteMode.String, MapReadMode.Dictionary, false); } diff --git a/ClickHouse.Driver/Types/JsonType.cs b/ClickHouse.Driver/Types/JsonType.cs index 5de588332..794f92013 100644 --- a/ClickHouse.Driver/Types/JsonType.cs +++ b/ClickHouse.Driver/Types/JsonType.cs @@ -70,6 +70,10 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) // which a column of plain scalar paths never has. HashSet objectValues = null; + // When set, an overlap keeps whichever value the row carries last and drops the other, + // rather than reporting that neither can be dropped without losing data. + var allowDuplicateKeys = TypeSettings.allowDuplicateJsonKeys; + var nfields = reader.Read7BitEncodedInt(); for (int i = 0; i < nfields; i++) { @@ -94,9 +98,10 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) depth++; var occupant = current[part]; - if (occupant is JsonObject subtree && !IsObjectValue(objectValues, subtree)) + if (occupant is JsonObject subtree && (allowDuplicateKeys || !IsObjectValue(objectValues, subtree))) { - // A container built earlier in this row for an overlapping deeper path. + // A container built earlier in this row for an overlapping deeper path, or — + // when duplicate keys are allowed — an object value this path merges into. current = subtree; } else if (occupant is null || (occupant is JsonObject empty && IsAllNull(empty, out _))) @@ -128,7 +133,7 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) continue; } - if (!IsAllNull(occupied, out var occupiedPath)) + if (!allowDuplicateKeys && !IsAllNull(occupied, out var occupiedPath)) { throw OverlappingPathsException(name, $"{name}.{occupiedPath}"); } @@ -136,10 +141,11 @@ private object ReadAsJsonObject(ExtendedBinaryReader reader) // The subtree carries no value either, so this path's value replaces it. } - if (jsonNode is JsonObject objectValue) + if (!allowDuplicateKeys && jsonNode is JsonObject objectValue) { // Mark it as a value so that an overlapping deeper path read later throws instead - // of descending into it and merging the two. + // of descending into it and merging the two. Only the throw consults this set, so + // it stays unallocated when duplicate keys are allowed. (objectValues ??= new HashSet(ObjectReferenceEqualityComparer.Instance)).Add(objectValue); } diff --git a/docs/overview.mdx b/docs/overview.mdx index 2e2b924ad..68afc63ee 100644 --- a/docs/overview.mdx +++ b/docs/overview.mdx @@ -110,6 +110,7 @@ Below is a full list of all the settings, their default values, and their effect | JsonReadMode | `JsonReadMode` | `Binary` | `JsonReadMode` | How JSON data is returned: `Binary` (returns `JsonObject`) or `String` (returns raw JSON string) | | JsonWriteMode | `JsonWriteMode` | `String` | `JsonWriteMode` | How JSON data is sent: `String` (serializes via `JsonSerializer`, accepts all inputs) or `Binary` (registered POCOs only with type hints) | | MapReadMode | `MapReadMode` | `Dictionary` | `MapReadMode` | How `Map(K, V)` data is returned: `Dictionary` (returns `Dictionary`; a repeated key keeps only its last value) or `KeyValuePairs` (returns `List>`, keeping every pair). See [Map type](#type-map-reading-map) | +| AllowDuplicateJsonKeys | `bool` | `false` | `AllowDuplicateJsonKeys` | How to read a `JSON` row whose overlapping paths both hold a value. `false` throws, because keeping either value means dropping the other; `true` keeps whichever the row carries last. See [Overlapping paths](#type-map-reading-json) | ### Session management {#session-management} @@ -1523,6 +1524,21 @@ expected. Read such a column with `JsonReadMode.String` to get the server's JSON text unchanged, duplicate key included. +Set `AllowDuplicateJsonKeys` to keep reading the column as a `JsonObject` instead of throwing. The +driver then keeps whichever of the two values the row carries last and drops the other, so the +result is lossy: `JSON(a Int64, a.b Int64)` holding `{"a.b":7}` reads as `{"a":0}`. A path whose +parent holds a scalar or an array still throws, because a subtree cannot be placed under either. + +```csharp +var settings = new ClickHouseClientSettings("Host=localhost") +{ + AllowDuplicateJsonKeys = true +}; + +// Or via connection string +// "Host=localhost;AllowDuplicateJsonKeys=true" +``` + --- #### Map type {#type-map-reading-map}