diff --git a/ClickHouse.Driver.Tests/SQL/FormDataLargeQueryTests.cs b/ClickHouse.Driver.Tests/SQL/FormDataLargeQueryTests.cs new file mode 100644 index 000000000..f2d3a5764 --- /dev/null +++ b/ClickHouse.Driver.Tests/SQL/FormDataLargeQueryTests.cs @@ -0,0 +1,114 @@ +using System.Collections.Generic; +using System.Linq; +using System.Net; +using System.Net.Http; +using System.Text; +using System.Threading; +using System.Threading.Tasks; +using ClickHouse.Driver.ADO; +using ClickHouse.Driver.ADO.Parameters; +using ClickHouse.Driver.Tests.Attributes; +using ClickHouse.Driver.Utility; +using NUnit.Framework; + +namespace ClickHouse.Driver.Tests.SQL; + +/// +/// SQL text larger than the server's default form-field limit (http_max_field_value_size, 128 KiB) +/// sent with UseFormDataParameters=true. +/// +public class FormDataLargeQueryTests : AbstractConnectionTestFixture +{ + private const int FormFieldLimit = 128 * 1024; + + private readonly ClickHouseClient formDataClient = TestUtilities.GetTestClickHouseClient(useFormDataParameters: true); + + // About 200 KB of SQL: over the form-field limit, under the default max_query_size of 256 KiB. + // The padding moves the field boundary to each byte position of a 4-byte UTF-8 character. + [RequiredFeature(Feature.ParamsInMultipartFormData)] + [TestCase("x", 0)] + [TestCase("😀", 0)] + [TestCase("😀", 1)] + [TestCase("😀", 2)] + [TestCase("😀", 3)] + public async Task ExecuteReaderAsync_FormDataWithSqlOverFieldLimit_SendsValidFieldsAndReturnsLiteralUnchanged(string unit, int padding) + { + var payload = new string('x', padding) + string.Concat(Enumerable.Repeat(unit, 200_000 / Encoding.UTF8.GetByteCount(unit))); + var sql = $"SELECT {{p:Int32}}, '{payload}'"; + var parameters = new ClickHouseParameterCollection(); + parameters.AddParameter("p", 42); + + var capture = new QueryFieldCaptureHandler(); + using var httpClient = new HttpClient(capture); + using var client = new ClickHouseClient(new ClickHouseClientSettings(TestUtilities.GetTestClickHouseClientSettings(useFormDataParameters: true)) + { + HttpClient = httpClient, + }); + + using var reader = await client.ExecuteReaderAsync(sql, parameters); + var row = reader.GetEnsureSingleRow(); + + var strictUtf8 = new UTF8Encoding(encoderShouldEmitUTF8Identifier: false, throwOnInvalidBytes: true); + Assert.Multiple(() => + { + Assert.That(row[0], Is.EqualTo(42)); + Assert.That(row[1], Is.EqualTo(payload)); + Assert.That(capture.QueryFields.Select(field => field.Length), Has.All.LessThanOrEqualTo(FormFieldLimit)); + Assert.That(string.Concat(capture.QueryFields.Select(strictUtf8.GetString)), Is.EqualTo(sql)); + }); + } + + [Test] + public async Task ExecuteNonQueryAsync_FormDataInsertWithInlineRowsOverFieldLimit_InsertsAllRows() + { + const int rows = 20_000; + var table = CreateTableName(); + await client.ExecuteNonQueryAsync($"CREATE TABLE {table} (id UInt32, s String) ENGINE = Memory"); + + // About 760 KiB of inline rows, which spans several form fields. + var sql = new StringBuilder($"INSERT INTO {table} FORMAT JSONEachRow\n"); + for (var i = 0; i < rows; i++) + { + sql.Append("{\"id\":").Append(i).Append(",\"s\":\"row-").Append(i).Append("-0123456789\"}\n"); + } + + await formDataClient.ExecuteNonQueryAsync(sql.ToString()); + + using var reader = await client.ExecuteReaderAsync( + $"SELECT count(), sum(id), countIf(s = concat('row-', toString(id), '-0123456789')) FROM {table}"); + var row = reader.GetEnsureSingleRow(); + + Assert.That(row[0], Is.EqualTo(rows)); + Assert.That(row[1], Is.EqualTo((ulong)rows * (rows - 1) / 2)); + Assert.That(row[2], Is.EqualTo(rows)); + } + + [OneTimeTearDown] + public void DisposeFormDataClient() => formDataClient.Dispose(); + + /// + /// Records the raw bytes of each query form field, then forwards the request to the server. + /// + private sealed class QueryFieldCaptureHandler : DelegatingHandler + { + public QueryFieldCaptureHandler() + : base(new HttpClientHandler { AutomaticDecompression = DecompressionMethods.GZip | DecompressionMethods.Deflate }) + { + } + + public List QueryFields { get; } = new(); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request.Content is MultipartFormDataContent form) + { + foreach (var field in form.Where(part => part.Headers.ContentDisposition?.Name?.Trim('"') == "query")) + { + QueryFields.Add(await field.ReadAsByteArrayAsync(cancellationToken)); + } + } + + return await base.SendAsync(request, cancellationToken); + } + } +} diff --git a/ClickHouse.Driver/ClickHouseClient.cs b/ClickHouse.Driver/ClickHouseClient.cs index e3591eecf..b7578a550 100644 --- a/ClickHouse.Driver/ClickHouseClient.cs +++ b/ClickHouse.Driver/ClickHouseClient.cs @@ -54,6 +54,9 @@ public sealed class ClickHouseClient : IClickHouseClient private const int DefaultMaxSmallPoolFreeBytes = 128 * 1024 * 1024; // 128 MB private const int DefaultMaxLargePoolFreeBytes = 512 * 1024 * 1024; // 512 MB + // Default of the server setting http_max_field_value_size, its limit on the size of one form field value. + private const int MaxFormFieldValueSize = 128 * 1024; // 128 KB + private readonly List disposables = new(); private readonly ConcurrentDictionary> loggerCache = new(); private readonly SchemaResolver schemaResolver; @@ -408,9 +411,7 @@ private HttpRequestMessage BuildHttpRequestMessageWithFormData(string sqlQuery, } } - content.Add( - content: new StringContent(sqlQuery), - name: "query"); + AddQueryFormFields(content, sqlQuery); var uri = uriBuilder.ToString(); @@ -423,6 +424,31 @@ private HttpRequestMessage BuildHttpRequestMessageWithFormData(string sqlQuery, return postMessage; } + /// + /// Adds the SQL text as one or more query form fields of at most + /// bytes each. The server concatenates all query fields in order, so the SQL text is not limited + /// to the size of one field. + /// + private static void AddQueryFormFields(MultipartFormDataContent content, string sqlQuery) + { + var bytes = Encoding.UTF8.GetBytes(sqlQuery); + var offset = 0; + do + { + var end = Math.Min(offset + MaxFormFieldValueSize, bytes.Length); + + // Do not split a multi-byte character: UTF-8 continuation bytes have the form 10xxxxxx. + while (end < bytes.Length && (bytes[end] & 0xC0) == 0x80) + end--; + + var field = new ByteArrayContent(bytes, offset, end - offset); + field.Headers.ContentType = new MediaTypeHeaderValue("text/plain") { CharSet = "utf-8" }; + content.Add(field, "query"); + offset = end; + } + while (offset < bytes.Length); + } + private static void LogQuerySuccess(Stopwatch stopwatch, string queryId, ILogger logger, QueryStats queryStats) { stopwatch.Stop(); diff --git a/changelog.d/700-form-data-large-sql.fixes.md b/changelog.d/700-form-data-large-sql.fixes.md new file mode 100644 index 000000000..d3358c9fe --- /dev/null +++ b/changelog.d/700-form-data-large-sql.fixes.md @@ -0,0 +1 @@ +* Fixed queries with SQL text larger than 128 KiB failing with `HTML Form Exception: Field value too long` when `UseFormDataParameters` is enabled (issue #700). diff --git a/docs/http.mdx b/docs/http.mdx index 1e1462472..29b2100a9 100644 --- a/docs/http.mdx +++ b/docs/http.mdx @@ -102,7 +102,7 @@ Below is a full list of all the settings, their default values, and their effect | AcceptEncoding | `string` | `null` | `AcceptEncoding` | `Accept-Encoding` sent with every request, replacing the codecs the driver advertises by default (`zstd, lz4, gzip, deflate`). Whatever the server answers with is decoded transparently. See [Response decompression](#response-decompression) | | UseCustomDecimals | `bool` | `true` | `UseCustomDecimals` | Use `ClickHouseDecimal` for arbitrary precision; if false, uses .NET `decimal` (128-bit limit) | | ReadStringsAsByteArrays | `bool` | `false` | `ReadStringsAsByteArrays` | Read `String` and `FixedString` columns as `byte[]` instead of `string`; useful for binary data | -| UseFormDataParameters | `bool` | `false` | `UseFormDataParameters` | Send parameters as form data instead of URL query string | +| UseFormDataParameters | `bool` | `false` | `UseFormDataParameters` | Send parameters as form data instead of URL query string. In both modes the server limits each parameter value to `http_max_field_value_size` (128 KiB by default; in the URL it applies to the encoded value), and a query setting cannot raise it | | ReadBufferSize | `int` | `65536` (64 KiB) | `ReadBufferSize` | Size in bytes of the buffer that reads HTTP query responses. The driver rents the buffer from a shared pool and returns it when it disposes the reader, so it is not an allocation for each query. Increase it to reduce buffer refills on large result sets. The driver holds one buffer for each concurrent reader, so memory use increases with the buffer size and the number of concurrent readers. See [Buffers](#perf-buffers). | | ParameterTypeResolver | `IParameterTypeResolver` | `null` | — | Custom resolver for `@`-style parameter type mapping; see [Custom parameter type mapping](#parameter-type-mapping) | | ParameterFormatter | `IParameterFormatter` | `null` | — | Custom formatter for parameter value serialization; see [Custom parameter value formatting](#parameter-value-formatting) |