From e77fc7fa565fe5e4f004a4244e22abb0ee17d326 Mon Sep 17 00:00:00 2001 From: Cosmin Staicu Date: Fri, 13 Nov 2026 15:18:10 +0200 Subject: [PATCH] feat(messagepack): MessagePack value serializer package Adds the UiPath.Caching.MessagePack package, a value serializer that a policy picks with Serializer.Write = "messagepack". It is registered with AddMessagePackSerializer, which takes an optional name and a configure callback for MessagePackValueSerializerOptions. Values below CompressionThresholdBytes (default 1024) are written plain with format id 0x02. At or above it the value is tried with LZ4, stored as id 0x03 when the result is smaller and as plain 0x02 otherwise. A threshold of 0 tries every value. Reads accept both ids and reject a compressed value that claims more than 255 times its stored size or more than MaxDecompressedBytes (default 512 MiB), which is also applied as MessagePack's own decompression limit. A value larger than that cap is written plain so it stays readable, and an OutOfMemoryException wrapped by MessagePack is rethrown unwrapped. The default serializer options use a contractless resolver with MessagePackSecurity.UntrustedData, and supplied options that carry TrustedData are made untrusted. The serializer is internal and owns only ids 0x02 and 0x03, registered through an internal method that UiPath.Caching exposes to the package with InternalsVisibleTo. The public AddValueSerializer still accepts consumer ids only, so the package depends on the exact same UiPath.Caching version. Nothing changes until an app registers the package and a policy names it. Once values are written as MessagePack, every reader of those keys needs the package registered, readers first in a rolling deploy. Signed-off-by: Cosmin Staicu --- CHANGELOG.md | 1 + README.md | 1 + UiPath.Caching.slnx | 1 + .../FakeRedisConnector.cs | 3 +- .../MessagePackCompressionBenchmark.cs | 119 +++++ .../SerializerBenchmark.cs | 83 ++++ .../UiPath.Caching.Benchmarks.csproj | 1 + docs/how-to/extending.md | 38 ++ docs/index.md | 1 + docs/reference/settings.md | 12 + .../GlobalUsings.cs | 1 + .../MessagePackCachingBuilderExtensions.cs | 21 + .../MessagePackValueSerializer.cs | 196 ++++++++ .../MessagePackValueSerializerOptions.cs | 15 + .../PackageGeneratedResolver.cs | 7 + .../PooledBufferWriter.cs | 46 ++ .../PublicAPI.Shipped.txt | 1 + .../PublicAPI.Unshipped.txt | 12 + .../RawMessagePack.cs | 6 + .../RawMessagePackResolver.cs | 29 ++ .../ResilientMessagePackSecurity.cs | 46 ++ .../UiPath.Caching.MessagePack.csproj | 25 + src/UiPath.Caching/GlobalUsings.cs | 1 + .../MessagePackAtomicWriteTests.cs | 46 ++ .../MessagePackCompressionTests.cs | 457 ++++++++++++++++++ .../MessagePackIntegrationTests.cs | 212 ++++++++ .../MessagePackOutOfMemoryTests.cs | 126 +++++ .../MessagePackValueSerializerTests.cs | 276 +++++++++++ .../MessagePack/PooledBufferWriterTests.cs | 72 +++ .../UiPath.Caching.Tests.csproj | 1 + 30 files changed, 1855 insertions(+), 1 deletion(-) create mode 100644 benchmarks/UiPath.Caching.Benchmarks/MessagePackCompressionBenchmark.cs create mode 100644 src/UiPath.Caching.MessagePack/GlobalUsings.cs create mode 100644 src/UiPath.Caching.MessagePack/MessagePackCachingBuilderExtensions.cs create mode 100644 src/UiPath.Caching.MessagePack/MessagePackValueSerializer.cs create mode 100644 src/UiPath.Caching.MessagePack/MessagePackValueSerializerOptions.cs create mode 100644 src/UiPath.Caching.MessagePack/PackageGeneratedResolver.cs create mode 100644 src/UiPath.Caching.MessagePack/PooledBufferWriter.cs create mode 100644 src/UiPath.Caching.MessagePack/PublicAPI.Shipped.txt create mode 100644 src/UiPath.Caching.MessagePack/PublicAPI.Unshipped.txt create mode 100644 src/UiPath.Caching.MessagePack/RawMessagePack.cs create mode 100644 src/UiPath.Caching.MessagePack/RawMessagePackResolver.cs create mode 100644 src/UiPath.Caching.MessagePack/ResilientMessagePackSecurity.cs create mode 100644 src/UiPath.Caching.MessagePack/UiPath.Caching.MessagePack.csproj create mode 100644 tests/UiPath.Caching.Tests/MessagePack/MessagePackAtomicWriteTests.cs create mode 100644 tests/UiPath.Caching.Tests/MessagePack/MessagePackCompressionTests.cs create mode 100644 tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs create mode 100644 tests/UiPath.Caching.Tests/MessagePack/MessagePackOutOfMemoryTests.cs create mode 100644 tests/UiPath.Caching.Tests/MessagePack/MessagePackValueSerializerTests.cs create mode 100644 tests/UiPath.Caching.Tests/MessagePack/PooledBufferWriterTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index 484e5789..e69c2fe6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Added +- **`UiPath.Caching.MessagePack`, a MessagePack value serializer.** `AddMessagePackSerializer()` registers it as `messagepack`, and a policy opts in per policy with `Serializer.Write = "messagepack"`; every other policy keeps writing JSON. It writes format id `0x02`, or `0x03` when the value's MessagePack bytes reach `CompressionThresholdBytes` (default 1024) and LZ4 compression makes them smaller, so an incompressible value stays `0x02`; it uses MessagePack's own LZ4 and adds no dependency. The default options are the contractless resolver with `MessagePackSecurity.UntrustedData`, and `SerializerOptions` replaces them. `MaxDecompressedBytes` defaults to 64 MiB: deserializing can allocate several times the decompressed size, so size it to available memory. Key types MessagePack has no collision-resistant comparer for (`TimeSpan`, `decimal`, tuples, custom structs) fall back to the default comparer. Register it on every node and finish that deploy before any policy writes it, since a node that cannot read the format misses and regenerates the value. Like JSON, contractless MessagePack writes property names, so renaming or retyping a property silently loses that data for values written before the change; `[MessagePackObject]` with `[Key]` avoids that. The package accepts MessagePack `[3.1.12, 4.0)`. See [MessagePack](docs/how-to/extending.md#messagepack). - **A seam for value serializers.** `ICacheValueSerializer` and `CacheValueFormats` define a buffer-based serializer whose output starts with a one-byte format id, `AddValueSerializer` registers one by name, and `CachePolicy.Serializer` (`Write`, `Read`, `MigrateOnRead`) names it per policy. `AddValueSerializer` throws at the call for a name used twice on one builder or for `json`. A policy that names an unregistered or blank serializer, has an empty `Read`, or writes a format its `Read` leaves out fails with an `OptionsValidationException`, as does a serializer whose ids are outside the consumer ranges; these are checked when the policy factory is first resolved, on first cache use rather than at host start. A custom `ICachePolicyFactory` whose policy names an unregistered serializer fails the same way when the Redis provider builds its cache. Serializers the container created are disposed with it, through `DisposeAsync` when they only implement `IAsyncDisposable`, so a factory must return a new instance rather than one resolved from DI. A policy that sets `Write` makes `RedisCache` and `RedisHashCache` write their values, hash fields and hash metadata with that serializer, and every read picks the serializer by the format id, whatever the policy writes. A value in a format no registered serializer owns is a miss, counted in `cache.serializer.unreadable` and removed only if the key still holds the bytes that were read, so a newer value written in between survives; a hash with such a field misses whole and is left in place. A value whose registered serializer throws is a logged miss and the key stays, like one that is not headered and fails to parse as JSON, because the bytes may be good for another version of the type during a rolling deploy; this departs from the design spec, which removed it, and an `OutOfMemoryException` from its payload is never swallowed, while one from a value with no header (JSON) is a logged miss as it is with no serializer registered. `RedisHashCache.TryDecodeFields` now treats any exception except `OutOfMemoryException` as a refresh, since a registered serializer can throw any type, where it caught four. A node with no serializer registered logs the `JsonException` and keeps the key. A policy passed to a single call that names an unregistered serializer throws `OptionsValidationException` before anything is written. The reader only looks for a header with the default JSON serializer or under a policy whose effective `Write` is a serializer other than `json` (`Read` does not count), so `byte[]` values stored through `RawByteSerializerProxy` or `AddDistributedCache` are never reinterpreted. A policy that sets no serializer, or only `Read`, takes the default policy's `Write`. The default stays JSON with no header, and `AddDistributedCache` ignores `Serializer`: its own policy always writes and reads JSON with no migration, so the caller's bytes never get a format header. A hash write that names a serializer leaves `_metadata_` out when the metadata is empty and nulls are not cached, and a `HashReplace` one deletes the field an earlier write left. Hash system field names are compared as bytes instead of strings. See [value serializers](docs/how-to/extending.md#value-serializers) and [rolling a serializer out](docs/concepts.md#rolling-a-serializer-out). ### Changed diff --git a/README.md b/README.md index b39ff867..ed6bb1fe 100644 --- a/README.md +++ b/README.md @@ -161,6 +161,7 @@ flowchart LR | `UiPath.Caching.OpenTelemetry` | OpenTelemetry adapter for `ICachingTelemetryProvider` — `ActivitySource` + `Meter` named `UiPath.Caching`. | | `UiPath.Caching.Azure` | Microsoft Entra ID authentication for Azure Managed Redis / Azure Cache for Redis. Adds `AddAzureEntraAuthentication()`. | | `UiPath.Caching.Queue` | Set ("queue") caches — `ISetCache` / `ISetCache`, registered per backing via `AddQueueMemory` / `AddQueueRedis` / `AddQueueInMemoryRedis`. | +| `UiPath.Caching.MessagePack` | MessagePack value serializer with LZ4 compression above a size threshold; selected per policy through `Serializer.Write`. | ## Documentation diff --git a/UiPath.Caching.slnx b/UiPath.Caching.slnx index b60d906d..68afa875 100644 --- a/UiPath.Caching.slnx +++ b/UiPath.Caching.slnx @@ -18,6 +18,7 @@ + diff --git a/benchmarks/UiPath.Caching.Benchmarks/FakeRedisConnector.cs b/benchmarks/UiPath.Caching.Benchmarks/FakeRedisConnector.cs index b841ec4c..85d33b3a 100644 --- a/benchmarks/UiPath.Caching.Benchmarks/FakeRedisConnector.cs +++ b/benchmarks/UiPath.Caching.Benchmarks/FakeRedisConnector.cs @@ -110,7 +110,8 @@ public static T Create(FakeRedisStore store) if (method.Name == "StringSetAsync" && args is [RedisKey setKey, RedisValue value, ..]) { - _store.Values[setKey] = value; + // The caller returns a pooled buffer to its pool once the call completes, so the store keeps its own bytes. + _store.Values[setKey] = value.IsNull ? value : (byte[])value!; _store.LastSet = setKey; return Task.FromResult(true); } diff --git a/benchmarks/UiPath.Caching.Benchmarks/MessagePackCompressionBenchmark.cs b/benchmarks/UiPath.Caching.Benchmarks/MessagePackCompressionBenchmark.cs new file mode 100644 index 00000000..286f22c0 --- /dev/null +++ b/benchmarks/UiPath.Caching.Benchmarks/MessagePackCompressionBenchmark.cs @@ -0,0 +1,119 @@ +using System.Buffers; +using BenchmarkDotNet.Attributes; +using BenchmarkDotNet.Configs; +using Microsoft.Extensions.DependencyInjection; +using UiPath.Caching.Config; +using UiPath.Caching.MessagePack; +using UiPath.Caching.Serialization; + +namespace UiPath.Caching.Benchmarks; + +/// The MessagePack serializer with compression off against its default threshold, at three payload sizes of compressible rows. +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[CategoriesColumn] +public class MessagePackCompressionBenchmark +{ + private readonly ArrayBufferWriter _target = new(); + private ServiceProvider _plainServices = default!; + private ServiceProvider _compressedServices = default!; + private ICacheValueSerializer _plain = default!; + private ICacheValueSerializer _compressed = default!; + private Payload _value = default!; + private byte[] _plainBytes = default!; + private byte[] _compressedBytes = default!; + private int _compressedId; + + [Params(512, 1024, 16 * 1024, 128 * 1024)] + public int Bytes { get; set; } + + [GlobalSetup] + public void Setup() + { + _plainServices = Build(o => o.CompressionThresholdBytes = int.MaxValue); + _compressedServices = Build(_ => { }); + _plain = ValueSerializerRegistration.BuildRegistry(_plainServices).Entries.Single().Serializer; + _compressed = ValueSerializerRegistration.BuildRegistry(_compressedServices).Entries.Single().Serializer; + _value = Payload.OfSize(Bytes); + _plainBytes = Write(_plain, out _); + _compressedBytes = Write(_compressed, out _compressedId); + Console.WriteLine($"size {Bytes}: plain {_plainBytes.Length}, compressed {_compressedBytes.Length} (id 0x{_compressedId:X2})"); + } + + [GlobalCleanup] + public void Cleanup() + { + _plainServices.Dispose(); + _compressedServices.Dispose(); + } + + [BenchmarkCategory("Write"), Benchmark(Baseline = true)] + public int Write_Plain() + { + _target.ResetWrittenCount(); + return _plain.Serialize(_value, _target); + } + + [BenchmarkCategory("Write"), Benchmark] + public int Write_Compressed() + { + _target.ResetWrittenCount(); + return _compressed.Serialize(_value, _target); + } + + [BenchmarkCategory("Read"), Benchmark(Baseline = true)] + public Payload? Read_Plain() => _plain.Deserialize(CacheValueFormats.MessagePack, new ReadOnlySequence(_plainBytes)); + + [BenchmarkCategory("Read"), Benchmark] + public Payload? Read_Compressed() => _compressed.Deserialize(_compressedId, new ReadOnlySequence(_compressedBytes)); + + private static ServiceProvider Build(Action configure) => + new ServiceCollection() + .AddCaching(b => b.AddMessagePackSerializer(configure: configure), o => o.AppShortName = "bench") + .BuildServiceProvider(); + + private byte[] Write(ICacheValueSerializer serializer, out int id) + { + var target = new ArrayBufferWriter(); + id = serializer.Serialize(_value, target); + return target.WrittenMemory.ToArray(); + } + + public sealed class Payload + { + public List Rows { get; set; } = []; + + public static Payload OfSize(int bytes) + { + var payload = new Payload(); + var random = new Random(42); + string[] statuses = ["active", "pending", "suspended", "closed"]; + for (var size = 0; size < bytes; size += 60) + { + payload.Rows.Add(new Row + { + Id = payload.Rows.Count, + Name = $"customer-{random.Next(100_000)}", + Status = statuses[random.Next(statuses.Length)], + Amount = Math.Round(random.NextDouble() * 1000, 2), + Created = new DateTime(2026, 1, 1, 0, 0, 0, DateTimeKind.Utc).AddMinutes(random.Next(500_000)), + }); + } + + return payload; + } + } + + public sealed class Row + { + public int Id { get; set; } + + public string? Name { get; set; } + + public string? Status { get; set; } + + public double Amount { get; set; } + + public DateTime Created { get; set; } + } +} diff --git a/benchmarks/UiPath.Caching.Benchmarks/SerializerBenchmark.cs b/benchmarks/UiPath.Caching.Benchmarks/SerializerBenchmark.cs index 2e336b1e..895341c9 100644 --- a/benchmarks/UiPath.Caching.Benchmarks/SerializerBenchmark.cs +++ b/benchmarks/UiPath.Caching.Benchmarks/SerializerBenchmark.cs @@ -2,9 +2,11 @@ using System.Text.Json; using BenchmarkDotNet.Attributes; using BenchmarkDotNet.Configs; +using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging.Abstractions; using StackExchange.Redis; using UiPath.Caching.Config; +using UiPath.Caching.MessagePack; using UiPath.Caching.Policies; using UiPath.Caching.Redis; using UiPath.Caching.Serialization; @@ -109,3 +111,84 @@ private sealed class UnusedSerializer : ICacheValueSerializer public T? Deserialize(int formatId, ReadOnlySequence source) => throw new NotSupportedException(); } } + +/// The same cache read and write under a policy that names json or messagepack, so the rows compare size, time and allocation of the two formats through . +[MemoryDiagnoser] +[GroupBenchmarksBy(BenchmarkLogicalGroupRule.ByCategory)] +[CategoriesColumn] +public class MessagePackSerializerBenchmark +{ + private static readonly TimeSpan Expiration = TimeSpan.FromMinutes(30); + + private CustomObject _obj = default!; + private FakeRedisConnector _redis = default!; + private RedisCache _cache = default!; + private ServiceProvider _services = default!; + private CachePolicy _policy = default!; + + [Params("Small", "Medium", "Large")] + public string Size { get; set; } = "Medium"; + + [Params("json", "messagepack")] + public string Format { get; set; } = "json"; + + [GlobalSetup] + public void Setup() + { + _obj = Size switch + { + "Small" => CustomObject.RandomSmall(), + "Large" => CustomObject.RandomLarge(), + _ => CustomObject.RandomMedium(), + }; + _services = new ServiceCollection() + .AddCaching(b => b.AddMessagePackSerializer(), o => o.AppShortName = "bench") + .BuildServiceProvider(); + var registry = ValueSerializerRegistration.BuildRegistry(_services); + _policy = new CachePolicy { Serializer = new SerializerPolicy { Write = Format } }; + _redis = new FakeRedisConnector(); + _cache = new RedisCache( + _redis, + new SystemJsonByteSerializerProxy(), + EmptyResiliencePipelineProvider.Instance, + NullTelemetryProvider.Instance, + new RedisCacheOptions(), + new CacheOptions { AppShortName = "bench" }, + new DefaultCachePolicyFactory(null, null, valueSerializers: registry), + TimeProvider.System, + NullLogger.Instance) + { + ValueSerializers = registry, + }; + _cache.SetAsync("bench:format", _obj, Expiration, _policy).AsTask().GetAwaiter().GetResult(); + var written = new ArrayBufferWriter(); + if (Format == "json") + { + written.Write(JsonSerializer.SerializeToUtf8Bytes(_obj)); + } + else + { + registry.Entries.Single().Serializer.Serialize(_obj, written); + } + + if (_cache.GetAsync("bench:format", _policy).AsTask().GetAwaiter().GetResult() is null) + { + throw new InvalidOperationException("The fake connector did not serve the read"); + } + + Console.WriteLine($"{Size} {Format}: {written.WrittenCount} bytes"); + } + + [GlobalCleanup] + public void Cleanup() + { + _redis.Dispose(); + _services.Dispose(); + } + + [BenchmarkCategory("Get"), Benchmark] + public ValueTask Get() => _cache.GetAsync("bench:format", _policy); + + [BenchmarkCategory("Set"), Benchmark] + public ValueTask Set() => _cache.SetAsync("bench:format", _obj, Expiration, _policy); +} diff --git a/benchmarks/UiPath.Caching.Benchmarks/UiPath.Caching.Benchmarks.csproj b/benchmarks/UiPath.Caching.Benchmarks/UiPath.Caching.Benchmarks.csproj index 7828de16..e1ecad3e 100644 --- a/benchmarks/UiPath.Caching.Benchmarks/UiPath.Caching.Benchmarks.csproj +++ b/benchmarks/UiPath.Caching.Benchmarks/UiPath.Caching.Benchmarks.csproj @@ -20,6 +20,7 @@ + diff --git a/docs/how-to/extending.md b/docs/how-to/extending.md index c985dd35..f6085e93 100644 --- a/docs/how-to/extending.md +++ b/docs/how-to/extending.md @@ -518,6 +518,44 @@ inspected under them. A policy that sets no `Serializer` takes the default policy's `Write`, and one that sets only `Read` does too. +### MessagePack + +`UiPath.Caching.MessagePack` is a ready `ICacheValueSerializer`: MessagePack, LZ4-compressed above a size threshold. It +needs no attributes on your types. In this repo's benchmarks a small object went from 172 bytes as JSON to 102 as MessagePack, a medium one from 799 to 645, and a large one from 388,780 to 277,208. Compression then cuts repetitive rows to roughly a third at 16 KB and above: 1 KB of rows 1,257 to 626 bytes, 16 KB 19,327 to 7,124, 128 KB 156,651 to 56,385. + +```csharp +services.AddCaching( + b => + { + b.AddRedis(_ => { }); + b.AddMessagePackSerializer(); // registered as "messagepack"; pass a name and a callback to change either + }, + o => o.Policies["Sessions"] = new CachePolicy { Serializer = new SerializerPolicy { Write = "messagepack" } }); +``` + +Or in configuration, with the serializer registered in code: `"Policies": { "Sessions": { "Serializer": { "Write": "messagepack" } } }`. +Only the policies that name it write MessagePack; every other policy keeps writing JSON. +The package depends on the exact `UiPath.Caching` version it was built with, so upgrade the two together. + +| Option | Default | Notes | +|---|---|---| +| `CompressionThresholdBytes` | `1024` | A value whose MessagePack bytes reach this size is compressed when that makes it smaller, so an incompressible value is stored plain. `0` tries every value, `int.MaxValue` none. | +| `MaxDecompressedBytes` | `67108864` (64 MiB) | A compressed value that claims more than this uncompressed is rejected on read, as is any claim above 255 times its compressed size. Deserializing can allocate several times the decompressed size (measured up to about 8x for typed collections, more for an `object[]` of nils), so size it to available memory. It replaces `MaximumDecompressedSize` on a supplied `MessagePackSecurity`. | +| `SerializerOptions` | contractless resolver, `MessagePackSecurity.UntrustedData` | Replace it to use your own resolvers. `UntrustedData` adds hash-collision-resistant comparers; the object-graph depth limit is the same as `TrustedData`. A supplied instance that carries `MessagePackSecurity.TrustedData` is switched to `UntrustedData`; to run trusted, pass your own `MessagePackSecurity` instance. Where MessagePack has no resistant comparer for a key type (`TimeSpan`, `decimal`, `Uri`, tuples, custom structs), the default comparer is used instead of failing the read. Compression set on it is ignored: the threshold decides. A resolver you supply that resolves `object` by type name (Typeless) or polymorphically reopens type confusion on bytes read from Redis. | + +The serializer owns format ids `0x02` (plain) and `0x03` (LZ4); the id stored with each value says which, so a node reads +both whatever its threshold. + +Roll it out readers first. A node reads a format only once the serializer is registered on it, so register +`AddMessagePackSerializer` on every node and finish that deploy before any policy sets `Write = "messagepack"`; a node that +cannot read the value misses and regenerates it (see [rolling a serializer out](../concepts.md#rolling-a-serializer-out)). +Setting `Write` back to `json` keeps values already stored as MessagePack readable while the default JSON proxy is in use; with a custom `ISerializerProxy` registered, headers are only recognized under a policy that writes a non-JSON format, so under `Write = "json"` those values go to your proxy and read as misses. + +The default resolver is contractless, which writes a type's public property names, as JSON does. Renaming a +property, or changing its type, therefore silently loses that data for values written before the change, exactly as +it would with JSON: the old values read back with the new property empty. Mark a type with `[MessagePackObject]` and +`[Key(n)]` to make the format independent of the names. + ## Swapping the default factories `ICacheFactory` and `ICachePolicyFactory` each have a default DI registration set up by `AddCaching` — the concrete `CacheFactory` and `DefaultCachePolicyFactory` respectively. When you need to substitute either, use the fluent `Use*Factory` extensions on `ICachingBuilder`. They internally call `Services.Replace(...)` so the swap survives the rest of the `AddCaching` pipeline (which uses `TryAddSingleton` and would otherwise lose to the default registration). diff --git a/docs/index.md b/docs/index.md index b4e9d5d4..19c5eff3 100644 --- a/docs/index.md +++ b/docs/index.md @@ -13,6 +13,7 @@ UiPath's internal multilayer caching library. L1 in-memory + L2 Redis, cross-nod | `UiPath.Caching.OpenTelemetry` | OpenTelemetry wiring for `ICachingTelemetryProvider` (`ActivitySource` + `Meter` named `UiPath.Caching`). | | `UiPath.Caching.Azure` | Microsoft Entra ID authentication for Azure Managed Redis / Azure Cache for Redis. | | `UiPath.Caching.Queue` | Set ("queue") caches on Redis sets — `ISetCache` / `ISetCache`. | +| `UiPath.Caching.MessagePack` | MessagePack value serializer with LZ4 compression above a size threshold. | ## Pick your path diff --git a/docs/reference/settings.md b/docs/reference/settings.md index 80d02732..d8f07ab3 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -484,3 +484,15 @@ Nested under a `CachePolicy` entry. Field-level merged against the default polic | `MigrateOnRead` | `bool?` | `null` | Per-policy | Rewrite a value read in another format into the write format; `null` = inherit, and under that off. Accepted and merged, but nothing reads it yet: a value is never rewritten in this release. | **`Read` and layered configuration.** A `Read` set in code replaces the default policy's list as a whole, but .NET configuration providers merge arrays by index. With a base layer of `"Read": ["a", "b"]` and an override layer of `"Read": ["json"]`, the bound list is `["json", "b"]`, not `["json"]`. An override of `"Read": []` binds as `null`, so it inherits and the policy reads every registered format. To override safely, set every index the base layer sets (or repeat the full list), or declare `Read` in one layer only. + +--- + +## MessagePackValueSerializerOptions (`UiPath.Caching.MessagePack`) + +Passed to `AddMessagePackSerializer(name, configure)` in code; not bound from configuration. A policy selects the serializer with `Serializer.Write` set to the registered name (default `messagepack`). See [MessagePack](../how-to/extending.md#messagepack). + +| Property | Type | Default | Scope | Notes | +|---|---|---|---|---| +| `CompressionThresholdBytes` | `int` | `1024` | Per-registration | A value whose MessagePack bytes reach this size is LZ4-compressed when that makes it smaller; otherwise stored plain. `0` tries every value, `int.MaxValue` compresses none, negative throws. Written as format id `0x03` when compressed, `0x02` when not. | +| `MaxDecompressedBytes` | `int` | `67108864` | Per-registration | A compressed value claiming more than this many bytes uncompressed (64 MiB by default) is rejected on read (also any claim above 255 times its compressed size); zero or negative throws. Deserializing can allocate several times the decompressed size (up to about 8x for typed collections, more for an `object[]` of nils), so size it to available memory. It replaces `MaximumDecompressedSize` on a supplied `MessagePackSecurity`. | +| `SerializerOptions` | `MessagePackSerializerOptions?` | contractless resolver, `MessagePackSecurity.UntrustedData` | Per-registration | Replaces the default options; compression set on it is ignored. `UntrustedData` adds hash-collision-resistant comparers (same depth limit as `TrustedData`); a supplied `MessagePackSecurity.TrustedData` is switched to `UntrustedData`, so pass a custom `MessagePackSecurity` to run trusted. A key type MessagePack has no resistant comparer for (`TimeSpan`, `decimal`, tuples, custom structs) uses the default comparer instead of failing the read. | diff --git a/src/UiPath.Caching.MessagePack/GlobalUsings.cs b/src/UiPath.Caching.MessagePack/GlobalUsings.cs new file mode 100644 index 00000000..0954f66c --- /dev/null +++ b/src/UiPath.Caching.MessagePack/GlobalUsings.cs @@ -0,0 +1 @@ +[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("UiPath.Caching.Tests")] diff --git a/src/UiPath.Caching.MessagePack/MessagePackCachingBuilderExtensions.cs b/src/UiPath.Caching.MessagePack/MessagePackCachingBuilderExtensions.cs new file mode 100644 index 00000000..fb977979 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/MessagePackCachingBuilderExtensions.cs @@ -0,0 +1,21 @@ +using UiPath.Caching.Config; + +namespace UiPath.Caching.MessagePack; + +public static class MessagePackCachingBuilderExtensions +{ + public const string DefaultName = "messagepack"; + + /// Registers the MessagePack serializer as , selectable per policy through Serializer.Write. + public static ICachingBuilder AddMessagePackSerializer( + this ICachingBuilder builder, + string name = DefaultName, + Action? configure = null) + { + ArgumentNullException.ThrowIfNull(builder); + + var options = new MessagePackValueSerializerOptions(); + configure?.Invoke(options); + return builder.AddLibraryValueSerializer(name, _ => new MessagePackValueSerializer(options)); + } +} diff --git a/src/UiPath.Caching.MessagePack/MessagePackValueSerializer.cs b/src/UiPath.Caching.MessagePack/MessagePackValueSerializer.cs new file mode 100644 index 00000000..34940e29 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/MessagePackValueSerializer.cs @@ -0,0 +1,196 @@ +using System.Buffers; +using System.Runtime.ExceptionServices; +using MessagePack; +using MessagePack.Resolvers; + +namespace UiPath.Caching.MessagePack; + +internal sealed class MessagePackValueSerializer : ICacheValueSerializer +{ + // MessagePack's reserved extension type for an LZ4 block array: the extension holds each block's uncompressed length. + private const sbyte Lz4BlockArrayTypeCode = 98; + + // LZ4 cannot expand a block by more than this factor, so a larger claim is corrupt input, not data. + private const long MaxExpansion = 255; + + private static readonly MessagePackSerializerOptions Default = + ContractlessStandardResolver.Options.WithSecurity(ResilientMessagePackSecurity.Instance); + + private readonly MessagePackSerializerOptions _plain; + private readonly MessagePackSerializerOptions _compressing; + private readonly MessagePackSerializerOptions _compressRaw; + private readonly int _threshold; + private readonly int _maxDecompressedBytes; + + public MessagePackValueSerializer(MessagePackValueSerializerOptions options) + { + ArgumentNullException.ThrowIfNull(options); + ArgumentOutOfRangeException.ThrowIfNegative(options.CompressionThresholdBytes); + ArgumentOutOfRangeException.ThrowIfNegativeOrZero(options.MaxDecompressedBytes); + + var user = options.SerializerOptions ?? Default; + if (ReferenceEquals(user.Security, MessagePackSecurity.TrustedData)) + { + user = user.WithSecurity(ResilientMessagePackSecurity.Instance); + } + + _plain = user.WithCompression(MessagePackCompression.None); + _threshold = Math.Max(options.CompressionThresholdBytes, 1); + _maxDecompressedBytes = options.MaxDecompressedBytes; + _compressing = _plain + .WithCompression(MessagePackCompression.Lz4BlockArray) + .WithCompressionMinLength(1) + .WithSecurity(_plain.Security.WithMaximumDecompressedSize(options.MaxDecompressedBytes)); + _compressRaw = _compressing.WithResolver(RawMessagePackResolver.Instance); + } + + public IReadOnlyList FormatIds { get; } = [CacheValueFormats.MessagePack, CacheValueFormats.MessagePackLz4]; + + internal MessagePackSerializerOptions Options => _plain; + + public int Serialize(T value, IBufferWriter target) + { + if (_threshold == int.MaxValue) + { + try + { + MessagePackSerializer.Serialize(target, value, _plain); + } + catch (MessagePackSerializationException e) + { + RethrowOutOfMemory(e); + throw; + } + + return CacheValueFormats.MessagePack; + } + + using var plain = new PooledBufferWriter(); + try + { + MessagePackSerializer.Serialize(plain, value, _plain); + } + catch (MessagePackSerializationException e) + { + RethrowOutOfMemory(e); + throw; + } + + var plainBytes = plain.WrittenMemory; + // A value above the read limit would be stored compressed and then never read back, so it stays plain. + if (plainBytes.Length >= _threshold && plainBytes.Length <= _maxDecompressedBytes) + { + // MessagePack keeps a compressed result even when it is larger, so the sizes decide which form is stored. + using var compressed = new PooledBufferWriter(); + try + { + MessagePackSerializer.Serialize(compressed, new RawMessagePack(plainBytes), _compressRaw); + } + catch (MessagePackSerializationException e) + { + RethrowOutOfMemory(e); + throw; + } + + if (compressed.WrittenMemory.Length < plainBytes.Length) + { + target.Write(compressed.WrittenMemory.Span); + return CacheValueFormats.MessagePackLz4; + } + } + + target.Write(plainBytes.Span); + return CacheValueFormats.MessagePack; + } + + public T? Deserialize(int formatId, ReadOnlySequence source) + { + try + { + return DeserializeCore(formatId, source); + } + catch (MessagePackSerializationException e) + { + RethrowOutOfMemory(e); + throw; + } + } + + // MessagePack wraps an allocation failure; a cache must not read it as a miss and run the generator under memory pressure. + private static void RethrowOutOfMemory(Exception e) + { + for (var inner = e.InnerException; inner is not null; inner = inner.InnerException) + { + if (inner is OutOfMemoryException) + { + ExceptionDispatchInfo.Capture(inner).Throw(); + } + } + } + + private static bool TryReadCompressedHeader(ReadOnlySequence bytes, out long plainLength) + { + plainLength = 0; + try + { + var reader = new MessagePackReader(bytes); + if (reader.NextMessagePackType != MessagePackType.Array) + { + return false; + } + + var blocks = reader.ReadArrayHeader() - 1; + if (blocks < 1 || reader.NextMessagePackType != MessagePackType.Extension) + { + return false; + } + + var header = reader.ReadExtensionFormatHeader(); + if (header.TypeCode != Lz4BlockArrayTypeCode) + { + return false; + } + + var lengths = new MessagePackReader(reader.ReadRaw(header.Length)); + for (var i = 0; i < blocks; i++) + { + var blockLength = lengths.ReadInt64(); + if (blockLength < 0) + { + return false; + } + + plainLength = checked(plainLength + blockLength); + } + + return lengths.End && reader.NextMessagePackType == MessagePackType.Binary; + } + catch (Exception e) when (e is MessagePackSerializationException or EndOfStreamException or OverflowException or InvalidOperationException) + { + return false; + } + } + + private T? DeserializeCore(int formatId, ReadOnlySequence source) + { + switch (formatId) + { + case CacheValueFormats.MessagePack: + return MessagePackSerializer.Deserialize(source, _plain); + case CacheValueFormats.MessagePackLz4: + if (!TryReadCompressedHeader(source, out var plainLength)) + { + throw new InvalidDataException("A value stored as MessagePack with LZ4 does not hold a compressed block array."); + } + + if (plainLength > source.Length * MaxExpansion || plainLength > _maxDecompressedBytes) + { + throw new InvalidDataException($"A compressed value of {source.Length} bytes claims {plainLength} bytes uncompressed."); + } + + return MessagePackSerializer.Deserialize(source, _compressing); + default: + throw new InvalidOperationException($"The MessagePack serializer does not own format id 0x{formatId:X2}."); + } + } +} diff --git a/src/UiPath.Caching.MessagePack/MessagePackValueSerializerOptions.cs b/src/UiPath.Caching.MessagePack/MessagePackValueSerializerOptions.cs new file mode 100644 index 00000000..53766447 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/MessagePackValueSerializerOptions.cs @@ -0,0 +1,15 @@ +using MessagePack; + +namespace UiPath.Caching.MessagePack; + +public sealed class MessagePackValueSerializerOptions +{ + /// A value whose MessagePack bytes reach this size is LZ4-compressed when that makes it smaller; zero compresses every value, none. + public int CompressionThresholdBytes { get; set; } = 1024; + + /// A compressed value that claims more than this many bytes uncompressed is rejected on read; the default is 64 MiB. A claim above 255 times the compressed size is rejected whatever this is. Deserializing can allocate several times the decompressed size (up to about 8x for typed collections, more for an object[] of nils), so size this to available memory. It replaces on a supplied . + public int MaxDecompressedBytes { get; set; } = 64 * 1024 * 1024; + + /// Defaults to the contractless resolver with , so a value that is not marked for MessagePack still serializes and hash-based collections use collision-resistant comparers, falling back to the default comparer for a key type MessagePack has none for (, , tuples, custom structs). A supplied instance that carries is switched to the default security; pass your own to opt out. Any compression set on it is ignored: the serializer compresses by . + public MessagePackSerializerOptions? SerializerOptions { get; set; } +} diff --git a/src/UiPath.Caching.MessagePack/PackageGeneratedResolver.cs b/src/UiPath.Caching.MessagePack/PackageGeneratedResolver.cs new file mode 100644 index 00000000..599cb577 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/PackageGeneratedResolver.cs @@ -0,0 +1,7 @@ +using MessagePack; + +namespace UiPath.Caching.MessagePack; + +// Names this assembly's source-generated resolver, so it cannot clash with MessagePack.GeneratedMessagePackResolver in an assembly that sees our internals. +[GeneratedMessagePackResolver] +internal partial class PackageGeneratedResolver; diff --git a/src/UiPath.Caching.MessagePack/PooledBufferWriter.cs b/src/UiPath.Caching.MessagePack/PooledBufferWriter.cs new file mode 100644 index 00000000..87831630 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/PooledBufferWriter.cs @@ -0,0 +1,46 @@ +using System.Buffers; + +namespace UiPath.Caching.MessagePack; + +internal sealed class PooledBufferWriter : IBufferWriter, IDisposable +{ + private byte[] _buffer = ArrayPool.Shared.Rent(4096); + private int _written; + + public ReadOnlyMemory WrittenMemory => _buffer.AsMemory(0, _written); + + public void Advance(int count) + { + ArgumentOutOfRangeException.ThrowIfNegative(count); + ArgumentOutOfRangeException.ThrowIfGreaterThan(count, _buffer.Length - _written); + _written += count; + } + + public Memory GetMemory(int sizeHint = 0) + { + Grow(sizeHint); + return _buffer.AsMemory(_written); + } + + public Span GetSpan(int sizeHint = 0) + { + Grow(sizeHint); + return _buffer.AsSpan(_written); + } + + public void Dispose() => ArrayPool.Shared.Return(_buffer, clearArray: true); + + private void Grow(int sizeHint) + { + var needed = _written + Math.Max(sizeHint, 1); + if (needed <= _buffer.Length) + { + return; + } + + var bigger = ArrayPool.Shared.Rent(Math.Max(needed, _buffer.Length * 2)); + _buffer.AsSpan(0, _written).CopyTo(bigger); + ArrayPool.Shared.Return(_buffer, clearArray: true); + _buffer = bigger; + } +} diff --git a/src/UiPath.Caching.MessagePack/PublicAPI.Shipped.txt b/src/UiPath.Caching.MessagePack/PublicAPI.Shipped.txt new file mode 100644 index 00000000..7dc5c581 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/PublicAPI.Shipped.txt @@ -0,0 +1 @@ +#nullable enable diff --git a/src/UiPath.Caching.MessagePack/PublicAPI.Unshipped.txt b/src/UiPath.Caching.MessagePack/PublicAPI.Unshipped.txt new file mode 100644 index 00000000..e6daecc1 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/PublicAPI.Unshipped.txt @@ -0,0 +1,12 @@ +#nullable enable +const UiPath.Caching.MessagePack.MessagePackCachingBuilderExtensions.DefaultName = "messagepack" -> string! +static UiPath.Caching.MessagePack.MessagePackCachingBuilderExtensions.AddMessagePackSerializer(this UiPath.Caching.Config.ICachingBuilder! builder, string! name = "messagepack", System.Action? configure = null) -> UiPath.Caching.Config.ICachingBuilder! +UiPath.Caching.MessagePack.MessagePackCachingBuilderExtensions +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions.CompressionThresholdBytes.get -> int +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions.CompressionThresholdBytes.set -> void +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions.MaxDecompressedBytes.get -> int +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions.MaxDecompressedBytes.set -> void +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions.MessagePackValueSerializerOptions() -> void +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions.SerializerOptions.get -> MessagePack.MessagePackSerializerOptions? +UiPath.Caching.MessagePack.MessagePackValueSerializerOptions.SerializerOptions.set -> void diff --git a/src/UiPath.Caching.MessagePack/RawMessagePack.cs b/src/UiPath.Caching.MessagePack/RawMessagePack.cs new file mode 100644 index 00000000..4145b650 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/RawMessagePack.cs @@ -0,0 +1,6 @@ +namespace UiPath.Caching.MessagePack; + +internal readonly struct RawMessagePack(ReadOnlyMemory bytes) +{ + public ReadOnlyMemory Bytes { get; } = bytes; +} diff --git a/src/UiPath.Caching.MessagePack/RawMessagePackResolver.cs b/src/UiPath.Caching.MessagePack/RawMessagePackResolver.cs new file mode 100644 index 00000000..45cae551 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/RawMessagePackResolver.cs @@ -0,0 +1,29 @@ +using MessagePack; +using MessagePack.Formatters; + +namespace UiPath.Caching.MessagePack; + +// Hands already serialized MessagePack to the serializer's compression, so a large value is not serialized twice. +internal sealed class RawMessagePackResolver : IFormatterResolver +{ + public static readonly RawMessagePackResolver Instance = new(); + + public IMessagePackFormatter? GetFormatter() => + typeof(T) == typeof(RawMessagePack) ? (IMessagePackFormatter)(object)RawFormatter.Instance : null; + + internal sealed class RawFormatter : IMessagePackFormatter + { + public static readonly RawFormatter Instance = new(); + + // WriteRaw(span) copies through 4 KiB scratch segments, which LZ4 would compress as separate blocks. + public void Serialize(ref MessagePackWriter writer, RawMessagePack value, MessagePackSerializerOptions options) + { + var length = value.Bytes.Length; + value.Bytes.Span.CopyTo(writer.GetSpan(length)); + writer.Advance(length); + } + + public RawMessagePack Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => + throw new NotSupportedException(); + } +} diff --git a/src/UiPath.Caching.MessagePack/ResilientMessagePackSecurity.cs b/src/UiPath.Caching.MessagePack/ResilientMessagePackSecurity.cs new file mode 100644 index 00000000..bbca495a --- /dev/null +++ b/src/UiPath.Caching.MessagePack/ResilientMessagePackSecurity.cs @@ -0,0 +1,46 @@ +using System.Collections; +using MessagePack; + +namespace UiPath.Caching.MessagePack; + +internal sealed class ResilientMessagePackSecurity : MessagePackSecurity +{ + public static readonly ResilientMessagePackSecurity Instance = new(); + + private ResilientMessagePackSecurity() + : base(UntrustedData) + { + } + + private ResilientMessagePackSecurity(ResilientMessagePackSecurity copyFrom) + : base(copyFrom) + { + } + + protected override MessagePackSecurity Clone() => new ResilientMessagePackSecurity(this); + + // MessagePack throws for a key type it has no resistant comparer for, which would turn every read of such a value into a miss. + protected override IEqualityComparer GetHashCollisionResistantEqualityComparer() + { + try + { + return base.GetHashCollisionResistantEqualityComparer(); + } + catch (TypeAccessException) + { + return EqualityComparer.Default; + } + } + + protected override IEqualityComparer GetHashCollisionResistantEqualityComparer() + { + try + { + return base.GetHashCollisionResistantEqualityComparer(); + } + catch (TypeAccessException) + { + return EqualityComparer.Default; + } + } +} diff --git a/src/UiPath.Caching.MessagePack/UiPath.Caching.MessagePack.csproj b/src/UiPath.Caching.MessagePack/UiPath.Caching.MessagePack.csproj new file mode 100644 index 00000000..b2f5e7f7 --- /dev/null +++ b/src/UiPath.Caching.MessagePack/UiPath.Caching.MessagePack.csproj @@ -0,0 +1,25 @@ + + + + $(TargetFrameworks) + true + false + A MessagePack value serializer, with LZ4 compression above a size threshold, for UiPath.Caching. + UiPath;Caching;MessagePack + + + + + + + + + + + + + + <_ProjectReferencesWithVersions Update="@(_ProjectReferencesWithVersions)" ProjectVersion="[%(ProjectVersion)]" /> + + + diff --git a/src/UiPath.Caching/GlobalUsings.cs b/src/UiPath.Caching/GlobalUsings.cs index 1df048e4..30d094fd 100644 --- a/src/UiPath.Caching/GlobalUsings.cs +++ b/src/UiPath.Caching/GlobalUsings.cs @@ -16,6 +16,7 @@ [assembly: System.Runtime.CompilerServices.InternalsVisibleTo("UiPath.Caching.Tests")] [assembly: System.Runtime.CompilerServices.InternalsVisibleTo("UiPath.Caching.Benchmarks")] +[assembly: System.Runtime.CompilerServices.InternalsVisibleTo("UiPath.Caching.MessagePack")] // Castle DynamicProxy (used by NSubstitute via AutoFixture in Caching.Tests) needs access to the // internal concrete cache classes to generate dynamic proxies for closed generics like // ILogger. Without this, mocking those proxies fails at test fixture time. diff --git a/tests/UiPath.Caching.Tests/MessagePack/MessagePackAtomicWriteTests.cs b/tests/UiPath.Caching.Tests/MessagePack/MessagePackAtomicWriteTests.cs new file mode 100644 index 00000000..4d3a7931 --- /dev/null +++ b/tests/UiPath.Caching.Tests/MessagePack/MessagePackAtomicWriteTests.cs @@ -0,0 +1,46 @@ +using System.Buffers; +using UiPath.Caching.MessagePack; + +namespace UiPath.Caching.Tests.MessagePack; + +public class MessagePackAtomicWriteTests +{ + [Theory] + [InlineData(int.MaxValue)] + [InlineData(1024)] + public void Bytes_advanced_before_a_throw_do_not_reach_the_next_stored_value(int threshold) + { + var inner = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { CompressionThresholdBytes = threshold }); + var serializer = new ThrowingOnce(inner); + int[] ids = [.. serializer.FormatIds]; + + var act = () => PooledJsonWriter.SerializeValue(serializer, ids, "first"); + act.Should().Throw(); + using var payload = PooledJsonWriter.SerializeValue(serializer, ids, "second"); + + var expected = new ArrayBufferWriter(); + inner.Serialize("second", expected); + payload.Memory.Span[1..].ToArray().Should().Equal(expected.WrittenMemory.ToArray()); + } + + private sealed class ThrowingOnce(MessagePackValueSerializer inner) : ICacheValueSerializer + { + private bool _thrown; + + public IReadOnlyList FormatIds => inner.FormatIds; + + public int Serialize(T value, IBufferWriter target) + { + if (!_thrown) + { + _thrown = true; + inner.Serialize(value, target); + throw new InvalidOperationException("the formatter failed after writing"); + } + + return inner.Serialize(value, target); + } + + public T? Deserialize(int formatId, ReadOnlySequence source) => inner.Deserialize(formatId, source); + } +} diff --git a/tests/UiPath.Caching.Tests/MessagePack/MessagePackCompressionTests.cs b/tests/UiPath.Caching.Tests/MessagePack/MessagePackCompressionTests.cs new file mode 100644 index 00000000..4bbe4bf1 --- /dev/null +++ b/tests/UiPath.Caching.Tests/MessagePack/MessagePackCompressionTests.cs @@ -0,0 +1,457 @@ +using System.Buffers; +using MessagePack; +using UiPath.Caching.MessagePack; + +namespace UiPath.Caching.Tests.MessagePack; + +public class MessagePackCompressionTests +{ + private const int Threshold = 1024; + + private readonly MessagePackValueSerializer _serializer = new(new MessagePackValueSerializerOptions { CompressionThresholdBytes = Threshold }); + + private delegate void WriteAction(ref MessagePackWriter writer); + + [Fact] + public void Below_threshold_writes_id_0x02() + { + var (id, bytes) = Write(new string('a', Threshold - 4)); + + id.Should().Be(0x02); + bytes.Length.Should().Be(Threshold - 1); + } + + [Fact] + public void At_threshold_writes_id_0x03_when_smaller() + { + // A str16 header is three bytes, so this value serializes to exactly the threshold. + var (id, bytes) = Write(new string('a', Threshold - 3)); + + id.Should().Be(0x03); + bytes.Length.Should().BeLessThan(Threshold); + } + + [Fact] + public void Incompressible_value_stays_id_0x02() + { + var random = new byte[4 * Threshold]; + new Random(7).NextBytes(random); + + var (id, bytes) = Write(random); + + id.Should().Be(0x02); + Read(id, bytes).Should().Equal(random); + } + + [Fact] + public void Compressed_value_round_trips() + { + var value = Enumerable.Range(0, 2000).Select(i => $"row-{i % 17}").ToList(); + + var (id, bytes) = Write(value); + + id.Should().Be(0x03); + Read>(id, bytes).Should().Equal(value); + } + + [Fact] + public void A_negative_threshold_is_rejected() + { + var act = () => new MessagePackValueSerializer(new MessagePackValueSerializerOptions { CompressionThresholdBytes = -1 }); + + act.Should().Throw(); + } + + [Fact] + public void A_truncated_compressed_value_throws() + { + var (id, bytes) = Write(Enumerable.Range(0, 2000).Select(i => $"row-{i % 17}").ToList()); + + var act = () => Read>(id, bytes[..(bytes.Length / 2)]); + + act.Should().Throw(); + } + + [Fact] + public void A_compressed_value_that_claims_int_MaxValue_is_rejected_by_the_ratio() + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { MaxDecompressedBytes = int.MaxValue }); + + var act = () => serializer.Deserialize(0x03, new ReadOnlySequence(Hostile(int.MaxValue, 3))); + + act.Should().Throw().WithMessage("*claims*"); + } + + [Fact] + public void A_claim_just_over_255_times_the_compressed_size_is_rejected() + { + var size = Hostile(1_000_000, 1000).Length; + + var act = () => Read(0x03, Hostile((size * 255L) + 1, 1000)); + + act.Should().Throw().WithMessage("*claims*"); + } + + [Fact] + public void A_claim_of_exactly_255_times_the_compressed_size_passes_the_guard() + { + var size = Hostile(1_000_000, 1000).Length; + + var act = () => Read(0x03, Hostile(size * 255L, 1000)); + + act.Should().Throw(); + } + + [Fact] + public void A_zero_filled_value_with_a_ratio_over_100x_round_trips() + { + var zeros = new byte[16 * 1024 * 1024]; + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions()); + var target = new ArrayBufferWriter(); + + var id = serializer.Serialize(zeros, target); + + id.Should().Be(0x03); + (zeros.Length / target.WrittenCount).Should().BeGreaterThan(100); + serializer.Deserialize(id, new ReadOnlySequence(target.WrittenMemory)).Should().Equal(zeros); + } + + [Fact] + public void A_claim_above_MaxDecompressedBytes_is_rejected_though_the_ratio_is_fine() + { + var target = new ArrayBufferWriter(); + var id = _serializer.Serialize(new string('a', 10_000), target); + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { MaxDecompressedBytes = 1000 }); + + var act = () => serializer.Deserialize(id, new ReadOnlySequence(target.WrittenMemory)); + + id.Should().Be(0x03); + act.Should().Throw(); + } + + [Fact] + public void A_value_above_MaxDecompressedBytes_is_written_plain_so_it_can_be_read_back() + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { MaxDecompressedBytes = 4096 }); + var value = new string('a', 10_000); + var target = new ArrayBufferWriter(); + + var id = serializer.Serialize(value, target); + + id.Should().Be(0x02); + serializer.Deserialize(id, new ReadOnlySequence(target.WrittenMemory)).Should().Be(value); + } + + [Fact] + public void A_compressible_value_above_the_64_MiB_default_round_trips_when_the_limit_is_raised() + { + var zeros = new byte[70 * 1024 * 1024]; + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { MaxDecompressedBytes = 128 * 1024 * 1024 }); + var target = new ArrayBufferWriter(); + + var id = serializer.Serialize(zeros, target); + + id.Should().Be(0x03); + serializer.Deserialize(id, new ReadOnlySequence(target.WrittenMemory)).Should().Equal(zeros); + } + + [Fact] + public void MaxDecompressedBytes_defaults_to_64_MiB_and_must_be_positive() + { + new MessagePackValueSerializerOptions().MaxDecompressedBytes.Should().Be(64 * 1024 * 1024); + var act = () => new MessagePackValueSerializer(new MessagePackValueSerializerOptions { MaxDecompressedBytes = 0 }); + + act.Should().Throw(); + } + + [Theory] + [InlineData(0)] + [InlineData(1)] + public void A_threshold_of_zero_or_one_tries_every_value(int threshold) + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { CompressionThresholdBytes = threshold }); + var target = new ArrayBufferWriter(); + + var id = serializer.Serialize(new string('a', 100), target); + + id.Should().Be(0x03); + serializer.Deserialize(id, new ReadOnlySequence(target.WrittenMemory)).Should().Be(new string('a', 100)); + } + + [Fact] + public void A_threshold_of_zero_keeps_a_value_that_does_not_shrink_as_plain() + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { CompressionThresholdBytes = 0 }); + var target = new ArrayBufferWriter(); + + var id = serializer.Serialize(7, target); + + id.Should().Be(0x02); + target.WrittenCount.Should().Be(1); + } + + [Fact] + public void A_threshold_of_int_MaxValue_never_compresses_yet_reads_both_formats() + { + var never = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { CompressionThresholdBytes = int.MaxValue }); + var value = new string('a', 10 * Threshold); + var (compressedId, compressed) = Write(value); + var target = new ArrayBufferWriter(); + + var id = never.Serialize(value, target); + + compressedId.Should().Be(0x03); + id.Should().Be(0x02); + never.Deserialize(compressedId, new ReadOnlySequence(compressed)).Should().Be(value); + } + + [Fact] + public void A_large_compressible_value_is_0x03_and_a_large_incompressible_one_is_0x02() + { + var random = new byte[10 * Threshold]; + new Random(3).NextBytes(random); + + Write(new string('a', 10 * Threshold)).Id.Should().Be(0x03); + Write(random).Id.Should().Be(0x02); + } + + [Fact] + public void The_compressed_bytes_are_what_MessagePack_reads_as_an_Lz4BlockArray() + { + var value = new string('a', 10 * Threshold); + var (id, bytes) = Write(value); + + var read = MessagePackSerializer.Deserialize(bytes, MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray), TestContext.Current.CancellationToken); + + id.Should().Be(0x03); + read.Should().Be(value); + } + + [Fact] + public void A_compressed_value_reads_from_a_sequence_of_small_segments() + { + var value = Enumerable.Range(0, 2000).Select(i => $"row-{i % 17}").ToList(); + var (id, bytes) = Write(value); + + var read = _serializer.Deserialize>(id, Segmented(bytes, 7)); + + id.Should().Be(0x03); + read.Should().Equal(value); + } + + [Fact] + public void A_claim_above_the_default_MaxDecompressedBytes_is_rejected_though_the_ratio_is_fine() + { + var claim = (64L * 1024 * 1024) + 1; + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions()); + + var act = () => serializer.Deserialize(0x03, new ReadOnlySequence(Hostile(claim, (int)(claim / 200)))); + + act.Should().Throw().WithMessage("*claims*"); + } + + [Fact] + public void A_128_KB_compressible_value_compresses_as_one_LZ4_block() + { + var value = string.Concat(Enumerable.Range(0, 16_000).Select(i => $"row-{i % 17};")); + var (id, bytes) = Write(value); + + var reader = new MessagePackReader(bytes); + id.Should().Be(0x03); + reader.ReadArrayHeader().Should().Be(2); + reader.ReadExtensionFormatHeader().TypeCode.Should().Be(98); + } + + [Fact] + public void A_compressed_value_with_fewer_than_one_block_is_rejected() + { + var bytes = Build((ref MessagePackWriter w) => + { + w.WriteArrayHeader(1); + w.WriteExtensionFormatHeader(new ExtensionHeader(98, 0)); + w.Write(new byte[3]); + }); + + var act = () => Read(0x03, bytes); + + act.Should().Throw(); + } + + [Fact] + public void A_compressed_value_whose_first_element_is_not_an_extension_is_rejected() + { + var bytes = Build((ref MessagePackWriter w) => + { + w.WriteArrayHeader(2); + w.Write(5); + w.Write(new byte[3]); + }); + + var act = () => Read(0x03, bytes); + + act.Should().Throw(); + } + + [Fact] + public void A_compressed_value_with_another_extension_type_code_is_rejected() + { + var bytes = Build((ref MessagePackWriter w) => + { + w.WriteArrayHeader(2); + w.WriteExtensionFormatHeader(new ExtensionHeader(99, 1)); + w.WriteRaw([0x05]); + w.Write(new byte[3]); + }); + + var act = () => Read(0x03, bytes); + + act.Should().Throw(); + } + + [Fact] + public void A_compressed_value_with_a_negative_block_length_is_rejected() + { + var bytes = Build((ref MessagePackWriter w) => + { + w.WriteArrayHeader(2); + w.WriteExtensionFormatHeader(new ExtensionHeader(98, 1)); + w.WriteRaw([0xFF]); + w.Write(new byte[3]); + }); + + var act = () => Read(0x03, bytes); + + act.Should().Throw(); + } + + [Fact] + public void A_compressed_value_whose_block_lengths_overflow_when_summed_is_rejected() + { + var lengths = Build((ref MessagePackWriter w) => + { + w.Write(long.MaxValue); + w.Write(long.MaxValue); + }); + var bytes = Build((ref MessagePackWriter w) => + { + w.WriteArrayHeader(3); + w.WriteExtensionFormatHeader(new ExtensionHeader(98, (uint)lengths.Length)); + w.WriteRaw(lengths); + w.Write(new byte[3]); + w.Write(new byte[3]); + }); + + var act = () => Read(0x03, bytes); + + act.Should().Throw(); + } + + [Theory] + [InlineData(1)] + [InlineData(3)] + public void A_compressed_value_whose_extension_does_not_match_its_block_count_is_rejected(int lengthCount) + { + var lengths = Build((ref MessagePackWriter w) => + { + for (var i = 0; i < lengthCount; i++) + { + w.Write(100); + } + }); + var bytes = Build((ref MessagePackWriter w) => + { + w.WriteArrayHeader(3); + w.WriteExtensionFormatHeader(new ExtensionHeader(98, (uint)lengths.Length)); + w.WriteRaw(lengths); + w.Write(new byte[3]); + w.Write(new byte[3]); + }); + + var act = () => Read(0x03, bytes); + + act.Should().Throw(); + } + + [Fact] + public void A_compressed_value_whose_first_block_is_not_binary_is_rejected() + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + writer.WriteArrayHeader(2); + writer.WriteExtensionFormatHeader(new ExtensionHeader(98, 1)); + writer.WriteRaw([0x05]); + writer.Write("not a block"); + writer.Flush(); + + var act = () => Read(0x03, buffer.WrittenMemory.ToArray()); + + act.Should().Throw(); + } + + private static byte[] Build(WriteAction write) + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + write(ref writer); + writer.Flush(); + return buffer.WrittenMemory.ToArray(); + } + + private static byte[] Hostile(long claim, int blockLength) + { + var buffer = new ArrayBufferWriter(); + var writer = new MessagePackWriter(buffer); + var lengths = new ArrayBufferWriter(); + var lengthWriter = new MessagePackWriter(lengths); + lengthWriter.Write(claim); + lengthWriter.Flush(); + writer.WriteArrayHeader(2); + writer.WriteExtensionFormatHeader(new ExtensionHeader(98, (uint)lengths.WrittenCount)); + writer.WriteRaw(lengths.WrittenSpan); + writer.Write(new byte[blockLength]); + writer.Flush(); + return buffer.WrittenMemory.ToArray(); + } + + private static ReadOnlySequence Segmented(byte[] bytes, int size) + { + Segment? first = null; + Segment? last = null; + for (var offset = 0; offset < bytes.Length; offset += size) + { + var next = new Segment(bytes.AsMemory(offset, Math.Min(size, bytes.Length - offset))); + if (last is null) + { + first = next; + } + else + { + last.Append(next); + } + + last = next; + } + + return new ReadOnlySequence(first!, 0, last!, last!.Memory.Length); + } + + private (int Id, byte[] Bytes) Write(T value) + { + var target = new ArrayBufferWriter(); + var id = _serializer.Serialize(value, target); + return (id, target.WrittenMemory.ToArray()); + } + + private T? Read(int id, byte[] bytes) => _serializer.Deserialize(id, new ReadOnlySequence(bytes)); + + private sealed class Segment : ReadOnlySequenceSegment + { + public Segment(ReadOnlyMemory memory) => Memory = memory; + + public void Append(Segment next) + { + next.RunningIndex = RunningIndex + Memory.Length; + Next = next; + } + } +} diff --git a/tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs b/tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs new file mode 100644 index 00000000..8c4582a2 --- /dev/null +++ b/tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs @@ -0,0 +1,212 @@ +using Microsoft.Extensions.DependencyInjection; +using StackExchange.Redis; +using UiPath.Caching.Config; +using UiPath.Caching.MessagePack; +using UiPath.Caching.Tests.Redis; + +namespace UiPath.Caching.Tests.MessagePack; + +[Collection("RedisIntegration")] +[Trait("Category", "Integration")] +public sealed class MessagePackIntegrationTests(RedisContainerFixture fixture) : IDisposable +{ + private const string Policy = "mp"; + + private readonly string _app = $"mp{Guid.NewGuid():N}"[..10]; + private readonly List _nodes = []; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task Key_switches_formats_over_time() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var cache = Cache(Node("a"), KnownCacheProviderNames.Redis); + var mp = PolicyOf(_nodes[0]); + var key = Unique(); + + await cache.SetAsync(key, new Profile("json", 1), policy: null, Ct); + (await FirstByte(key)).Should().Be((byte)'{'); + (await cache.GetAsync(key, mp, Ct)).Should().Be(new Profile("json", 1)); + + await cache.SetAsync(key, new Profile("msgpack", 2), mp, Ct); + (await FirstByte(key)).Should().Be(CacheValueFormats.MessagePack); + (await cache.GetAsync(key, policy: null, Ct)).Should().Be(new Profile("msgpack", 2)); + + await cache.SetAsync(key, new Profile("json again", 3), policy: null, Ct); + (await FirstByte(key)).Should().Be((byte)'{'); + (await cache.GetAsync(key, mp, Ct)).Should().Be(new Profile("json again", 3)); + } + + [Fact] + public async Task A_large_compressible_value_is_stored_compressed_and_reads_back() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var cache = Cache(Node("a"), KnownCacheProviderNames.Redis); + var key = Unique(); + var value = new Profile(new string('x', 50_000), 7); + + await cache.SetAsync(key, value, PolicyOf(_nodes[0]), Ct); + + (await FirstByte(key)).Should().Be(CacheValueFormats.MessagePackLz4); + (await cache.GetAsync(key, policy: null, Ct)).Should().Be(value); + } + + [Fact] + public async Task Multilayer_peer_invalidation_works_with_msgpack() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var writer = Cache(Node("a"), KnownCacheProviderNames.InMemoryRedis); + var reader = Cache(Node("b"), KnownCacheProviderNames.InMemoryRedis); + var mp = PolicyOf(_nodes[0]); + var key = Unique(); + await writer.SetAsync(key, new Profile("first", 1), mp, Ct); + (await reader.GetAsync(key, mp, Ct)).Should().Be(new Profile("first", 1)); + + await writer.SetAsync(key, new Profile("second", 2), mp, Ct); + + (await UntilAsync(async () => await reader.GetAsync(key, mp, Ct) == new Profile("second", 2))) + .Should().BeTrue("the peer's invalidation drops the reader's copy and it reads the new value from Redis"); + } + + [Fact] + public async Task Hash_cache_round_trips_with_msgpack() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var node = Node("a"); + var hashes = Hashes(node, KnownCacheProviderNames.Redis); + var mp = PolicyOf(node); + var key = Unique(); + var fields = new Dictionary { ["a"] = new("one", 1), ["b"] = new(new string('y', 5_000), 2) }; + var options = new HashCacheEntryOptions(TimeToLive: TimeSpan.FromMinutes(5), Metadata: new Dictionary { ["m"] = "v" }); + + (await hashes.SetAsync(key, fields, options, mp, Ct)).Should().BeTrue(); + var entry = await hashes.GetCacheEntryAsync(key, mp, Ct); + + entry.Value.Should().BeEquivalentTo(fields); + entry.Metadata.Should().BeEquivalentTo(options.Metadata); + (await RawField(key, "a"))![0].Should().Be(CacheValueFormats.MessagePack); + (await RawField(key, "b"))![0].Should().Be(CacheValueFormats.MessagePackLz4); + } + + [Fact] + public async Task Batch_set_and_get_with_msgpack() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var node = Node("a"); + var cache = Cache(node, KnownCacheProviderNames.Redis); + var mp = PolicyOf(node); + CacheKey[] keys = [Unique(), Unique(), Unique()]; + Profile[] values = [new("small", 1), new(new string('z', 4_000), 2), new("other", 3)]; + + await cache.SetAsync([.. keys.Zip(values, (k, v) => new KeyValuePair(k, v))], mp, Ct); + + (await FirstByte(keys[0])).Should().Be(CacheValueFormats.MessagePack); + (await FirstByte(keys[1])).Should().Be(CacheValueFormats.MessagePackLz4); + (await cache.GetAsync(keys, mp, Ct)).Select(v => v.Value).Should().Equal(values); + (await cache.GetAsync(keys, policy: null, Ct)).Select(v => v.Value).Should().Equal(values); + } + + [Fact] + public async Task Peer_field_update_with_msgpack_is_applied() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var writer = Hashes(Node("a", peerFieldUpdates: true, defaultPolicy: true), KnownCacheProviderNames.InMemoryRedis); + var peer = Hashes(Node("b", peerFieldUpdates: true, defaultPolicy: true), KnownCacheProviderNames.InMemoryRedis); + var key = Unique(); + (await writer.SetAsync(key, Fields(("a", "1")), TimeSpan.FromMinutes(10), Ct)).Should().BeTrue(); + (await peer.GetAsync(key, Ct)).Should().HaveCount(1); + await Task.Delay(1000, Ct); + + (await writer.SetAsync(key, Fields(("b", "2")), new HashCacheEntryOptions(TimeToLive: TimeSpan.FromMinutes(10), SetOption: HashCacheSetOption.HashReplace), Ct)).Should().BeTrue(); + + (await UntilAsync(async () => (await peer.GetAsync(key, Ct)).ContainsKey("b"))).Should().BeTrue("the peer applies the field from the event"); + (await peer.GetAsync(key, Ct)).Keys.Should().BeEquivalentTo(["a", "b"]); + (await RawField(key, "b"))![0].Should().Be(CacheValueFormats.MessagePack); + } + + public void Dispose() + { + foreach (var node in _nodes) + { + node.Dispose(); + } + } + + private static string Unique() => $"it-{Guid.NewGuid():N}"; + + private static ICache Cache(ServiceProvider node, string provider) => + node.GetRequiredService().CreateCache(provider); + + private static IHashCache Hashes(ServiceProvider node, string provider) => + node.GetRequiredService().CreateHashCache(provider); + + private static CachePolicy PolicyOf(ServiceProvider node) => node.GetRequiredService().Resolve(Policy)!; + + private static Dictionary Fields(params (string Field, string Value)[] fields) => + fields.ToDictionary(f => f.Field, f => (Profile?)new Profile(f.Value, int.Parse(f.Value, System.Globalization.CultureInfo.InvariantCulture))); + + private static async Task UntilAsync(Func> condition) + { + var until = DateTime.UtcNow.AddSeconds(15); + while (DateTime.UtcNow < until) + { + if (await condition()) + { + return true; + } + + await Task.Delay(50, Ct); + } + + return false; + } + + private async Task FirstByte(string key) + { + await using var redis = await ConnectionMultiplexer.ConnectAsync(fixture.ConnectionString); + var found = redis.GetServer(redis.GetEndPoints()[0]).Keys(pattern: $"*{key}*").Single(); + return ((byte[])(await redis.GetDatabase().StringGetAsync(found))!)[0]; + } + + private async Task RawField(string key, string field) + { + await using var redis = await ConnectionMultiplexer.ConnectAsync(fixture.ConnectionString); + var found = redis.GetServer(redis.GetEndPoints()[0]).Keys(pattern: $"*{key}*").Single(); + return await redis.GetDatabase().HashGetAsync(found, field); + } + + private ServiceProvider Node(string source, bool peerFieldUpdates = false, bool defaultPolicy = false) + { + var provider = new ServiceCollection() + .AddCaching( + b => + { + b.AddRedisConnection(o => o.ConnectionString = fixture.ConnectionString); + b.AddRedis(_ => { }); + b.AddBroadcast(); + b.AddInMemoryRedis(o => o.PeerFieldUpdates = peerFieldUpdates); + b.AddMessagePackSerializer(); + }, + o => + { + o.AppShortName = _app; + o.SourceUri = new Uri($"urn:{source}-{_app}"); + var policy = new CachePolicy + { + DistributedExpiration = TimeSpan.FromMinutes(5), + Serializer = new SerializerPolicy { Write = MessagePackCachingBuilderExtensions.DefaultName }, + }; + o.Policies[Policy] = policy; + if (defaultPolicy) + { + o.DefaultCachePolicy = policy; + } + }) + .BuildServiceProvider(); + _nodes.Add(provider); + return provider; + } + + public sealed record Profile(string Name, int Version); +} diff --git a/tests/UiPath.Caching.Tests/MessagePack/MessagePackOutOfMemoryTests.cs b/tests/UiPath.Caching.Tests/MessagePack/MessagePackOutOfMemoryTests.cs new file mode 100644 index 00000000..eb551e56 --- /dev/null +++ b/tests/UiPath.Caching.Tests/MessagePack/MessagePackOutOfMemoryTests.cs @@ -0,0 +1,126 @@ +using System.Buffers; +using MessagePack; +using MessagePack.Formatters; +using MessagePack.Resolvers; +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using UiPath.Caching.Config; +using UiPath.Caching.MessagePack; +using UiPath.Caching.Policies; +using UiPath.Caching.Serialization; +using UiPath.Caching.Telemetry; +using UiPath.Caching.Tests.Fakes; + +namespace UiPath.Caching.Tests.MessagePack; + +[Collection(WriterPoolCollection.Name)] +public class MessagePackOutOfMemoryTests(ITestContextAccessor testContextAccessor) +{ + private static readonly byte[] EmptyArray = [0x02, 0x90]; + + private readonly MessagePackValueSerializer _serializer = new(new MessagePackValueSerializerOptions + { + SerializerOptions = MessagePackSerializerOptions.Standard.WithResolver( + CompositeResolver.Create([new ExhaustedFormatter()], [ContractlessStandardResolver.Instance])), + }); + + private CancellationToken Token => testContextAccessor.Current.CancellationToken; + + [Fact] + public void Deserializing_rethrows_an_out_of_memory_failure_instead_of_a_serialization_error() + { + var act = () => _serializer.Deserialize(0x02, new ReadOnlySequence(EmptyArray)); + + act.Should().Throw(); + } + + [Fact] + public void Serializing_rethrows_an_out_of_memory_failure_instead_of_a_serialization_error() + { + var act = () => _serializer.Serialize(new Exhausted(), new ArrayBufferWriter()); + + act.Should().Throw(); + } + + [Fact] + public void Serializing_without_compression_rethrows_an_out_of_memory_failure() + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions + { + CompressionThresholdBytes = int.MaxValue, + SerializerOptions = _serializer.Options, + }); + + var act = () => serializer.Serialize(new Exhausted(), new ArrayBufferWriter()); + + act.Should().Throw(); + } + + [Fact] + public void A_corrupt_value_is_still_a_serialization_error() + { + var act = () => _serializer.Deserialize(0x02, new ReadOnlySequence([0x02, 0xC1])); + + act.Should().Throw(); + } + + [Fact] + public async Task Get_and_GetOrAdd_propagate_the_failure_and_do_not_run_the_generator() + { + var database = Substitute.For(); + database.StringGetAsync(Arg.Any(), Arg.Any()).Returns((RedisValue)EmptyArray); + var sut = Create(database); + var generated = 0; + Task Generate(CancellationToken token) + { + generated++; + return Task.FromResult(new Exhausted()); + } + + await FluentActions.Awaiting(() => sut.GetAsync("k", policy: null, Token).AsTask()).Should().ThrowAsync(); + await FluentActions.Awaiting(() => sut.GetOrAddAsync("k", Generate, TimeSpan.FromMinutes(1), policy: null, Token).AsTask()).Should().ThrowAsync(); + + generated.Should().Be(0); + } + + private RedisCache Create(IDatabase database) + { + var connector = Substitute.For(); + connector.Database.Returns(database); + connector.Version.Returns(new Version(6, 0)); + connector.IsConnected.Returns(true); + database.Multiplexer.GetHashSlot(Arg.Any()).Returns(0); + var pipelines = Substitute.For(); + pipelines.Get(Arg.Any()).Returns(EmptyResiliencePipeline.Instance); + var strategy = Substitute.For(); + strategy.GetRedisKey(Arg.Any()).Returns(call => (RedisKey)$"test:str:{call.Arg()}"); + var strategies = Substitute.For(); + strategies.Create(Arg.Any(), Arg.Any()).Returns(strategy); + var registry = new ValueSerializerRegistry([("messagepack", _serializer, true)]); + + return new RedisCache( + connector, + new SystemJsonByteSerializerProxy(), + pipelines, + NullTelemetryProvider.Instance, + new RedisCacheOptions { RedisKeyStrategyFactory = strategies, ReadScriptsEnabled = false }, + new CacheOptions { AppShortName = "test", ConnectionMonitorEnabled = true }, + new DefaultCachePolicyFactory(null, null, valueSerializers: registry), + TimeProvider.System, + NullLogger.Instance) + { + ValueSerializers = registry, + }; + } + + public sealed class Exhausted + { + } + + public sealed class ExhaustedFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, Exhausted? value, MessagePackSerializerOptions options) => throw new OutOfMemoryException(); + + public Exhausted? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) => throw new OutOfMemoryException(); + } +} diff --git a/tests/UiPath.Caching.Tests/MessagePack/MessagePackValueSerializerTests.cs b/tests/UiPath.Caching.Tests/MessagePack/MessagePackValueSerializerTests.cs new file mode 100644 index 00000000..2eb63bcb --- /dev/null +++ b/tests/UiPath.Caching.Tests/MessagePack/MessagePackValueSerializerTests.cs @@ -0,0 +1,276 @@ +using System.Buffers; +using System.Text.Json; +using MessagePack; +using MessagePack.Formatters; +using MessagePack.Resolvers; +using UiPath.Caching.MessagePack; + +namespace UiPath.Caching.Tests.MessagePack; + +public class MessagePackValueSerializerTests +{ + private readonly MessagePackValueSerializer _serializer = new(new MessagePackValueSerializerOptions()); + + [Fact] + public void Round_trips_a_contractless_type() + { + var value = Plain.Sample(); + + var (id, bytes) = Write(value); + + id.Should().Be(CacheValueFormats.MessagePack); + Read(id, bytes).Should().BeEquivalentTo(value); + } + + [Fact] + public void Round_trips_an_int_key_type() + { + var value = new Keyed { Name = "n", Count = 7 }; + + var (id, bytes) = Write(value); + + Read(id, bytes).Should().BeEquivalentTo(value); + } + + [Fact] + public void Null_and_empty_collections_round_trip_with_non_empty_payload() + { + var (nullId, nullBytes) = Write(null); + var (listId, listBytes) = Write(new List()); + var (mapId, mapBytes) = Write(new Dictionary()); + + nullBytes.Should().NotBeEmpty(); + listBytes.Should().NotBeEmpty(); + mapBytes.Should().NotBeEmpty(); + Read(nullId, nullBytes).Should().BeNull(); + Read>(listId, listBytes).Should().BeEmpty(); + Read>(mapId, mapBytes).Should().BeEmpty(); + } + + [Fact] + public void Byte_array_round_trips() + { + byte[] value = [0x02, 0, 1, 255]; + + var (id, bytes) = Write(value); + + Read(id, bytes).Should().Equal(value); + } + + [Fact] + public void Value_is_smaller_than_json() + { + var value = Plain.Sample(); + + var (_, bytes) = Write(value); + + bytes.Length.Should().BeLessThan((int)(JsonSerializer.SerializeToUtf8Bytes(value).Length * 0.7)); + } + + [Fact] + public void Deeply_nested_data_is_rejected() + { + var nested = new ArrayBufferWriter(); + var writer = new MessagePackWriter(nested); + for (var i = 0; i < 1000; i++) + { + writer.WriteArrayHeader(1); + } + + writer.Write(1); + writer.Flush(); + + var act = () => Read(CacheValueFormats.MessagePack, nested.WrittenMemory.ToArray()); + + act.Should().Throw(); + } + + [Fact] + public void Default_options_treat_the_data_as_untrusted() + { + _serializer.Options.Security.Should().BeSameAs(ResilientMessagePackSecurity.Instance); + _serializer.Options.Security.HashCollisionResistant.Should().BeTrue(); + } + + [Fact] + public void Supplied_options_that_carry_trusted_data_are_made_untrusted() + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { SerializerOptions = MessagePackSerializerOptions.Standard }); + + serializer.Options.Security.Should().BeSameAs(ResilientMessagePackSecurity.Instance); + } + + [Fact] + public void A_custom_security_instance_is_kept() + { + var custom = MessagePackSecurity.TrustedData.WithMaximumObjectGraphDepth(10); + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions + { + SerializerOptions = MessagePackSerializerOptions.Standard.WithSecurity(custom), + }); + + serializer.Options.Security.Should().BeSameAs(custom); + serializer.Options.Security.MaximumObjectGraphDepth.Should().Be(10); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Collections_keyed_by_types_without_a_resistant_comparer_round_trip(bool trustedOptions) + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions + { + SerializerOptions = trustedOptions ? MessagePackSerializerOptions.Standard.WithResolver(ContractlessStandardResolver.Instance) : null, + }); + var spans = new Dictionary { [TimeSpan.FromSeconds(3)] = 1 }; + var decimals = new Dictionary { [1.5m] = 2 }; + var tuples = new HashSet<(int, int)> { (1, 2) }; + + Round(serializer, spans).Should().Equal(spans); + Round(serializer, decimals).Should().Equal(decimals); + Round(serializer, tuples).Should().BeEquivalentTo(tuples); + } + + [Fact] + public void String_keys_still_use_the_resistant_comparer() + { + _serializer.Options.Security.GetEqualityComparer().Should().NotBeSameAs(EqualityComparer.Default); + _serializer.Options.Security.HashCollisionResistant.Should().BeTrue(); + } + + [Fact] + public void Compression_set_on_the_supplied_options_is_overridden() + { + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions + { + CompressionThresholdBytes = int.MaxValue, + SerializerOptions = MessagePackSerializerOptions.Standard.WithCompression(MessagePackCompression.Lz4BlockArray).WithCompressionMinLength(1), + }); + var target = new ArrayBufferWriter(); + + var id = serializer.Serialize(new string('a', 10_000), target); + + id.Should().Be(CacheValueFormats.MessagePack); + serializer.Options.Compression.Should().Be(MessagePackCompression.None); + target.WrittenCount.Should().BeGreaterThan(10_000); + } + + [Fact] + public void Wrong_id_payloads_are_rejected() + { + var big = new string('a', 10_000); + var compressed = new ArrayBufferWriter(); + var compressedId = _serializer.Serialize(big, compressed); + var (plainId, plainBytes) = Write("small"); + + compressedId.Should().Be(CacheValueFormats.MessagePackLz4); + plainId.Should().Be(CacheValueFormats.MessagePack); + var asPlain = () => Read(CacheValueFormats.MessagePack, compressed.WrittenMemory.ToArray()); + var asCompressed = () => Read(CacheValueFormats.MessagePackLz4, plainBytes); + asPlain.Should().Throw(); + asCompressed.Should().Throw(); + } + + [Fact] + public void A_plain_value_that_starts_with_the_compression_extension_is_stored_as_plain() + { + var options = MessagePackSerializerOptions.Standard.WithResolver(CompositeResolver.Create([new LookalikeFormatter()], [StandardResolver.Instance])); + var serializer = new MessagePackValueSerializer(new MessagePackValueSerializerOptions { SerializerOptions = options }); + var target = new ArrayBufferWriter(); + + var id = serializer.Serialize(new Lookalike(), target); + + id.Should().Be(CacheValueFormats.MessagePack); + target.WrittenSpan[..2].ToArray().Should().Equal(0x92, 0xD4); + serializer.Deserialize(id, new ReadOnlySequence(target.WrittenMemory.ToArray())).Should().NotBeNull(); + } + + [Fact] + public void Large_value_round_trips() + { + var value = new Plain { Name = new string('x', 200 * 1024), Numbers = [1, 2, 3] }; + + var (id, bytes) = Write(value); + + Read(id, bytes).Should().BeEquivalentTo(value); + } + + [Fact] + public void An_id_it_does_not_own_throws() + { + var (_, bytes) = Write(Plain.Sample()); + + var act = () => Read(0x10, bytes); + + act.Should().Throw(); + } + + private static T? Round(MessagePackValueSerializer serializer, T value) + { + var target = new ArrayBufferWriter(); + var id = serializer.Serialize(value, target); + return serializer.Deserialize(id, new ReadOnlySequence(target.WrittenMemory)); + } + + private (int Id, byte[] Bytes) Write(T value) + { + var target = new ArrayBufferWriter(); + var id = _serializer.Serialize(value, target); + return (id, target.WrittenMemory.ToArray()); + } + + private T? Read(int id, byte[] bytes) => _serializer.Deserialize(id, new ReadOnlySequence(bytes)); + + public sealed class Plain + { + public string? Name { get; set; } + + public int[]? Numbers { get; set; } + + public double[]? Measures { get; set; } + + public DateTime When { get; set; } + + public Guid Id { get; set; } + + public static Plain Sample() => new() + { + Name = "a medium sized object", + Numbers = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10], + Measures = [.. Enumerable.Range(0, 10).Select(i => Math.PI * (i + 1) / 7)], + When = new DateTime(2026, 1, 2, 3, 4, 5, DateTimeKind.Utc), + Id = Guid.NewGuid(), + }; + } + + public sealed class Lookalike + { + } + + public sealed class LookalikeFormatter : IMessagePackFormatter + { + public void Serialize(ref MessagePackWriter writer, Lookalike? value, MessagePackSerializerOptions options) + { + writer.WriteArrayHeader(2); + writer.WriteExtensionFormatHeader(new ExtensionHeader(98, 1)); + writer.WriteRaw([0x05]); + writer.Write("not a block"); + } + + public Lookalike? Deserialize(ref MessagePackReader reader, MessagePackSerializerOptions options) + { + reader.Skip(); + return new Lookalike(); + } + } + + [MessagePackObject] + public sealed class Keyed + { + [Key(0)] + public string? Name { get; set; } + + [Key(1)] + public int Count { get; set; } + } +} diff --git a/tests/UiPath.Caching.Tests/MessagePack/PooledBufferWriterTests.cs b/tests/UiPath.Caching.Tests/MessagePack/PooledBufferWriterTests.cs new file mode 100644 index 00000000..5e664208 --- /dev/null +++ b/tests/UiPath.Caching.Tests/MessagePack/PooledBufferWriterTests.cs @@ -0,0 +1,72 @@ +using System.Buffers; +using UiPath.Caching.MessagePack; + +namespace UiPath.Caching.Tests.MessagePack; + +public class PooledBufferWriterTests +{ + [Fact] + public void Advance_rejects_a_negative_count() + { + using var writer = new PooledBufferWriter(); + + var act = () => writer.Advance(-1); + + act.Should().Throw(); + } + + [Fact] + public void Advance_rejects_a_count_past_the_buffer() + { + using var writer = new PooledBufferWriter(); + var capacity = writer.GetSpan().Length; + + var act = () => writer.Advance(capacity + 1); + + act.Should().Throw(); + } + + [Fact] + public void Dispose_clears_the_array_it_returns() + { + var writer = new PooledBufferWriter(); + var span = writer.GetSpan(); + span.Fill(0x55); + writer.Advance(span.Length); + var capacity = span.Length; + + writer.Dispose(); + + var rented = ArrayPool.Shared.Rent(capacity); + try + { + rented.Should().OnlyContain(b => b == 0); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } + + [Fact] + public void Growing_clears_the_array_it_replaces() + { + using var writer = new PooledBufferWriter(); + var span = writer.GetSpan(); + span.Fill(0x55); + writer.Advance(span.Length); + var capacity = span.Length; + + writer.GetSpan(1); + + var rented = ArrayPool.Shared.Rent(capacity); + try + { + rented.Should().OnlyContain(b => b == 0); + } + finally + { + ArrayPool.Shared.Return(rented); + } + } +} diff --git a/tests/UiPath.Caching.Tests/UiPath.Caching.Tests.csproj b/tests/UiPath.Caching.Tests/UiPath.Caching.Tests.csproj index 13592829..d973624b 100644 --- a/tests/UiPath.Caching.Tests/UiPath.Caching.Tests.csproj +++ b/tests/UiPath.Caching.Tests/UiPath.Caching.Tests.csproj @@ -39,6 +39,7 @@ +