From 19fd942fd26aa69388f5217e1c6be0157402bd6a Mon Sep 17 00:00:00 2001 From: Cosmin Staicu Date: Fri, 13 Nov 2026 15:18:18 +0200 Subject: [PATCH] feat(negotiation): switch the value format once every node can read it A policy's non-JSON value format is now written only once every live node is known to read it. The switch happens in one deploy with no operator step. Every node with a non-JSON serializer registered advertises, every 15 seconds, the formats its reads decode, stamped with Redis TIME in a hash. A policy whose reads do not look for a header advertises json only, which blocks the switch. Reader-only and Force nodes advertise too, and a stopping node marks its advertisement as left. One decision is kept per registered serializer, refreshed every 15 seconds from the advertisements and the Streams consumer groups. The format is written when every live node advertises it and no node joined within UpgradeHold. A node that leaves or crashes keeps the format. A node that cannot read it, or a live group with no advertisement, sends writes back to JSON at once. An unreadable node list keeps the decision for MaxViewAge and then falls back to JSON. A node that cannot see its peers, as with broadcast off or Pub/Sub only, writes JSON unless AssumeAllNodesCanRead is set. Mode = Force writes the policy's format as before, and raw or custom proxy policies always write their own format. UpgradeHold defaults to 1 hour and 0 is allowed. A rollback within the hold never wrote the new format, and after the switch it makes old nodes miss and rewrite those keys as JSON, never read wrong values. FormatFor reads a cached decision with no Redis call. An app with no serializer registered writes no key, starts no timer and allocates nothing extra. New options live under RedisCacheOptions.FormatNegotiation: Mode, AdvertiseInterval, ViewInterval, LivenessWindow, UpgradeHold, MaxViewAge and AssumeAllNodesCanRead. New metrics are cache.format.effective, cache.format.fallback, cache.format.nodes and cache.format.view.age. Signed-off-by: Cosmin Staicu --- CHANGELOG.md | 1 + docs/concepts.md | 38 +- docs/how-to/extending.md | 2 +- docs/reference/settings.md | 7 + .../appsettings.all.json | 18 +- .../Redis/RedisStreamHealthMaintainer.cs | 13 +- .../Redis/RedisStreamsTopicProvider.cs | 26 +- src/UiPath.Caching/Config/CachingBuilder.cs | 8 + src/UiPath.Caching/Metrics.cs | 5 + .../Negotiation/DesiredWrite.cs | 7 + .../Negotiation/FormatNegotiation.cs | 69 +++ .../Negotiation/FormatNegotiationService.cs | 211 +++++++++ .../Negotiation/FormatNegotiator.cs | 235 ++++++++++ .../Negotiation/IEffectiveFormat.cs | 8 + .../Negotiation/IStreamGroupSource.cs | 11 + .../Negotiation/NegotiationPlan.cs | 77 ++++ .../Negotiation/NodeAdvertisement.cs | 125 ++++++ .../Negotiation/NodeListStore.cs | 179 ++++++++ .../Negotiation/NodeListView.cs | 8 + .../Negotiation/RedisStreamGroupSource.cs | 33 ++ .../Negotiation/StoredAdvertisement.cs | 4 + .../Negotiation/StreamNodeList.cs | 37 ++ .../Negotiation/TickSchedule.cs | 59 +++ src/UiPath.Caching/PublicAPI.Unshipped.txt | 25 ++ .../Redis/FormatNegotiationMode.cs | 10 + .../Redis/FormatNegotiationOptions.cs | 47 ++ src/UiPath.Caching/Redis/KeyEscaping.cs | 15 + src/UiPath.Caching/Redis/RedisCache.cs | 13 +- src/UiPath.Caching/Redis/RedisCacheOptions.cs | 9 +- .../Redis/RedisCacheProvider.cs | 11 +- src/UiPath.Caching/Redis/RedisHashCache.cs | 13 +- .../Serialization/ValueFormatResolver.cs | 19 +- .../Serialization/ValueReader.cs | 3 + .../RedisStreamsTopicProviderTests.cs | 31 ++ .../Config/AppSettingsAllJsonBindingTests.cs | 1 + .../MessagePackIntegrationTests.cs | 2 +- .../Negotiation/EffectiveWriteTests.cs | 424 ++++++++++++++++++ .../Negotiation/FakeNodeListServer.cs | 86 ++++ .../FormatNegotiationIntegrationTests.cs | 336 ++++++++++++++ .../FormatNegotiationServiceTests.cs | 346 ++++++++++++++ .../Negotiation/FormatNegotiatorTests.cs | 422 +++++++++++++++++ .../Negotiation/NodeAdvertisementTests.cs | 186 ++++++++ .../RedisStreamGroupSourceTests.cs | 24 + .../Negotiation/StreamNodeListTests.cs | 106 +++++ .../Negotiation/TickScheduleTests.cs | 45 ++ .../Redis/PeerFieldUpdateIntegrationTests.cs | 1 + .../Redis/SerializerIntegrationTests.cs | 6 +- 47 files changed, 3337 insertions(+), 25 deletions(-) create mode 100644 src/UiPath.Caching/Negotiation/DesiredWrite.cs create mode 100644 src/UiPath.Caching/Negotiation/FormatNegotiation.cs create mode 100644 src/UiPath.Caching/Negotiation/FormatNegotiationService.cs create mode 100644 src/UiPath.Caching/Negotiation/FormatNegotiator.cs create mode 100644 src/UiPath.Caching/Negotiation/IEffectiveFormat.cs create mode 100644 src/UiPath.Caching/Negotiation/IStreamGroupSource.cs create mode 100644 src/UiPath.Caching/Negotiation/NegotiationPlan.cs create mode 100644 src/UiPath.Caching/Negotiation/NodeAdvertisement.cs create mode 100644 src/UiPath.Caching/Negotiation/NodeListStore.cs create mode 100644 src/UiPath.Caching/Negotiation/NodeListView.cs create mode 100644 src/UiPath.Caching/Negotiation/RedisStreamGroupSource.cs create mode 100644 src/UiPath.Caching/Negotiation/StoredAdvertisement.cs create mode 100644 src/UiPath.Caching/Negotiation/StreamNodeList.cs create mode 100644 src/UiPath.Caching/Negotiation/TickSchedule.cs create mode 100644 src/UiPath.Caching/Redis/FormatNegotiationMode.cs create mode 100644 src/UiPath.Caching/Redis/FormatNegotiationOptions.cs create mode 100644 src/UiPath.Caching/Redis/KeyEscaping.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/EffectiveWriteTests.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/FakeNodeListServer.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationIntegrationTests.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationServiceTests.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/FormatNegotiatorTests.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/NodeAdvertisementTests.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/RedisStreamGroupSourceTests.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/StreamNodeListTests.cs create mode 100644 tests/UiPath.Caching.Tests/Negotiation/TickScheduleTests.cs diff --git a/CHANGELOG.md b/CHANGELOG.md index e621b1e4..b6cdc763 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -15,6 +15,7 @@ The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/) ### Changed +- **A policy's non-JSON `Serializer.Write` is written only while every node can read it.** With the new `RedisCacheOptions.FormatNegotiation` (`Mode` `Auto` by default), each node advertises the formats it reads in a Redis hash stamped with the server's clock and reads the node list, with the Streams consumer groups, on a 15-second timer, off the read and write paths. A policy writes its format once every fresh advertisement lists it, no live node is unaccounted for, and that has held for `UpgradeHold` (an hour by default; zero switches at once); a node that cannot read it stops the writes at once, and an unreadable node list keeps the decision for `MaxViewAge` (two minutes), then writes JSON. A node with a serializer registered advertises what it reads in either mode, even if none of its policies writes a non-JSON format, so a reader-only or `Force` node never blocks the writers (an app with no Streams transport neither advertises nor reads advertisements); in `Auto` it decides for every registered serializer, so a policy built at the call site is held to it too. An advertisement outlives the consumer group of a node that crashed, and a node that stops writes a departing advertisement, so a node that leaves never drops the fleet to JSON or restarts the hold, which a join or a restart does. After a Redis outage the decision is kept for two advertise intervals while peers advertise again. `AssumeAllNodesCanRead` covers only nodes that advertise nothing; with no Streams transport it switches after the hold with nothing that can stop it. An advertisement older than its group's last read does not account for the group, so a restart under the same id is seen at once, and a node list that fails to read restarts the hold. The advertisement follows what a node's reads really do: on a custom proxy a policy that writes no non-JSON format advertises JSON only, which keeps writers on JSON, and a policy on such a proxy always writes its configured format, since raw bytes have no JSON to fall back to and could read back as a header. Advertise and node list reads run on their own intervals, and a topic's stream is forgotten when the topic is removed. An app with no serializer registered does nothing new. An app that cannot see its peers (`BroadcastEnabled` off, or Pub/Sub only) writes JSON unless `AssumeAllNodesCanRead` is set. `Mode = Force` writes exactly what each policy configures, as before. New metrics `cache.format.effective`, `cache.format.fallback`, `cache.format.nodes` and `cache.format.view.age`. See [choosing when a format is written](docs/concepts.md#choosing-when-a-format-is-written). - **An event in a known format that cannot be decoded is a topic loss.** The Streams reader acknowledges the entry and reports a loss, and counts such an entry as one it would have acted on when it checks what a new group skipped; the Pub/Sub reader expires through the coalesced dropped-event path, off the delivery thread, so it holds for caches that keep their tier over a subscription gap. The Streams reader reports once per batch. This applies to a body whose first byte is `0x02`, `0x03` or a consumer format id (`0x10`-`0x1E`, `0x1F`), such as a binary event on a node that cannot read it. Any other body that does not decode, another producer's first byte included, is still skipped as junk. - **Cache event properties are read as typed values.** The built-in event formatters parse `_version_`, the expirations, the metadata and the peer fields straight into `long`, `DateTimeOffset`, string dictionaries and byte arrays instead of a `JsonElement` that was parsed again by the value serializer, which saves a clone, a box and a second parse per property of a peer-update event. The event bytes on the wire are unchanged. A custom `ISerializerProxy` no longer sees these properties in `TryDeserialize(object)` when the built-in formatters decode the event. - **Decoded `ICacheEvent.Data.Properties` hold typed values.** From the built-in formatter and the CloudEvents fast path they are `long`, `DateTimeOffset`, `Dictionary` and byte-valued peer fields instead of `JsonElement`, so a custom subscriber that casts a property to `JsonElement` must handle those types. diff --git a/docs/concepts.md b/docs/concepts.md index 9903e55c..ef8673ef 100644 --- a/docs/concepts.md +++ b/docs/concepts.md @@ -516,9 +516,10 @@ the key's policy is not known to the peer, it does not apply the update and read ### Rolling a serializer out -Reading comes first, writing second. Every node from 3.2.0 reads every registered format; nothing writes a +With `FormatNegotiation:Mode` at `Force`, reading comes first, writing second. Every node from 3.2.0 reads every registered format; nothing writes a new one until a policy sets `Serializer`, so set it only after every node runs 3.2.0 or later with the -serializer registered. +serializer registered. With the default `Auto`, a node that sees an older node keeps writing JSON by itself; see +[choosing when a format is written](#choosing-when-a-format-is-written). A node that cannot read a format treats the value as a miss, and what happens to the key depends on why: @@ -543,6 +544,39 @@ See [MessagePack events](how-to/extending.md#messagepack-events). All of it shows up as factory load and churn during the deploy, never as a wrong value. Switching a policy back to JSON is safe at any time: new writes are JSON and the older values still read. +### Choosing when a format is written + +A policy's `Serializer.Write` names the format it *wants*. With `FormatNegotiation:Mode` at `Auto` (the default), a node writes it +only while every node that shares the Streams transport is known to read it, and writes JSON until then; this replaces the two-deploy +order for values, so a node that registers the serializer and sets `Write` can be deployed in one step. A policy that sets `Write = "json"` +is pinned to JSON, and `Mode = Force` writes exactly what each policy configures, as before. + +How it works. An app with a serializer registered advertises what it reads, whether or not any policy of its own writes a non-JSON format and in either mode, so a node that only reads, or runs `Force`, never blocks the writers. In `Auto` it also reads the node list and decides, for every registered serializer, so a policy built at the call site that names one is held to the same decision as a registered policy. An app with no serializer registered does none of this: it advertises nothing, and so correctly counts as a node that cannot read. + +The advertisement says what the node's reads actually do. With the default JSON proxy a read looks for a format header whatever the policy writes. With a custom proxy, a subclass of the default or `RawByteSerializerProxy`, a read looks for one only under a policy that itself writes a non-JSON format; for every other policy (and the default policy, when it is not one) the node advertises that it reads JSON only, and no writer upgrades while such a node is on the node list. A policy on such a proxy never falls back to JSON either, since its bytes are not JSON and could read back as a header: it always writes the format it names, as in `Force`. + +- Each node writes an advertisement (the formats it reads, and the read list of each policy that sets one) to a hash in Redis every + 15 seconds, stamped with the server's clock, so a node's own clock never matters. It advertises first when its hosted service starts, before it reads the + node list; the service is registered with the caching, so an app hosted service that touches a cache earlier can subscribe first and be briefly unaccounted + for. An advertisement counts for `LivenessWindow` plus two advertise intervals, so it outlives the consumer group of a node that crashed. A node that stops + writes a departing advertisement (`left=1`) instead of deleting its own: its group is then a node that left, neither a reader nor an unaccounted one. +- Every node also reads that hash and the consumer groups of its Streams topics. A group has the node's source id as its name; one whose + consumer read within `LivenessWindow` and that no fresh advertisement accounts for is a node that cannot say what it reads (a + node before 3.2, for one), and it keeps every writer on JSON. A node that went quiet for longer than `LivenessWindow` no longer counts. +- The format is written once every advertisement lists it, no live node is unaccounted for, and that has held for `UpgradeHold` (an hour by default) since a node + last joined or restarted (each process advertises its own id, so a restart under the same name is a join). A node that leaves never restarts the hold. A node that cannot read it, or an unaccounted node that appears, stops the writes at once. Redis being unreachable keeps the decision + for `MaxViewAge` since the last good read, then writes JSON, and for two advertise intervals after it returns, so peers can advertise again; an advertisement that lacks the format is never excused. All of this runs on a timer; a read or a write sends no extra command, only a read of the cached decision and its age. + +The hold trades speed for rollback safety. After a rollback to a build that cannot read the format, nodes miss on the values written in that +format and the generator rewrites them as JSON, never a wrong value. With the default hour, an app rolled back within the hour never wrote the +format at all. A shorter `UpgradeHold` switches sooner and widens that rollback window; zero switches as soon as the node list agrees. + +The node list sees only nodes on the Streams transport. An app that has `BroadcastEnabled` off, or that publishes events over Pub/Sub only, cannot see the others and writes JSON; set +`FormatNegotiation:AssumeAllNodesCanRead` to attest that every node that says nothing reads the desired formats. The attestation covers nodes that advertise nothing, not one whose advertisement lacks the format, and the hold still applies. Every node judges the others by its own `AdvertiseInterval` and `LivenessWindow`, so keep them alike across the fleet. Group +names that are not node ids (for example another app sharing the stream key) also count as live nodes, a safe false positive. +Telemetry (`cache.format.effective`, `cache.format.fallback`, `cache.format.nodes` and `cache.format.view.age`) is emitted only while this runs, and +a node that is blocking is named in a Warning log. The advertisement needs a host that runs hosted services: without one, nothing advertises and values stay JSON. + See also: [how-to/extending.md#value-serializers](how-to/extending.md#value-serializers). --- diff --git a/docs/how-to/extending.md b/docs/how-to/extending.md index ebeb0f5a..0d1bd810 100644 --- a/docs/how-to/extending.md +++ b/docs/how-to/extending.md @@ -546,7 +546,7 @@ The package depends on the exact `UiPath.Caching` version it was built with, so The serializer owns format ids `0x02` (plain) and `0x03` (LZ4); the id stored with each value says which, so a node reads both whatever its threshold. -Roll it out readers first. A node reads a format only once the serializer is registered on it, so register +By default (`FormatNegotiation:Mode` `Auto`) a policy that sets `Write = "messagepack"` writes it only while every node on the Streams transport is known to read it, and JSON until then, so this is safe to deploy in one step; see [choosing when a format is written](../concepts.md#choosing-when-a-format-is-written). With `Force`, roll it out readers first. A node reads a format only once the serializer is registered on it, so register `AddMessagePackSerializer` on every node and finish that deploy before any policy sets `Write = "messagepack"`; a node that cannot read the value misses and regenerates it (see [rolling a serializer out](../concepts.md#rolling-a-serializer-out)). Setting `Write` back to `json` keeps values already stored as MessagePack readable while the default JSON proxy is in use; with a custom `ISerializerProxy` registered, headers are only recognized under a policy that writes a non-JSON format, so under `Write = "json"` those values go to your proxy and read as misses. diff --git a/docs/reference/settings.md b/docs/reference/settings.md index d8f07ab3..de885079 100644 --- a/docs/reference/settings.md +++ b/docs/reference/settings.md @@ -215,6 +215,13 @@ Per-topic overrides: add entries to `Topics[]` under `Broadcast:RedisPubSub`. Ea | `ReadScriptsEnabled` | `bool` | `false` | Per-provider | By default a value and its expiration are read with a pipelined `GET` and `PTTL` (one round trip, no transaction); the pair is not atomic, so a key that expires between the two reads as a value without a TTL. Set to `true` to read on Redis 7 and later with one `EVAL_RO` script instead (a batch read runs one script for all its keys; less allocation, but several times the server time of a plain read on Azure Managed Redis). The connection's user then needs `EVAL_RO`/`EVALSHA_RO` (ACL category `@scripting`); if the first script fails for any reason, or the server later refuses one (`NOPERM`, or a disabled or renamed command), reads switch to the `GET`/`PTTL` pair for good with one warning per cache instance, so a server or proxy that cannot run Lua never turns reads into misses. Before Redis 7 no script is sent. Command statistics then show `EVAL_RO` instead of `GET`/`PTTL`. | | `ReadFrom` | `RedisReadFrom` | `Master` | Per-provider | `Master` sends reads to the primary (`PreferMaster`); `Replica` sends them to a replica when one exists (`PreferReplica`). On Azure Redis 6 a replica round trip measured about 3 times the primary's (1.6 ms against 0.55 ms), and a replica can still hold a key the primary has already removed, so a read right after a write or an invalidation could return, or refill the entry with, stale data. With `Replica`, a key this node wrote or removed (counted from when the write completes), or heard an invalidation for through a live local entry, within `MasterReadWindow` still reads from the primary, as does every read for that window after a lost-events clear on a node holding local entries and, if more than 10,000 keys were touched within one window, every read until the window passes. A key with no local entry is not guarded against another node's write. Turn it on only to take read load off the primary. Writes and refreshes always go to the primary. | | `MasterReadWindow` | `TimeSpan` | `00:00:01` | Per-provider | With `ReadFrom` set to `Replica`, how long a key this node wrote, removed or heard invalidated keeps reading from the master. Must be positive, and must exceed the expected replica lag: a replica that is further behind can still return the old value once the window ends. A fire-and-forget TTL write (`AwaitRefresh` off) starts the window when it is queued, not when the server applies it. Measured on a monotonic clock. Ignored with `Master`. | +| `FormatNegotiation:Mode` | `FormatNegotiationMode` | `Auto` | Per-provider | `Auto` decides which formats this node writes: a registered serializer is written only while every node on the Streams transport is known to read it, whichever policy asks for it (a registered policy or one built at the call site), and JSON otherwise. `Force` writes exactly what each policy configures, with no node list and no decision. A node with a serializer registered advertises the formats it reads in either mode, so a `Force` node does not keep an `Auto` neighbour on JSON, unless the app has no Streams transport (see `AssumeAllNodesCanRead`), where a node neither advertises nor reads advertisements. A policy with `Write = "json"` is never affected, and neither is one on a custom `ISerializerProxy` (or a subclass of the default, or `RawByteSerializerProxy`): its bytes are not JSON, so there is nothing safe to fall back to, and it always writes the format the policy names, as `Force` does. See [choosing when a format is written](../concepts.md#choosing-when-a-format-is-written). | +| `FormatNegotiation:AdvertiseInterval` | `TimeSpan` | `00:00:15` | Per-provider | How often this node rewrites its advertisement (the formats it reads) in Redis, stamped with the server's clock. Every node judges the others' advertisements by its own settings, so set `AdvertiseInterval`, `ViewInterval` and `LivenessWindow` alike across the fleet. An advertisement counts for `LivenessWindow` plus two of these, so it outlives the consumer group of a node that crashed. Must be positive. | +| `FormatNegotiation:ViewInterval` | `TimeSpan` | `00:00:15` | Per-provider | How often this node reads the advertisements and the Streams consumer groups; it advertises first, on the same tick. Must be positive. | +| `FormatNegotiation:LivenessWindow` | `TimeSpan` | `00:02:00` | Per-provider | A consumer group no advertisement accounts for (a node before 3.2) counts as a live node, and blocks the switch, while its consumer read within this window. A group no consumer ever read from counts as nobody. An advertisement accounts for its group only if it is at least as fresh as the group's last read (within two `AdvertiseInterval`s), so a node restarted under the same id is not hidden by its previous run's advertisement. Must be positive. | +| `FormatNegotiation:UpgradeHold` | `TimeSpan` | `01:00:00` | Per-provider | How long every node must have been accounted for before a format is written, counted from the last time a node joined or restarted (a node that leaves never restarts it). Zero switches at once; negative is rejected. A shorter hold switches sooner, but if the app is rolled back after it, nodes that cannot read the format miss and rewrite values as JSON, never a wrong value. A longer one means an app rolled back within it never wrote the format, so the older nodes see no misses. A node that cannot read the format stops the writes at once, whatever the hold. A node list that fails to read (other than a group that vanished mid-read) restarts the hold. | +| `FormatNegotiation:MaxViewAge` | `TimeSpan` | `00:02:00` | Per-provider | While Redis cannot be read, the last decision stands for this long since the last good read, then values are written as JSON; a decision older than this is never used, even if a read hangs. After an outage shorter than this, the decision is also kept for two `AdvertiseInterval`s of successful ticks, so peers can advertise again, unless an advertisement says a node cannot read the format. Must be positive. | +| `FormatNegotiation:AssumeAllNodesCanRead` | `bool` | `false` | Per-provider | Attests that the nodes that say nothing read the desired formats: a live Streams consumer group with no advertisement, and, for an app with `BroadcastEnabled` off or without a Streams topic (Pub/Sub only), the nodes it cannot see at all. For an app that can list its peers, an advertisement that lacks the format still stops the writes. Without a Streams transport the node neither advertises nor reads advertisements, so nothing can stop the switch once `UpgradeHold` has passed, counted from the first decision. Without the setting such an app writes JSON. | *Code-only seams:* `EntryFactory`, `CacheKeyStrategy`, `RedisKeyStrategyFactory`. diff --git a/samples/UiPath.Caching.Sample/appsettings.all.json b/samples/UiPath.Caching.Sample/appsettings.all.json index 228e138a..fb596f4a 100644 --- a/samples/UiPath.Caching.Sample/appsettings.all.json +++ b/samples/UiPath.Caching.Sample/appsettings.all.json @@ -375,7 +375,23 @@ // ReadFrom: Master reads from the master, which is faster and never stale after an invalidation; Replica reads from a replica when one exists, except for keys just written or invalidated "ReadFrom": "Master", // MasterReadWindow: with ReadFrom Replica, how long a just-written or invalidated key keeps reading from the master; must exceed the replica lag - "MasterReadWindow": "0:00:01" + "MasterReadWindow": "0:00:01", + "FormatNegotiation": { + // Mode: Auto writes a registered serializer's format only while every node on the Streams transport is known to read it; Force writes exactly what each policy configures. Either way the node advertises what it reads + "Mode": "Auto", + // AdvertiseInterval: how often this node refreshes its advertisement; an advertisement lives LivenessWindow plus two of these. Keep it alike across the fleet + "AdvertiseInterval": "0:00:15", + // ViewInterval: how often this node reads the node list + "ViewInterval": "0:00:15", + // LivenessWindow: how long a node without an advertisement counts as live after its consumer last read its stream. Keep it alike across the fleet + "LivenessWindow": "0:02:00", + // UpgradeHold: how long every node must have been accounted for before the format is written, counted from the last join or restart; zero switches at once, a node that cannot read it stops writes immediately + "UpgradeHold": "1:00:00", + // MaxViewAge: how long a decision survives a node list that cannot be read, before writes fall back to JSON + "MaxViewAge": "0:02:00", + // AssumeAllNodesCanRead: attest that nodes that advertise nothing (live Streams groups without an advertisement; broadcast off, Pub/Sub only) read the desired formats; an advertisement that lacks the format still blocks + "AssumeAllNodesCanRead": false + } }, // ---- ResiliencePolicies (UiPath.Caching.Polly.ResiliencePoliciesOptions) ---- diff --git a/src/UiPath.Caching/Broadcast/Redis/RedisStreamHealthMaintainer.cs b/src/UiPath.Caching/Broadcast/Redis/RedisStreamHealthMaintainer.cs index 9ae4c754..cb9a106f 100644 --- a/src/UiPath.Caching/Broadcast/Redis/RedisStreamHealthMaintainer.cs +++ b/src/UiPath.Caching/Broadcast/Redis/RedisStreamHealthMaintainer.cs @@ -55,6 +55,8 @@ public RedisStreamHealthMaintainer( internal Task Task { get; private set; } = Task.CompletedTask; + internal RedisKey LockKey => _lockKey; + private IDatabase Database => _redis.Database; public Task StartAsync(CancellationToken cancellationToken) @@ -149,16 +151,7 @@ internal RedisKey QuarantineKey(RedisKey streamKey) => EscapeBraces(_quarantineMarker + streamKey.ToString()) + StreamKeyTerminator, CacheKeyCasing.Sensitive)); - // The composed name goes through the configured strategy, which under ShardKeyEnabled refuses a name whose - // braces form no tag -- so a single stream named 'app:st:{}topic' would abort the pass for every other - // stream. The quarantine hash has no need of the stream's tag, the two being deleted one command at a time, - // so the braces are escaped away. The whole name, not just the stream key: CacheOptions.Separator is only - // required to be non-whitespace, so the marker can carry braces too. Escaping '%' first keeps this injective. - private static string EscapeBraces(string streamKey) => - streamKey - .Replace("%", "%25", StringComparison.Ordinal) - .Replace("{", "%7B", StringComparison.Ordinal) - .Replace("}", "%7D", StringComparison.Ordinal); + private static string EscapeBraces(string streamKey) => KeyEscaping.EscapeBraces(streamKey); private static bool TryParseDeliveredIdToDatetimeOffset(string? entryId, out DateTimeOffset? dateTimeOffset) { diff --git a/src/UiPath.Caching/Broadcast/Redis/RedisStreamsTopicProvider.cs b/src/UiPath.Caching/Broadcast/Redis/RedisStreamsTopicProvider.cs index 00c51331..5617e573 100644 --- a/src/UiPath.Caching/Broadcast/Redis/RedisStreamsTopicProvider.cs +++ b/src/UiPath.Caching/Broadcast/Redis/RedisStreamsTopicProvider.cs @@ -1,3 +1,4 @@ +using System.Collections.Concurrent; using UiPath.Caching.Policies; using UiPath.Caching.Telemetry; @@ -12,6 +13,7 @@ public class RedisStreamsTopicProvider : RedisTopicProviderBase private readonly PerTopicOptionsRegistry _registry; private readonly CancellationTokenSource _stopTokenSource = new(); private readonly ILogger _logger; + private readonly ConcurrentDictionary _streamKeys = new(); private bool _disposed; public RedisStreamsTopicProvider( @@ -39,10 +41,28 @@ public RedisStreamsTopicProvider( public override bool Enabled { get; } + internal int StreamKeyCount => _streamKeys.Count; + + /// The stream of each topic this provider has made, for the node node list. + internal IReadOnlyList StreamKeys() + { + List streams = []; + foreach (var key in Keys) + { + if (_streamKeys.TryGetValue(key, out var stream)) + { + streams.Add(stream); + } + } + + return streams; + } + protected override ITopic CreateInternalTopic(TopicKey topicKey) { var resolved = ResolveOptions(topicKey) ?? _redisStreamsTopicOptions; - return new RedisStreamsTopic( + _streamKeys[topicKey] = (resolved.RedisStreamKeyStrategy ?? new PrefixStrategy(RedisKeyspaces.Streams, _cacheOptions)).GetRedisKey(topicKey); + var topic = new RedisStreamsTopic( topicKey, ConnectionState, Redis, @@ -55,6 +75,10 @@ protected override ITopic CreateInternalTopic(TopicKey topicKey) Telemetry, Profiler, _stopTokenSource.Token); + + // The topic's stream goes with it, so a provider that makes and drops many topics keeps no record of the dropped ones. + topic.OnDisposed += (_, _) => _streamKeys.TryRemove(topicKey, out _); + return topic; } protected override void Dispose(bool disposing) diff --git a/src/UiPath.Caching/Config/CachingBuilder.cs b/src/UiPath.Caching/Config/CachingBuilder.cs index 2acd0efb..d0c769eb 100644 --- a/src/UiPath.Caching/Config/CachingBuilder.cs +++ b/src/UiPath.Caching/Config/CachingBuilder.cs @@ -1,5 +1,7 @@ using System.Text.Json; +using Microsoft.Extensions.Hosting; using UiPath.Caching.Locking; +using UiPath.Caching.Negotiation; using UiPath.Caching.Policies; using UiPath.Caching.Serialization; using UiPath.Caching.Telemetry; @@ -56,6 +58,12 @@ internal void Complete() ServiceDescriptor.Singleton, CacheKeyCasingSeeder>()); Services.TryAddSingleton>(sp => new SystemJsonByteSerializerProxy(sp.GetService())); Services.TryAddSingleton(ValueSerializerRegistration.BuildRegistry); + if (Services.Any(d => d.ServiceType == typeof(ValueSerializerRegistration))) + { + Services.TryAddSingleton(FormatNegotiation.Create); + Services.TryAddEnumerable(ServiceDescriptor.Singleton()); + } + Services.TryAddSingleton(EmptyResiliencePipelineProvider.Instance); Services.TryAddSingleton(NullChangeTokenFactory.Instance); Services.TryAddSingleton(NullTopicFactory.Instance); diff --git a/src/UiPath.Caching/Metrics.cs b/src/UiPath.Caching/Metrics.cs index f26e4813..104d808e 100644 --- a/src/UiPath.Caching/Metrics.cs +++ b/src/UiPath.Caching/Metrics.cs @@ -15,6 +15,11 @@ public static class Metrics public const string Read = $".{nameof(Read)}"; public const string MessageId = $".{nameof(MessageId)}"; + public const string FormatEffective = "cache.format.effective"; + public const string FormatFallback = "cache.format.fallback"; + public const string FormatViewAge = "cache.format.view.age"; + public const string FormatNodes = "cache.format.nodes"; + public const string TopicName = nameof(TopicName); public const string SequenceNumber = nameof(SequenceNumber); public const string Invalid = nameof(Invalid); diff --git a/src/UiPath.Caching/Negotiation/DesiredWrite.cs b/src/UiPath.Caching/Negotiation/DesiredWrite.cs new file mode 100644 index 00000000..2e00b6a9 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/DesiredWrite.cs @@ -0,0 +1,7 @@ +namespace UiPath.Caching.Negotiation; + +/// +/// A registered serializer a policy may write, and the tokens a node must read to be sent it. are the registered policies +/// that write it, and the registered policies that write something else, whose read lists say nothing about it. +/// +internal sealed record DesiredWrite(ICacheValueSerializer Writer, IReadOnlyList Tokens, IReadOnlyCollection Policies, IReadOnlyCollection Others); diff --git a/src/UiPath.Caching/Negotiation/FormatNegotiation.cs b/src/UiPath.Caching/Negotiation/FormatNegotiation.cs new file mode 100644 index 00000000..a1085b99 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/FormatNegotiation.cs @@ -0,0 +1,69 @@ +using System.Reflection; +using UiPath.Caching.Broadcast.Redis; +using UiPath.Caching.Serialization; +using UiPath.Caching.Telemetry; + +namespace UiPath.Caching.Negotiation; + +/// Everything one stack needs to advertise what it reads and, in , to decide what it writes; for an app with no serializer registered: then nothing is written, read or scheduled. +internal sealed class FormatNegotiation +{ + public static FormatNegotiation Inactive { get; } = new(); + + /// Null in : the node advertises, and writes exactly what its policies say. + public FormatNegotiator? Negotiator { get; init; } + + public NodeAdvertisement? Advertisement { get; init; } + + public string NodeId { get; init; } = string.Empty; + + public NodeListStore Store { get; init; } = default!; + + public StreamNodeList NodeList { get; init; } = default!; + + /// The streams this node's topics read; when the app has no Streams transport, so the node list is blind. + public Func>? StreamKeys { get; init; } + + public FormatNegotiationOptions Options { get; init; } = new(); + + public TimeProvider Clock { get; init; } = TimeProvider.System; + + public ILogger Logger { get; init; } = NullLogger.Instance; + + public bool IsActive => Advertisement is not null; + + public static FormatNegotiation Create(IServiceProvider services) + { + var redisOptions = services.GetRequiredService>().Value; + var options = redisOptions.FormatNegotiation; + var registry = services.GetRequiredService(); + if (registry.IsEmpty || services.GetService() is not { } connector) + { + return Inactive; + } + + options.Validate(); + var policies = services.GetRequiredService(); + var cacheOptions = services.GetRequiredService>().Value; + var loggers = services.GetRequiredService(); + var clock = services.GetRequiredService(); + var streams = cacheOptions.BroadcastEnabled + ? services.GetServices().OfType().FirstOrDefault(p => p.Enabled) + : null; + var library = typeof(FormatNegotiation).Assembly.GetCustomAttribute()?.InformationalVersion ?? string.Empty; + return new() + { + Negotiator = options.Mode == FormatNegotiationMode.Force + ? null + : new FormatNegotiator(NegotiationPlan.Desired(registry, policies), options, clock, services.GetRequiredService(), loggers.CreateLogger()), + Advertisement = NegotiationPlan.Advertisement(registry, policies, new ValueReader(services.GetRequiredService>(), registry), library), + NodeId = (cacheOptions.SourceUri ?? CacheOptions.MachineUri).ToString().ToLowerInvariant(), + Store = new NodeListStore(connector, NodeListStore.KeyFor(cacheOptions, redisOptions)), + NodeList = new StreamNodeList(new RedisStreamGroupSource(connector), options), + StreamKeys = streams is null ? null : streams.StreamKeys, + Options = options, + Clock = clock, + Logger = loggers.CreateLogger(), + }; + } +} diff --git a/src/UiPath.Caching/Negotiation/FormatNegotiationService.cs b/src/UiPath.Caching/Negotiation/FormatNegotiationService.cs new file mode 100644 index 00000000..0490a75b --- /dev/null +++ b/src/UiPath.Caching/Negotiation/FormatNegotiationService.cs @@ -0,0 +1,211 @@ +using Microsoft.Extensions.Hosting; + +namespace UiPath.Caching.Negotiation; + +/// +/// Advertises this node and, where it decides what it writes, reads the node list, on timers off the read and write paths. It advertises first +/// when it starts, and writes a departing advertisement when it stops, so the nodes that outlive it count it as gone and not as unaccounted for. +/// +internal sealed class FormatNegotiationService(FormatNegotiation negotiation) : IHostedService +{ + private static readonly TimeSpan Budget = TimeSpan.FromSeconds(2); + + private readonly CancellationTokenSource _stop = new(); + private Task[] _loops = []; + private int _stopped; + + public async Task StartAsync(CancellationToken cancellationToken) + { + if (!negotiation.IsActive) + { + return; + } + + var options = negotiation.Options; + if (negotiation.StreamKeys is null) + { + StartBlind(); + return; + } + + using (var startup = new CancellationTokenSource(Budget, negotiation.Clock)) + using (var linked = CancellationTokenSource.CreateLinkedTokenSource(startup.Token, cancellationToken)) + { + await TickAsync(linked.Token).ConfigureAwait(false); + } + + // A node that only reads has no decision to make, so it never reads the node list. + _loops = negotiation.Negotiator is null + ? [Task.Run(() => Loop(options.AdvertiseInterval, AdvertiseAsync), CancellationToken.None)] + : [Task.Run(ScheduledLoop, CancellationToken.None)]; + } + + public async Task StopAsync(CancellationToken cancellationToken) + { + if (Interlocked.Exchange(ref _stopped, 1) == 1) + { + return; + } + + await _stop.CancelAsync().ConfigureAwait(false); + try + { + await Task.WhenAll(_loops).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + negotiation.Logger.LogDebug(ex, "A format negotiation loop ended with an error while stopping."); + } + + _stop.Dispose(); + if (negotiation.IsActive && negotiation.StreamKeys is not null) + { + try + { + using var timeout = new CancellationTokenSource(Budget, negotiation.Clock); + await negotiation.Store.WriteAsync(negotiation.NodeId, negotiation.Advertisement! with { Left = true }, timeout.Token).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + negotiation.Logger.LogDebug(ex, "This node's departing advertisement could not be written; its advertisement expires on its own."); + } + } + } + + internal async Task TickAsync(CancellationToken cancellationToken) + { + // Advertise before the node list is read, so a node that has just come back from an outage is not counted as unaccounted for by itself. + await AdvertiseAsync(cancellationToken).ConfigureAwait(false); + if (negotiation.Negotiator is not null) + { + await RefreshAsync(cancellationToken).ConfigureAwait(false); + } + } + + internal async Task AdvertiseAsync(CancellationToken cancellationToken) + { + try + { + await negotiation.Store.WriteAsync(negotiation.NodeId, negotiation.Advertisement!, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OutOfMemoryException && !Stopping()) + { + negotiation.Logger.LogWarning(ex, "This node's format advertisement could not be written."); + } + } + + internal async Task RefreshAsync(CancellationToken cancellationToken) + { + var negotiator = negotiation.Negotiator!; + var options = negotiation.Options; + try + { + // An advertisement outlives the group of the node that wrote it, so a node that crashes is gone from the node list before it stops counting as one. + var fresh = options.LivenessWindow + TimeSpan.FromTicks(options.AdvertiseInterval.Ticks * 2); + var gone = TimeSpan.FromTicks(Math.Max(options.AdvertiseInterval.Ticks * 10, fresh.Ticks * 2)); + var all = await negotiation.Store.ReadAsync(cancellationToken).ConfigureAwait(false); + var current = all.Where(a => a.Age <= fresh).ToList(); + var readers = current.Where(a => !a.Advertisement.Left).ToList(); + var streams = negotiation.StreamKeys!(); + var blockers = streams.Count == 0 + ? [] + : await negotiation.NodeList.BlockersAsync(streams, AgesByNode(current), cancellationToken).ConfigureAwait(false); + negotiator.Observe(new NodeListView(readers, blockers, Complete: streams.Count > 0)); + await Prune(all.Where(a => a.Age > gone).ToList(), cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OutOfMemoryException && !Stopping()) + { + negotiation.Logger.LogWarning(ex, "The format node list could not be read; the last decision stands until it is too old."); + negotiator.ObservationFailed(); + } + } + + private static Dictionary AgesByNode(IEnumerable ads) => + ads.GroupBy(a => a.NodeId, StringComparer.OrdinalIgnoreCase).ToDictionary(g => g.Key, g => g.Min(a => a.Age), StringComparer.OrdinalIgnoreCase); + + // No Streams topic, or broadcast off: nothing to read, so only the operator's word can switch, and the hold still applies. + private void StartBlind() + { + if (negotiation.Negotiator is not { } negotiator) + { + return; + } + + if (!negotiation.Options.AssumeAllNodesCanRead) + { + negotiation.Logger.LogInformation("Value formats stay JSON: the app has no Streams transport, so the nodes cannot be listed. Set FormatNegotiation:AssumeAllNodesCanRead to attest that they all read the desired formats."); + return; + } + + var blind = new NodeListView([], [], Complete: false); + negotiator.Observe(blind); + _loops = [Task.Run(() => Loop(negotiation.Options.ViewInterval, _ => { negotiator.Observe(blind); return Task.CompletedTask; }), CancellationToken.None)]; + } + + // Only this node stopping is quiet: a step that outran its budget is a failed step, and is logged and counted as one. + private bool Stopping() => _stop.IsCancellationRequested; + + private async Task Prune(IReadOnlyCollection gone, CancellationToken cancellationToken) + { + try + { + await negotiation.Store.PruneAsync(gone, cancellationToken).ConfigureAwait(false); + } + catch (Exception ex) when (ex is not OutOfMemoryException && !Stopping()) + { + negotiation.Logger.LogDebug(ex, "Advertisements of departed nodes could not be removed."); + } + } + + // The advertisement and the node list read keep their own cadences; a read that is due advertises first. + private async Task ScheduledLoop() + { + var options = negotiation.Options; + var schedule = new TickSchedule(options.AdvertiseInterval, options.ViewInterval); + var started = negotiation.Clock.GetTimestamp(); + while (!_stop.IsCancellationRequested) + { + try + { + await Task.Delay(schedule.Wait(negotiation.Clock.GetElapsedTime(started)), negotiation.Clock, _stop.Token).ConfigureAwait(false); + var (advertise, refresh) = schedule.Take(negotiation.Clock.GetElapsedTime(started)); + if (refresh) + { + await TickAsync(_stop.Token).ConfigureAwait(false); + } + else if (advertise) + { + await AdvertiseAsync(_stop.Token).ConfigureAwait(false); + } + } + catch (OperationCanceledException) when (_stop.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + negotiation.Logger.LogWarning(ex, "A format negotiation step failed; it runs again when it is next due."); + } + } + } + + private async Task Loop(TimeSpan interval, Func step) + { + while (!_stop.IsCancellationRequested) + { + try + { + await Task.Delay(interval * (0.8 + (Random.Shared.NextDouble() * 0.4)), negotiation.Clock, _stop.Token).ConfigureAwait(false); + await step(_stop.Token).ConfigureAwait(false); + } + catch (OperationCanceledException) when (_stop.IsCancellationRequested) + { + return; + } + catch (Exception ex) when (ex is not OutOfMemoryException) + { + negotiation.Logger.LogWarning(ex, "A format negotiation step failed; it runs again at the next tick."); + } + } + } +} diff --git a/src/UiPath.Caching/Negotiation/FormatNegotiator.cs b/src/UiPath.Caching/Negotiation/FormatNegotiator.cs new file mode 100644 index 00000000..56f08feb --- /dev/null +++ b/src/UiPath.Caching/Negotiation/FormatNegotiator.cs @@ -0,0 +1,235 @@ +using UiPath.Caching.Telemetry; + +namespace UiPath.Caching.Negotiation; + +/// +/// Turns what the node list shows into which serializers are written now, whatever policy asks for them. A format is written only while every +/// advertisement lists it and no live node is unaccounted for, and only after that has held for +/// since the reading nodes last gained a member. The first node that cannot read it stops the writes at once; a node that leaves never does. +/// +internal sealed class FormatNegotiator : IEffectiveFormat +{ + public const string ReasonUnaccountedGroup = "unaccounted_group"; + public const string ReasonMissingFormat = "missing_format"; + public const string ReasonNoView = "no_view"; + public const string ReasonViewStale = "view_stale"; + public const string ReasonTransportUnknown = "transport_unknown"; + + private readonly DesiredWrite[] _desired; + private readonly FormatNegotiationOptions _options; + private readonly TimeProvider _clock; + private readonly ICachingTelemetryProvider _telemetry; + private readonly ILogger _logger; + private readonly Lock _gate = new(); + private readonly long?[] _candidateSince; + private readonly string?[] _reasons; + private HashSet _readers = []; + private volatile Snapshot _snapshot; + private long _lastView; + private long? _graceStart; + private bool _hasView; + private bool _failed; + + public FormatNegotiator( + IEnumerable desired, + FormatNegotiationOptions options, + TimeProvider clock, + ICachingTelemetryProvider? telemetry = null, + ILogger? logger = null) + { + options.Validate(); + _desired = [.. desired]; + _options = options; + _clock = clock; + _telemetry = telemetry ?? NullTelemetryProvider.Instance; + _logger = logger ?? NullLogger.Instance; + _candidateSince = new long?[_desired.Length]; + _reasons = new string?[_desired.Length]; + Array.Fill(_reasons, ReasonNoView); + _snapshot = Publish(new bool[_desired.Length], 0); + } + + public bool Allows(ICacheValueSerializer writer) + { + var snapshot = _snapshot; + var writers = snapshot.Writers; + for (var i = 0; i < writers.Length; i++) + { + if (ReferenceEquals(writers[i].Writer, writer)) + { + // A view that stopped refreshing, without failing, decays on its own: the age is checked here, off the cached stamp and the injected clock. + return writers[i].Allowed && _clock.GetElapsedTime(snapshot.Stamp) <= _options.MaxViewAge; + } + } + + return false; + } + + public void Observe(NodeListView view) + { + lock (_gate) + { + var now = _clock.GetTimestamp(); + if (_failed) + { + // Peers need a moment to advertise again after Redis came back; what they failed to say meanwhile is not an answer. + _graceStart = now; + _failed = false; + } + + var grace = _graceStart is { } start && _clock.GetElapsedTime(start, now) < TimeSpan.FromTicks(_options.AdvertiseInterval.Ticks * 2); + var joined = view.Readers.Any(reader => !_readers.Contains(reader)); + _readers = [.. view.Readers]; + _hasView = true; + _lastView = now; + var allowed = new bool[_desired.Length]; + var previous = _snapshot.Writers; + for (var i = 0; i < _desired.Length; i++) + { + _reasons[i] = Block(_desired[i], view); + if (_reasons[i] is ReasonNoView or ReasonUnaccountedGroup && grace && previous[i].Allowed) + { + _reasons[i] = null; + allowed[i] = true; + } + else if (_reasons[i] is not null) + { + _candidateSince[i] = null; + } + else if (previous[i].Allowed) + { + allowed[i] = true; + } + else + { + if (_candidateSince[i] is null || joined) + { + _candidateSince[i] = now; + } + + allowed[i] = _clock.GetElapsedTime(_candidateSince[i]!.Value, now) >= _options.UpgradeHold; + } + } + + Transition(previous, allowed); + _snapshot = Publish(allowed, now); + Report(view); + } + } + + /// The node list could not be read: the decision stands until has passed since the last view, then JSON. + public void ObservationFailed() + { + lock (_gate) + { + if (!_hasView) + { + return; + } + + _failed = true; + var age = _clock.GetElapsedTime(_lastView); + _telemetry.TrackMetric(Metrics.FormatViewAge, age.TotalSeconds); + Array.Fill(_candidateSince, null); + if (age <= _options.MaxViewAge) + { + return; + } + + _hasView = false; + Array.Fill(_reasons, ReasonViewStale); + var allowed = new bool[_desired.Length]; + Transition(_snapshot.Writers, allowed); + _snapshot = Publish(allowed, _snapshot.Stamp); + } + } + + private string? Block(DesiredWrite want, NodeListView view) + { + // What an advertisement says it cannot read comes first: no grace after an outage and no attestation excuses it. + foreach (var stored in view.Ads) + { + var ad = stored.Advertisement; + if (!want.Tokens.All(ad.ValueReads.Contains)) + { + return ReasonMissingFormat; + } + + foreach (var (policy, list) in ad.Strict) + { + if (list.Contains(NodeAdvertisement.NoSniff) || (!want.Others.Contains(policy) && !want.Tokens.All(list.Contains))) + { + return ReasonMissingFormat; + } + } + } + + var attested = _options.AssumeAllNodesCanRead; + if (!view.Complete) + { + if (!attested) + { + return ReasonTransportUnknown; + } + } + else if (view.Ads.Count == 0) + { + return ReasonNoView; + } + + // The attestation covers nodes that say nothing; a node that advertises without the format is not one of them. + return view.Blockers.Count > 0 && !attested ? ReasonUnaccountedGroup : null; + } + + private void Transition(WriterState[] previous, bool[] allowed) + { + for (var i = 0; i < allowed.Length; i++) + { + var policies = string.Join(',', _desired[i].Policies); + if (previous[i].Allowed && !allowed[i]) + { + _telemetry.TrackMetric(Metrics.FormatFallback, 1, [new("reason", _reasons[i] ?? ReasonNoView), new("policy", policies)]); + _logger.LogWarning("Value format {Format} falls back to JSON: {Reason}.", string.Join('+', _desired[i].Tokens), _reasons[i]); + } + else if (!previous[i].Allowed && allowed[i]) + { + _logger.LogInformation("Value format {Format} is now written.", string.Join('+', _desired[i].Tokens)); + } + } + } + + private void Report(NodeListView view) + { + _telemetry.TrackMetric(Metrics.FormatViewAge, 0); + _telemetry.TrackMetric(Metrics.FormatNodes, view.Ads.Count, [new("state", "live")]); + _telemetry.TrackMetric(Metrics.FormatNodes, view.Blockers.Count, [new("state", "blocking")]); + if (view.Blockers.Count > 0) + { + _logger.LogWarning("Nodes live on a stream without an advertisement, so not sent a binary format: {Nodes}.", string.Join(", ", view.Blockers)); + } + + var current = _snapshot.Writers; + for (var i = 0; i < _desired.Length; i++) + { + _telemetry.TrackMetric( + Metrics.FormatEffective, + current[i].Allowed ? 1 : 0, + [new("kind", "value"), new("policy", string.Join(',', _desired[i].Policies)), new("format", string.Join('+', _desired[i].Tokens))]); + } + } + + private Snapshot Publish(bool[] allowed, long stamp) + { + var writers = new WriterState[_desired.Length]; + for (var i = 0; i < writers.Length; i++) + { + writers[i] = new(_desired[i].Writer, allowed[i]); + } + + return new(writers, stamp); + } + + private sealed record WriterState(ICacheValueSerializer Writer, bool Allowed); + + private sealed record Snapshot(WriterState[] Writers, long Stamp); +} diff --git a/src/UiPath.Caching/Negotiation/IEffectiveFormat.cs b/src/UiPath.Caching/Negotiation/IEffectiveFormat.cs new file mode 100644 index 00000000..19e92382 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/IEffectiveFormat.cs @@ -0,0 +1,8 @@ +namespace UiPath.Caching.Negotiation; + +/// Whether a serializer's format may be written now; a format that may not is written as JSON. +internal interface IEffectiveFormat +{ + /// Reads one volatile reference and the injected clock, with no allocation and no call to Redis. + bool Allows(ICacheValueSerializer writer); +} diff --git a/src/UiPath.Caching/Negotiation/IStreamGroupSource.cs b/src/UiPath.Caching/Negotiation/IStreamGroupSource.cs new file mode 100644 index 00000000..2481c0d0 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/IStreamGroupSource.cs @@ -0,0 +1,11 @@ +namespace UiPath.Caching.Negotiation; + +/// The consumer groups of a stream and how recently each was read, as the node list needs them. +internal interface IStreamGroupSource +{ + /// The names of the stream's groups; none for a stream that does not exist. + Task> GroupsAsync(RedisKey stream, CancellationToken cancellationToken); + + /// How long ago a consumer of the group last read, the most recent of them; when it has no consumer or no longer exists. + Task IdleAsync(RedisKey stream, string group, CancellationToken cancellationToken); +} diff --git a/src/UiPath.Caching/Negotiation/NegotiationPlan.cs b/src/UiPath.Caching/Negotiation/NegotiationPlan.cs new file mode 100644 index 00000000..38ea8407 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/NegotiationPlan.cs @@ -0,0 +1,77 @@ +using UiPath.Caching.Serialization; + +namespace UiPath.Caching.Negotiation; + +/// What this node can write and what it can read, from its registered serializers and policies. +internal static class NegotiationPlan +{ + public const string DefaultPolicyName = "default"; + + /// One entry per registered serializer, so a policy built at the call site that names one is held to the same decision as a registered one. + public static IReadOnlyList Desired(ValueSerializerRegistry registry, ICachePolicyFactory policies) + { + var named = Policies(policies).ToList(); + var desired = new List(); + foreach (var (_, writer, _) in registry.Entries) + { + desired.Add(new( + writer, + [.. registry.FormatIdsOf(writer).ToArray().Select(NodeAdvertisement.Token)], + [.. named.Where(p => ReferenceEquals(p.Policy.WriteSerializer, writer)).Select(p => p.Name.ToLowerInvariant())], + [.. named.Where(p => !ReferenceEquals(p.Policy.WriteSerializer, writer)).Select(p => p.Name.ToLowerInvariant())])); + } + + return desired; + } + + public static NodeAdvertisement Advertisement(ValueSerializerRegistry registry, ICachePolicyFactory policies, ValueReader reader, string library) + { + var reads = new List { NodeAdvertisement.Json }; + foreach (var (_, serializer, _) in registry.Entries) + { + reads.AddRange(registry.FormatIdsOf(serializer).ToArray().Select(NodeAdvertisement.Token)); + } + + var strict = new Dictionary>(StringComparer.Ordinal); + foreach (var (name, policy) in Policies(policies)) + { + if (!reader.SniffsFor(policy)) + { + // Its reads never look for a header, so a value in another format would be handed to it as bytes or fail: it reads JSON only. + strict[name.ToLowerInvariant()] = [NodeAdvertisement.Json, NodeAdvertisement.NoSniff]; + } + else if (policy.Serializer?.Read is { } read) + { + strict[name.ToLowerInvariant()] = [.. read.SelectMany(format => Tokens(registry, format)).Distinct()]; + } + } + + if (policies.Default is null && !reader.SniffsFor(null)) + { + strict[DefaultPolicyName] = [NodeAdvertisement.Json, NodeAdvertisement.NoSniff]; + } + + return new(library, Guid.NewGuid().ToString("N"), reads.Distinct().ToList(), strict); + } + + private static IEnumerable Tokens(ValueSerializerRegistry registry, string format) => + registry.Resolve(format) is { } serializer + ? registry.FormatIdsOf(serializer).ToArray().Select(NodeAdvertisement.Token) + : [NodeAdvertisement.Json]; + + private static IEnumerable<(string Name, CachePolicy Policy)> Policies(ICachePolicyFactory policies) + { + if (policies.Default is { } fallback) + { + yield return (DefaultPolicyName, fallback); + } + + foreach (var name in policies.Keys) + { + if (policies.Resolve(name) is { } policy) + { + yield return (name, policy); + } + } + } +} diff --git a/src/UiPath.Caching/Negotiation/NodeAdvertisement.cs b/src/UiPath.Caching/Negotiation/NodeAdvertisement.cs new file mode 100644 index 00000000..bcf588e3 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/NodeAdvertisement.cs @@ -0,0 +1,125 @@ +using System.Globalization; +using System.Text; + +namespace UiPath.Caching.Negotiation; + +/// What a node tells the others about itself: the value formats it reads, per-policy read lists, a field reserved for migration capabilities, and whether the node is leaving. +internal sealed record NodeAdvertisement( + string Library, + string Instance, + IReadOnlyList ValueReads, + IReadOnlyDictionary> Strict, + string? Migration = null, + bool Left = false) +{ + public const string Json = "json"; + + /// Marks a read list of a policy whose reads do not look for a header: it reads JSON only, whatever it lists. + public const string NoSniff = "nosniff"; + + public static string Token(int formatId) => formatId.ToString("X2", CultureInfo.InvariantCulture); + + public string Encode() + { + var text = new StringBuilder(); + text.Append("lib=").Append(Escape(Library)).Append(";inst=").Append(Escape(Instance)).Append(";vr=").AppendJoin(',', ValueReads); + if (Strict.Count > 0) + { + text.Append(";strict="); + var first = true; + foreach (var (policy, formats) in Strict.OrderBy(p => p.Key, StringComparer.Ordinal)) + { + text.Append(first ? string.Empty : ",").Append(Escape(policy)).Append(':').AppendJoin('|', formats); + first = false; + } + } + + if (Migration is not null) + { + text.Append(";mig=").Append(Escape(Migration)); + } + + if (Left) + { + text.Append(";left=1"); + } + + return text.ToString(); + } + + /// A field it does not know is skipped, so a later version can add fields; a text without a readable vr is not an advertisement. + public static bool TryParse(string text, [NotNullWhen(true)] out NodeAdvertisement? advertisement) + { + advertisement = null; + string library = string.Empty, instance = string.Empty; + string? migration = null; + var left = false; + IReadOnlyList? reads = null; + Dictionary> strict = []; + foreach (var field in text.Split(';')) + { + var separator = field.IndexOf('=', StringComparison.Ordinal); + if (separator < 0) + { + continue; + } + + var value = field[(separator + 1)..]; + switch (field[..separator]) + { + case "lib": + library = Unescape(value); + break; + case "inst": + instance = Unescape(value); + break; + case "vr": + reads = value.Split(',', StringSplitOptions.RemoveEmptyEntries); + break; + case "mig": + migration = Unescape(value); + break; + case "left": + left = value == "1"; + break; + case "strict": + foreach (var entry in value.Split(',', StringSplitOptions.RemoveEmptyEntries)) + { + var colon = entry.LastIndexOf(':'); + if (colon < 0) + { + return false; + } + + strict[Unescape(entry[..colon])] = entry[(colon + 1)..].Split('|', StringSplitOptions.RemoveEmptyEntries); + } + + break; + } + } + + if (reads is null) + { + return false; + } + + advertisement = new NodeAdvertisement(library, instance, reads, strict, migration, left); + return true; + } + + private static string Escape(string text) => + text.Replace("%", "%25", StringComparison.Ordinal) + .Replace(";", "%3B", StringComparison.Ordinal) + .Replace(",", "%2C", StringComparison.Ordinal) + .Replace(":", "%3A", StringComparison.Ordinal) + .Replace("|", "%7C", StringComparison.Ordinal) + .Replace("=", "%3D", StringComparison.Ordinal); + + private static string Unescape(string text) => + text.Replace("%3D", "=", StringComparison.Ordinal) + .Replace("%7C", "|", StringComparison.Ordinal) + .Replace("%3A", ":", StringComparison.Ordinal) + .Replace("%2C", ",", StringComparison.Ordinal) + .Replace("%3B", ";", StringComparison.Ordinal) + .Replace("%25", "%", StringComparison.Ordinal); +} diff --git a/src/UiPath.Caching/Negotiation/NodeListStore.cs b/src/UiPath.Caching/Negotiation/NodeListStore.cs new file mode 100644 index 00000000..8d78eb79 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/NodeListStore.cs @@ -0,0 +1,179 @@ +using System.Globalization; +using UiPath.Caching.Broadcast.Redis; + +namespace UiPath.Caching.Negotiation; + +/// +/// One hash per keyspace of advertisements, each stamped with the server's clock when written and aged against the server's clock when read, +/// so a node's own clock never enters. +/// +internal sealed class NodeListStore(IRedisConnector redis, RedisKey key) +{ + private const char StampSeparator = '|'; + + private const string WriteScript = """ + local t = redis.call('TIME') + local ms = t[1] * 1000 + math.floor(t[2] / 1000) + redis.call('HSET', KEYS[1], ARGV[1], string.format('%d', ms) .. '|' .. ARGV[2]) + return ms + """; + + private const string ReadScript = """ + local t = redis.call('TIME') + return { t[1] * 1000 + math.floor(t[2] / 1000), redis.call('HGETALL', KEYS[1]) } + """; + + // Removes a field only while it still holds the stamp that was read, so a node that advertised again in between keeps its ad. + private const string PruneScript = """ + local removed = 0 + for i = 1, #ARGV, 2 do + local value = redis.call('HGET', KEYS[1], ARGV[i]) + if value and string.sub(value, 1, #ARGV[i + 1] + 1) == ARGV[i + 1] .. '|' then + redis.call('HDEL', KEYS[1], ARGV[i]) + removed = removed + 1 + end + end + return removed + """; + + // Off for good once the server refuses scripts; the same steps then run as commands. A transient error (READONLY, LOADING, BUSY) is not a refusal. + private volatile bool _scriptsRefused; + + public RedisKey Key { get; } = key; + + private IDatabase Database => redis.Database; + + /// The same construction as the stream maintainer's lock key, so the connector's prefix and a cluster slot follow it. + public static RedisKey KeyFor(CacheOptions cacheOptions, RedisCacheOptions redisOptions) + { + var factory = redisOptions.RedisKeyStrategyFactory ?? new DefaultRedisKeyStrategyFactory(); + return factory.Create(cacheOptions, RedisKeyspaces.String) + .GetRedisKey(KeyEscaping.EscapeBraces(string.Join(cacheOptions.Separator, "Caching", "Nodes"))); + } + + public static bool IsScriptRefusal(Exception ex) => + ex is RedisCommandException + || (ex is RedisServerException server + && (server.Message.StartsWith("NOPERM", StringComparison.OrdinalIgnoreCase) + || server.Message.StartsWith("NOSCRIPT", StringComparison.OrdinalIgnoreCase) + || server.Message.Contains("unknown command", StringComparison.OrdinalIgnoreCase) + || server.Message.Contains("disabled", StringComparison.OrdinalIgnoreCase))); + + public async Task WriteAsync(string nodeId, NodeAdvertisement advertisement, CancellationToken cancellationToken) + { + var text = advertisement.Encode(); + await ConnectAsync(cancellationToken).ConfigureAwait(false); + if (!_scriptsRefused) + { + try + { + await Database.ScriptEvaluateAsync(WriteScript, [Key], [nodeId, text], CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false); + return; + } + catch (Exception ex) when (IsScriptRefusal(ex)) + { + _scriptsRefused = true; + } + } + + var now = await ServerTimeAsync(cancellationToken).ConfigureAwait(false); + await Database.HashSetAsync(Key, nodeId, string.Create(CultureInfo.InvariantCulture, $"{now}{StampSeparator}{text}"), flags: CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false); + } + + /// Entries that do not parse are left out: another version's field is not this node's to judge. + public async Task> ReadAsync(CancellationToken cancellationToken) + { + await ConnectAsync(cancellationToken).ConfigureAwait(false); + long now; + HashEntry[] entries; + if (_scriptsRefused) + { + (now, entries) = await ReadWithCommandsAsync(cancellationToken).ConfigureAwait(false); + } + else + { + try + { + var result = (RedisResult[])(await Database.ScriptEvaluateAsync(ReadScript, [Key], flags: CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false))!; + now = (long)result[0]; + var flat = (RedisResult[])result[1]!; + entries = new HashEntry[flat.Length / 2]; + for (var i = 0; i < entries.Length; i++) + { + entries[i] = new HashEntry((string)flat[2 * i]!, (string)flat[(2 * i) + 1]!); + } + } + catch (Exception ex) when (IsScriptRefusal(ex)) + { + _scriptsRefused = true; + (now, entries) = await ReadWithCommandsAsync(cancellationToken).ConfigureAwait(false); + } + } + + var ads = new List(entries.Length); + foreach (var entry in entries) + { + var text = (string?)entry.Value; + var separator = text?.IndexOf(StampSeparator, StringComparison.Ordinal) ?? -1; + if (separator > 0 + && long.TryParse(text![..separator], NumberStyles.None, CultureInfo.InvariantCulture, out var stamp) + && NodeAdvertisement.TryParse(text[(separator + 1)..], out var ad)) + { + ads.Add(new((string)entry.Name!, ad, TimeSpan.FromMilliseconds(Math.Max(0, now - stamp)), stamp)); + } + } + + return ads; + } + + /// Removes the advertisements that still hold the stamp they were read with. + public async Task PruneAsync(IReadOnlyCollection stale, CancellationToken cancellationToken) + { + if (stale.Count == 0) + { + return; + } + + await ConnectAsync(cancellationToken).ConfigureAwait(false); + if (!_scriptsRefused) + { + try + { + RedisValue[] arguments = [.. stale.SelectMany(a => new RedisValue[] { a.NodeId, a.Stamp.ToString(CultureInfo.InvariantCulture) })]; + await Database.ScriptEvaluateAsync(PruneScript, [Key], arguments, CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false); + return; + } + catch (Exception ex) when (IsScriptRefusal(ex)) + { + _scriptsRefused = true; + } + } + + // Without scripts the check and the delete are two commands; a node that advertises between them loses its ad until the next tick. + foreach (var ad in stale) + { + var held = (string?)await Database.HashGetAsync(Key, ad.NodeId, CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false); + if (held is not null && held.StartsWith(ad.Stamp.ToString(CultureInfo.InvariantCulture) + StampSeparator, StringComparison.Ordinal)) + { + await Database.HashDeleteAsync(Key, ad.NodeId, CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false); + } + } + } + + // The connector's database blocks on the first connect; connecting under the token first keeps a start with Redis down inside its budget. + private Task ConnectAsync(CancellationToken cancellationToken) => + redis.ConnectAsync(cancellationToken).AsTask().WaitAsync(cancellationToken); + + private async Task ServerTimeAsync(CancellationToken cancellationToken) + { + var time = (RedisResult[])(await Database.ExecuteAsync("TIME", args: null, flags: CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false))!; + return ((long)time[0] * 1000) + ((long)time[1] / 1000); + } + + private async Task<(long Now, HashEntry[] Entries)> ReadWithCommandsAsync(CancellationToken cancellationToken) + { + var entries = Database.HashGetAllAsync(Key, CommandFlags.DemandMaster); + var now = ServerTimeAsync(cancellationToken); + return (await now.ConfigureAwait(false), await entries.WaitAsync(cancellationToken).ConfigureAwait(false)); + } +} diff --git a/src/UiPath.Caching/Negotiation/NodeListView.cs b/src/UiPath.Caching/Negotiation/NodeListView.cs new file mode 100644 index 00000000..15233828 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/NodeListView.cs @@ -0,0 +1,8 @@ +namespace UiPath.Caching.Negotiation; + +/// What one node list read found: the advertisements of the nodes that read, the live nodes without one, and whether the node list can see every node. +internal sealed record NodeListView(IReadOnlyList Ads, IReadOnlyList Blockers, bool Complete) +{ + /// Each reading node and the process it advertised from, so a node that joins or restarts is told from one that merely stays. + public IReadOnlyCollection Readers { get; } = [.. Ads.Select(a => a.NodeId + "/" + a.Advertisement.Instance)]; +} diff --git a/src/UiPath.Caching/Negotiation/RedisStreamGroupSource.cs b/src/UiPath.Caching/Negotiation/RedisStreamGroupSource.cs new file mode 100644 index 00000000..d0e9191d --- /dev/null +++ b/src/UiPath.Caching/Negotiation/RedisStreamGroupSource.cs @@ -0,0 +1,33 @@ +namespace UiPath.Caching.Negotiation; + +internal sealed class RedisStreamGroupSource(IRedisConnector redis) : IStreamGroupSource +{ + public async Task> GroupsAsync(RedisKey stream, CancellationToken cancellationToken) + { + await redis.ConnectAsync(cancellationToken).AsTask().WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var groups = await redis.Database.StreamGroupInfoAsync(stream, CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false); + return [.. groups.Select(g => g.Name)]; + } + catch (RedisServerException ex) when (ex.Message.Contains("no such key", StringComparison.OrdinalIgnoreCase)) + { + return []; + } + } + + public async Task IdleAsync(RedisKey stream, string group, CancellationToken cancellationToken) + { + await redis.ConnectAsync(cancellationToken).AsTask().WaitAsync(cancellationToken).ConfigureAwait(false); + try + { + var consumers = await redis.Database.StreamConsumerInfoAsync(stream, group, CommandFlags.DemandMaster).WaitAsync(cancellationToken).ConfigureAwait(false); + return consumers.Length == 0 ? null : TimeSpan.FromMilliseconds(consumers.Min(c => c.IdleTimeInMilliseconds)); + } + catch (RedisServerException ex) when (ex.Message.Contains("NOGROUP", StringComparison.OrdinalIgnoreCase) || ex.Message.Contains("no such key", StringComparison.OrdinalIgnoreCase)) + { + // The maintainer deleted the group between listing and reading it: it has no consumer. + return null; + } + } +} diff --git a/src/UiPath.Caching/Negotiation/StoredAdvertisement.cs b/src/UiPath.Caching/Negotiation/StoredAdvertisement.cs new file mode 100644 index 00000000..c83ea471 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/StoredAdvertisement.cs @@ -0,0 +1,4 @@ +namespace UiPath.Caching.Negotiation; + +/// A node's advertisement, the server stamp it was written under, and how long ago that was. +internal readonly record struct StoredAdvertisement(string NodeId, NodeAdvertisement Advertisement, TimeSpan Age, long Stamp = 0); diff --git a/src/UiPath.Caching/Negotiation/StreamNodeList.cs b/src/UiPath.Caching/Negotiation/StreamNodeList.cs new file mode 100644 index 00000000..8771e69d --- /dev/null +++ b/src/UiPath.Caching/Negotiation/StreamNodeList.cs @@ -0,0 +1,37 @@ +namespace UiPath.Caching.Negotiation; + +/// Finds the live nodes of the Streams transport that advertise nothing: a group whose consumer read recently and that no advertisement as fresh as that read accounts for, even one of a node that has left. +internal sealed class StreamNodeList(IStreamGroupSource groups, FormatNegotiationOptions options) +{ + public async Task> BlockersAsync(IEnumerable streams, IReadOnlyDictionary advertisementAges, CancellationToken cancellationToken) + { + var ages = new Dictionary(advertisementAges, StringComparer.OrdinalIgnoreCase); + var blockers = new SortedSet(StringComparer.Ordinal); + foreach (var stream in streams) + { + foreach (var group in await groups.GroupsAsync(stream, cancellationToken).ConfigureAwait(false)) + { + if (blockers.Contains(group)) + { + continue; + } + + // A group no consumer ever read from belongs to no running node, so it counts as nobody. + var idle = await groups.IdleAsync(stream, group, cancellationToken).ConfigureAwait(false); + if (idle is not { } sinceRead || sinceRead > options.LivenessWindow) + { + continue; + } + + // An advertisement older than the group's last read is a previous run's under the same id, which may read less. + var accountedFor = ages.TryGetValue(group, out var age) && age <= sinceRead + TimeSpan.FromTicks(options.AdvertiseInterval.Ticks * 2); + if (!accountedFor) + { + blockers.Add(group); + } + } + } + + return [.. blockers]; + } +} diff --git a/src/UiPath.Caching/Negotiation/TickSchedule.cs b/src/UiPath.Caching/Negotiation/TickSchedule.cs new file mode 100644 index 00000000..4b911b19 --- /dev/null +++ b/src/UiPath.Caching/Negotiation/TickSchedule.cs @@ -0,0 +1,59 @@ +namespace UiPath.Caching.Negotiation; + +/// +/// When this node advertises and when it reads the node list, each on its own interval. A read is always preceded by an advertisement, and the +/// advertisement that comes with it counts as the next one. +/// +internal sealed class TickSchedule +{ + private readonly TimeSpan _advertise; + private readonly TimeSpan _view; + private readonly Func _jitter; + private TimeSpan _advertiseDue; + private TimeSpan _viewDue; + + public TickSchedule(TimeSpan advertise, TimeSpan view, Func? jitter = null) + { + _advertise = advertise; + _view = view; + _jitter = jitter ?? (() => 0.8 + (Random.Shared.NextDouble() * 0.4)); + Reschedule(TimeSpan.Zero, advertisement: true, nodeList: true); + } + + public TimeSpan Wait(TimeSpan now) + { + var due = _advertiseDue < _viewDue ? _advertiseDue : _viewDue; + return due > now ? due - now : TimeSpan.Zero; + } + + /// What is due at , and counts it done; a due node list read brings the advertisement with it. + public (bool Advertise, bool Refresh) Take(TimeSpan now) + { + if (now >= _viewDue) + { + Reschedule(now, advertisement: true, nodeList: true); + return (true, true); + } + + if (now >= _advertiseDue) + { + Reschedule(now, advertisement: true, nodeList: false); + return (true, false); + } + + return (false, false); + } + + private void Reschedule(TimeSpan now, bool advertisement, bool nodeList) + { + if (advertisement) + { + _advertiseDue = now + (_advertise * _jitter()); + } + + if (nodeList) + { + _viewDue = now + (_view * _jitter()); + } + } +} diff --git a/src/UiPath.Caching/PublicAPI.Unshipped.txt b/src/UiPath.Caching/PublicAPI.Unshipped.txt index 68c2e9f8..bf65855f 100644 --- a/src/UiPath.Caching/PublicAPI.Unshipped.txt +++ b/src/UiPath.Caching/PublicAPI.Unshipped.txt @@ -1,3 +1,28 @@ #nullable enable static UiPath.Caching.Config.ValueSerializerBuilderExtensions.AddValueSerializer(this UiPath.Caching.Config.ICachingBuilder! builder, string! name, System.Func! factory) -> UiPath.Caching.Config.ICachingBuilder! UiPath.Caching.Config.ValueSerializerBuilderExtensions +UiPath.Caching.Redis.FormatNegotiationMode +UiPath.Caching.Redis.FormatNegotiationMode.Auto = 0 -> UiPath.Caching.Redis.FormatNegotiationMode +UiPath.Caching.Redis.FormatNegotiationMode.Force = 1 -> UiPath.Caching.Redis.FormatNegotiationMode +UiPath.Caching.Redis.FormatNegotiationOptions +UiPath.Caching.Redis.FormatNegotiationOptions.AdvertiseInterval.get -> System.TimeSpan +UiPath.Caching.Redis.FormatNegotiationOptions.AdvertiseInterval.set -> void +UiPath.Caching.Redis.FormatNegotiationOptions.AssumeAllNodesCanRead.get -> bool +UiPath.Caching.Redis.FormatNegotiationOptions.AssumeAllNodesCanRead.set -> void +UiPath.Caching.Redis.FormatNegotiationOptions.FormatNegotiationOptions() -> void +UiPath.Caching.Redis.FormatNegotiationOptions.LivenessWindow.get -> System.TimeSpan +UiPath.Caching.Redis.FormatNegotiationOptions.LivenessWindow.set -> void +UiPath.Caching.Redis.FormatNegotiationOptions.MaxViewAge.get -> System.TimeSpan +UiPath.Caching.Redis.FormatNegotiationOptions.MaxViewAge.set -> void +UiPath.Caching.Redis.FormatNegotiationOptions.Mode.get -> UiPath.Caching.Redis.FormatNegotiationMode +UiPath.Caching.Redis.FormatNegotiationOptions.Mode.set -> void +UiPath.Caching.Redis.FormatNegotiationOptions.UpgradeHold.get -> System.TimeSpan +UiPath.Caching.Redis.FormatNegotiationOptions.UpgradeHold.set -> void +UiPath.Caching.Redis.FormatNegotiationOptions.ViewInterval.get -> System.TimeSpan +UiPath.Caching.Redis.FormatNegotiationOptions.ViewInterval.set -> void +UiPath.Caching.Redis.RedisCacheOptions.FormatNegotiation.get -> UiPath.Caching.Redis.FormatNegotiationOptions! +UiPath.Caching.Redis.RedisCacheOptions.FormatNegotiation.set -> void +const UiPath.Caching.Metrics.FormatEffective = "cache.format.effective" -> string! +const UiPath.Caching.Metrics.FormatFallback = "cache.format.fallback" -> string! +const UiPath.Caching.Metrics.FormatNodes = "cache.format.nodes" -> string! +const UiPath.Caching.Metrics.FormatViewAge = "cache.format.view.age" -> string! diff --git a/src/UiPath.Caching/Redis/FormatNegotiationMode.cs b/src/UiPath.Caching/Redis/FormatNegotiationMode.cs new file mode 100644 index 00000000..3796aa03 --- /dev/null +++ b/src/UiPath.Caching/Redis/FormatNegotiationMode.cs @@ -0,0 +1,10 @@ +namespace UiPath.Caching.Redis; + +public enum FormatNegotiationMode +{ + /// Write a non-JSON format only while every node on the transport is known to read it. + Auto, + + /// Write exactly what each policy configures, with no node list. + Force, +} diff --git a/src/UiPath.Caching/Redis/FormatNegotiationOptions.cs b/src/UiPath.Caching/Redis/FormatNegotiationOptions.cs new file mode 100644 index 00000000..ff0f3b9f --- /dev/null +++ b/src/UiPath.Caching/Redis/FormatNegotiationOptions.cs @@ -0,0 +1,47 @@ +namespace UiPath.Caching.Redis; + +/// Decides when a policy's desired format is safe to write, from what the nodes advertise. +public sealed class FormatNegotiationOptions +{ + public FormatNegotiationMode Mode { get; set; } = FormatNegotiationMode.Auto; + + /// How often this node refreshes its advertisement; an advertisement counts for plus two of these. + public TimeSpan AdvertiseInterval { get; set; } = TimeSpan.FromSeconds(15); + + /// How often this node reads the node list. + public TimeSpan ViewInterval { get; set; } = TimeSpan.FromSeconds(15); + + /// How long a node without an advertisement counts as live after its consumer last read its stream. + public TimeSpan LivenessWindow { get; set; } = TimeSpan.FromSeconds(120); + + /// How long every node must have been accounted for, unchanged, before a format is written; zero switches at once, and a node that cannot read it stops writes at once. An hour by default, so a rollback within it never wrote the format. + public TimeSpan UpgradeHold { get; set; } = TimeSpan.FromHours(1); + + /// How long a decision survives a node list that cannot be read, before writes fall back to JSON. + public TimeSpan MaxViewAge { get; set; } = TimeSpan.FromMinutes(2); + + /// Attests that every node the node list cannot see (broadcast off, or no Streams topic) reads the desired formats; off by default, so such an app writes JSON. + public bool AssumeAllNodesCanRead { get; set; } + + internal FormatNegotiationOptions Clone() => (FormatNegotiationOptions)MemberwiseClone(); + + internal void Validate() + { + Positive(AdvertiseInterval, nameof(AdvertiseInterval)); + Positive(ViewInterval, nameof(ViewInterval)); + Positive(LivenessWindow, nameof(LivenessWindow)); + Positive(MaxViewAge, nameof(MaxViewAge)); + if (UpgradeHold < TimeSpan.Zero) + { + throw new InvalidOperationException($"RedisCacheOptions.FormatNegotiation.{nameof(UpgradeHold)} must not be negative; it is {UpgradeHold}."); + } + } + + private static void Positive(TimeSpan value, string name) + { + if (value <= TimeSpan.Zero) + { + throw new InvalidOperationException($"RedisCacheOptions.FormatNegotiation.{name} must be positive; it is {value}."); + } + } +} diff --git a/src/UiPath.Caching/Redis/KeyEscaping.cs b/src/UiPath.Caching/Redis/KeyEscaping.cs new file mode 100644 index 00000000..c1736dca --- /dev/null +++ b/src/UiPath.Caching/Redis/KeyEscaping.cs @@ -0,0 +1,15 @@ +namespace UiPath.Caching.Redis; + +internal static class KeyEscaping +{ + // The composed name goes through the configured strategy, which under ShardKeyEnabled refuses a name whose + // braces form no tag -- so a single stream named 'app:st:{}topic' would abort the pass for every other + // stream. The quarantine hash has no need of the stream's tag, the two being deleted one command at a time, + // so the braces are escaped away. The whole name, not just the stream key: CacheOptions.Separator is only + // required to be non-whitespace, so the marker can carry braces too. Escaping '%' first keeps this injective. + public static string EscapeBraces(string name) => + name + .Replace("%", "%25", StringComparison.Ordinal) + .Replace("{", "%7B", StringComparison.Ordinal) + .Replace("}", "%7D", StringComparison.Ordinal); +} diff --git a/src/UiPath.Caching/Redis/RedisCache.cs b/src/UiPath.Caching/Redis/RedisCache.cs index b099861c..b0570e79 100644 --- a/src/UiPath.Caching/Redis/RedisCache.cs +++ b/src/UiPath.Caching/Redis/RedisCache.cs @@ -1,5 +1,6 @@ using System.Collections.Concurrent; using System.Runtime.CompilerServices; +using UiPath.Caching.Negotiation; using UiPath.Caching.Policies; using UiPath.Caching.Serialization; using UiPath.Caching.Telemetry; @@ -29,6 +30,7 @@ internal sealed partial class RedisCache : RedisCacheBase, ICache, IInvalidation private readonly ConcurrentDictionary _generations = new(); private ValueSerializerRegistry _valueSerializers = ValueSerializerRegistry.Empty; private ValueFormatResolver _formats; + private IEffectiveFormat? _effectiveFormat; public RedisCache( IRedisConnector redis, @@ -70,7 +72,16 @@ internal ValueSerializerRegistry ValueSerializers init { _valueSerializers = value; - _formats = new ValueFormatResolver(_serializer, value, DefaultPolicy); + _formats = new ValueFormatResolver(_serializer, value, DefaultPolicy, _effectiveFormat); + } + } + + internal IEffectiveFormat? EffectiveFormat + { + init + { + _effectiveFormat = value; + _formats = new ValueFormatResolver(_serializer, _valueSerializers, DefaultPolicy, value); } } diff --git a/src/UiPath.Caching/Redis/RedisCacheOptions.cs b/src/UiPath.Caching/Redis/RedisCacheOptions.cs index ff01fb28..eff71fa4 100644 --- a/src/UiPath.Caching/Redis/RedisCacheOptions.cs +++ b/src/UiPath.Caching/Redis/RedisCacheOptions.cs @@ -54,6 +54,13 @@ public class RedisCacheOptions : ICacheOptions /// How many fields of one hash a single Redis command carries; a hash call over more is split into commands of this size, so none blocks the server for long. public int HashFieldBatchSize { get; set; } = 1000; + public FormatNegotiationOptions FormatNegotiation { get; set; } = new(); + /// Member-wise copy, so a caller can vary one setting without mutating the DI singleton. - internal RedisCacheOptions ShallowCopy() => (RedisCacheOptions)MemberwiseClone(); + internal RedisCacheOptions ShallowCopy() + { + var copy = (RedisCacheOptions)MemberwiseClone(); + copy.FormatNegotiation = FormatNegotiation.Clone(); + return copy; + } } diff --git a/src/UiPath.Caching/Redis/RedisCacheProvider.cs b/src/UiPath.Caching/Redis/RedisCacheProvider.cs index adb4c509..1275dfd5 100644 --- a/src/UiPath.Caching/Redis/RedisCacheProvider.cs +++ b/src/UiPath.Caching/Redis/RedisCacheProvider.cs @@ -1,3 +1,4 @@ +using UiPath.Caching.Negotiation; using UiPath.Caching.Policies; using UiPath.Caching.Serialization; using UiPath.Caching.Telemetry; @@ -17,6 +18,7 @@ public sealed class RedisCacheProvider : ICacheProvider private readonly TimeProvider _clock; private readonly IKeyMaskingPolicy? _keyMaskingPolicy; private readonly ValueSerializerRegistry _valueSerializers; + private readonly IEffectiveFormat? _effectiveFormat; private readonly Lazy _cache; private readonly Lazy _hashCache; @@ -57,9 +59,11 @@ internal RedisCacheProvider( ICachePolicyFactory policyFactory, TimeProvider clock, IKeyMaskingPolicy? keyMaskingPolicy, - ValueSerializerRegistry valueSerializers) + ValueSerializerRegistry valueSerializers, + IEffectiveFormat? effectiveFormat = null) { _valueSerializers = valueSerializers; + _effectiveFormat = effectiveFormat; _keyMaskingPolicy = keyMaskingPolicy; _clock = clock; _redisCacheOptions = redisCacheOptions.Value; @@ -110,7 +114,8 @@ internal static RedisCacheProvider Create(IServiceProvider services) => services.GetRequiredService(), services.GetRequiredService(), services.GetService(), - services.GetRequiredService()); + services.GetRequiredService(), + services.GetService()?.Negotiator); private RedisCache BuildCache() { @@ -128,6 +133,7 @@ private RedisCache BuildCache() _keyMaskingPolicy) { ValueSerializers = _valueSerializers, + EffectiveFormat = _effectiveFormat, }; } @@ -147,6 +153,7 @@ private RedisHashCache BuildHashCache() _keyMaskingPolicy) { ValueSerializers = _valueSerializers, + EffectiveFormat = _effectiveFormat, }; } } diff --git a/src/UiPath.Caching/Redis/RedisHashCache.cs b/src/UiPath.Caching/Redis/RedisHashCache.cs index b96cbe78..36c31eef 100644 --- a/src/UiPath.Caching/Redis/RedisHashCache.cs +++ b/src/UiPath.Caching/Redis/RedisHashCache.cs @@ -1,5 +1,6 @@ using System.Collections.Immutable; using System.Runtime.CompilerServices; +using UiPath.Caching.Negotiation; using UiPath.Caching.Policies; using UiPath.Caching.Serialization; using UiPath.Caching.Telemetry; @@ -32,6 +33,7 @@ internal sealed partial class RedisHashCache : RedisCacheBase, IHashCache, ISpan private readonly int _fieldBatchSize; private ValueSerializerRegistry _valueSerializers = ValueSerializerRegistry.Empty; private ValueFormatResolver _formats; + private IEffectiveFormat? _effectiveFormat; public RedisHashCache( IRedisConnector redis, @@ -70,7 +72,16 @@ internal ValueSerializerRegistry ValueSerializers init { _valueSerializers = value; - _formats = new ValueFormatResolver(_serializer, value, DefaultPolicy); + _formats = new ValueFormatResolver(_serializer, value, DefaultPolicy, _effectiveFormat); + } + } + + internal IEffectiveFormat? EffectiveFormat + { + init + { + _effectiveFormat = value; + _formats = new ValueFormatResolver(_serializer, _valueSerializers, DefaultPolicy, value); } } diff --git a/src/UiPath.Caching/Serialization/ValueFormatResolver.cs b/src/UiPath.Caching/Serialization/ValueFormatResolver.cs index beb1b406..7e331a64 100644 --- a/src/UiPath.Caching/Serialization/ValueFormatResolver.cs +++ b/src/UiPath.Caching/Serialization/ValueFormatResolver.cs @@ -1,3 +1,5 @@ +using UiPath.Caching.Negotiation; + namespace UiPath.Caching.Serialization; /// Decides per call which serializer a write uses and whether a read looks for a format header. @@ -5,18 +7,20 @@ internal sealed class ValueFormatResolver { private readonly ValueSerializerRegistry _registry; private readonly CachePolicy _defaultPolicy; + private readonly IEffectiveFormat? _effective; - public ValueFormatResolver(ISerializerProxy proxy, ValueSerializerRegistry registry, CachePolicy defaultPolicy) + public ValueFormatResolver(ISerializerProxy proxy, ValueSerializerRegistry registry, CachePolicy defaultPolicy, IEffectiveFormat? effective = null) { _registry = registry; _defaultPolicy = defaultPolicy; + _effective = effective; Reader = registry.IsEmpty ? null : new ValueReader(proxy, registry); } /// Null while no serializer is registered: nothing then reads by header. public ValueReader? Reader { get; } - /// A call without a policy takes the default policy's, and a policy that sets no write serializer falls back to the default's. A write that names a serializer nobody registered throws. + /// A write the node node list does not yet allow is JSON. A call without a policy takes the default policy's, and a policy that sets no write serializer falls back to the default's. A write that names a serializer nobody registered throws. public ValueFormat For(CachePolicy? policy, bool forWrite) { var effective = WriteSource(policy); @@ -25,7 +29,16 @@ public ValueFormat For(CachePolicy? policy, bool forWrite) return default; } - return new(forWrite ? WriterOf(effective) : null, Reader is { } reader && reader.SniffsFor(effective)); + var writer = forWrite ? WriterOf(effective) : null; + + // The policy still names the format, so a read still looks for headers; only the write is held to what every node reads. + // A custom proxy has no JSON to fall back to: what it writes can read back as a header, so such a policy always writes its own format. + if (writer is not null && _effective is { } gate && Reader is { JsonProxyIsDefault: true } && !gate.Allows(writer)) + { + writer = null; + } + + return new(writer, Reader is { } reader && reader.SniffsFor(effective)); } // Per field, as PolicyDuration does, without building a merged policy: the one that supplies Write supplies its resolved slot too. diff --git a/src/UiPath.Caching/Serialization/ValueReader.cs b/src/UiPath.Caching/Serialization/ValueReader.cs index 54763a55..75c04875 100644 --- a/src/UiPath.Caching/Serialization/ValueReader.cs +++ b/src/UiPath.Caching/Serialization/ValueReader.cs @@ -11,6 +11,9 @@ internal sealed class ValueReader(ISerializerProxy json, ValueSerializer private readonly bool _jsonProxyIsDefault = json.GetType() == typeof(SystemJsonByteSerializerProxy); // Only a policy that writes a non-JSON format puts headers where a custom proxy could not tell them from its own bytes; Read lists never open the gate. + /// False for a custom proxy or a subclass: its bytes are not known to be JSON, so a write it makes cannot stand in for one with a header. + public bool JsonProxyIsDefault => _jsonProxyIsDefault; + public bool SniffsFor(CachePolicy? policy) => !registry.IsEmpty && (_jsonProxyIsDefault || WritesHeaders(policy)); diff --git a/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicProviderTests.cs b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicProviderTests.cs index 28a8e0b9..a6359827 100644 --- a/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicProviderTests.cs +++ b/tests/UiPath.Caching.Tests/Broadcast/RedisStreamsTopicProviderTests.cs @@ -42,6 +42,37 @@ public async Task Disposing_topic_removes_it_from_provider() Sut.Keys.Should().BeEmpty(); } + [Fact] + public async Task A_provider_that_makes_and_drops_topics_keeps_no_record_of_the_dropped_ones() + { + var token = testContextAccessor.Current.CancellationToken; + TopicKey[] keys = [.. Enumerable.Range(0, 20).Select(i => (TopicKey)$"topic-{i}")]; + foreach (var key in keys) + { + Sut.Create(key); + } + + Sut.StreamKeyCount.Should().Be(keys.Length); + Sut.StreamKeys().Should().HaveCount(keys.Length); + + foreach (var key in keys.Take(10)) + { + Sut.Remove(key); + } + + foreach (var key in keys.Skip(10)) + { + Sut.Create(key).Dispose(); + } + + for (var attempt = 0; Sut.StreamKeyCount > 0 && attempt < 50; attempt++) + { + await Task.Delay(20, token); + } + + Sut.StreamKeyCount.Should().Be(0); + } + [Fact] public void Remove_topic_from_provider() { diff --git a/tests/UiPath.Caching.Tests/Config/AppSettingsAllJsonBindingTests.cs b/tests/UiPath.Caching.Tests/Config/AppSettingsAllJsonBindingTests.cs index b77cb163..7b0f01cd 100644 --- a/tests/UiPath.Caching.Tests/Config/AppSettingsAllJsonBindingTests.cs +++ b/tests/UiPath.Caching.Tests/Config/AppSettingsAllJsonBindingTests.cs @@ -32,6 +32,7 @@ public class AppSettingsAllJsonBindingTests { "Caching:Broadcast:RedisPubSub", [typeof(RedisPubSubTopicOptions)], ["Topics"] }, { "Caching:InMemoryRedis", [typeof(InMemoryRedisCacheOptions)], [] }, { "Caching:Redis", [typeof(RedisCacheOptions)], [] }, + { "Caching:Redis:FormatNegotiation", [typeof(FormatNegotiationOptions)], [] }, { "Caching:InMemory", [typeof(InMemoryCacheOptions)], [] }, { "Caching:ResiliencePolicies", [typeof(ResiliencePoliciesOptions)], [] }, { "Caching:Queue:Redis", [typeof(RedisSetCacheOptions)], [] }, diff --git a/tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs b/tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs index 8c4582a2..7a5c1531 100644 --- a/tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs +++ b/tests/UiPath.Caching.Tests/MessagePack/MessagePackIntegrationTests.cs @@ -183,7 +183,7 @@ private ServiceProvider Node(string source, bool peerFieldUpdates = false, bool b => { b.AddRedisConnection(o => o.ConnectionString = fixture.ConnectionString); - b.AddRedis(_ => { }); + b.AddRedis(o => o.FormatNegotiation.Mode = FormatNegotiationMode.Force); b.AddBroadcast(); b.AddInMemoryRedis(o => o.PeerFieldUpdates = peerFieldUpdates); b.AddMessagePackSerializer(); diff --git a/tests/UiPath.Caching.Tests/Negotiation/EffectiveWriteTests.cs b/tests/UiPath.Caching.Tests/Negotiation/EffectiveWriteTests.cs new file mode 100644 index 00000000..8ee86238 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/EffectiveWriteTests.cs @@ -0,0 +1,424 @@ +using System.Text; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Logging; +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using UiPath.Caching.Config; +using UiPath.Caching.Negotiation; +using UiPath.Caching.Policies; +using UiPath.Caching.Serialization; +using UiPath.Caching.Telemetry; +using UiPath.Caching.Tests.Fakes; +using UiPath.Caching.Tests.Telemetry; + +namespace UiPath.Caching.Tests.Negotiation; + +[Collection(WriterPoolCollection.Name)] +public class EffectiveWriteTests(ITestContextAccessor testContextAccessor) +{ + private static readonly string[] Binary = ["10"]; + + private readonly IDatabase _database = Substitute.For(); + private readonly ITransaction _transaction = Substitute.For(); + private readonly RecordingResiliencePipeline _write = new(); + private readonly PassthroughValueSerializer _fast = new(0x10); + private readonly Gate _gate = new(); + + private CancellationToken Token => testContextAccessor.Current.CancellationToken; + + [Fact] + public async Task Write_follows_effective_format_for_the_simple_cache() + { + var sut = SimpleCache(out var registry); + var policy = ResolvedPolicy(registry); + var written = CaptureSimpleWrites(); + + await sut.SetAsync("k", "hello", TimeSpan.FromMinutes(1), policy, Token); + _gate.Open = true; + await sut.SetAsync("k", "hello", TimeSpan.FromMinutes(1), policy, Token); + await sut.GetOrAddAsync("g", _ => Task.FromResult("hello"), TimeSpan.FromMinutes(1), policy, Token); + _gate.Open = false; + await sut.SetAsync("k", "hello", TimeSpan.FromMinutes(1), policy, Token); + + written.Select(w => w[0] == 0x10).Should().Equal(false, true, true, false); + written[0].Should().Equal(Encoding.UTF8.GetBytes("\"hello\""), "a closed gate writes the JSON the default policy writes"); + } + + [Fact] + public async Task Write_follows_effective_format_for_hash_fields_and_metadata() + { + var sut = HashCache(out var registry); + var policy = ResolvedPolicy(registry); + var written = CaptureHashWrites(); + var options = new HashCacheEntryOptions(DateTimeOffset.UtcNow.AddMinutes(5), Metadata: new Dictionary { ["m"] = "v" }); + + await sut.SetAsync("k", new Dictionary { ["a"] = "x" }, options, policy, Token); + _gate.Open = true; + await sut.SetAsync("k", new Dictionary { ["a"] = "x" }, options, policy, Token); + + written.Should().HaveCount(2); + Raw(written[0], "a").Should().Equal(Encoding.UTF8.GetBytes("\"x\"")); + Raw(written[0], "_metadata_").Should().Equal(Encoding.UTF8.GetBytes("{\"m\":\"v\"}")); + Raw(written[1], "a")[0].Should().Be(0x10); + Raw(written[1], "_metadata_")[0].Should().Be(0x10); + } + + [Fact] + public async Task Write_follows_effective_format_for_a_hash_replace_and_for_metadata_alone() + { + var sut = HashCache(out var registry, new CachePolicy { Serializer = new SerializerPolicy { Write = "fast" } }); + var policy = ResolvedPolicy(registry); + var written = CaptureHashWrites(); + var replace = new HashCacheEntryOptions(DateTimeOffset.UtcNow.AddMinutes(5), Metadata: new Dictionary { ["m"] = "v" }, SetOption: HashCacheSetOption.HashReplace); + var metadata = new Dictionary { ["m"] = "v" }; + _database.HashSetAsync(Arg.Any(), Arg.Any(), Arg.Do(v => written.Add([new HashEntry("_metadata_", v)])), Arg.Any(), Arg.Any()).Returns(true); + + await sut.SetAsync("k", new Dictionary { ["a"] = "x" }, replace, policy, Token); + await sut.SetMetadataAsync("k", metadata, Token); + + written.Should().NotBeEmpty(); + written.SelectMany(e => e).Should().OnlyContain(e => ((byte[])e.Value!)[0] != 0x10); + written.Clear(); + + _gate.Open = true; + await sut.SetAsync("k", new Dictionary { ["a"] = "x" }, replace, policy, Token); + await sut.SetMetadataAsync("k", metadata, Token); + + written.Should().NotBeEmpty(); + written.SelectMany(e => e).Should().OnlyContain(e => ((byte[])e.Value!)[0] == 0x10); + } + + [Fact] + public async Task A_closed_gate_still_reads_a_headered_value() + { + var sut = SimpleCache(out var registry); + _database.StringGetAsync(Arg.Any(), Arg.Any()).Returns((RedisValue)new byte[] { 0x10, (byte)'h', (byte)'i' }); + + (await sut.GetAsync("k", ResolvedPolicy(registry), Token)).Should().Be("hi"); + } + + [Fact] + public async Task Hot_path_issues_no_extra_redis_call() + { + var nodeList = new FakeNodeListServer(); + var policy = ResolvedPolicy(new ValueSerializerRegistry([("fast", _fast, false)])); + var negotiator = new FormatNegotiator([new DesiredWrite(_fast, Binary, ["default"], [])], new FormatNegotiationOptions { UpgradeHold = TimeSpan.Zero }, TimeProvider.System); + negotiator.Observe(new NodeListView([new("me", new("3.2.0", "i", ["json", "10"], new Dictionary>()), TimeSpan.Zero)], [], Complete: true)); + negotiator.Allows(_fast).Should().BeTrue(); + var without = SimpleCache(out _, gate: false); + var with = SimpleCache(out _, negotiator: negotiator); + _database.StringSetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + _transaction.StringSetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + _database.StringGetAsync(Arg.Any(), Arg.Any()).Returns(RedisValue.Null); + + _database.ClearReceivedCalls(); + await Operations(without, policy); + var baseline = _database.ReceivedCalls().Count() + _transaction.ReceivedCalls().Count(); + _database.ClearReceivedCalls(); + _transaction.ClearReceivedCalls(); + await Operations(with, policy); + + // A set, a get and a get-or-add miss: one write, one read, one read and one write. + baseline.Should().Be(4); + (_database.ReceivedCalls().Count() + _transaction.ReceivedCalls().Count()).Should().Be(4); + nodeList.Database.ReceivedCalls().Should().BeEmpty("the node list is read and written by the service on its timer, never by a cache call"); + } + + [Fact] + public async Task Force_mode_ignores_node_list_and_still_advertises_what_it_reads() + { + var options = new RedisCacheOptions { FormatNegotiation = { Mode = FormatNegotiationMode.Force } }; + var services = Build(options, new ValueSerializerRegistry([("fast", _fast, false)]), new CachePolicy { Serializer = new SerializerPolicy { Write = "fast" } }); + + var forced = FormatNegotiation.Create(services); + forced.Negotiator.Should().BeNull("Force writes exactly what is configured, so there is nothing to decide"); + forced.IsActive.Should().BeTrue("it reads every format, so the nodes that decide must be told"); + forced.Advertisement!.ValueReads.Should().Equal("json", "10"); + var sut = SimpleCache(out var registry, gate: false); + var written = CaptureSimpleWrites(); + await sut.SetAsync("k", "hello", TimeSpan.FromMinutes(1), ResolvedPolicy(registry), Token); + written.Should().ContainSingle().Which[0].Should().Be(0x10); + + options.FormatNegotiation.Mode = FormatNegotiationMode.Auto; + FormatNegotiation.Create(services).Negotiator.Should().NotBeNull(); + } + + [Fact] + public void A_reader_only_app_advertises_and_decides_for_a_policy_built_at_the_call_site_and_one_with_no_serializer_does_neither() + { + var reader = FormatNegotiation.Create(Build(new RedisCacheOptions(), new ValueSerializerRegistry([("fast", _fast, false)]), null)); + reader.IsActive.Should().BeTrue(); + reader.Negotiator.Should().NotBeNull("an ad-hoc policy may still name a registered serializer"); + reader.Advertisement!.ValueReads.Should().Equal("json", "10"); + + FormatNegotiation.Create(Build(new RedisCacheOptions(), ValueSerializerRegistry.Empty, null)).IsActive.Should().BeFalse(); + } + + [Theory] + [InlineData(0)] + [InlineData(-1)] + public void A_node_that_only_advertises_validates_its_options(int seconds) + { + var options = new RedisCacheOptions { FormatNegotiation = { Mode = FormatNegotiationMode.Force, AdvertiseInterval = TimeSpan.FromSeconds(seconds) } }; + + FluentActions.Invoking(() => FormatNegotiation.Create(Build(options, new ValueSerializerRegistry([("fast", _fast, false)]), null))) + .Should().Throw().WithMessage("*AdvertiseInterval*"); + } + + [Fact] + public async Task A_policy_built_at_the_call_site_is_held_to_the_decision_of_the_serializer_it_names() + { + var other = new PassthroughValueSerializer(0x11); + var negotiator = new FormatNegotiator( + [new DesiredWrite(_fast, Binary, ["default"], []), new DesiredWrite(other, ["11"], [], ["default"])], + new FormatNegotiationOptions { UpgradeHold = TimeSpan.Zero }, + TimeProvider.System); + // Every node reads 11 and one node does not read 10. + negotiator.Observe(new NodeListView([new("me", new("3.2.0", "i", ["json", "11"], new Dictionary>()), TimeSpan.Zero)], [], Complete: true)); + var registry = new ValueSerializerRegistry([("fast", _fast, false), ("other", other, false)]); + var sut = SimpleCache(registry, negotiator); + var written = CaptureSimpleWrites(); + + await sut.SetAsync("k", "hello", TimeSpan.FromMinutes(1), new CachePolicy { Serializer = new SerializerPolicy { Write = "fast" } }, Token); + await sut.SetAsync("k", "hello", TimeSpan.FromMinutes(1), new CachePolicy { Serializer = new SerializerPolicy { Write = "other" } }, Token); + + written[0][0].Should().NotBe(0x10, "no node on the node list is known to read 10"); + written[1][0].Should().Be(0x11, "the node list says every node reads 11"); + } + + [Fact] + public void Reading_the_decision_through_the_resolver_allocates_nothing() + { + var negotiator = new FormatNegotiator([new DesiredWrite(_fast, Binary, ["default"], [])], new FormatNegotiationOptions { UpgradeHold = TimeSpan.Zero }, TimeProvider.System); + negotiator.Observe(new NodeListView([new("me", new("3.2.0", "i", ["json", "10"], new Dictionary>()), TimeSpan.Zero)], [], Complete: true)); + var registry = new ValueSerializerRegistry([("fast", _fast, false)]); + var resolver = new ValueFormatResolver(new SystemJsonByteSerializerProxy(), registry, new CachePolicy(), negotiator); + var policy = ResolvedPolicy(registry); + resolver.For(policy, forWrite: true).Writer.Should().BeSameAs(_fast); + + // Steady state: after a warm-up, so the JIT and tiering do not count, the best of a few passes allocates nothing. + var best = long.MaxValue; + for (var attempt = 0; attempt < 5; attempt++) + { + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 10_000; i++) + { + _ = resolver.For(policy, forWrite: true); + } + + best = Math.Min(best, GC.GetAllocatedBytesForCurrentThread() - before); + } + + best.Should().Be(0); + } + + [Fact] + public void A_copy_of_the_options_does_not_share_the_negotiation_settings() + { + var original = new RedisCacheOptions(); + + var copy = original.ShallowCopy(); + copy.FormatNegotiation.UpgradeHold = TimeSpan.Zero; + copy.FormatNegotiation.Mode = FormatNegotiationMode.Force; + + original.FormatNegotiation.UpgradeHold.Should().Be(TimeSpan.FromHours(1)); + original.FormatNegotiation.Mode.Should().Be(FormatNegotiationMode.Auto); + } + + [Fact] + public void Every_registered_serializer_is_planned_whichever_policies_write_it() + { + var other = new PassthroughValueSerializer(0x11); + var registry = new ValueSerializerRegistry([("fast", _fast, false), ("other", other, false)]); + var policies = new DefaultCachePolicyFactory( + new Dictionary { ["Orders"] = new() { Serializer = new SerializerPolicy { Write = "fast" } } }, + null, + valueSerializers: registry); + + var desired = NegotiationPlan.Desired(registry, policies); + + desired.Should().HaveCount(2, "a policy built at the call site may name either"); + desired.Single(d => ReferenceEquals(d.Writer, _fast)).Policies.Should().Equal("orders"); + var unused = desired.Single(d => ReferenceEquals(d.Writer, other)); + unused.Policies.Should().BeEmpty(); + unused.Others.Should().Equal("orders"); + unused.Tokens.Should().Equal("11"); + } + + [Fact] + public async Task A_policy_on_a_raw_byte_proxy_writes_its_own_format_whatever_the_gate_says_so_nothing_reads_back_differently() + { + var registry = new ValueSerializerRegistry([("fast", _fast, false)]); + var sut = SimpleCache(registry, _gate, new RawByteSerializerProxy()); + var written = CaptureSimpleWrites(); + byte[] looksLikeAHeader = [0x02, 0xC4, 0x01, 0x41]; + var policy = ResolvedPolicy(registry); + + // Closed: a JSON fallback does not exist for raw bytes, so these would be stored as they are and read back as one byte. + await sut.SetAsync("k", looksLikeAHeader, TimeSpan.FromMinutes(1), policy, Token); + + var stored = written.Should().ContainSingle().Subject; + stored[0].Should().Be(0x10, "the policy's own format, which keeps the bytes whole behind a header"); + _database.StringGetAsync(Arg.Any(), Arg.Any()).Returns((RedisValue)stored); + (await sut.GetAsync("k", policy, Token)).Should().Equal(looksLikeAHeader); + } + + [Fact] + public void A_node_that_does_not_sniff_advertises_json_only_for_the_policies_that_do_not() + { + var registry = new ValueSerializerRegistry([("fast", _fast, false)]); + var policies = new DefaultCachePolicyFactory( + new Dictionary { ["Orders"] = new() { Serializer = new SerializerPolicy { Write = "fast" } }, ["Plain"] = new() }, + null, + valueSerializers: registry); + + var raw = NegotiationPlan.Advertisement(registry, policies, new ValueReader(new RawByteSerializerProxy(), registry), "3.2"); + var normal = NegotiationPlan.Advertisement(registry, policies, new ValueReader(new SystemJsonByteSerializerProxy(), registry), "3.2"); + + raw.Strict.Should().NotContainKey("orders", "its own reads look for the header it writes"); + raw.Strict["plain"].Should().Equal("json", "nosniff"); + raw.Strict["default"].Should().Equal("json", "nosniff"); + normal.Strict.Should().BeEmpty("the default proxy reads a header whatever the policy writes"); + } + + [Fact] + public void A_default_app_registers_nothing_to_negotiate() + { + var services = new ServiceCollection(); + var builder = new CachingBuilder(services); + builder.Complete(); + + services.Should().NotContain(d => d.ServiceType == typeof(FormatNegotiation)); + services.Should().NotContain(d => d.ImplementationType == typeof(FormatNegotiationService)); + } + + [Fact] + public void An_app_with_a_serializer_and_no_json_only_policy_registers_the_service() + { + var services = new ServiceCollection(); + var builder = new CachingBuilder(services); + builder.AddValueSerializer("fast", _ => _fast); + builder.Complete(); + + services.Should().Contain(d => d.ImplementationType == typeof(FormatNegotiationService)); + } + + private static byte[] Raw(HashEntry[] entries, string field) => (byte[])entries.Single(e => e.Name == field).Value!; + + private static async Task Operations(RedisCache cache, CachePolicy policy) + { + await cache.SetAsync("k", "hello", TimeSpan.FromMinutes(1), policy, TestContext.Current.CancellationToken); + await cache.GetAsync("k", policy, TestContext.Current.CancellationToken); + await cache.GetOrAddAsync("k", _ => Task.FromResult("hello"), TimeSpan.FromMinutes(1), policy, TestContext.Current.CancellationToken); + } + + private ServiceProvider Build(RedisCacheOptions options, ValueSerializerRegistry registry, CachePolicy? defaultPolicy) => + new ServiceCollection() + .AddSingleton(Options.Create(options)) + .AddSingleton(Options.Create(new CacheOptions { AppShortName = "t" })) + .AddSingleton(Substitute.For()) + .AddSingleton(NullLoggerFactory.Instance) + .AddSingleton(NullTelemetryProvider.Instance) + .AddSingleton(TimeProvider.System) + .AddSingleton>(new SystemJsonByteSerializerProxy()) + .AddSingleton(registry) + .AddSingleton(sp => new DefaultCachePolicyFactory(null, defaultPolicy, valueSerializers: registry)) + .BuildServiceProvider(); + + private CachePolicy ResolvedPolicy(ValueSerializerRegistry registry) => + new DefaultCachePolicyFactory(null, new CachePolicy { Serializer = new SerializerPolicy { Write = "fast" } }, valueSerializers: registry).Default!; + + private List CaptureSimpleWrites() + { + var captured = new List(); + _database.StringSetAsync(Arg.Any(), Arg.Do(v => captured.Add(((byte[]?)v)!)), Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + _database.StringGetAsync(Arg.Any(), Arg.Any()).Returns(RedisValue.Null); + _transaction.StringSetAsync(Arg.Any(), Arg.Do(v => captured.Add(((byte[]?)v)!)), Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + return captured; + } + + private List CaptureHashWrites() + { + var written = new List(); + _transaction.HashSetAsync(Arg.Any(), Arg.Do(written.Add), Arg.Any()).Returns(Task.CompletedTask); + _transaction.KeyExpireAsync(Arg.Any(), Arg.Any(), Arg.Any()).Returns(true); + _database.KeyDeleteAsync(Arg.Any(), Arg.Any()).Returns(true); + return written; + } + + private IRedisConnector Connector() + { + var connector = Substitute.For(); + connector.Database.Returns(_database); + connector.Version.Returns(new Version(6, 0)); + connector.IsConnected.Returns(true); + _database.Multiplexer.GetHashSlot(Arg.Any()).Returns(0); + _transaction.ExecuteAsync(Arg.Any()).Returns(true); + _database.CreateTransaction(Arg.Any()).Returns(_transaction); + return connector; + } + + private (IResiliencePipelineProvider Pipelines, IRedisKeyStrategyFactory Strategies) Plumbing(string keyspace) + { + var pipelines = Substitute.For(); + pipelines.Get(ResiliencePipelineNames.Read).Returns(EmptyResiliencePipeline.Instance); + pipelines.Get(ResiliencePipelineNames.Write).Returns(_write); + var strategy = Substitute.For(); + strategy.GetRedisKey(Arg.Any()).Returns(call => (RedisKey)$"test:{keyspace}:{call.Arg()}"); + var strategies = Substitute.For(); + strategies.Create(Arg.Any(), Arg.Any()).Returns(strategy); + return (pipelines, strategies); + } + + private RedisCache SimpleCache(out ValueSerializerRegistry registry, bool gate = true, IEffectiveFormat? negotiator = null) + { + registry = new ValueSerializerRegistry([("fast", _fast, false)]); + return SimpleCache(registry, negotiator ?? (gate ? _gate : null)); + } + + private RedisCache SimpleCache(ValueSerializerRegistry registry, IEffectiveFormat? effective, ISerializerProxy? proxy = null) + { + var (pipelines, strategies) = Plumbing("str"); + return new RedisCache( + Connector(), + proxy ?? new SystemJsonByteSerializerProxy(), + pipelines, + new RecordingTelemetryProvider(), + new RedisCacheOptions { RedisKeyStrategyFactory = strategies, ReadScriptsEnabled = false }, + new CacheOptions { AppShortName = "test", ConnectionMonitorEnabled = true }, + new DefaultCachePolicyFactory(null, null, valueSerializers: registry), + TimeProvider.System, + NullLogger.Instance) + { + ValueSerializers = registry, + EffectiveFormat = effective, + }; + } + + private RedisHashCache HashCache(out ValueSerializerRegistry registry, CachePolicy? defaultPolicy = null) + { + registry = new ValueSerializerRegistry([("fast", _fast, false)]); + var (pipelines, strategies) = Plumbing("hash"); + return new RedisHashCache( + Connector(), + new SystemJsonByteSerializerProxy(), + pipelines, + new RecordingTelemetryProvider(), + new RedisCacheOptions { RedisKeyStrategyFactory = strategies, ReadScriptsEnabled = false }, + new CacheOptions { AppShortName = "test", ConnectionMonitorEnabled = true }, + new DefaultCachePolicyFactory(null, defaultPolicy, valueSerializers: registry), + TimeProvider.System, + NullLogger.Instance) + { + ValueSerializers = registry, + EffectiveFormat = _gate, + }; + } + + private sealed class Gate : IEffectiveFormat + { + public bool Open { get; set; } + + public bool Allows(ICacheValueSerializer writer) => Open; + } +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/FakeNodeListServer.cs b/tests/UiPath.Caching.Tests/Negotiation/FakeNodeListServer.cs new file mode 100644 index 00000000..ad808c13 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/FakeNodeListServer.cs @@ -0,0 +1,86 @@ +using System.Globalization; +using StackExchange.Redis; + +namespace UiPath.Caching.Tests.Negotiation; + +/// A server whose clock a test sets, holding one hash, answering the node list's scripts or, with scripts denied, its plain commands. +internal sealed class FakeNodeListServer +{ + private readonly Dictionary _hash = []; + + public FakeNodeListServer(bool scripts = true) + { + Database = Substitute.For(); + Connector = Substitute.For(); + Connector.Database.Returns(Database); + Database.ScriptEvaluateAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => Failure is { } failure ? Task.FromException(failure) : scripts + ? Task.FromResult(Script(call.ArgAt(0), call.ArgAt(2))) +#pragma warning disable CS0618 // The only public constructor of a server error. + : Task.FromException(new RedisServerException("ERR unknown command 'EVAL'"))); +#pragma warning restore CS0618 + Database.ExecuteAsync("TIME", Arg.Any?>(), Arg.Any()) + .Returns(_ => Task.FromResult(RedisResult.Create([(RedisValue)(NowMs / 1000), (RedisValue)((NowMs % 1000) * 1000)]))); + Database.HashSetAsync(Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => + { + _hash[(string)call.ArgAt(1)!] = (string)call.ArgAt(2)!; + return Task.FromResult(true); + }); + Database.HashGetAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => Task.FromResult(_hash.TryGetValue((string)call.ArgAt(1)!, out var held) ? (RedisValue)held : RedisValue.Null)); + Database.HashGetAllAsync(Arg.Any(), Arg.Any()) + .Returns(_ => Task.FromResult(_hash.Select(p => new HashEntry(p.Key, p.Value)).ToArray())); + Database.HashDeleteAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => Task.FromResult(_hash.Remove((string)call.ArgAt(1)!))); + Database.HashDeleteAsync(Arg.Any(), Arg.Any(), Arg.Any()) + .Returns(call => Task.FromResult((long)call.ArgAt(1).Count(f => _hash.Remove((string)f!)))); + } + + public IDatabase Database { get; } + + /// When set, every script fails with it, as an unreachable Redis does. + public Exception? Failure { get; set; } + + public IRedisConnector Connector { get; } + + /// The server's clock, in milliseconds since the epoch. + public long NowMs { get; set; } = 1_000_000_000_000; + + public IReadOnlyDictionary Hash => _hash; + + /// The node list commands that ran, in order: write and read. + public List Events { get; } = []; + + /// Stores a raw field, as another node or version would have written it. + public void Put(string field, string value) => _hash[field] = value; + + private RedisResult Script(string script, RedisValue[] arguments) + { + if (script.Contains("HDEL", StringComparison.Ordinal)) + { + var removed = 0; + for (var i = 0; i < arguments.Length; i += 2) + { + if (_hash.TryGetValue((string)arguments[i]!, out var held) && held.StartsWith((string)arguments[i + 1]! + "|", StringComparison.Ordinal)) + { + _hash.Remove((string)arguments[i]!); + removed++; + } + } + + return RedisResult.Create((RedisValue)removed); + } + + if (script.Contains("HSET", StringComparison.Ordinal)) + { + Events.Add("write"); + _hash[(string)arguments[0]!] = NowMs.ToString(CultureInfo.InvariantCulture) + "|" + (string)arguments[1]!; + return RedisResult.Create((RedisValue)NowMs); + } + + Events.Add("read"); + var flat = _hash.SelectMany(p => new[] { RedisResult.Create((RedisValue)p.Key, ResultType.BulkString), RedisResult.Create((RedisValue)p.Value, ResultType.BulkString) }).ToArray(); + return RedisResult.Create([RedisResult.Create((RedisValue)NowMs), RedisResult.Create(flat)]); + } +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationIntegrationTests.cs b/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationIntegrationTests.cs new file mode 100644 index 00000000..745a0b6a --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationIntegrationTests.cs @@ -0,0 +1,336 @@ +using System.Net; +using Microsoft.Extensions.DependencyInjection; +using Microsoft.Extensions.Hosting; +using Microsoft.Extensions.Logging.Abstractions; +using Microsoft.Extensions.Options; +using StackExchange.Redis; +using StackExchange.Redis.KeyspaceIsolation; +using UiPath.Caching.Config; +using UiPath.Caching.Negotiation; +using UiPath.Caching.Redis; +using UiPath.Caching.Serialization; +using UiPath.Caching.Telemetry; +using UiPath.Caching.Tests.Fakes; +using UiPath.Caching.Tests.Redis; + +namespace UiPath.Caching.Tests.Negotiation; + +[Collection("RedisIntegration")] +[Trait("Category", "Integration")] +public class FormatNegotiationIntegrationTests(RedisContainerFixture fixture) +{ + private const string Policy = "fast"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task The_scripts_stamp_with_the_servers_clock_under_the_connectors_prefix_and_prune_by_stamp() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await using var mux = await ConnectionMultiplexer.ConnectAsync(fixture.ConnectionString); + var redisOptions = new RedisCacheOptions { KeyPrefix = "a:" }; + var key = NodeListStore.KeyFor(new CacheOptions { AppShortName = $"fn{Guid.NewGuid():N}"[..12] }, redisOptions); + var store = new NodeListStore(new PrefixingConnector(mux, redisOptions.KeyPrefix), key); + var ad = new NodeAdvertisement("3.2.0", "i", ["json", "10"], new Dictionary> { ["p"] = ["json"] }); + var db = mux.GetDatabase(); + try + { + var before = ServerMs(await db.ExecuteAsync("TIME")); + await store.WriteAsync("node-a", ad, Ct); + await store.WriteAsync("node-b", ad, Ct); + await Task.Delay(1100, Ct); + var after = ServerMs(await db.ExecuteAsync("TIME")); + + var read = await store.ReadAsync(Ct); + + // The stamp is the server's TIME, taken inside the script, and the age is the server's TIME again. + read.Should().HaveCount(2); + read[0].Stamp.Should().BeInRange(before, after); + read[0].Age.Should().BeGreaterThan(TimeSpan.FromSeconds(1)).And.BeLessThan(TimeSpan.FromSeconds(10)); + read.Single(a => a.NodeId == "node-a").Advertisement.Strict["p"].Should().Equal("json"); + (await db.HashExistsAsync($"a:{key}", "node-a")).Should().BeTrue("the connector's prefix is the one the key lives under"); + + // node-b advertises again after it was read: pruning by the stamp it was read with must leave it alone. + var stale = read.Where(a => a.NodeId is "node-a" or "node-b").ToList(); + await store.WriteAsync("node-b", ad with { Left = true }, Ct); + await store.PruneAsync(stale, Ct); + + var left = await store.ReadAsync(Ct); + left.Select(a => a.NodeId).Should().Equal("node-b"); + left[0].Advertisement.Left.Should().BeTrue(); + } + finally + { + await db.KeyDeleteAsync($"a:{key}"); + } + } + + [Fact] + public async Task With_redis_unreachable_the_store_stays_inside_its_budget() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var options = Options.Create(new RedisConnectionOptions { ConnectionString = "127.0.0.1:1,connectTimeout=30000", EnableHangDetection = false }); + using var connector = new RedisConnector( + NullTelemetryProvider.Instance, + new RedisConfigurationOptionsProvider(NullLoggerFactory.Instance, options), + new ConnectionMultiplexerFactory(options, NullRedisProfiler.Instance), + options); + var store = new NodeListStore(connector, "ads"); + var ad = new NodeAdvertisement("3.2.0", "i", ["json"], new Dictionary>()); + using var budget = new CancellationTokenSource(TimeSpan.FromSeconds(1)); + var started = DateTime.UtcNow; + + var write = () => store.WriteAsync("node", ad, budget.Token); + + await write.Should().ThrowAsync(); + (DateTime.UtcNow - started).Should().BeLessThan(TimeSpan.FromSeconds(5), "the lazy connect must not be waited on past the budget"); + } + + [Fact] + public async Task A_stream_that_does_not_exist_has_no_groups_and_an_idle_group_reports_its_idle() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + await using var mux = await ConnectionMultiplexer.ConnectAsync(fixture.ConnectionString); + var source = new RedisStreamGroupSource(new PrefixingConnector(mux, null)); + RedisKey stream = $"fn-{Guid.NewGuid():N}"; + var db = mux.GetDatabase(); + try + { + (await source.GroupsAsync(stream, Ct)).Should().BeEmpty(); + + await db.StreamAddAsync(stream, "f", "v"); + await db.StreamCreateConsumerGroupAsync(stream, "g", "0-0"); + (await source.IdleAsync(stream, "g", Ct)).Should().BeNull("a group nobody has read has no consumer"); + await db.StreamReadGroupAsync(stream, "g", "c", ">", 1); + await Task.Delay(300, Ct); + + (await source.GroupsAsync(stream, Ct)).Should().Equal("g"); + (await source.IdleAsync(stream, "g", Ct)).Should().BeGreaterThan(TimeSpan.FromMilliseconds(200)).And.BeLessThan(TimeSpan.FromSeconds(5)); + } + finally + { + await db.KeyDeleteAsync(stream); + } + } + + [Fact] + public async Task An_old_node_keeps_the_writer_on_json_until_it_goes_idle_and_a_returning_one_stops_the_writes_at_once() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var app = $"fn{Guid.NewGuid():N}"[..12]; + var topic = new TopicKey($"orders-{Guid.NewGuid():N}"); + await using var raw = await ConnectionMultiplexer.ConnectAsync(fixture.ConnectionString); + var db = raw.GetDatabase(); + using var node = Node(app, "node-a"); + using var oldNode = Node(app, "node-old", registerSerializer: false); + var service = node.GetServices().OfType().Single(); + var negotiation = node.GetRequiredService(); + var policy = node.GetRequiredService().Resolve(Policy)!; + var cache = node.GetRequiredService().CreateCache(KnownCacheProviderNames.Redis); + var hashes = node.GetRequiredService().CreateHashCache(KnownCacheProviderNames.Redis); + var oldCache = oldNode.GetRequiredService().CreateCache(KnownCacheProviderNames.Redis); + var oldHashes = oldNode.GetRequiredService().CreateHashCache(KnownCacheProviderNames.Redis); + var streamKey = new PrefixStrategy(RedisKeyspaces.Streams, new CacheOptions { AppShortName = app }).GetRedisKey(topic); + var nodeListKey = NodeListStore.KeyFor(new CacheOptions { AppShortName = app }, new RedisCacheOptions()); + var writer = negotiation.Negotiator!; + var serializer = node.GetRequiredService().Resolve("fast")!; + try + { + // The simulated old node: a consumer group named by its source id, read just now, and no advertisement. + await db.StreamAddAsync(streamKey, "f", "v"); + await db.StreamCreateConsumerGroupAsync(streamKey, "urn:node-old", "0-0"); + await db.StreamReadGroupAsync(streamKey, "urn:node-old", "urn:node-old", ">", 1); + + node.GetRequiredService().Get(KnownTopicNames.RedisStreams).Create(topic); + await service.StartAsync(Ct); + + // The old node keeps reading for the length of a hold and a liveness window, so it stays live throughout. + for (var i = 0; i < 10; i++) + { + await db.StreamReadGroupAsync(streamKey, "urn:node-old", "urn:node-old", ">", 1); + await Task.Delay(500, Ct); + } + + writer.Allows(serializer).Should().BeFalse("a live group without an advertisement may not read the format"); + await cache.SetAsync("k1", "json-while-blocked", TimeSpan.FromMinutes(1), policy, Ct); + var blocked = await RawAsync(db, app, "k1"); + blocked.Should().NotBeNull(); + blocked![0].Should().NotBe(0x10); + + // The old node stops reading: past the liveness window, and then past the hold with the same nodes, the writer switches. + await WaitAsync(() => writer.Allows(serializer), TimeSpan.FromSeconds(30)); + await cache.SetAsync("k2", "headered", TimeSpan.FromMinutes(1), policy, Ct); + (await RawAsync(db, app, "k2"))![0].Should().Be(0x10); + await hashes.SetAsync("h2", new Dictionary { ["a"] = "x" }, new HashCacheEntryOptions(TimeToLive: TimeSpan.FromMinutes(1), Metadata: new Dictionary { ["m"] = "v" }), policy, Ct); + (await db.HashGetAsync(HashKey(raw, app, "h2"), "a")).ToString().Should().StartWith("\u0010"); + + // Peer fields travel as the bytes the hash holds, so they follow the same decision. + var versioned = (RedisHashCache)hashes; + var peer = await versioned.SetVersionedAsync("h2p", new Dictionary { ["a"] = "x" }, new HashCacheEntryOptions(TimeToLive: TimeSpan.FromMinutes(1), SetOption: HashCacheSetOption.HashReplace), policy, new PeerFieldLimits(10, 1024), Ct); + peer.Fields!["a"]![0].Should().Be(0x10); + + // An old node that joined in the window reads the new value as a miss, never as another value. + (await oldCache.GetAsync("k2", policy: null, Ct)).Should().BeNull(); + (await oldHashes.GetAsync("h2", policy: null, Ct)).Should().BeNullOrEmpty(); + + // It comes back: fresh read, no advertisement. The next view stops the writes, without a hold. + await db.StreamReadGroupAsync(streamKey, "urn:node-old", "urn:node-old", ">", 1); + await WaitAsync(() => !writer.Allows(serializer), TimeSpan.FromSeconds(5)); + var peerAfter = await versioned.SetVersionedAsync("h3p", new Dictionary { ["a"] = "x" }, new HashCacheEntryOptions(TimeToLive: TimeSpan.FromMinutes(1), SetOption: HashCacheSetOption.HashReplace), policy, new PeerFieldLimits(10, 1024), Ct); + peerAfter.Fields!["a"]![0].Should().NotBe(0x10); + await cache.SetAsync("k3", "json-again", TimeSpan.FromMinutes(1), policy, Ct); + await hashes.SetAsync("h3", new Dictionary { ["a"] = "x" }, new HashCacheEntryOptions(TimeToLive: TimeSpan.FromMinutes(1)), policy, Ct); + + (await oldCache.GetAsync("k3", policy: null, Ct)).Should().Be("json-again"); + (await oldHashes.GetAsync("h3", policy: null, Ct)).Should().ContainKey("a").WhoseValue.Should().Be("x"); + } + finally + { + await service.StopAsync(CancellationToken.None); + await db.KeyDeleteAsync([streamKey, nodeListKey]); + } + } + + [Fact] + public async Task A_node_that_stops_gracefully_never_makes_the_rest_fall_back() + { + Assert.SkipUnless(fixture.Enabled, "Set RUN_REDIS_INTEGRATION_TESTS=1 (Docker required) to run."); + var app = $"fn{Guid.NewGuid():N}"[..12]; + var topic = new TopicKey($"orders-{Guid.NewGuid():N}"); + await using var raw = await ConnectionMultiplexer.ConnectAsync(fixture.ConnectionString); + var db = raw.GetDatabase(); + using var stays = Node(app, "node-stays"); + using var leaves = Node(app, "node-leaves"); + var staysService = stays.GetServices().OfType().Single(); + var leavesService = leaves.GetServices().OfType().Single(); + var writer = stays.GetRequiredService().Negotiator!; + var serializer = stays.GetRequiredService().Resolve("fast")!; + var streamKey = new PrefixStrategy(RedisKeyspaces.Streams, new CacheOptions { AppShortName = app }).GetRedisKey(topic); + var nodeListKey = NodeListStore.KeyFor(new CacheOptions { AppShortName = app }, new RedisCacheOptions()); + try + { + stays.GetRequiredService().Get(KnownTopicNames.RedisStreams).Create(topic); + leaves.GetRequiredService().Get(KnownTopicNames.RedisStreams).Create(topic); + await staysService.StartAsync(Ct); + await leavesService.StartAsync(Ct); + await WaitAsync(() => writer.Allows(serializer), TimeSpan.FromSeconds(30)); + + // Its group is read a moment ago and its ad is gone from the readers: both are the departed's, and neither may count against the fleet. + await leavesService.StopAsync(Ct); + leaves.Dispose(); + for (var i = 0; i < 8; i++) + { + writer.Allows(serializer).Should().BeTrue("a node that left is accounted for, not an unaccounted one"); + await Task.Delay(500, Ct); + } + } + finally + { + await staysService.StopAsync(CancellationToken.None); + await db.KeyDeleteAsync([streamKey, nodeListKey]); + } + } + + private static long ServerMs(RedisResult time) + { + var parts = (RedisResult[])time!; + return ((long)parts[0] * 1000) + ((long)parts[1] / 1000); + } + + private static async Task RawAsync(IDatabase db, string app, string key) + { + var server = db.Multiplexer.GetServer(db.Multiplexer.GetEndPoints()[0]); + var found = server.Keys(pattern: $"{app}:s:{key}").Cast().FirstOrDefault(); + return found is null ? null : (byte[]?)await db.StringGetAsync(found.Value); + } + + private static RedisKey HashKey(ConnectionMultiplexer raw, string app, string key) => + raw.GetServer(raw.GetEndPoints()[0]).Keys(pattern: $"{app}:h:{key}").Single(); + + private static async Task WaitAsync(Func condition, TimeSpan timeout) + { + var until = DateTime.UtcNow + timeout; + while (!condition()) + { + Assert.True(DateTime.UtcNow < until, "The condition did not hold in time."); + await Task.Delay(200, Ct); + } + } + + private ServiceProvider Node(string app, string source, bool registerSerializer = true) => + new ServiceCollection() + .AddLogging() + .AddCaching( + b => + { + b.AddRedisConnection(o => o.ConnectionString = fixture.ConnectionString); + b.AddRedis(o => + { + o.FormatNegotiation.AdvertiseInterval = TimeSpan.FromSeconds(1); + o.FormatNegotiation.ViewInterval = TimeSpan.FromSeconds(1); + o.FormatNegotiation.LivenessWindow = TimeSpan.FromSeconds(3); + o.FormatNegotiation.UpgradeHold = TimeSpan.FromSeconds(2); + o.FormatNegotiation.MaxViewAge = TimeSpan.FromSeconds(10); + }); + b.AddRedisStreams(o => { o.Enabled = true; o.MaintainerEnabled = false; }); + b.AddBroadcast(); + b.AddInMemoryRedis(_ => { }); + if (registerSerializer) + { + b.AddValueSerializer("fast", _ => new PassthroughValueSerializer(0x10)); + } + }, + o => + { + o.AppShortName = app; + o.SourceUri = new Uri($"urn:{source}"); + if (registerSerializer) + { + o.Policies[Policy] = new CachePolicy + { + DistributedExpiration = TimeSpan.FromMinutes(5), + Serializer = new SerializerPolicy { Write = "fast" }, + }; + } + }) + .BuildServiceProvider(); + + private sealed class PrefixingConnector(IConnectionMultiplexer mux, string? prefix) : IRedisConnector + { + public event EventHandler? OnConnectionFailed + { + add { } + remove { } + } + + public event EventHandler? OnConnectionRestored + { + add { } + remove { } + } + + public event EventHandler? OnReconnected + { + add { } + remove { } + } + + public bool IsConnected => mux.IsConnected; + + public Version Version => mux.GetServer(mux.GetEndPoints()[0]).Version; + + public IDatabase Database => prefix is null ? mux.GetDatabase() : mux.GetDatabase().WithKeyPrefix(prefix); + + public ISubscriber Subscriber => mux.GetSubscriber(); + + public void ForceReconnect() + { + } + + public EndPoint[] GetEndPoints(bool configuredOnly = false) => mux.GetEndPoints(configuredOnly); + + public void Dispose() + { + } + } +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationServiceTests.cs b/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationServiceTests.cs new file mode 100644 index 00000000..f5e4c6b7 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiationServiceTests.cs @@ -0,0 +1,346 @@ +using StackExchange.Redis; +using UiPath.Caching.Negotiation; +using UiPath.Caching.Tests.Fakes; + +namespace UiPath.Caching.Tests.Negotiation; + +public class FormatNegotiationServiceTests +{ + private static readonly string[] Binary = ["10"]; + private static readonly RedisKey Stream = "app:st:orders"; + + private readonly ManualTimeProvider _clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + private readonly FakeNodeListServer _server = new(); + private readonly FormatNegotiationOptions _options = new() { UpgradeHold = TimeSpan.Zero }; + private readonly FakeValueSerializer _writer = new(0x10); + private readonly List _events; + private readonly TestGroups _groups; + + public FormatNegotiationServiceTests() + { + _events = _server.Events; + _groups = new TestGroups(_events); + } + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public async Task New_pod_advertises_before_it_reads_the_node_list() + { + var sut = Service(streams: [Stream]); + + await sut.StartAsync(Ct); + + // The write comes before the first read and before the first look at any group, so this node is never an unaccounted one. + _events.Should().StartWith(["write", "read", "groups"]); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_graceful_stop_writes_a_departing_ad_instead_of_deleting_it_and_is_idempotent() + { + var sut = Service(streams: [Stream]); + await sut.StartAsync(Ct); + + await sut.StopAsync(Ct); + await sut.StopAsync(Ct); + + _server.Hash["me"].Should().Contain("left=1").And.Contain("vr=json,10"); + } + + [Fact] + public async Task A_stop_with_redis_down_does_not_throw() + { + var sut = Service(streams: [Stream]); + await sut.StartAsync(Ct); + _server.Failure = new TimeoutException(); + + await FluentActions.Awaiting(() => sut.StopAsync(Ct)).Should().NotThrowAsync(); + } + + [Fact] + public async Task A_node_that_registers_nothing_touches_neither_redis_nor_a_timer() + { + var sut = new FormatNegotiationService(FormatNegotiation.Inactive); + + await sut.StartAsync(Ct); + await sut.StopAsync(Ct); + + _server.Database.ReceivedCalls().Should().BeEmpty(); + _events.Should().BeEmpty(); + } + + [Fact] + public async Task A_legacy_group_keeps_the_writer_on_json_until_it_goes_idle() + { + _groups.Add("legacy", TimeSpan.FromSeconds(10)); + var sut = Service(streams: [Stream], out var negotiator); + await sut.StartAsync(Ct); + + negotiator.Allows(_writer).Should().BeFalse(); + + _groups.Add("legacy", _options.LivenessWindow + TimeSpan.FromSeconds(1)); + await sut.RefreshAsync(Ct); + + negotiator.Allows(_writer).Should().BeTrue(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_group_no_consumer_ever_read_is_nobody() + { + _groups.Add("made-by-tooling", idle: null); + var sut = Service(streams: [Stream], out var negotiator); + + await sut.StartAsync(Ct); + + negotiator.Allows(_writer).Should().BeTrue(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_crashed_node_whose_ad_outlives_its_group_never_makes_the_fleet_fall_back() + { + _options.UpgradeHold = TimeSpan.FromMinutes(1); + var sut = Service(streams: [Stream], out var negotiator); + _server.Put("urn:dead", _server.NowMs + "|lib=3.2;inst=d;vr=json,10"); + _groups.Add("urn:dead", TimeSpan.FromSeconds(1)); + await sut.StartAsync(Ct); + _clock.Advance(TimeSpan.FromMinutes(1)); + await sut.RefreshAsync(Ct); + negotiator.Allows(_writer).Should().BeTrue(); + + // 60 s on: the crashed node's ad is a minute old and its group has been silent for a minute. Neither the ad, which has not + // expired, nor the group, which is not yet idle long enough to be nobody, may count it as unaccounted for. + _server.NowMs += 60_000; + _groups.Add("urn:dead", TimeSpan.FromSeconds(60)); + await sut.AdvertiseAsync(Ct); + await sut.RefreshAsync(Ct); + + negotiator.Allows(_writer).Should().BeTrue(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_restart_under_the_same_id_is_not_hidden_by_the_previous_runs_departing_ad() + { + var sut = Service(streams: [Stream], out var negotiator); + _server.Put("urn:x", _server.NowMs - 100_000 + "|lib=3.2;inst=x;vr=json,10;left=1"); + _groups.Add("urn:x", TimeSpan.FromSeconds(100)); + await sut.StartAsync(Ct); + negotiator.Allows(_writer).Should().BeTrue(); + + _groups.Add("urn:x", TimeSpan.FromSeconds(1)); + await sut.RefreshAsync(Ct); + + negotiator.Allows(_writer).Should().BeFalse("a node read under that id after its ad was written, and may be a build that cannot read the format"); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_node_that_left_gracefully_is_neither_a_reader_nor_unaccounted_for() + { + _options.UpgradeHold = TimeSpan.FromMinutes(1); + _server.Put("urn:leaver", _server.NowMs + "|lib=3.2;inst=l;vr=json;left=1"); + _groups.Add("urn:leaver", TimeSpan.FromSeconds(3)); + var sut = Service(streams: [Stream], out var negotiator); + await sut.StartAsync(Ct); + _clock.Advance(TimeSpan.FromMinutes(1)); + + await sut.RefreshAsync(Ct); + + // Its ad lists no format, but a node that has left reads nothing, so it must not hold the format back either. + negotiator.Allows(_writer).Should().BeTrue(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_departure_does_not_restart_the_hold() + { + _options.UpgradeHold = TimeSpan.FromMinutes(1); + _server.Put("urn:other", _server.NowMs + "|lib=3.2;inst=o;vr=json,10"); + var sut = Service(streams: [Stream], out var negotiator); + await sut.StartAsync(Ct); + _clock.Advance(TimeSpan.FromSeconds(50)); + _server.NowMs += 50_000; + _server.Put("urn:other", _server.NowMs - 50_000 + "|lib=3.2;inst=o;vr=json,10;left=1"); + await sut.RefreshAsync(Ct); + _clock.Advance(TimeSpan.FromSeconds(11)); + + await sut.RefreshAsync(Ct); + + negotiator.Allows(_writer).Should().BeTrue(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task After_an_outage_longer_than_the_ad_lifetime_the_decision_is_kept_while_peers_advertise_again() + { + _options.UpgradeHold = TimeSpan.FromMinutes(1); + _server.Put("urn:peer", _server.NowMs + "|lib=3.2;inst=p;vr=json,10"); + var sut = Service(streams: [Stream], out var negotiator); + await sut.StartAsync(Ct); + _clock.Advance(TimeSpan.FromMinutes(1)); + await sut.RefreshAsync(Ct); + negotiator.Allows(_writer).Should().BeTrue(); + + _server.Failure = new TimeoutException(); + await sut.TickAsync(Ct); + _clock.Advance(TimeSpan.FromSeconds(60)); + _server.NowMs += 200_000; + _server.Failure = null; + _groups.Add("urn:peer", TimeSpan.FromSeconds(70)); + + // Redis is back after longer than an ad lives. This node advertises first; the peer has not yet. + await sut.TickAsync(Ct); + + negotiator.Allows(_writer).Should().BeTrue(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_stale_ad_is_ignored_and_a_long_departed_one_is_removed() + { + _server.Put("old-no-format", _server.NowMs - 1_000_000 + "|lib=1;inst=x;vr=json"); + _server.Put("departed", _server.NowMs - 1_000_000_000 + "|lib=1;inst=x;vr=json"); + var sut = Service(streams: [Stream], out var negotiator); + + await sut.StartAsync(Ct); + + negotiator.Allows(_writer).Should().BeTrue(); + _server.Hash.Keys.Should().NotContain("departed"); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_fresh_ad_without_the_format_stops_the_writes_at_the_next_refresh() + { + var sut = Service(streams: [Stream], out var negotiator); + await sut.StartAsync(Ct); + negotiator.Allows(_writer).Should().BeTrue(); + + _server.Put("old", _server.NowMs + "|lib=3.1;inst=x;vr=json"); + await sut.RefreshAsync(Ct); + + negotiator.Allows(_writer).Should().BeFalse(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task Without_a_streams_transport_the_node_list_is_blind_and_only_the_operators_word_switches() + { + var sut = Service(streams: null, out var negotiator); + await sut.StartAsync(Ct); + negotiator.Allows(_writer).Should().BeFalse(); + _server.Database.ReceivedCalls().Should().BeEmpty("a blind node list has nothing to read or write"); + + _options.AssumeAllNodesCanRead = true; + sut = Service(streams: null, out negotiator); + await sut.StartAsync(Ct); + negotiator.Allows(_writer).Should().BeTrue("the hold is zero here"); + _server.Database.ReceivedCalls().Should().BeEmpty(); + await sut.StopAsync(Ct); + + _options.UpgradeHold = TimeSpan.FromMinutes(1); + sut = Service(streams: null, out negotiator); + await sut.StartAsync(Ct); + negotiator.Allows(_writer).Should().BeFalse("the hold starts at the first decision even when the operator attests"); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_node_list_that_cannot_be_read_does_not_fail_start() + { + _server.Failure = new TimeoutException(); + var sut = Service(streams: [Stream], out var negotiator); + + await sut.StartAsync(Ct); + + negotiator.Allows(_writer).Should().BeFalse(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_reader_only_node_advertises_what_it_reads_and_never_reads_the_node_list() + { + var sut = ReaderOnly(); + + await sut.StartAsync(Ct); + + _server.Hash.Should().ContainKey("me").WhoseValue.Should().Contain("vr=json,10"); + _events.Should().Equal("write"); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_writer_and_a_reader_only_node_switch_after_the_hold() + { + _options.UpgradeHold = TimeSpan.FromMinutes(1); + _server.Put("reader", _server.NowMs + "|lib=3.2;inst=r;vr=json,10"); + var sut = Service(streams: [Stream], out var negotiator); + await sut.StartAsync(Ct); + negotiator.Allows(_writer).Should().BeFalse(); + + _clock.Advance(TimeSpan.FromMinutes(1)); + _server.Put("reader", _server.NowMs + "|lib=3.2;inst=r;vr=json,10"); + await sut.RefreshAsync(Ct); + + negotiator.Allows(_writer).Should().BeTrue(); + await sut.StopAsync(Ct); + } + + [Fact] + public async Task A_node_in_force_mode_still_advertises_what_it_reads() + { + var sut = ReaderOnly(); + + await sut.StartAsync(Ct); + + _server.Hash.Should().ContainKey("me"); + await sut.StopAsync(Ct); + } + + private FormatNegotiationService ReaderOnly() => new(new FormatNegotiation + { + Advertisement = new("3.2.0", "me", ["json", "10"], new Dictionary>()), + NodeId = "me", + Store = new NodeListStore(_server.Connector, "ads"), + NodeList = new StreamNodeList(_groups, _options), + StreamKeys = () => [Stream], + Options = _options, + Clock = _clock, + }); + + private FormatNegotiationService Service(RedisKey[]? streams) => Service(streams, out _); + + private FormatNegotiationService Service(RedisKey[]? streams, out FormatNegotiator negotiator) + { + negotiator = new FormatNegotiator([new DesiredWrite(_writer, Binary, ["default"], [])], _options, _clock); + return new FormatNegotiationService(new FormatNegotiation + { + Negotiator = negotiator, + Advertisement = new("3.2.0", "me", ["json", "10"], new Dictionary>()), + NodeId = "me", + Store = new NodeListStore(_server.Connector, "ads"), + NodeList = new StreamNodeList(_groups, _options), + StreamKeys = streams is null ? null : () => streams, + Options = _options, + Clock = _clock, + }); + } + + private sealed class TestGroups(List events) : IStreamGroupSource + { + private readonly Dictionary _idle = []; + + public void Add(string group, TimeSpan? idle) => _idle[group] = idle; + + public Task> GroupsAsync(RedisKey stream, CancellationToken cancellationToken) + { + events.Add("groups"); + return Task.FromResult>([.. _idle.Keys]); + } + + public Task IdleAsync(RedisKey stream, string group, CancellationToken cancellationToken) => Task.FromResult(_idle[group]); + } +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiatorTests.cs b/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiatorTests.cs new file mode 100644 index 00000000..83288711 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/FormatNegotiatorTests.cs @@ -0,0 +1,422 @@ +using Microsoft.Extensions.Configuration; +using UiPath.Caching.Negotiation; +using UiPath.Caching.Tests.Fakes; +using UiPath.Caching.Tests.Telemetry; + +namespace UiPath.Caching.Tests.Negotiation; + +public class FormatNegotiatorTests +{ + private static readonly string[] Binary = ["10"]; + + private readonly ManualTimeProvider _clock = new(new DateTimeOffset(2026, 1, 1, 0, 0, 0, TimeSpan.Zero)); + private readonly RecordingTelemetryProvider _telemetry = new(); + private readonly FakeValueSerializer _writer = new(0x10); + private readonly FormatNegotiationOptions _options = new(); + + [Fact] + public void No_view_means_json() + { + var sut = Negotiator(); + + sut.Allows(_writer).Should().BeFalse(); + sut.ObservationFailed(); + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void All_fresh_ads_list_format_gives_desired_once_the_hold_has_passed() + { + var sut = Negotiator(); + var view = View(Node("a", Binary), Node("b", Binary)); + + sut.Observe(view); + sut.Allows(_writer).Should().BeFalse("the node list has only just agreed"); + _clock.Advance(_options.UpgradeHold); + sut.Observe(view); + + sut.Allows(_writer).Should().BeTrue(); + } + + [Fact] + public void One_ad_without_format_gives_json_immediately() + { + var sut = Negotiator(); + Settle(sut, View(Node("a", Binary), Node("b", Binary))); + sut.Allows(_writer).Should().BeTrue(); + + sut.Observe(View(Node("a", Binary), Node("b", Binary), Node("old", []))); + + sut.Allows(_writer).Should().BeFalse(); + _telemetry.Metrics.Should().Contain(m => m.Name == Metrics.FormatFallback && m.Properties!["reason"] == FormatNegotiator.ReasonMissingFormat); + } + + [Fact] + public void One_ad_without_format_blocks_even_when_the_rest_list_it() + { + var sut = Negotiator(); + + Settle(sut, View(Node("a", Binary), Node("b", Binary), Node("old", ["json"]))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void The_upgrade_hold_defaults_to_an_hour() + { + new FormatNegotiationOptions().UpgradeHold.Should().Be(TimeSpan.FromHours(1)); + } + + [Fact] + public void A_configured_hold_is_bound_from_configuration() + { + var configuration = new ConfigurationBuilder() + .AddInMemoryCollection(new Dictionary { ["FormatNegotiation:UpgradeHold"] = "00:20:00", ["FormatNegotiation:Mode"] = "Force" }) + .Build(); + var options = new RedisCacheOptions(); + + configuration.Bind(options); + + options.FormatNegotiation.UpgradeHold.Should().Be(TimeSpan.FromMinutes(20)); + options.FormatNegotiation.Mode.Should().Be(FormatNegotiationMode.Force); + } + + [Fact] + public void No_upgrade_before_the_configured_hold_and_an_upgrade_after_it() + { + _options.UpgradeHold = TimeSpan.FromMinutes(20); + var sut = Negotiator(); + var view = View(Node("a", Binary)); + + sut.Observe(view); + _clock.Advance(TimeSpan.FromMinutes(20) - TimeSpan.FromSeconds(1)); + sut.Observe(view); + sut.Allows(_writer).Should().BeFalse(); + + _clock.Advance(TimeSpan.FromSeconds(1)); + sut.Observe(view); + sut.Allows(_writer).Should().BeTrue(); + } + + [Fact] + public void A_zero_hold_switches_at_once_and_a_negative_one_is_rejected() + { + _options.UpgradeHold = TimeSpan.Zero; + var sut = Negotiator(); + sut.Observe(View(Node("a", Binary))); + sut.Allows(_writer).Should().BeTrue(); + + _options.UpgradeHold = TimeSpan.FromSeconds(-1); + FluentActions.Invoking(() => Negotiator()).Should().Throw().WithMessage("*UpgradeHold*negative*"); + } + + [Fact] + public void Upgrade_waits_for_hold_since_the_last_node_joined() + { + var sut = Negotiator(); + sut.Observe(View(Node("a", Binary))); + _clock.Advance(_options.UpgradeHold - TimeSpan.FromSeconds(1)); + + // A node joins late in the hold: the clock starts again. + sut.Observe(View(Node("a", Binary), Node("b", Binary))); + _clock.Advance(TimeSpan.FromSeconds(30)); + sut.Observe(View(Node("a", Binary), Node("b", Binary))); + sut.Allows(_writer).Should().BeFalse(); + + _clock.Advance(_options.UpgradeHold); + sut.Observe(View(Node("a", Binary), Node("b", Binary))); + sut.Allows(_writer).Should().BeTrue(); + } + + [Fact] + public void A_node_that_leaves_never_resets_the_hold() + { + var sut = Negotiator(); + sut.Observe(View(Node("a", Binary), Node("b", Binary))); + _clock.Advance(_options.UpgradeHold - TimeSpan.FromSeconds(1)); + + sut.Observe(View(Node("a", Binary))); + _clock.Advance(TimeSpan.FromSeconds(2)); + sut.Observe(View(Node("a", Binary))); + + sut.Allows(_writer).Should().BeTrue(); + } + + [Fact] + public void A_node_that_restarts_under_the_same_id_is_a_join() + { + var sut = Negotiator(); + sut.Observe(View(Node("a", Binary, instance: "first"))); + _clock.Advance(_options.UpgradeHold - TimeSpan.FromSeconds(1)); + + sut.Observe(View(Node("a", Binary, instance: "second"))); + _clock.Advance(TimeSpan.FromSeconds(2)); + sut.Observe(View(Node("a", Binary, instance: "second"))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void A_blocker_resets_the_hold() + { + var sut = Negotiator(); + sut.Observe(View(Node("a", Binary))); + _clock.Advance(_options.UpgradeHold - TimeSpan.FromSeconds(1)); + sut.Observe(View([Node("a", Binary)], ["legacy"])); + _clock.Advance(TimeSpan.FromSeconds(10)); + + sut.Observe(View(Node("a", Binary))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void Redis_failure_keeps_decision_until_max_view_age_then_json() + { + var sut = Negotiator(); + Settle(sut, View(Node("a", Binary))); + + _clock.Advance(_options.MaxViewAge - TimeSpan.FromSeconds(1)); + sut.ObservationFailed(); + sut.Allows(_writer).Should().BeTrue(); + + _clock.Advance(TimeSpan.FromSeconds(2)); + sut.ObservationFailed(); + sut.Allows(_writer).Should().BeFalse(); + _telemetry.Metrics.Should().Contain(m => m.Name == Metrics.FormatFallback && m.Properties!["reason"] == FormatNegotiator.ReasonViewStale); + } + + [Fact] + public void A_view_that_stops_refreshing_without_failing_decays_to_json_on_its_own() + { + var sut = Negotiator(); + Settle(sut, View(Node("a", Binary))); + sut.Allows(_writer).Should().BeTrue(); + + _clock.Advance(_options.MaxViewAge + TimeSpan.FromSeconds(1)); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void After_an_outage_the_decision_is_kept_while_peers_advertise_again_and_then_judged() + { + var sut = Negotiator(); + Settle(sut, View(Node("a", Binary), Node("b", Binary))); + _clock.Advance(TimeSpan.FromSeconds(60)); + sut.ObservationFailed(); + + // Redis is back but nobody has advertised yet, and the consumers read nothing meanwhile. + sut.Observe(View([], ["b"])); + sut.Allows(_writer).Should().BeTrue("peers get two advertise intervals to say what they read"); + sut.Observe(View([Node("a", Binary)], ["b"])); + sut.Allows(_writer).Should().BeTrue(); + + _clock.Advance(TimeSpan.FromTicks(_options.AdvertiseInterval.Ticks * 2)); + sut.Observe(View([Node("a", Binary)], ["b"])); + + sut.Allows(_writer).Should().BeFalse("a node still unaccounted for once the grace is over"); + } + + [Fact] + public void After_an_outage_a_fresh_ad_without_the_format_still_stops_the_writes_at_once() + { + var sut = Negotiator(); + Settle(sut, View(Node("a", Binary))); + _clock.Advance(TimeSpan.FromSeconds(60)); + sut.ObservationFailed(); + + sut.Observe(View(Node("a", Binary), Node("old", []))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void After_an_outage_a_view_with_both_an_unaccounted_group_and_an_ad_lacking_the_format_goes_json_at_once() + { + var sut = Negotiator(); + Settle(sut, View(Node("a", Binary), Node("b", Binary))); + _clock.Advance(TimeSpan.FromSeconds(60)); + sut.ObservationFailed(); + + sut.Observe(View([Node("a", Binary), Node("old", [])], ["b"])); + + sut.Allows(_writer).Should().BeFalse("the grace excuses silence, not an advertisement that says no"); + _telemetry.Metrics.Should().Contain(m => m.Name == Metrics.FormatFallback && m.Properties!["reason"] == FormatNegotiator.ReasonMissingFormat); + } + + [Fact] + public void A_peer_that_does_not_look_for_a_header_keeps_the_writer_on_json_even_for_a_policy_the_writer_knows() + { + var sut = Negotiator(policies: ["orders"], others: ["default"]); + var peer = new Dictionary> { ["default"] = ["json", "nosniff"] }; + + Settle(sut, View(Node("a", Binary), Node("raw-peer", Binary, peer))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void A_pending_upgrade_does_not_survive_a_failed_read() + { + var sut = Negotiator(); + sut.Observe(View(Node("a", Binary))); + _clock.Advance(_options.UpgradeHold - TimeSpan.FromSeconds(1)); + sut.ObservationFailed(); + _clock.Advance(TimeSpan.FromSeconds(5)); + + sut.Observe(View(Node("a", Binary))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void Strict_read_list_blocks_a_format_it_excludes() + { + var sut = Negotiator(policies: ["orders"]); + var strict = new Dictionary> { ["orders"] = ["json"] }; + + Settle(sut, View(Node("a", Binary), Node("b", Binary, strict))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void Strict_read_list_of_a_policy_that_writes_something_else_does_not_block() + { + var sut = Negotiator(policies: ["orders"], others: ["users"]); + var strict = new Dictionary> { ["users"] = ["json"] }; + + Settle(sut, View(Node("a", Binary), Node("b", Binary, strict))); + + sut.Allows(_writer).Should().BeTrue(); + } + + [Fact] + public void Strict_read_list_of_a_policy_this_node_does_not_know_blocks() + { + var sut = Negotiator(policies: ["orders"]); + var strict = new Dictionary> { ["elsewhere"] = ["json"] }; + + Settle(sut, View(Node("a", Binary), Node("b", Binary, strict))); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void A_live_group_without_an_ad_blocks_unless_the_operator_attests() + { + var blocked = Negotiator(); + Settle(blocked, View([Node("a", Binary)], ["legacy"])); + blocked.Allows(_writer).Should().BeFalse(); + + _options.AssumeAllNodesCanRead = true; + var attested = Negotiator(); + Settle(attested, View([Node("a", Binary)], ["legacy"])); + attested.Allows(_writer).Should().BeTrue(); + } + + [Fact] + public void The_attestation_never_covers_an_ad_that_lacks_the_format() + { + _options.AssumeAllNodesCanRead = true; + var sut = Negotiator(); + + Settle(sut, View([Node("a", Binary), Node("old", ["json"])], ["legacy"])); + sut.Allows(_writer).Should().BeFalse(); + + var blind = Negotiator(); + Settle(blind, View([Node("a", Binary), Node("old", ["json"])], [], complete: false)); + blind.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void A_node_list_that_cannot_see_every_node_means_json_unless_the_operator_attests_and_the_hold_still_applies() + { + var blind = Negotiator(); + Settle(blind, View([Node("a", Binary)], [], complete: false)); + blind.Allows(_writer).Should().BeFalse(); + + _options.AssumeAllNodesCanRead = true; + var attested = Negotiator(); + var nobody = View([], [], complete: false); + attested.Observe(nobody); + attested.Allows(_writer).Should().BeFalse("the hold starts at the first decision"); + _clock.Advance(_options.UpgradeHold); + attested.Observe(nobody); + + attested.Allows(_writer).Should().BeTrue(); + } + + [Fact] + public void An_ad_naming_no_node_is_not_a_view() + { + var sut = Negotiator(); + + Settle(sut, View()); + + sut.Allows(_writer).Should().BeFalse(); + } + + [Fact] + public void Every_registered_serializer_is_decided_whatever_policy_asks_for_it() + { + var other = new FakeValueSerializer(0x11); + var sut = new FormatNegotiator( + [ + new DesiredWrite(_writer, Binary, ["orders"], []), + new DesiredWrite(other, ["11"], [], ["orders"]), + ], + _options, + _clock, + _telemetry); + var view = View(Node("a", ["10", "11"]), Node("b", ["10"])); + + Settle(sut, view); + + sut.Allows(_writer).Should().BeTrue(); + sut.Allows(other).Should().BeFalse("one node does not read 11"); + sut.Allows(new FakeValueSerializer(0x12)).Should().BeFalse("a serializer nobody registered is never allowed"); + } + + [Fact] + public void A_decision_is_read_without_allocating() + { + var sut = Negotiator(); + Settle(sut, View(Node("a", Binary))); + sut.Allows(_writer).Should().BeTrue(); + + // Steady state: after a warm-up, so the JIT and tiering do not count, the best of a few passes allocates nothing. + var best = long.MaxValue; + for (var attempt = 0; attempt < 5; attempt++) + { + var before = GC.GetAllocatedBytesForCurrentThread(); + for (var i = 0; i < 10_000; i++) + { + _ = sut.Allows(_writer); + } + + best = Math.Min(best, GC.GetAllocatedBytesForCurrentThread() - before); + } + + best.Should().Be(0); + } + + private static (string Id, NodeAdvertisement Ad) Node(string id, string[] formats, IReadOnlyDictionary>? strict = null, string? instance = null) => + (id, new("3.2.0", instance ?? id, ["json", .. formats], strict ?? new Dictionary>())); + + private static NodeListView View(params (string Id, NodeAdvertisement Ad)[] nodes) => View(nodes, [], true); + + private static NodeListView View((string Id, NodeAdvertisement Ad)[] nodes, string[] blockers, bool complete = true) => + new([.. nodes.Select(n => new StoredAdvertisement(n.Id, n.Ad, TimeSpan.Zero))], blockers, complete); + + private FormatNegotiator Negotiator(string[]? policies = null, string[]? others = null) => + new([new DesiredWrite(_writer, Binary, policies ?? ["default"], others ?? [])], _options, _clock, _telemetry); + + private void Settle(FormatNegotiator sut, NodeListView view) + { + sut.Observe(view); + _clock.Advance(_options.UpgradeHold); + sut.Observe(view); + } +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/NodeAdvertisementTests.cs b/tests/UiPath.Caching.Tests/Negotiation/NodeAdvertisementTests.cs new file mode 100644 index 00000000..03d50457 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/NodeAdvertisementTests.cs @@ -0,0 +1,186 @@ +using Microsoft.Extensions.Logging.Abstractions; +using StackExchange.Redis; +using UiPath.Caching.Negotiation; + +namespace UiPath.Caching.Tests.Negotiation; + +public class NodeAdvertisementTests +{ + private static readonly RedisKey Key = "ads"; + + private static CancellationToken Ct => TestContext.Current.CancellationToken; + + [Fact] + public void Ad_round_trips_formats_version_and_strict_lists() + { + var ad = new NodeAdvertisement( + "3.2.0", + "9f1c", + ["json", "02", "03", "10"], + new Dictionary> { ["a;b:c"] = ["json", "02"], ["plain"] = ["json"] }, + Migration: "orders:dualdelete", + Left: true); + + NodeAdvertisement.TryParse(ad.Encode(), out var parsed).Should().BeTrue(); + + parsed!.Library.Should().Be("3.2.0"); + parsed.Instance.Should().Be("9f1c"); + parsed.ValueReads.Should().Equal("json", "02", "03", "10"); + parsed.Strict.Should().BeEquivalentTo(ad.Strict); + parsed.Migration.Should().Be("orders:dualdelete"); + parsed.Left.Should().BeTrue(); + } + + [Fact] + public void An_ad_that_is_not_leaving_does_not_say_so() + { + var ad = new NodeAdvertisement("3.2.0", "i", ["json"], new Dictionary>()); + + ad.Encode().Should().NotContain("left"); + NodeAdvertisement.TryParse(ad.Encode(), out var parsed).Should().BeTrue(); + parsed!.Left.Should().BeFalse(); + } + + [Fact] + public void An_unknown_field_is_skipped_and_a_text_without_reads_is_no_ad() + { + NodeAdvertisement.TryParse("lib=3.3.0;inst=x;vr=json,02;er=json,02;sub=c:1", out var parsed).Should().BeTrue(); + parsed!.ValueReads.Should().Equal("json", "02"); + parsed.Migration.Should().BeNull(); + + NodeAdvertisement.TryParse("lib=3.3.0;inst=x", out _).Should().BeFalse(); + NodeAdvertisement.TryParse("garbage", out _).Should().BeFalse(); + } + + [Theory] + [InlineData(false)] + [InlineData(true)] + public void Ad_key_is_built_like_the_maintainers_lock_key_and_carries_its_hash_tag(bool shardKeys) + { + var cacheOptions = new CacheOptions { AppShortName = "app" }; +#pragma warning disable CS0618 // The shard strategy is the one a cluster app sets. + cacheOptions.ShardKeyEnabled = shardKeys; +#pragma warning restore CS0618 + var redisOptions = new RedisCacheOptions(); + var maintainer = new RedisStreamHealthMaintainer( + Substitute.For(), + Substitute.For(), + Options.Create(new RedisStreamsTopicOptions()), + Options.Create(redisOptions), + Options.Create(cacheOptions), + NullLogger.Instance, + TimeProvider.System); + maintainer.Initialize(); + + var key = NodeListStore.KeyFor(cacheOptions, redisOptions).ToString(); + var lockKey = maintainer.LockKey.ToString(); + + var prefix = key[..key.IndexOf("caching", StringComparison.Ordinal)]; + lockKey.Should().StartWith(prefix, "the ad key is built by the lock key's strategy, so the keyspace and tag rules are the same"); + prefix.Should().Be(shardKeys ? "app:s:{" : "app:s:"); + RedisHashTag.HasValidTag(key).Should().Be(shardKeys); + RedisHashTag.HasValidTag(lockKey).Should().Be(shardKeys); + } + + [Fact] + public async Task Age_uses_server_time_not_local_clock() + { + var server = new FakeNodeListServer { NowMs = 946_684_800_000 }; + var store = new NodeListStore(server.Connector, Key); + + await store.WriteAsync("node-a", Ad(), Ct); + server.NowMs += 10_000; + var ads = await store.ReadAsync(Ct); + + // A stamp from this machine's clock would be decades away from the server's, and the age with it. + ads.Should().ContainSingle().Which.Age.Should().Be(TimeSpan.FromSeconds(10)); + ads[0].Stamp.Should().Be(946_684_800_000); + server.Hash["node-a"].Should().StartWith("946684800000|"); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Works_with_scripts_disabled(bool scripts) + { + var server = new FakeNodeListServer(scripts); + var store = new NodeListStore(server.Connector, Key); + + await store.WriteAsync("node-a", Ad(), Ct); + server.NowMs += 4_000; + await store.WriteAsync("node-b", Ad(), Ct); + server.NowMs += 1_000; + var ads = await store.ReadAsync(Ct); + + ads.Should().HaveCount(2); + ads.Single(a => a.NodeId == "node-a").Age.Should().Be(TimeSpan.FromSeconds(5)); + ads.Single(a => a.NodeId == "node-b").Age.Should().Be(TimeSpan.FromSeconds(1)); + ads[0].Advertisement.ValueReads.Should().Equal("json", "02"); + } + + [Theory] + [InlineData(true)] + [InlineData(false)] + public async Task Prune_removes_only_an_ad_that_still_holds_the_stamp_it_was_read_with(bool scripts) + { + var server = new FakeNodeListServer(scripts); + var store = new NodeListStore(server.Connector, Key); + await store.WriteAsync("old", Ad(), Ct); + await store.WriteAsync("raced", Ad(), Ct); + server.NowMs += 600_000; + await store.WriteAsync("fresh", Ad(), Ct); + var read = await store.ReadAsync(Ct); + + // "raced" advertised again between the read and the prune. + server.NowMs += 1_000; + await store.WriteAsync("raced", Ad(), Ct); + await store.PruneAsync([.. read.Where(a => a.Age > TimeSpan.FromMinutes(5))], Ct); + + (await store.ReadAsync(Ct)).Select(a => a.NodeId).Should().BeEquivalentTo(["raced", "fresh"]); + } + + [Fact] + public async Task Fields_that_do_not_parse_are_left_out() + { + var server = new FakeNodeListServer(); + var store = new NodeListStore(server.Connector, Key); + server.Put("junk", "not-a-stamp|vr=json"); + server.Put("other", "1|no reads here"); + await store.WriteAsync("fresh", Ad(), Ct); + + (await store.ReadAsync(Ct)).Select(a => a.NodeId).Should().Equal("fresh"); + } + + [Fact] + public async Task A_transient_server_error_is_not_a_script_refusal_and_scripts_are_tried_again() + { + var server = new FakeNodeListServer(); + var store = new NodeListStore(server.Connector, Key); +#pragma warning disable CS0618 // The only public constructor of a server error. + server.Failure = new RedisServerException("READONLY You can't write against a read only replica."); +#pragma warning restore CS0618 + + await FluentActions.Awaiting(() => store.WriteAsync("a", Ad(), Ct)).Should().ThrowAsync(); + server.Failure = null; + await store.WriteAsync("a", Ad(), Ct); + + server.Events.Should().Equal("write"); + await server.Database.DidNotReceive().ExecuteAsync("TIME", Arg.Any?>(), Arg.Any()); + } + + [Theory] + [InlineData("NOPERM this user has no permissions to run the 'eval' command")] + [InlineData("ERR unknown command 'EVAL'")] + [InlineData("NOSCRIPT No matching script.")] + public void A_refusal_to_run_scripts_is_told_from_a_transient_error(string message) + { +#pragma warning disable CS0618 // The only public constructor of a server error. + NodeListStore.IsScriptRefusal(new RedisServerException(message)).Should().BeTrue(); + NodeListStore.IsScriptRefusal(new RedisServerException("LOADING Redis is loading the dataset in memory")).Should().BeFalse(); + NodeListStore.IsScriptRefusal(new RedisServerException("BUSY Redis is busy running a script.")).Should().BeFalse(); + NodeListStore.IsScriptRefusal(new RedisServerException("READONLY You can't write against a read only replica.")).Should().BeFalse(); +#pragma warning restore CS0618 + } + + private static NodeAdvertisement Ad() => new("3.2.0", "i", ["json", "02"], new Dictionary>()); +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/RedisStreamGroupSourceTests.cs b/tests/UiPath.Caching.Tests/Negotiation/RedisStreamGroupSourceTests.cs new file mode 100644 index 00000000..fea0488d --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/RedisStreamGroupSourceTests.cs @@ -0,0 +1,24 @@ +using StackExchange.Redis; +using UiPath.Caching.Negotiation; + +namespace UiPath.Caching.Tests.Negotiation; + +public class RedisStreamGroupSourceTests +{ + private static readonly RedisKey Stream = "app:st:orders"; + + [Theory] + [InlineData("NOGROUP No such consumer group 'g' for key name 'app:st:orders'")] + [InlineData("ERR no such key")] + public async Task A_group_deleted_between_listing_and_reading_has_no_consumer(string error) + { + var database = Substitute.For(); + var connector = Substitute.For(); + connector.Database.Returns(database); +#pragma warning disable CS0618 // The only public constructor of a server error. + database.StreamConsumerInfoAsync(Stream, "g", Arg.Any()).Returns(Task.FromException(new RedisServerException(error))); +#pragma warning restore CS0618 + + (await new RedisStreamGroupSource(connector).IdleAsync(Stream, "g", TestContext.Current.CancellationToken)).Should().BeNull(); + } +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/StreamNodeListTests.cs b/tests/UiPath.Caching.Tests/Negotiation/StreamNodeListTests.cs new file mode 100644 index 00000000..75d0a1b9 --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/StreamNodeListTests.cs @@ -0,0 +1,106 @@ +using StackExchange.Redis; +using UiPath.Caching.Negotiation; + +namespace UiPath.Caching.Tests.Negotiation; + +public class StreamNodeListTests +{ + private static readonly RedisKey Stream = "app:st:orders"; + + private readonly FakeGroups _groups = new(); + private readonly FormatNegotiationOptions _options = new(); + + [Fact] + public async Task Group_with_fresh_ad_is_accounted() + { + _groups.Add(Stream, "node-a", TimeSpan.FromSeconds(1)); + + var blockers = await new StreamNodeList(_groups, _options).BlockersAsync([Stream], Ads("node-a"), TestContext.Current.CancellationToken); + + blockers.Should().BeEmpty(); + } + + [Fact] + public async Task Group_read_after_its_ad_was_written_is_a_restart_under_the_same_id_and_blocks() + { + _groups.Add(Stream, "node-a", TimeSpan.FromSeconds(1)); + var tooOld = TimeSpan.FromSeconds(1) + TimeSpan.FromTicks(_options.AdvertiseInterval.Ticks * 2) + TimeSpan.FromSeconds(1); + + var blockers = await new StreamNodeList(_groups, _options).BlockersAsync([Stream], new Dictionary { ["node-a"] = tooOld }, TestContext.Current.CancellationToken); + + blockers.Should().Equal("node-a"); + } + + [Fact] + public async Task Group_without_ad_and_low_idle_blocks_and_names_the_node() + { + _groups.Add(Stream, "node-a", TimeSpan.FromSeconds(1)); + _groups.Add(Stream, "legacy-1", TimeSpan.FromSeconds(30)); + + var blockers = await new StreamNodeList(_groups, _options).BlockersAsync([Stream], Ads("node-a"), TestContext.Current.CancellationToken); + + blockers.Should().Equal("legacy-1"); + } + + [Fact] + public async Task Group_with_high_idle_is_dead_and_does_not_block() + { + _groups.Add(Stream, "gone", _options.LivenessWindow + TimeSpan.FromSeconds(1)); + _groups.Add(Stream, "edge", _options.LivenessWindow); + + var blockers = await new StreamNodeList(_groups, _options).BlockersAsync([Stream], Ads(), TestContext.Current.CancellationToken); + + blockers.Should().Equal("edge"); + } + + [Fact] + public async Task Group_without_a_consumer_is_nobody() + { + _groups.Add(Stream, "joining", idle: null); + + (await new StreamNodeList(_groups, _options).BlockersAsync([Stream], Ads(), TestContext.Current.CancellationToken)).Should().BeEmpty("no consumer ever read from it, so no running node owns it"); + } + + [Fact] + public async Task A_group_on_two_streams_is_named_once_and_ids_match_regardless_of_case() + { + RedisKey other = "app:st:users"; + _groups.Add(Stream, "legacy", TimeSpan.Zero); + _groups.Add(other, "legacy", TimeSpan.Zero); + _groups.Add(other, "Node-A", TimeSpan.Zero); + + var blockers = await new StreamNodeList(_groups, _options).BlockersAsync([Stream, other], Ads("node-a"), TestContext.Current.CancellationToken); + + blockers.Should().Equal("legacy"); + } + + private static Dictionary Ads(params string[] nodes) => nodes.ToDictionary(n => n, _ => TimeSpan.Zero); + + private sealed class FakeGroups : IStreamGroupSource + { + private readonly Dictionary<(string, string), TimeSpan?> _idle = []; + private readonly Dictionary> _names = []; + + public List IdleQueries { get; } = []; + + public void Add(RedisKey stream, string group, TimeSpan? idle) + { + _idle[(stream.ToString(), group)] = idle; + if (!_names.TryGetValue(stream.ToString(), out var names)) + { + _names[stream.ToString()] = names = []; + } + + names.Add(group); + } + + public Task> GroupsAsync(RedisKey stream, CancellationToken cancellationToken) => + Task.FromResult>(_names.TryGetValue(stream.ToString(), out var names) ? names : []); + + public Task IdleAsync(RedisKey stream, string group, CancellationToken cancellationToken) + { + IdleQueries.Add(group); + return Task.FromResult(_idle[(stream.ToString(), group)]); + } + } +} diff --git a/tests/UiPath.Caching.Tests/Negotiation/TickScheduleTests.cs b/tests/UiPath.Caching.Tests/Negotiation/TickScheduleTests.cs new file mode 100644 index 00000000..7cef4abc --- /dev/null +++ b/tests/UiPath.Caching.Tests/Negotiation/TickScheduleTests.cs @@ -0,0 +1,45 @@ +using UiPath.Caching.Negotiation; + +namespace UiPath.Caching.Tests.Negotiation; + +public class TickScheduleTests +{ + [Fact] + public void The_node_list_is_read_at_its_own_cadence_when_the_advertisement_is_much_faster() + { + var sut = new TickSchedule(TimeSpan.FromSeconds(1), TimeSpan.FromMinutes(1), () => 1.0); + var now = TimeSpan.Zero; + int advertisements = 0, reads = 0; + + while (now < TimeSpan.FromMinutes(5)) + { + now += sut.Wait(now); + var (advertise, refresh) = sut.Take(now); + advertisements += advertise ? 1 : 0; + reads += refresh ? 1 : 0; + if (refresh) + { + advertise.Should().BeTrue("a read is always preceded by an advertisement"); + } + } + + reads.Should().Be(5); + advertisements.Should().BeInRange(299, 301); + } + + [Fact] + public void A_slow_advertisement_does_not_slow_the_node_list_read() + { + var sut = new TickSchedule(TimeSpan.FromMinutes(1), TimeSpan.FromSeconds(15), () => 1.0); + var now = TimeSpan.Zero; + var reads = 0; + + while (now < TimeSpan.FromMinutes(1)) + { + now += sut.Wait(now); + reads += sut.Take(now).Refresh ? 1 : 0; + } + + reads.Should().Be(4); + } +} diff --git a/tests/UiPath.Caching.Tests/Redis/PeerFieldUpdateIntegrationTests.cs b/tests/UiPath.Caching.Tests/Redis/PeerFieldUpdateIntegrationTests.cs index fa87ed97..26a7ef12 100644 --- a/tests/UiPath.Caching.Tests/Redis/PeerFieldUpdateIntegrationTests.cs +++ b/tests/UiPath.Caching.Tests/Redis/PeerFieldUpdateIntegrationTests.cs @@ -509,6 +509,7 @@ private Peer Node(string source, bool peerFieldUpdates, int fieldBatch = 1000, b { o.HashFieldBatchSize = fieldBatch; o.ReadScriptsEnabled = readScripts; + o.FormatNegotiation.Mode = FormatNegotiationMode.Force; }); b.AddBroadcast(); b.AddInMemoryRedis(o => o.PeerFieldUpdates = peerFieldUpdates); diff --git a/tests/UiPath.Caching.Tests/Redis/SerializerIntegrationTests.cs b/tests/UiPath.Caching.Tests/Redis/SerializerIntegrationTests.cs index 628d88ac..a5d877dc 100644 --- a/tests/UiPath.Caching.Tests/Redis/SerializerIntegrationTests.cs +++ b/tests/UiPath.Caching.Tests/Redis/SerializerIntegrationTests.cs @@ -333,7 +333,11 @@ private ServiceProvider Build(bool distributed = false, string registry = "fast" b => { b.AddRedisConnection(o => o.ConnectionString = fixture.ConnectionString); - b.AddRedis(o => o.HashFieldBatchSize = fieldBatchSize); + b.AddRedis(o => + { + o.HashFieldBatchSize = fieldBatchSize; + o.FormatNegotiation.Mode = FormatNegotiationMode.Force; + }); switch (registry) { case "fast":