diff --git a/src/Opc.Ua.Redundancy.Kubernetes/LeaderElection/KubernetesLeaseLeaderElection.cs b/src/Opc.Ua.Redundancy.Kubernetes/LeaderElection/KubernetesLeaseLeaderElection.cs index 8ce012c8ff..4f252d1f04 100644 --- a/src/Opc.Ua.Redundancy.Kubernetes/LeaderElection/KubernetesLeaseLeaderElection.cs +++ b/src/Opc.Ua.Redundancy.Kubernetes/LeaderElection/KubernetesLeaseLeaderElection.cs @@ -28,9 +28,11 @@ * ======================================================================*/ using System; +using System.Collections.Generic; using System.Globalization; using System.Net; using System.Net.Http; +using System.Runtime.ExceptionServices; using System.Threading; using System.Threading.Tasks; using Microsoft.Extensions.Logging; @@ -74,6 +76,11 @@ internal KubernetesLeaseLeaderElection( m_namespace = KubernetesApiClientFactory.ResolveNamespace(m_options.Kubernetes, m_apiClient); m_timeProvider = timeProvider ?? TimeProvider.System; m_logger = logger; + m_watchdog = m_timeProvider.CreateTimer( + static state => ((KubernetesLeaseLeaderElection)state!).OnWatchdogElapsed(), + this, + Timeout.InfiniteTimeSpan, + Timeout.InfiniteTimeSpan); } /// @@ -94,58 +101,16 @@ public bool IsLeader /// public async ValueTask TryAcquireOrRenewAsync(CancellationToken ct = default) { - if (!m_apiClient.IsInCluster) - { - SetLeader(false); - return false; - } - + ThrowIfDisposed(); + using var linkedCts = CancellationTokenSource.CreateLinkedTokenSource(ct, m_cts.Token); + await m_attemptGate.WaitAsync(linkedCts.Token).ConfigureAwait(false); try { - DateTimeOffset now = m_timeProvider.GetUtcNow(); - KubernetesLease? current = await m_apiClient - .GetLeaseAsync(m_namespace, m_options.LeaseName, ct) - .ConfigureAwait(false); - - if (current == null) - { - KubernetesLease created = NewLease(now); - await m_apiClient.CreateLeaseAsync(m_namespace, created, ct).ConfigureAwait(false); - SetLeader(true); - return true; - } - - if (!CanAcquireOrRenew(current, now)) - { - SetLeader(false); - return false; - } - - string? previousHolder = current.Spec.HolderIdentity; - current.Spec.HolderIdentity = m_options.Kubernetes.NodeId; - current.Spec.LeaseDurationSeconds = ToLeaseSeconds(m_options.LeaseDuration); - current.Spec.RenewTime = FormatTime(now); - if (!string.Equals(previousHolder, m_options.Kubernetes.NodeId, StringComparison.Ordinal)) - { - current.Spec.AcquireTime = FormatTime(now); - current.Spec.LeaseTransitions = (current.Spec.LeaseTransitions ?? 0) + 1; - } - - await m_apiClient - .ReplaceLeaseAsync(m_namespace, m_options.LeaseName, current, ct) - .ConfigureAwait(false); - SetLeader(true); - return true; + return await TryAcquireOrRenewCoreAsync(linkedCts.Token).ConfigureAwait(false); } - catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Conflict) + finally { - SetLeader(false); - return false; - } - catch - { - SetLeader(false); - throw; + m_attemptGate.Release(); } } @@ -166,6 +131,8 @@ public void Start() /// public async ValueTask DisposeAsync() { + bool leadershipChanged; + long notificationGeneration; lock (m_lock) { if (m_disposed) @@ -173,7 +140,12 @@ public async ValueTask DisposeAsync() return; } m_disposed = true; + m_fence++; + leadershipChanged = m_isLeader; + m_isLeader = false; + notificationGeneration = m_fence; } + NotifyLeadershipChanged(false, notificationGeneration, leadershipChanged); m_cts.Cancel(); if (m_loop != null) @@ -188,8 +160,18 @@ public async ValueTask DisposeAsync() } } - await ReleaseIfOwnedAsync().ConfigureAwait(false); - m_cts.Dispose(); + await m_attemptGate.WaitAsync().ConfigureAwait(false); + try + { + await ReleaseIfOwnedAsync().ConfigureAwait(false); + m_watchdog.Dispose(); + m_cts.Dispose(); + } + finally + { + m_attemptGate.Release(); + m_attemptGate.Dispose(); + } } /// @@ -203,6 +185,77 @@ internal static bool IsReadyServiceLevel(byte serviceLevel, byte readyMinimumSer return serviceLevel >= readyMinimumServiceLevel; } + private async ValueTask TryAcquireOrRenewCoreAsync(CancellationToken ct) + { + long attemptFence = GetFence(); + try + { + if (!m_apiClient.IsInCluster) + { + LoseLeadership(); + return false; + } + + DateTimeOffset now = m_timeProvider.GetUtcNow(); + KubernetesLease? current = await ExecuteApiAttemptAsync( + token => m_apiClient.GetLeaseAsync(m_namespace, m_options.LeaseName, token), + ct) + .ConfigureAwait(false); + if (!IsAttemptCurrent(attemptFence)) + { + return false; + } + + if (current == null) + { + ClearObservedForeignLease(); + KubernetesLease created = NewLease(now); + _ = await ExecuteApiAttemptAsync( + token => m_apiClient.CreateLeaseAsync(m_namespace, created, token), + ct) + .ConfigureAwait(false); + return TryConfirmLeadership(attemptFence); + } + + if (!CanAcquireOrRenew(current)) + { + LoseLeadership(); + return false; + } + + string? previousHolder = current.Spec.HolderIdentity; + current.Spec.HolderIdentity = m_options.Kubernetes.NodeId; + current.Spec.LeaseDurationSeconds = ToLeaseSeconds(m_options.LeaseDuration); + current.Spec.RenewTime = FormatTime(now); + if (!string.Equals(previousHolder, m_options.Kubernetes.NodeId, StringComparison.Ordinal)) + { + current.Spec.AcquireTime = FormatTime(now); + current.Spec.LeaseTransitions = (current.Spec.LeaseTransitions ?? 0) + 1; + } + + _ = await ExecuteApiAttemptAsync( + token => m_apiClient.ReplaceLeaseAsync( + m_namespace, + m_options.LeaseName, + current, + token), + ct) + .ConfigureAwait(false); + return TryConfirmLeadership(attemptFence); + } + catch (HttpRequestException ex) when (ex.StatusCode == HttpStatusCode.Conflict) + { + ClearObservedForeignLease(); + LoseLeadership(); + return false; + } + catch + { + LoseLeadership(); + throw; + } + } + private static IKubernetesApiClient CreateApiClient(KubernetesLeaderElectionOptions options) { return KubernetesApiClientFactory.Create( @@ -230,20 +283,43 @@ private KubernetesLease NewLease(DateTimeOffset now) }; } - private bool CanAcquireOrRenew(KubernetesLease lease, DateTimeOffset now) + private bool CanAcquireOrRenew(KubernetesLease lease) { string? holder = lease.Spec.HolderIdentity; if (string.IsNullOrEmpty(holder) || string.Equals(holder, m_options.Kubernetes.NodeId, StringComparison.Ordinal)) { + ClearObservedForeignLease(); return true; } - DateTimeOffset renewTime = ParseTime(lease.Spec.RenewTime) ?? DateTimeOffset.MinValue; - int leaseSeconds = lease.Spec.LeaseDurationSeconds > 0 - ? lease.Spec.LeaseDurationSeconds - : ToLeaseSeconds(m_options.LeaseDuration); - return renewTime.AddSeconds(leaseSeconds) <= now; + string? resourceVersion = lease.Metadata.ResourceVersion; + long observedTimestamp = m_timeProvider.GetTimestamp(); + TimeSpan leaseDuration = lease.Spec.LeaseDurationSeconds > 0 + ? TimeSpan.FromSeconds(lease.Spec.LeaseDurationSeconds) + : GetLeaseDuration(); + // RenewTime is written by another process's wall clock. Treat the lease as expired only after its + // resource version remains unchanged for a full lease duration measured by this process. + lock (m_lock) + { + if (string.IsNullOrEmpty(resourceVersion)) + { + ClearObservedForeignLeaseCore(); + return false; + } + if (!string.Equals(m_observedResourceVersion, resourceVersion, StringComparison.Ordinal) || + !string.Equals(m_observedHolderIdentity, holder, StringComparison.Ordinal)) + { + m_observedResourceVersion = resourceVersion; + m_observedHolderIdentity = holder; + m_observedResourceVersionTimestamp = observedTimestamp; + m_observedLeaseDuration = leaseDuration; + return false; + } + + return m_timeProvider.GetElapsedTime(m_observedResourceVersionTimestamp) >= + m_observedLeaseDuration; + } } private async Task RenewLoopAsync(CancellationToken ct) @@ -265,7 +341,7 @@ private async Task RenewLoopAsync(CancellationToken ct) m_logger?.KubernetesLeaseElectionRenewFailed(ex, m_options.Kubernetes.NodeId); } - await Task.Delay(m_options.RenewInterval, ct).ConfigureAwait(false); + await m_timeProvider.Delay(m_options.RenewInterval, ct).ConfigureAwait(false); } } catch (OperationCanceledException) @@ -283,16 +359,24 @@ private async Task ReleaseIfOwnedAsync() try { - KubernetesLease? current = await m_apiClient - .GetLeaseAsync(m_namespace, m_options.LeaseName, CancellationToken.None) + KubernetesLease? current = await ExecuteApiAttemptAsync( + token => m_apiClient.GetLeaseAsync(m_namespace, m_options.LeaseName, token), + CancellationToken.None, + stopOnDispose: false) .ConfigureAwait(false); if (current != null && string.Equals(current.Spec.HolderIdentity, m_options.Kubernetes.NodeId, StringComparison.Ordinal)) { current.Spec.HolderIdentity = null; current.Spec.RenewTime = FormatTime(m_timeProvider.GetUtcNow()); - await m_apiClient - .ReplaceLeaseAsync(m_namespace, m_options.LeaseName, current, CancellationToken.None) + _ = await ExecuteApiAttemptAsync( + token => m_apiClient.ReplaceLeaseAsync( + m_namespace, + m_options.LeaseName, + current, + token), + CancellationToken.None, + stopOnDispose: false) .ConfigureAwait(false); } } @@ -302,36 +386,248 @@ await m_apiClient } } - private void SetLeader(bool value) + private long GetFence() + { + lock (m_lock) + { + return m_fence; + } + } + + private bool IsAttemptCurrent(long attemptFence) + { + lock (m_lock) + { + return !m_disposed && attemptFence == m_fence; + } + } + + private bool TryConfirmLeadership(long attemptFence) { bool changed; + long notificationGeneration; lock (m_lock) { - changed = m_isLeader != value; - m_isLeader = value; + if (m_disposed || attemptFence != m_fence) + { + return false; + } + + changed = !m_isLeader; + m_isLeader = true; + m_lastConfirmedLeadershipTimestamp = m_timeProvider.GetTimestamp(); + ClearObservedForeignLeaseCore(); + m_watchdog.Change(GetLeaseDuration(), Timeout.InfiniteTimeSpan); + notificationGeneration = m_fence; } - if (changed) + NotifyLeadershipChanged(true, notificationGeneration, changed); + return true; + } + + private void LoseLeadership() + { + bool changed; + long notificationGeneration; + lock (m_lock) { - LeadershipChanged?.Invoke(value); + m_fence++; + changed = m_isLeader; + m_isLeader = false; + if (!m_disposed) + { + m_watchdog.Change(Timeout.InfiniteTimeSpan, Timeout.InfiniteTimeSpan); + } + notificationGeneration = m_fence; } + NotifyLeadershipChanged(false, notificationGeneration, changed); } - private static string FormatTime(DateTimeOffset time) + private void OnWatchdogElapsed() { - return time.UtcDateTime.ToString("O", CultureInfo.InvariantCulture); + bool changed = false; + long notificationGeneration = 0; + lock (m_lock) + { + if (m_disposed || !m_isLeader) + { + return; + } + + TimeSpan leaseDuration = GetLeaseDuration(); + TimeSpan elapsed = m_timeProvider.GetElapsedTime(m_lastConfirmedLeadershipTimestamp); + if (elapsed < leaseDuration) + { + m_watchdog.Change(leaseDuration - elapsed, Timeout.InfiniteTimeSpan); + return; + } + + m_fence++; + m_isLeader = false; + changed = true; + notificationGeneration = m_fence; + } + NotifyLeadershipChanged(false, notificationGeneration, changed); + } + + private void NotifyLeadershipChanged( + bool value, + long notificationGeneration, + bool stateChanged) + { + lock (m_notificationLock) + { + lock (m_lock) + { + if (notificationGeneration != m_fence || + m_isLeader != value || + (value && m_disposed)) + { + return; + } + } + + if (notificationGeneration < m_lastNotificationGeneration) + { + return; + } + m_lastNotificationGeneration = notificationGeneration; + if (m_hasLeadershipNotification && m_lastNotifiedLeadership == value) + { + return; + } + if (!m_hasLeadershipNotification && !stateChanged) + { + return; + } + + m_hasLeadershipNotification = true; + m_lastNotifiedLeadership = value; + m_pendingLeadershipNotifications.Enqueue( + new LeadershipNotification(LeadershipChanged, value)); + if (m_dispatchingLeadershipNotifications) + { + return; + } + m_dispatchingLeadershipNotifications = true; + } + DispatchLeadershipNotifications(); + } + + private void DispatchLeadershipNotifications() + { + Exception? dispatchException = null; + while (true) + { + LeadershipNotification notification; + lock (m_notificationLock) + { + if (!m_pendingLeadershipNotifications.TryDequeue(out notification)) + { + m_dispatchingLeadershipNotifications = false; + break; + } + } + + try + { + notification.Handler?.Invoke(notification.Value); + } + catch (Exception ex) + { + dispatchException ??= ex; + } + } + + if (dispatchException != null) + { + ExceptionDispatchInfo.Capture(dispatchException).Throw(); + } + } + + private void ClearObservedForeignLease() + { + lock (m_lock) + { + ClearObservedForeignLeaseCore(); + } + } + + private void ClearObservedForeignLeaseCore() + { + m_observedResourceVersion = null; + m_observedHolderIdentity = null; + m_observedResourceVersionTimestamp = 0; + m_observedLeaseDuration = default; } - private static DateTimeOffset? ParseTime(string? value) + private async ValueTask ExecuteApiAttemptAsync( + Func> operation, + CancellationToken ct, + bool stopOnDispose = true) { - if (DateTimeOffset.TryParse( - value, - CultureInfo.InvariantCulture, - DateTimeStyles.AssumeUniversal, - out DateTimeOffset parsed)) + using CancellationTokenSource operationCts = stopOnDispose + ? CancellationTokenSource.CreateLinkedTokenSource(ct, m_cts.Token) + : CancellationTokenSource.CreateLinkedTokenSource(ct); + using CancellationTokenSource delayCts = stopOnDispose + ? CancellationTokenSource.CreateLinkedTokenSource(ct, m_cts.Token) + : CancellationTokenSource.CreateLinkedTokenSource(ct); + Task operationTask = operation(operationCts.Token).AsTask(); + TimeSpan timeout = GetApiAttemptTimeout(); + Task delayTask = m_timeProvider.Delay(timeout, delayCts.Token); + _ = await Task.WhenAny(operationTask, delayTask).ConfigureAwait(false); + if (operationTask.IsCompleted) { - return parsed.ToUniversalTime(); + delayCts.Cancel(); + return await operationTask.ConfigureAwait(false); + } + + operationCts.Cancel(); + ObserveFault(operationTask); + ct.ThrowIfCancellationRequested(); + if (stopOnDispose) + { + m_cts.Token.ThrowIfCancellationRequested(); + } + throw new TimeoutException( + $"The Kubernetes Lease API attempt did not complete within {timeout}."); + } + + private static void ObserveFault(Task task) + { + _ = task.ContinueWith( + static completed => _ = completed.Exception, + CancellationToken.None, + TaskContinuationOptions.ExecuteSynchronously | TaskContinuationOptions.OnlyOnFaulted, + TaskScheduler.Default); + } + + private TimeSpan GetApiAttemptTimeout() + { + if (m_options.RenewInterval > TimeSpan.Zero) + { + return m_options.RenewInterval; + } + return GetLeaseDuration(); + } + + private TimeSpan GetLeaseDuration() + { + return m_options.LeaseDuration > TimeSpan.Zero + ? m_options.LeaseDuration + : TimeSpan.FromSeconds(1); + } + + private void ThrowIfDisposed() + { + lock (m_lock) + { + ObjectDisposedException.ThrowIf(m_disposed, this); } - return null; + } + + private static string FormatTime(DateTimeOffset time) + { + return time.UtcDateTime.ToString("O", CultureInfo.InvariantCulture); } private static int ToLeaseSeconds(TimeSpan leaseDuration) @@ -345,11 +641,27 @@ private static int ToLeaseSeconds(TimeSpan leaseDuration) private readonly TimeProvider m_timeProvider; private readonly ILogger? m_logger; private readonly Lock m_lock = new(); + private readonly Lock m_notificationLock = new(); + private readonly Queue m_pendingLeadershipNotifications = []; private readonly CancellationTokenSource m_cts = new(); + private readonly SemaphoreSlim m_attemptGate = new(1, 1); + private readonly ITimer m_watchdog; private Task? m_loop; + private string? m_observedResourceVersion; + private string? m_observedHolderIdentity; + private long m_observedResourceVersionTimestamp; + private long m_lastConfirmedLeadershipTimestamp; + private long m_fence; + private long m_lastNotificationGeneration = -1; + private TimeSpan m_observedLeaseDuration; private bool m_isLeader; + private bool m_hasLeadershipNotification; + private bool m_lastNotifiedLeadership; + private bool m_dispatchingLeadershipNotifications; private bool m_started; private bool m_disposed; + + private readonly record struct LeadershipNotification(Action? Handler, bool Value); } /// diff --git a/tests/Opc.Ua.Redundancy.Kubernetes.Tests/LeaderElection/KubernetesLeaseLeaderElectionTests.cs b/tests/Opc.Ua.Redundancy.Kubernetes.Tests/LeaderElection/KubernetesLeaseLeaderElectionTests.cs index 42a2dcdf9d..ad2920142a 100644 --- a/tests/Opc.Ua.Redundancy.Kubernetes.Tests/LeaderElection/KubernetesLeaseLeaderElectionTests.cs +++ b/tests/Opc.Ua.Redundancy.Kubernetes.Tests/LeaderElection/KubernetesLeaseLeaderElectionTests.cs @@ -36,6 +36,7 @@ #pragma warning disable CA2007 using System; +using System.Collections.Generic; using System.Globalization; using System.Net; using System.Net.Http; @@ -113,16 +114,20 @@ public async Task LiveForeignHolderKeepsReplicaFollowerAsync() } [Test] - public async Task ExpiredForeignHolderCanBeTakenOverAsync() + public async Task ForeignHolderCanBeTakenOverAfterUnchangedResourceVersionDurationAsync() { Mock api = NewApi(); - KubernetesLease lease = HeldLease("pod-b", ParseUtc("2026-01-01T00:00:00Z")); - var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:31Z")); + KubernetesLease lease = HeldLease("pod-b", ParseUtc("2100-01-01T00:00:00Z")); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:00Z")); api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) .ReturnsAsync(lease); await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), time); + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.False); + time.Advance(TimeSpan.FromSeconds(29)); + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.False); + time.Advance(TimeSpan.FromSeconds(1)); bool acquired = await election.TryAcquireOrRenewAsync().ConfigureAwait(false); Assert.That(acquired, Is.True); @@ -130,6 +135,29 @@ public async Task ExpiredForeignHolderCanBeTakenOverAsync() Assert.That(lease.Spec.LeaseTransitions, Is.EqualTo(1)); } + [Test] + public async Task ResourceVersionChangeRestartsForeignLeaseObservationAsync() + { + Mock api = NewApi(); + KubernetesLease lease = HeldLease("pod-b", ParseUtc("2000-01-01T00:00:00Z")); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:00Z")); + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); + api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) + .ReturnsAsync(lease); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, NewOptions(), time); + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.False); + time.Advance(TimeSpan.FromSeconds(29)); + lease.Metadata.ResourceVersion = "2"; + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.False); + time.Advance(TimeSpan.FromSeconds(29)); + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.False); + time.Advance(TimeSpan.FromSeconds(1)); + + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.True); + Assert.That(lease.Spec.HolderIdentity, Is.EqualTo("pod-a")); + } + [Test] public async Task ConflictLosesLeadershipAsync() { @@ -150,6 +178,44 @@ public async Task ConflictLosesLeadershipAsync() Assert.That(election.IsLeader, Is.False); } + [Test] + public async Task LeadershipChangedHandlerCanDisposeWithoutDeadlockAsync() + { + Mock api = NewApi(); + api.SetupSequence(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())) + .ReturnsAsync((KubernetesLease?)null) + .ReturnsAsync((KubernetesLease?)null); + api.Setup(x => x.CreateLeaseAsync("ns", It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, KubernetesLease lease, CancellationToken _) => lease); + var election = new KubernetesLeaseLeaderElection( + api.Object, + NewOptions(), + new FakeTimeProvider(ParseUtc("2026-01-01T00:00:00Z"))); + var notifications = new List(); + Task? disposeTask = null; + election.LeadershipChanged += value => + { + notifications.Add(value); + if (value) + { + disposeTask = election.DisposeAsync().AsTask(); + } + }; + + Task acquireTask = election.TryAcquireOrRenewAsync().AsTask(); + bool acquired = await acquireTask + .WaitAsync(TimeSpan.FromSeconds(30)) + .ConfigureAwait(false); + + Assert.That(acquired, Is.True); + Assert.That(disposeTask, Is.Not.Null); + await disposeTask!.WaitAsync(TimeSpan.FromSeconds(30)).ConfigureAwait(false); + Assert.That(notifications, Has.Count.EqualTo(2)); + Assert.That(notifications[0], Is.True); + Assert.That(notifications[1], Is.False); + Assert.That(election.IsLeader, Is.False); + } + [Test] public async Task DisposeReleasesOwnedLeaseAsync() { @@ -198,7 +264,7 @@ public async Task EmptyLeaseHolderCanBeAcquiredAsync() } [Test] - public async Task MissingRenewTimeUsesExpiredFallbackAsync() + public async Task MissingRenewTimeStillRequiresLocalObservationWindowAsync() { Mock api = NewApi(); KubernetesLease lease = HeldLease("pod-b", ParseUtc("2026-01-01T00:00:00Z")); @@ -206,11 +272,14 @@ public async Task MissingRenewTimeUsesExpiredFallbackAsync() api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())).ReturnsAsync(lease); api.Setup(x => x.ReplaceLeaseAsync("ns", "opcua", lease, It.IsAny())) .ReturnsAsync(lease); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:05Z")); await using var election = new KubernetesLeaseLeaderElection( api.Object, NewOptions(), - new FakeTimeProvider(ParseUtc("2026-01-01T00:00:05Z"))); + time); + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.False); + time.Advance(TimeSpan.FromSeconds(30)); bool acquired = await election.TryAcquireOrRenewAsync().ConfigureAwait(false); Assert.That(acquired, Is.True); @@ -336,6 +405,92 @@ public async Task RenewalFailureWithoutHttpStatusClearsLeadershipAsync() Assert.That(election.IsLeader, Is.False); } + [Test] + public async Task HungApiAttemptTimesOutAtRenewIntervalAsync() + { + Mock api = NewApi(); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:00Z")); + var getStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hungGet = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + int getCalls = 0; + CancellationToken requestToken = default; + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())) + .Returns((string _, string _, CancellationToken token) => + { + int call = Interlocked.Increment(ref getCalls); + if (call == 1) + { + requestToken = token; + getStarted.TrySetResult(); + return new ValueTask(hungGet.Task); + } + return new ValueTask((KubernetesLease?)null); + }); + KubernetesLeaderElectionOptions options = NewOptions(); + options.RenewInterval = TimeSpan.FromSeconds(5); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, options, time); + Task attempt = election.TryAcquireOrRenewAsync().AsTask(); + await getStarted.Task.ConfigureAwait(false); + time.Advance(TimeSpan.FromSeconds(5)); + + Assert.That( + () => attempt, + Throws.TypeOf() + .With.Message.Contains(TimeSpan.FromSeconds(5).ToString())); + Assert.That(election.IsLeader, Is.False); + Assert.That(requestToken.IsCancellationRequested, Is.True); + hungGet.TrySetResult(null); + } + + [Test] + public async Task WatchdogFencesHungRenewalAndRejectsStaleCompletionAsync() + { + Mock api = NewApi(); + var time = new FakeTimeProvider(ParseUtc("2026-01-01T00:00:00Z")); + var renewStarted = new TaskCompletionSource(TaskCreationOptions.RunContinuationsAsynchronously); + var hungRenew = new TaskCompletionSource( + TaskCreationOptions.RunContinuationsAsynchronously); + int getCalls = 0; + api.Setup(x => x.GetLeaseAsync("ns", "opcua", It.IsAny())) + .Returns((string _, string _, CancellationToken _) => + { + int call = Interlocked.Increment(ref getCalls); + if (call == 1 || call > 2) + { + return new ValueTask((KubernetesLease?)null); + } + renewStarted.TrySetResult(); + return new ValueTask(hungRenew.Task); + }); + api.Setup(x => x.CreateLeaseAsync("ns", It.IsAny(), It.IsAny())) + .ReturnsAsync((string _, KubernetesLease lease, CancellationToken _) => lease); + KubernetesLeaderElectionOptions options = NewOptions(); + options.RenewInterval = TimeSpan.FromMinutes(1); + + await using var election = new KubernetesLeaseLeaderElection(api.Object, options, time); + Assert.That(await election.TryAcquireOrRenewAsync().ConfigureAwait(false), Is.True); + Task staleAttempt = election.TryAcquireOrRenewAsync().AsTask(); + await renewStarted.Task.ConfigureAwait(false); + time.Advance(TimeSpan.FromSeconds(29)); + Assert.That(election.IsLeader, Is.True); + + time.Advance(TimeSpan.FromSeconds(1)); + Assert.That(election.IsLeader, Is.False); + hungRenew.TrySetResult(HeldLease("pod-a", ParseUtc("2026-01-01T00:00:00Z"))); + + Assert.That(await staleAttempt.ConfigureAwait(false), Is.False); + Assert.That(election.IsLeader, Is.False); + api.Verify( + x => x.ReplaceLeaseAsync( + It.IsAny(), + It.IsAny(), + It.IsAny(), + It.IsAny()), + Times.Never); + } + [Test] public async Task RenewLoopStopsOnCanceledExceptionAsync() {