diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs index 3fbfc283b28..dbbd3985bdb 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/AzureDevOpsResultPublisher.cs @@ -476,7 +476,7 @@ private async Task SendWithRetryAsync( object? payload, CancellationToken cancellationToken) { - int triesLeft = 10; + int triesLeft = GetRetryCount(method, _azdoParameters.RetryWrites); string? body = payload is null ? null : JsonSerializer.Serialize(payload, s_serializerOptions); if (!string.IsNullOrEmpty(body)) { @@ -549,6 +549,9 @@ await _uploadClient.UploadAsync( } } + internal static int GetRetryCount(HttpMethod method, bool retryWrites) + => retryWrites || method == HttpMethod.Get ? 10 : 0; + private static TimeSpan? GetRetryDelay(HttpResponseMessage response) { RetryConditionHeaderValue? retryAfter = response.Headers.RetryAfter; diff --git a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs index 118c79dcc88..364cd7444cc 100644 --- a/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs +++ b/src/Microsoft.DotNet.Helix/AzureDevOpsTestPublisher/Model/AzureDevOpsReportingParameters.cs @@ -8,4 +8,7 @@ public sealed record AzureDevOpsReportingParameters( string TeamProject, string TestRunId, string? AccessToken = null, - bool UseFullyQualifiedTestName = false); + bool UseFullyQualifiedTestName = false) +{ + public bool RetryWrites { get; init; } = true; +} diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md index 6e37556f1cf..63bda759236 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.Design.md @@ -186,29 +186,23 @@ addresses each: wait" scheme. → Resubmission-not-possible is treated as an actionable hard failure so the invocation fails fast instead of hanging (§2.3.3). -Each of these cases is pinned by a pipeline-emulating test in -`JobMonitorRunnerTests` (the `AttemptScoped_*` suite): `RetryOnlyMonitor` -(case 1), `StrandedWaitingPreviousWork` (cases 2/3, mirroring #17156), -`FastRerun_CurrentIncarnationExists` (case 4), -`UnlinkedRerunDuplicates_HigherAttemptWinsOutcome` (case 5), and -`UnresubmittablePreviousWork_FailsFast` (case 6). The `MultiAttempt_*` tests -additionally exercise the end-to-end lineage across five stage attempts — -including an attempt in which the monitor never ran — verifying that only -still-unfinished streams are carried forward and that a stream which has passed -in any prior attempt is never resubmitted again. - ### 2.4 Upload invariants Upload is restart-resilient but logically independent from retry. -1. The same Helix job's test results are never uploaded twice. The durable +1. A Helix job has at most one durably completed, tagged test run. The durable deduplication signal is the Helix-job-name tag on completed AzDO test runs. -2. For every completed Helix job not already uploaded, all available test - results are uploaded. + An interrupted, untagged upload is intentionally replayed. +2. For every completed Helix job without a completed, tagged upload, all + available test results are uploaded. 3. Uploads happen in lineage order — oldest incarnation first. If both an original job and its resubmission have completed and neither has been uploaded, the original uploads first. -4. Upload failures are logged but never affect pass/fail. +4. Upload failures are logged as warnings but never affect pass/fail. Read-only + download failures are retried a bounded number of times. State-changing + operations are not replayed by the queue after an ambiguous failure because + they may have partially succeeded. A failed upload remains untagged so a + later monitor invocation may replay it in a new run. 5. A failed original Helix job may be resubmitted on entry and still have its original test results uploaded during the same invocation if those results were not uploaded earlier. @@ -235,15 +229,17 @@ Exit code is `0` only when both checks pass; otherwise `1`. Cancellation The runner must be safe to re-run after any abrupt termination. In particular: -- Partial uploads must not cause duplicate uploads on the next run. +- Completed, tagged uploads must not be repeated. Interrupted, untagged uploads + are intentionally replayed on the next run. - Retry candidates must be rediscovered from Helix job properties, not from prior in-memory state. -- Cancellation must drain in-flight uploads before exiting so partially - uploaded results are not lost. -- On cancellation, in-flight Helix jobs should receive a best-effort cancel - request even though the runner's own cancellation token has already - fired. This requires a fresh, short-lived cancellation budget for the - cleanup path. +- Cancellation must not wait for pending or in-flight uploads. An upload that + does not complete remains untagged and is rediscovered and re-uploaded by a + later invocation. +- On cancellation, immediately emit the timeout report, then best-effort cancel + the latest in-flight Helix job incarnation in each lineage even though the + runner's own cancellation token has already fired. This cleanup uses a fresh, + bounded cancellation token so it cannot extend shutdown indefinitely. Re-attach spans stage attempts. Within the same attempt (a monitor job-retry or a crashed-and-restarted monitor process) the runner re-attaches to the same @@ -289,7 +285,9 @@ behaviorally; method names are illustrative. - **List work items for a job** — return all work-item summaries. - **Download test results** — given a job and a set of work-item names, download recognized result files into a working directory. Individual - per-work-item failures must not abort the batch. + per-file failures must not prevent the remaining files from being attempted. + After the batch, transient failures cause the read-only download phase to be + retried; permanent failures are logged and omitted. - **Cancel a job** — best-effort cancellation. - **Resubmit failed work items** — given the original job and a set of failed (or unfinished) work items, submit a new Helix job that contains only @@ -326,8 +324,10 @@ or the runner will silently fail to see its own jobs. 3. Perform the one-shot retry pass (§5.3). 4. Enter the poll loop (§5.4) until the build finishes or cancellation fires. -5. On cancellation (timeout), drain pending uploads, emit a timeout report - (§5.6), best-effort cancel in-flight Helix jobs (§2.6), and exit `1`. +5. On cancellation (timeout), do not wait for pending uploads. Immediately emit + a timeout report (§5.6), use a fresh bounded token to best-effort cancel the + latest in-flight Helix jobs (§2.6), and exit `1`. Incomplete uploads remain + untagged and are retried by a later invocation. 6. On normal completion, emit the final summary and exit per §2.5. ### 5.3 Retry pass @@ -455,16 +455,25 @@ lines are plain logger output. ### 5.9 Test-result upload pipeline -Uploads are fire-and-forget tasks tracked for later draining: +Uploads are asynchronous tasks tracked for normal completion. Their in-memory +lifecycle distinguishes queued, in-progress, durably completed, and failed +uploads; only a completed, tagged test run is considered durable. - Each upload is queued asynchronously and tracked. Multiple uploads may proceed concurrently. -- An upload retries indefinitely on transient errors; only cancellation - exits the retry loop. -- Both the normal-termination and cancellation paths wait for queued - uploads to drain before exiting. The cancellation drain uses a fresh - cancellation budget so uploads in progress when the runner token fires - are not abandoned. +- Test results are downloaded before the AzDO test run is created. Transient + download failures are safe to retry and use a bounded retry budget. +- The queue invokes each state-changing phase — create the run, publish results, + and complete/tag the run — once. The monitor's adapters also disable automatic + retries for those writes because an error response may arrive after the + service has partially committed the request. Replaying it could duplicate + test runs, results, or attachments. +- Permanent failures, exhausted safe retries, and ambiguous write failures are + logged as warnings and stop the upload task without affecting pass/fail. +- The normal-termination path waits for queued uploads to drain before exiting. +- The cancellation path does not wait for pending or in-flight uploads. If an + upload has not completed and applied its Helix-job tag, it remains untagged; + durable-state discovery causes a later invocation to upload it again. The upload sequence per job is: create (or reuse) a test run with the plain `{TestRunName}`, download results, upload them, complete the test run and tag @@ -483,8 +492,8 @@ least one work item), or `Waiting` (no work items observed yet). ## 6. Externally observable formats -These shapes are observed by other tools, downstream parsers, or tests and -must be preserved: +These shapes are observed by other tools or downstream parsers and must be +preserved: - AzDO test-run tag: `helixjob`, applied to a completed test run as an object-form tag (`{ "name": "..." }`). diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs index e24c0e1cb88..b65c0304b73 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/JobMonitorRunner.cs @@ -304,7 +304,6 @@ await _azdo.GetTimelineRecordsAsync(cancellationToken), foreach (HelixJobInfo job in completedJobs.Where(j => !_state.IsHelixJobProcessed(j.JobName))) { await ReconcileCompletedJobAsync(job, queueUpload: true, cancellationToken); - _state.TryMarkHelixJobProcessed(job.JobName); } // Second pass: ensure outcomes for every completed job (any attempt) are reflected in @@ -395,7 +394,10 @@ private async Task ReconcileCompletedJobAsync( if (queueUpload && !alreadyUploadedByPriorAttempt) { - _uploads.Enqueue(helixJob, workItems, cancellationToken); + if (_state.TryQueueHelixJobUpload(helixJob.JobName)) + { + _uploads.Enqueue(helixJob, workItems, cancellationToken); + } } if (!alreadyUploadedByPriorAttempt) diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs index 6f100c7915a..4ea36f76d9e 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/MonitorState.cs @@ -11,6 +11,14 @@ namespace Microsoft.DotNet.Helix.JobMonitor { + internal enum TestResultUploadState + { + Queued, + InProgress, + DurablyCompleted, + Failed, + } + /// /// Thread-safe container for every piece of runtime state the monitor accumulates while /// observing a single invocation. Mutations from the main poll loop and from background @@ -28,9 +36,11 @@ internal sealed class MonitorState // by GetSubmitterChainKey to walk back through PreviousHelixJobName links across polls. private readonly Dictionary _associatedJobs = new(StringComparer.OrdinalIgnoreCase); - // Helix jobs whose results have been uploaded to Azure DevOps in this or a prior - // monitor invocation. Seeded on entry from the AzDO test-run tags. - private readonly HashSet _processedHelixJobs = new(StringComparer.OrdinalIgnoreCase); + // Upload lifecycle for each Helix job observed by this invocation. Jobs discovered from + // Azure DevOps completion tags are seeded as DurablyCompleted. A queued/in-progress job + // must not be treated as durable: cancellation may abandon it, in which case the absent + // tag causes a later invocation to replay the upload. + private readonly Dictionary _testResultUploadStates = new(StringComparer.OrdinalIgnoreCase); // Tracks the latest outcome for each logical work item, keyed by // (SubmitterChainKey, WorkItemName). See GetSubmitterChainKey for the keying rationale. @@ -156,21 +166,54 @@ public void AddProcessedHelixJobs(IEnumerable jobNames) { foreach (string jobName in jobNames) { - _processedHelixJobs.Add(jobName); + _testResultUploadStates[jobName] = TestResultUploadState.DurablyCompleted; } } } + public bool TryQueueHelixJobUpload(string jobName) + { + lock (_sync) + { + if (_testResultUploadStates.ContainsKey(jobName)) + { + return false; + } + + _testResultUploadStates[jobName] = TestResultUploadState.Queued; + return true; + } + } + + public void MarkHelixJobUploadInProgress(string jobName) + { + lock (_sync) + { + _testResultUploadStates[jobName] = TestResultUploadState.InProgress; + } + } + + public void MarkHelixJobUploadFailed(string jobName) + { + lock (_sync) + { + _testResultUploadStates[jobName] = TestResultUploadState.Failed; + } + } + /// - /// Marks the given Helix job as processed and increments . - /// Returns true if this is the first time the job was marked. + /// Marks the given Helix job as durably completed and increments + /// . This must only be called after Azure DevOps has + /// completed and tagged the test run. /// public bool TryMarkHelixJobProcessed(string jobName) { lock (_sync) { - if (_processedHelixJobs.Add(jobName)) + if (!_testResultUploadStates.TryGetValue(jobName, out TestResultUploadState state) + || state != TestResultUploadState.DurablyCompleted) { + _testResultUploadStates[jobName] = TestResultUploadState.DurablyCompleted; _processedJobCount++; return true; } @@ -183,7 +226,8 @@ public bool IsHelixJobProcessed(string jobName) { lock (_sync) { - return _processedHelixJobs.Contains(jobName); + return _testResultUploadStates.TryGetValue(jobName, out TestResultUploadState state) + && state == TestResultUploadState.DurablyCompleted; } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs index a7139ff30bb..0d0b8c37806 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/AzureDevOpsService.cs @@ -318,6 +318,7 @@ public async Task CreateTestRunAsync(string name, CancellationToken cancell ["name"] = name, ["state"] = "InProgress", }, + retryTransientFailures: false, cancellationToken: cancellationToken); return result?["id"]?.ToObject() ?? 0; } @@ -360,6 +361,7 @@ public async Task CompleteTestRunAsync( await SendAsync(new HttpMethod("PATCH"), $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/runs/{testRunId}?api-version=7.1", body, + retryTransientFailures: false, cancellationToken: cancellationToken); } @@ -391,6 +393,7 @@ await SendAsync( HttpMethod.Post, $"{_options.CollectionUri}{_options.TeamProject}/_apis/test/Runs/{testRunId}/attachments?api-version=7.1", body, + retryTransientFailures: false, cancellationToken: cancellationToken); } @@ -399,13 +402,17 @@ await SendAsync( IReadOnlyList results, CancellationToken cancellationToken) { - using var publisher = new AzureDevOpsResultPublisher( - new AzureDevOpsReportingParameters( + var reportingParameters = new AzureDevOpsReportingParameters( new Uri(_options.CollectionUri, UriKind.Absolute), _options.TeamProject, testRunId.ToString(CultureInfo.InvariantCulture), _options.SystemAccessToken, - _options.UseFullyQualifiedTestName), + _options.UseFullyQualifiedTestName) + { + RetryWrites = false, + }; + using var publisher = new AzureDevOpsResultPublisher( + reportingParameters, _logger); async Task UploadWorkItemAsync(WorkItemTestResults workItem) @@ -441,18 +448,28 @@ async Task UploadWorkItemAsync(WorkItemTestResults work return testSummaries.ToDictionary(t => (t.WorkItem.JobName, t.WorkItem.WorkItemName), t => t.Summary); } - private async Task SendAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) + private async Task SendAsync( + HttpMethod method, + string requestUri, + JToken body = null, + bool retryTransientFailures = true, + CancellationToken cancellationToken = default) { - string content = await SendForStringAsync(method, requestUri, body, cancellationToken); + string content = await SendForStringAsync(method, requestUri, body, retryTransientFailures, cancellationToken); return string.IsNullOrWhiteSpace(content) ? [] : JObject.Parse(content); } // Sends a request and returns the raw response body as a string. Used for endpoints // (such as test-run attachment downloads) that do not return JSON, where SendAsync's // JObject parsing would fail or discard the payload. - private async Task SendForStringAsync(HttpMethod method, string requestUri, JToken body = null, CancellationToken cancellationToken = default) + private async Task SendForStringAsync( + HttpMethod method, + string requestUri, + JToken body = null, + bool retryTransientFailures = true, + CancellationToken cancellationToken = default) { - return await RetryHelper.RetryAsync(async () => + async Task SendOnceAsync() { using var request = new HttpRequestMessage(method, requestUri); if (body != null) @@ -465,12 +482,19 @@ private async Task SendForStringAsync(HttpMethod method, string requestU if (!response.IsSuccessStatusCode) { await HonorRateLimitAsync(response, requestUri, cancellationToken); - throw new HttpRequestException($"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}"); + throw new HttpRequestException( + $"Request to {requestUri} failed with {(int)response.StatusCode} {response.ReasonPhrase}. {content}", + null, + response.StatusCode); } await HonorRateLimitAsync(response, requestUri, cancellationToken); return content; - }, cancellationToken); + } + + return retryTransientFailures + ? await RetryHelper.RetryAsync(SendOnceAsync, cancellationToken) + : await SendOnceAsync(); } // Honors Azure DevOps rate limiting guidance: diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs index 59832f6b750..bb2763d0381 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/Services/HelixService.cs @@ -4,6 +4,7 @@ using System; using System.Collections.Generic; using System.Collections.Immutable; +using System.IO; using System.Linq; using System.Threading; using System.Threading.Tasks; @@ -73,6 +74,7 @@ public async Task> DownloadTestResultsAsync( CancellationToken cancellationToken) { List downloadedFiles = []; + List transientFailures = []; string outputDirectory = _fileSystem.PathCombine(workingDirectory, SanitizeDirName(jobName)); _fileSystem.CreateDirectory(outputDirectory); @@ -110,6 +112,20 @@ public async Task> DownloadTestResultsAsync( await blobClient.DownloadToAsync(destinationFile, cancellationToken); workItemFiles.Add(destinationFile); } + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) + { + throw; + } + catch (Exception ex) when (TransientFailureDetector.IsTransient(ex)) + { + transientFailures.Add(ex); + _logger.LogWarning(ex, + "Transient failure downloading '{FileName}' for '{JobName}/{WorkItemName}'. " + + "The remaining files will still be attempted before the download is retried.", + file.Name, + jobName, + workItemName); + } catch (Exception ex) { _logger.LogWarning(ex, "Failed to download '{FileName}' for '{JobName}/{WorkItemName}'.", file.Name, jobName, workItemName); @@ -119,6 +135,13 @@ public async Task> DownloadTestResultsAsync( downloadedFiles.Add(new WorkItemTestResults(jobName, workItemName, workItemFiles)); } + if (transientFailures.Count > 0) + { + throw new IOException( + $"One or more transient test-result downloads failed for Helix job '{jobName}'.", + new AggregateException(transientFailures)); + } + return downloadedFiles; } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs index 248689bc094..74308498204 100644 --- a/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs +++ b/src/Microsoft.DotNet.Helix/JobMonitor/TestResultUploadQueue.cs @@ -23,6 +23,9 @@ namespace Microsoft.DotNet.Helix.JobMonitor /// internal sealed class TestResultUploadQueue { + private const int MaximumTransientAttempts = 3; + private const string AzdoWarningPrefix = "##vso[task.logissue type=warning]"; + private readonly ILogger _logger; private readonly JobMonitorOptions _options; private readonly IAzureDevOpsService _azdo; @@ -87,81 +90,143 @@ private async Task UploadAsync( IReadOnlyCollection workItemNames, CancellationToken cancellationToken) { - while (true) - { - cancellationToken.ThrowIfCancellationRequested(); - int testRunId = 0; + _monitorState.MarkHelixJobUploadInProgress(helixJob.JobName); - try - { - testRunId = await _azdo.CreateTestRunAsync(helixJob.TestRunName, cancellationToken); - IReadOnlyList downloaded = await _helix.DownloadTestResultsAsync( + (bool downloadedSuccessfully, IReadOnlyList downloaded) = await TryExecuteWithRetryAsync( + () => _helix.DownloadTestResultsAsync( helixJob.JobName, workItemNames, _options.WorkingDirectory, - cancellationToken); - - IReadOnlyDictionary<(string JobName, string WorkItemName), TestResultUploadSummary> testResults = await _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken); - if (_options.FailWorkItemsWithFailedTests) - { - _monitorState.ObserveTestResults(testResults); - } - - // Persist the list of work items whose tests failed as an attachment on the - // test run, so a subsequent monitor invocation can recover the resubmission - // set in a small fixed number of REST calls per run (see - // AzureDevOpsService.GetFailedTestWorkItemsAsync). - IReadOnlyCollection failedWorkItems = - [ - .. testResults - .Where(kv => !kv.Value.AllPassed) - .Select(kv => kv.Key.WorkItemName) - ]; - await CompleteTestRunAsync(testRunId, helixJob, failedWorkItems, cancellationToken); - - long uploadedCount = testResults.Values.Sum(r => r.UploadedCount); - _logger.LogInformation("{UploadedCount} test results for job '{JobName}' processed.", - uploadedCount, - helixJob.DisplayName); - return; - } - catch (OperationCanceledException) - { - throw; - } - catch (Exception ex) + cancellationToken), + "download the Helix test results", + helixJob, + testRunId: 0, + safeToRetry: true, + cancellationToken); + if (!downloadedSuccessfully) + { + _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); + return; + } + + (bool created, int testRunId) = await TryExecuteWithRetryAsync( + () => _azdo.CreateTestRunAsync(helixJob.TestRunName, cancellationToken), + "create the Azure DevOps test run", + helixJob, + testRunId: 0, + safeToRetry: false, + cancellationToken); + if (!created) + { + _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); + return; + } + + (bool uploadedSuccessfully, IReadOnlyDictionary<(string JobName, string WorkItemName), TestResultUploadSummary> testResults) + = await TryExecuteWithRetryAsync( + () => _azdo.UploadTestResultsAsync(testRunId, downloaded, cancellationToken), + "upload the test results to Azure DevOps", + helixJob, + testRunId, + safeToRetry: false, + cancellationToken); + if (!uploadedSuccessfully) + { + _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); + return; + } + + if (_options.FailWorkItemsWithFailedTests) + { + _monitorState.ObserveTestResults(testResults); + } + + IReadOnlyCollection failedWorkItems = + [ + .. testResults + .Where(kv => !kv.Value.AllPassed) + .Select(kv => kv.Key.WorkItemName) + ]; + + (bool completed, _) = await TryExecuteWithRetryAsync( + async () => { - _logger.LogDebug(ex, - "Failed to upload test results for job {JobName} to Azure DevOps. Test run ID was {TestRunId}. Retrying after delay.", - helixJob.DisplayName, - testRunId); - await _delay(cancellationToken); - } + await _azdo.CompleteTestRunAsync(testRunId, helixJob.JobName, failedWorkItems, cancellationToken); + return true; + }, + "complete and tag the Azure DevOps test run", + helixJob, + testRunId, + safeToRetry: false, + cancellationToken); + if (!completed) + { + _monitorState.MarkHelixJobUploadFailed(helixJob.JobName); + return; } + + _monitorState.TryMarkHelixJobProcessed(helixJob.JobName); + + long uploadedCount = testResults.Values.Sum(r => r.UploadedCount); + _logger.LogInformation("{UploadedCount} test results for job '{JobName}' processed.", + uploadedCount, + helixJob.DisplayName); } - private async Task CompleteTestRunAsync(int testRunId, HelixJobInfo helixJob, IReadOnlyCollection failedWorkItems, CancellationToken cancellationToken) + private async Task<(bool Success, T Result)> TryExecuteWithRetryAsync( + Func> operation, + string operationDescription, + HelixJobInfo helixJob, + int testRunId, + bool safeToRetry, + CancellationToken cancellationToken) { - while (true) + int maximumAttempts = safeToRetry ? MaximumTransientAttempts : 1; + for (int attempt = 1; attempt <= maximumAttempts; attempt++) { + cancellationToken.ThrowIfCancellationRequested(); + try { - await _azdo.CompleteTestRunAsync(testRunId, helixJob.JobName, failedWorkItems, cancellationToken); - return; + return (true, await operation()); } - catch (OperationCanceledException) + catch (OperationCanceledException) when (cancellationToken.IsCancellationRequested) { throw; } - catch (Exception ex) + catch (Exception ex) when (TransientFailureDetector.IsTransient(ex) && attempt < maximumAttempts) { _logger.LogDebug(ex, - "Failed to complete Azure DevOps test run {TestRunId} for job {JobName}. Retrying after delay.", + "Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. " + + "Transient attempt {Attempt} of {MaximumAttempts} failed; retrying after delay.", + operationDescription, + helixJob.DisplayName, testRunId, - helixJob.JobName); + attempt, + maximumAttempts); await _delay(cancellationToken); } + catch (Exception ex) + { + string failureKind = TransientFailureDetector.IsTransient(ex) + ? safeToRetry + ? "Transient retry limit reached." + : "The operation may have partially completed and is not safe to replay in this invocation." + : "The failure is not retryable."; + _logger.LogWarning(ex, + "{Prefix}Failed to {OperationDescription} for job {JobName}. Test run ID was {TestRunId}. " + + "{FailureKind} The run remains untagged and a later monitor invocation may retry the upload.", + AzdoWarningPrefix, + operationDescription, + helixJob.DisplayName, + testRunId, + failureKind); + return (false, default); + } } + + throw new InvalidOperationException("Upload retry loop exited unexpectedly."); } + } } diff --git a/src/Microsoft.DotNet.Helix/JobMonitor/TransientFailureDetector.cs b/src/Microsoft.DotNet.Helix/JobMonitor/TransientFailureDetector.cs new file mode 100644 index 00000000000..fc00cebf119 --- /dev/null +++ b/src/Microsoft.DotNet.Helix/JobMonitor/TransientFailureDetector.cs @@ -0,0 +1,36 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. + +using System; +using System.IO; +using System.Net; +using System.Net.Http; +using System.Net.Sockets; +using System.Threading.Tasks; +using Azure; + +namespace Microsoft.DotNet.Helix.JobMonitor +{ + internal static class TransientFailureDetector + { + public static bool IsTransient(Exception exception) + { + return exception switch + { + TaskCanceledException => true, + TimeoutException => true, + SocketException => true, + IOException => true, + HttpRequestException { StatusCode: null } => true, + HttpRequestException httpException => IsTransientStatusCode((int)httpException.StatusCode.Value), + RequestFailedException requestFailedException => IsTransientStatusCode(requestFailedException.Status), + _ => false, + }; + } + + private static bool IsTransientStatusCode(int statusCode) + => statusCode == (int)HttpStatusCode.RequestTimeout + || statusCode == (int)HttpStatusCode.TooManyRequests + || statusCode >= 500; + } +} diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs index dd50868fa81..f64de6e56ea 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsResultPublisherTests.cs @@ -29,5 +29,19 @@ public void Constructor_ConfiguresHttpClientTimeoutForLongUploads() Assert.Equal(TimeSpan.FromMinutes(5), client.Timeout); } + + [Theory] + [InlineData("GET", false, 10)] + [InlineData("POST", true, 10)] + [InlineData("POST", false, 0)] + public void GetRetryCount_DisablesAmbiguousWriteRetriesWhenConfigured( + string method, + bool retryWrites, + int expected) + { + Assert.Equal( + expected, + AzureDevOpsResultPublisher.GetRetryCount(new HttpMethod(method), retryWrites)); + } } } diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs index e6843a390e0..e2aeca84da7 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/AzureDevOpsServiceTests.cs @@ -48,6 +48,19 @@ public async Task CreateTestRunAsync_UsesPlainNameAndDoesNotTag() body["tags"].Should().BeNull(); } + [Fact] + public async Task CreateTestRunAsync_DoesNotRetryAmbiguousWrite() + { + var handler = new RecordingHttpMessageHandler(_ => + new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)); + using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler)); + + Func action = () => service.CreateTestRunAsync("Test Run", CancellationToken.None); + + await action.Should().ThrowAsync(); + handler.Requests.Should().ContainSingle(); + } + [Fact] public async Task CompleteTestRunAsync_SendsCompletedStateAndHelixJobTag() { @@ -73,6 +86,23 @@ public async Task CompleteTestRunAsync_SendsCompletedStateAndHelixJobTag() tags[0].Value("name").Should().Be(HelixJobTag); } + [Fact] + public async Task CompleteTestRunAsync_DoesNotRetryAmbiguousWrite() + { + var handler = new RecordingHttpMessageHandler(_ => + new HttpResponseMessage(HttpStatusCode.ServiceUnavailable)); + using var service = new AzureDevOpsService(CreateOptions(), NullLogger.Instance, new HttpClient(handler)); + + Func action = () => service.CompleteTestRunAsync( + 123, + HelixJobGuid, + [], + CancellationToken.None); + + await action.Should().ThrowAsync(); + handler.Requests.Should().ContainSingle(); + } + [Fact] public async Task CompleteTestRunAsync_UploadsFailedWorkItemsAttachmentBeforePatch() { diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs index 020ba792b10..5fce31eae8d 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeAzureDevOpsService.cs @@ -4,6 +4,8 @@ using System; using System.Collections.Generic; using System.Linq; +using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.Helix.AzureDevOpsTestPublisher; @@ -20,7 +22,9 @@ internal sealed class FakeAzureDevOpsService : IAzureDevOpsService private readonly object _sync = new(); private readonly List _timelineResponses = []; private readonly HashSet _previouslyProcessedJobs = new(StringComparer.OrdinalIgnoreCase); + private readonly Queue _createFailures = []; private readonly Queue _uploadFailures = []; + private readonly Queue _completeFailures = []; private readonly HashSet<(string JobName, string WorkItemName)> _recordedFailedTests = new(FailedTestWorkItemComparer.Instance); private readonly HashSet<(string JobName, string WorkItemName)> _uploadFailedTests @@ -35,6 +39,7 @@ internal sealed class FakeAzureDevOpsService : IAzureDevOpsService public List UploadedJobNames { get; } = []; public int CreateTestRunCallCount { get; private set; } public int UploadTestResultsCallCount { get; private set; } + public int CompleteTestRunCallCount { get; private set; } public TaskCompletionSource UploadStarted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public TaskCompletionSource UploadCompleted { get; } = new(TaskCreationOptions.RunContinuationsAsynchronously); public Task UploadBlocker { get; set; } = Task.CompletedTask; @@ -74,11 +79,31 @@ public FakeAzureDevOpsService WithPreviouslyProcessedJob(string jobName) return this; } + public FakeAzureDevOpsService FailNextCreate(Exception exception = null) + { + lock (_sync) + { + _createFailures.Enqueue(exception ?? CreateTransientFailure("Injected test-run creation failure.")); + } + + return this; + } + public FakeAzureDevOpsService FailNextUpload(Exception exception = null) { lock (_sync) { - _uploadFailures.Enqueue(exception ?? new InvalidOperationException("Injected upload failure.")); + _uploadFailures.Enqueue(exception ?? CreateTransientFailure("Injected test-result upload failure.")); + } + + return this; + } + + public FakeAzureDevOpsService FailNextComplete(Exception exception = null) + { + lock (_sync) + { + _completeFailures.Enqueue(exception ?? CreateTransientFailure("Injected test-run completion failure.")); } return this; @@ -158,11 +183,11 @@ public Task CreateTestRunAsync(string name, CancellationToken cancellationT lock (_sync) { CreateTestRunCallCount++; + if (_createFailures.Count > 0) + { + throw _createFailures.Dequeue(); + } - // Matches the real AzureDevOpsService: every call creates a brand new in-progress - // run. The service never reuses an existing run, so a transient upload failure that - // re-enters the retry loop leaves an orphaned (untagged) run behind — exactly the - // crash-resilience behavior described in the design. int id = Interlocked.Increment(ref _nextTestRunId); CreatedTestRuns.Add(name); return Task.FromResult(id); @@ -173,7 +198,14 @@ public Task CompleteTestRunAsync(int testRunId, string helixJobName, IReadOnlyCo { lock (_sync) { + CompleteTestRunCallCount++; + if (_completeFailures.Count > 0) + { + throw _completeFailures.Dequeue(); + } + CompletedTestRunIds.Add(testRunId); + _previouslyProcessedJobs.Add(helixJobName); foreach (string workItemName in failedWorkItems ?? []) { if (!string.IsNullOrEmpty(workItemName)) @@ -221,7 +253,6 @@ public Task CompleteTestRunAsync(int testRunId, string helixJobName, IReadOnlyCo foreach (string jobName in results.Select(r => r.JobName).Distinct(StringComparer.OrdinalIgnoreCase)) { UploadedJobNames.Add(jobName); - _previouslyProcessedJobs.Add(jobName); } foreach (WorkItemTestResults result in results) @@ -236,6 +267,9 @@ public Task CompleteTestRunAsync(int testRunId, string helixJobName, IReadOnlyCo return summaries; } + private static HttpRequestException CreateTransientFailure(string message) + => new(message, null, HttpStatusCode.ServiceUnavailable); + private sealed class FailedTestWorkItemComparer : IEqualityComparer<(string JobName, string WorkItemName)> { public static readonly FailedTestWorkItemComparer Instance = new(); diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs index d61932e8a9a..1b266aade5e 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/Fakes/FakeHelixService.cs @@ -5,6 +5,8 @@ using System.Collections.Concurrent; using System.Collections.Generic; using System.Linq; +using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.DotNet.Helix.Client.Models; @@ -17,6 +19,7 @@ internal sealed class FakeHelixService : IHelixService { private readonly List _responses = []; private readonly HashSet _downloadFailureJobs = new(StringComparer.OrdinalIgnoreCase); + private readonly Dictionary> _downloadFailures = new(StringComparer.OrdinalIgnoreCase); private readonly Dictionary> _customWorkItems = new(StringComparer.OrdinalIgnoreCase); private int _getJobsCallCount; @@ -41,6 +44,21 @@ public FakeHelixService AddResponse( public FakeHelixService FailDownloadForJob(string jobName) { _downloadFailureJobs.Add(jobName); return this; } public void ClearDownloadFailures() { _downloadFailureJobs.Clear(); } + public FakeHelixService FailNextDownloadForJob(string jobName, Exception exception = null) + { + if (!_downloadFailures.TryGetValue(jobName, out Queue failures)) + { + failures = []; + _downloadFailures[jobName] = failures; + } + + failures.Enqueue(exception ?? new HttpRequestException( + "Injected transient download failure.", + null, + HttpStatusCode.ServiceUnavailable)); + return this; + } + public FakeHelixService WithWorkItems(string jobName, IReadOnlyCollection workItems) { _customWorkItems[jobName] = workItems; @@ -82,6 +100,12 @@ public Task> DownloadTestResultsAsync( throw new InvalidOperationException($"Injected download failure for Helix job '{jobName}'."); } + if (_downloadFailures.TryGetValue(jobName, out Queue failures) + && failures.Count > 0) + { + throw failures.Dequeue(); + } + if (CurrentSnapshot.TestResultsByJob.TryGetValue(jobName, out List explicitResults)) { return Task.FromResult>(explicitResults); diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs index 60c76d71a00..6b2fca83d1b 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/HelixServiceTests.cs @@ -6,6 +6,8 @@ using System.Collections.Immutable; using System.IO; using System.Linq; +using System.Net; +using System.Net.Http; using System.Threading; using System.Threading.Tasks; using Microsoft.Arcade.Common; @@ -123,6 +125,30 @@ [new DownloadCall("https://storage/nested/testResults.xml", "?resultSas", expect blobClientFactory.Downloads); } + [Fact] + public async Task DownloadTestResultsAsync_ReportsTransientFileFailureAfterAttemptingBatch() + { + var api = CreateApi(); + api.Job + .Setup(j => j.ResultsAsync("job", It.IsAny())) + .ReturnsAsync(new JobResultsUri { ResultsUriRSAS = "?resultSas" }); + api.WorkItem + .Setup(w => w.ListFilesAsync("work-item", "job", false, It.IsAny())) + .ReturnsAsync(ImmutableList.Create( + new UploadedFile("first.trx", "https://storage/first.trx"), + new UploadedFile("second.trx", "https://storage/second.trx"))); + + var blobClientFactory = new FakeBlobClientFactory(); + blobClientFactory.DownloadFailures["https://storage/first.trx"] = + new HttpRequestException("Injected transient failure.", null, HttpStatusCode.ServiceUnavailable); + + Func action = () => CreateService(api.Api.Object, blobClientFactory, new MockFileSystem()) + .DownloadTestResultsAsync("job", ["work-item"], "work", CancellationToken.None); + + await Assert.ThrowsAsync(action); + Assert.Equal(2, blobClientFactory.Downloads.Count); + } + [Fact] public async Task CancelJobAsync_DelegatesToHelixApiJobCancelAsync() { @@ -422,6 +448,8 @@ private sealed class FakeBlobClientFactory : IBlobClientFactory public HashSet FailDownloadsFor { get; } = new(StringComparer.OrdinalIgnoreCase); + public Dictionary DownloadFailures { get; } = new(StringComparer.OrdinalIgnoreCase); + public string DownloadedText { get; set; } = "[]"; public string DownloadedTextUri { get; private set; } @@ -470,6 +498,11 @@ public FakeBlobClient(Uri uri, FakeBlobClientFactory factory, Uri containerUri, public Task DownloadToAsync(string destinationFile, CancellationToken cancellationToken) { _factory.Downloads.Add(new DownloadCall(_blobUri, _sasToken, destinationFile)); + if (_factory.DownloadFailures.TryGetValue(_blobUri, out Exception failure)) + { + throw failure; + } + if (_factory.FailDownloadsFor.Contains(_blobUri)) { throw new InvalidOperationException("Injected download failure."); diff --git a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs index a6d53dc6a04..95f34b21aad 100644 --- a/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs +++ b/src/Microsoft.DotNet.Helix/Sdk.Tests/Microsoft.DotNet.Helix.Sdk.Tests/JobMonitorRunnerTests.cs @@ -414,10 +414,11 @@ public async Task CompletedHelixJob_QueuesTestResultUploadWithoutBlockingNextPol } [Fact] - public async Task PassedHelixWork_UploadFailsOnce_RetriesAndProcessesResults() + public async Task PassedHelixWork_TransientUploadFailure_DoesNotReplayAmbiguousWrite() { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); + var logger = new RecordingLogger(); int delayCount = 0; azdo.FailNextUpload(); @@ -440,22 +441,129 @@ public async Task PassedHelixWork_UploadFailsOnce_RetriesAndProcessesResults() ], }); - var runner = new JobMonitorRunner(DefaultOptions(), NullLogger.Instance, azdo, helix, + var runner = new JobMonitorRunner(DefaultOptions(), logger, azdo, helix, + (_, _) => + { + delayCount++; + return Task.CompletedTask; + }); + int exitCode = await runner.RunAsync(CancellationToken.None); + + exitCode.Should().Be(0); + azdo.UploadTestResultsCallCount.Should().Be(1); + delayCount.Should().Be(0); + azdo.CreatedTestRuns.Should().ContainSingle(); + azdo.CompletedTestRunIds.Should().BeEmpty(); + azdo.UploadedJobNames.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("not safe to replay in this invocation", StringComparison.Ordinal)); + } + + [Fact] + public async Task PassedHelixWork_TransientCompletionFailure_DoesNotReplayCompletionSequence() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + int delayCount = 0; + + azdo.FailNextComplete(); + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1"]), + }); + + var runner = new JobMonitorRunner(DefaultOptions(), logger, azdo, helix, (_, _) => { delayCount++; return Task.CompletedTask; }); + int exitCode = await runner.RunAsync(CancellationToken.None); exitCode.Should().Be(0); - azdo.UploadTestResultsCallCount.Should().Be(2); + azdo.CreateTestRunCallCount.Should().Be(1); + azdo.UploadTestResultsCallCount.Should().Be(1); + azdo.CompleteTestRunCallCount.Should().Be(1); + delayCount.Should().Be(0); + azdo.CompletedTestRunIds.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("not safe to replay in this invocation", StringComparison.Ordinal)); + } + + [Fact] + public async Task PassedHelixWork_PermanentUploadFailure_DoesNotHangOrFailBuild() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + var logger = new RecordingLogger(); + + azdo.FailNextUpload(new InvalidOperationException("Injected permanent upload failure.")); + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1"]), + }); + + var runner = CreateRunner(azdo, helix, logger: logger); + int exitCode = await runner.RunAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10)); + + exitCode.Should().Be(0); + azdo.CreateTestRunCallCount.Should().Be(1); + azdo.UploadTestResultsCallCount.Should().Be(1); + azdo.CompleteTestRunCallCount.Should().Be(0); + azdo.CompletedTestRunIds.Should().BeEmpty(); + logger.Messages.Should().Contain(message => + message.Contains("The failure is not retryable", StringComparison.Ordinal) + && message.Contains("later monitor invocation may retry the upload", StringComparison.Ordinal)); + } + + [Fact] + public async Task PassedHelixWork_TransientDownloadFailure_RetriesBeforeCreatingTestRun() + { + var azdo = new FakeAzureDevOpsService(); + var helix = new FakeHelixService(); + int delayCount = 0; + + azdo.AddTimelineResponse( + MonitorJob(), + PipelineJob("Test Linux", "completed", "succeeded")); + + helix.FailNextDownloadForJob("helix-linux"); + helix.AddResponse( + jobs: [HelixJob("helix-linux", "finished")], + passFailByJob: new(StringComparer.OrdinalIgnoreCase) + { + ["helix-linux"] = PassFail(passed: ["workitem-1"]), + }); + + var runner = new JobMonitorRunner(DefaultOptions(), NullLogger.Instance, azdo, helix, + (_, _) => + { + delayCount++; + return Task.CompletedTask; + }); + + int exitCode = await runner.RunAsync(CancellationToken.None).WaitAsync(TimeSpan.FromSeconds(10)); + + exitCode.Should().Be(0); + azdo.CreateTestRunCallCount.Should().Be(1); + azdo.UploadTestResultsCallCount.Should().Be(1); + azdo.CompleteTestRunCallCount.Should().Be(1); delayCount.Should().Be(1); - // The first attempt creates a run then fails mid-upload, orphaning it (untagged, - // never completed). The retry creates a second run that uploads and completes. - azdo.CreatedTestRuns.Should().HaveCount(2); azdo.CompletedTestRunIds.Should().ContainSingle(); - azdo.UploadedJobNames.Should().BeEquivalentTo(["helix-linux"]); } /// @@ -1778,6 +1886,7 @@ public async Task MonitorTimesOut_DoesNotWaitForUploadDrainBeforeCancellingHelix { var azdo = new FakeAzureDevOpsService(); var helix = new FakeHelixService(); + var logger = new RecordingLogger(); // The upload of helix-finished starts but never completes and ignores cancellation, // simulating an upload stuck in a non-cancellable network call when the timeout fires. @@ -1801,7 +1910,7 @@ public async Task MonitorTimesOut_DoesNotWaitForUploadDrainBeforeCancellingHelix }); using var cts = new CancellationTokenSource(); - var runner = new JobMonitorRunner(DefaultOptions(), NullLogger.Instance, azdo, helix, + var runner = new JobMonitorRunner(DefaultOptions(), logger, azdo, helix, async (_, _) => { // Cancel once the (stuck) upload is genuinely in flight. @@ -1822,6 +1931,9 @@ public async Task MonitorTimesOut_DoesNotWaitForUploadDrainBeforeCancellingHelix // monitor invocation re-uploads it. azdo.UploadedJobNames.Should().BeEmpty(); azdo.UploadCompleted.Task.IsCompleted.Should().BeFalse(); + logger.Messages.Should().Contain(message => + message.Contains("Helix job(s) were unfinished or unprocessed", StringComparison.Ordinal) + && message.Contains("helix-finished", StringComparison.Ordinal)); // Release the stuck upload so the abandoned task can finish. uploadGate.TrySetResult();