Skip to content

Make dotnet-eng-status deployment notifications best-effort (fix intermittent 500) #6643

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
missymessa wants to merge 6 commits into
base: main
Choose a base branch
from
Open

Make dotnet-eng-status deployment notifications best-effort (fix intermittent 500) #6643

Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
1bf216e
Make deployment notifications best-effort so they never 500 the pipeline
missymessa Jul 17, 2026
b4f7c83
Address review: drop direct Grafana annotation writes, keep durable A…
missymessa Jul 20, 2026
f9716ba
Merge branch 'main' into missymessa/fix-deployment-notify-500
missymessa Jul 21, 2026
3560e7e
Address review: split deployment table config into dedicated Deployme…
missymessa Jul 22, 2026
c2027e9
Merge branch 'main' into missymessa/fix-deployment-notify-500
missymessa Jul 22, 2026
189ac85
Remove now-unused grafana-api-token secret from vault-config manifests
missymessa Jul 22, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
174 changes: 104 additions & 70 deletions src/DotNet.Status.Web/DotNet.Status.Web/Controllers/DeploymentController.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
using System.Collections.Immutable;
using System.ComponentModel.DataAnnotations;
using System.IO;
using System.Net;
using System.Net.Http;
using System.Net.Http.Headers;
using System.Text;
Expand Down Expand Up @@ -58,98 +59,131 @@ public async Task<IActionResult> MarkStart([Required] string service, [Required]
Tags = new[] {"deploy", $"deploy-{service}", service},
Time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
};

NewGrafanaAnnotationResponse annotation;
using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }))

// Recording a deployment annotation is best-effort telemetry and must never fail the
// calling deployment pipeline. Any failure talking to Grafana or table storage is logged
// and swallowed so the notification endpoint returns success instead of a 500.
try
{
annotation = await _retry.RetryAsync(async () =>
{
GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue;
_logger.LogInformation("Creating annotation to {url}", grafanaOptions.BaseUrl);
using (var request = new HttpRequestMessage(HttpMethod.Post,
$"{grafanaOptions.BaseUrl}/api/annotations"))
NewGrafanaAnnotationResponse annotation;
using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }))
{
annotation = await _retry.RetryAsync(async () =>
{
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
request.Headers.Authorization =
new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken);
request.Content = CreateObjectContent(content);

using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None))
GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue;
_logger.LogInformation("Creating annotation to {url}", grafanaOptions.BaseUrl);
using (var request = new HttpRequestMessage(HttpMethod.Post,
$"{grafanaOptions.BaseUrl}/api/annotations"))
{
_logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase);
response.EnsureSuccessStatusCode();
return await ReadObjectContent<NewGrafanaAnnotationResponse>(response.Content);
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
request.Headers.Authorization =
new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken);
request.Content = CreateObjectContent(content);

using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None))
{
_logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase);
response.EnsureSuccessStatusCode();
return await ReadObjectContent<NewGrafanaAnnotationResponse>(response.Content);
}
}
}
},
e => _logger.LogWarning(e, "Failed to send new annotation"),
e => true
);
},
e => _logger.LogWarning(e, "Failed to send new annotation"),
IsTransientFailure
);
}
_logger.LogInformation("Created annotation {annotationId}, inserting into table", annotation.Id);

TableClient table = await GetCloudTable();
Comment thread
missymessa marked this conversation as resolved.
await table.UpsertEntityAsync(new DotNet.Status.Web.Models.AnnotationEntity(service, id, annotation.Id)
{
ETag = new Azure.ETag("*")
});
}
_logger.LogInformation("Created annotation {annotationId}, inserting into table", annotation.Id);

TableClient table = await GetCloudTable();
await table.UpsertEntityAsync(new DotNet.Status.Web.Models.AnnotationEntity(service, id, annotation.Id)
catch (Exception ex)
{
ETag = new Azure.ETag("*")
});
_logger.LogError(ex, "Failed to record start of deployment of '{service}' with id '{id}'; continuing without blocking the deployment", service, id);
Comment thread
missymessa marked this conversation as resolved.
Outdated
}
return NoContent();
}

[HttpPost("{service}/{id}/end")]
public async Task<IActionResult> MarkEnd([Required] string service, [Required] string id)
{
_logger.LogInformation("Recording end of deployment of '{service}' with id '{id}'", service, id);
TableClient table = await GetCloudTable();
_logger.LogInformation("Looking for existing deployment");
var getResult = await table.GetEntityIfExistsAsync<StatusWebAnnotationEntity>(service, id);
_logger.LogTrace("Table response code {responseCode}", getResult.GetRawResponse().Status);
if (!getResult.HasValue)
{
return NotFound();
}

var annotation = getResult.Value;

_logger.LogTrace("Updating end time of deployment...");
annotation.Ended = DateTimeOffset.UtcNow;

var updateResult = await table.UpdateEntityAsync(annotation, annotation.ETag, TableUpdateMode.Replace);
_logger.LogInformation("Update response code {responseCode}", updateResult.Status);

var content = new NewGrafanaAnnotationRequest
// As with MarkStart, recording the end of a deployment is best-effort telemetry and must
// never fail the calling deployment pipeline. Any failure is logged and swallowed.
try
{
TimeEnd = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
};

using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }))
{
await _retry.RetryAsync(async () =>
{
GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue;
_logger.LogInformation("Updating annotation {annotationId} to {url}", annotation.GrafanaAnnotationId, grafanaOptions.BaseUrl);
using (var request = new HttpRequestMessage(HttpMethod.Patch,
$"{grafanaOptions.BaseUrl}/api/annotations/{annotation.GrafanaAnnotationId}"))
TableClient table = await GetCloudTable();
_logger.LogInformation("Looking for existing deployment");
var getResult = await table.GetEntityIfExistsAsync<StatusWebAnnotationEntity>(service, id);
_logger.LogTrace("Table response code {responseCode}", getResult.GetRawResponse().Status);
if (!getResult.HasValue)
{
_logger.LogWarning("No deployment start record found for '{service}' with id '{id}'; nothing to mark as ended", service, id);
return NoContent();
}

var annotation = getResult.Value;

_logger.LogTrace("Updating end time of deployment...");
annotation.Ended = DateTimeOffset.UtcNow;

var updateResult = await table.UpdateEntityAsync(annotation, annotation.ETag, TableUpdateMode.Replace);
_logger.LogInformation("Update response code {responseCode}", updateResult.Status);

var content = new NewGrafanaAnnotationRequest
{
TimeEnd = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
};

using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }))
Comment thread
missymessa marked this conversation as resolved.
Outdated
{
await _retry.RetryAsync(async () =>
{
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
request.Headers.Authorization =
new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken);
request.Content = CreateObjectContent(content);
using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None))
GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue;
_logger.LogInformation("Updating annotation {annotationId} to {url}", annotation.GrafanaAnnotationId, grafanaOptions.BaseUrl);
using (var request = new HttpRequestMessage(HttpMethod.Patch,
$"{grafanaOptions.BaseUrl}/api/annotations/{annotation.GrafanaAnnotationId}"))
{
_logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase);
response.EnsureSuccessStatusCode();
request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
request.Headers.Authorization =
new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken);
request.Content = CreateObjectContent(content);
using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None))
{
_logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase);
response.EnsureSuccessStatusCode();
}
}
}
},
e => _logger.LogWarning(e, "Failed to send new annotation"),
e => true
);
},
e => _logger.LogWarning(e, "Failed to send new annotation"),
IsTransientFailure
);
}
}
catch (Exception ex)
{
_logger.LogError(ex, "Failed to record end of deployment of '{service}' with id '{id}'; continuing without blocking the deployment", service, id);
Comment thread
missymessa marked this conversation as resolved.
Outdated
}

return NoContent();
}

private static bool IsTransientFailure(Exception e)
{
// Only retry transient failures (network errors, timeouts, or 5xx responses from Grafana).
// Non-transient failures such as 4xx responses should fail fast rather than being retried
// repeatedly and then rethrown, which previously surfaced as a 500 to the deployment pipeline.
if (e is HttpRequestException httpException)
{
return httpException.StatusCode is null or >= HttpStatusCode.InternalServerError;
}

return e is TaskCanceledException or OperationCanceledException;
}

private async Task<TableClient> GetCloudTable()
{
TableClient table;
Expand Down