Make dotnet-eng-status deployment notifications best-effort (fix intermittent 500)

.vault-config/dotneteng-status-local.yaml

@@ -19,11 +19,6 @@ secrets:
       hasWebhookSecret: true
       hasOAuthSecret: true
 
-  grafana-api-token:
-    type: azure-managed-grafana-api-key
-    parameters:
-      environment: dotnet-eng-grafana-staging.westus2.cloudapp.azure.com
-
   # Shared secret for Grafana webhook contact point Basic Auth
   grafana-webhook-secret:
     type: text

.vault-config/dotneteng-status-prod.yaml

@@ -19,12 +19,6 @@ keys:
 importSecretsFrom: shared/dotneteng-status-secrets.yaml
 
 secrets:
-  # Grafana API key with admin privileges
-  grafana-api-token:
-    type: azure-managed-grafana-api-key
-    parameters:
-      environment: dotnet-eng-grafana.westus2.cloudapp.azure.com
-
   # Shared secret for Grafana webhook contact point Basic Auth
   grafana-webhook-secret:
     type: text

.vault-config/dotneteng-status-staging.yaml

@@ -19,12 +19,6 @@ keys:
 importSecretsFrom: shared/dotneteng-status-secrets.yaml
 
 secrets:
-  # Grafana API key with admin privileges
-  grafana-api-token:
-    type: azure-managed-grafana-api-key
-    parameters:
-      environment: dotnet-eng-grafana-staging.westus2.cloudapp.azure.com
-
   # Shared secret for Grafana webhook contact point Basic Auth
   grafana-webhook-secret:
     type: text

src/DotNet.Status.Web/DotNet.Status.Web.Tests/AnnotationsControllerTests.cs

@@ -122,7 +122,7 @@ public TestData()
                 services.AddControllers()
                     .AddApplicationPart(typeof(AnnotationsController).Assembly);
 
-                services.Configure<GrafanaOptions>(options =>
+                services.Configure<DeploymentTableOptions>(options =>
                 {
                     options.TableUri = "https://127.0.0.1:10002/devstoreaccount1/deployments";
                 });

src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Development.json

@@ -49,7 +49,7 @@
     "StorageAccountConnectionString": "",
     "KeyIdentifier": ""
   },
-  "Grafana": {
+  "Deployments": {
     "TableUri": ""
   },
   "Kusto": {

src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json

@@ -23,8 +23,7 @@
   "AzureTableTokenStore": {
     "TableUri": "https://dotnetengstatusprod.table.core.windows.net"
   },
-  "Grafana": {
-    "BaseUrl": "https://dotnet-eng-grafana.westus2.cloudapp.azure.com",
+  "Deployments": {
     "TableUri": "https://dotnetengstatusprod.table.core.windows.net"
   },
   "Kusto": {

src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json

@@ -25,7 +25,7 @@
   "AzureTableTokenStore": {
     "TableUri": "https://dotnetengstatusstaging.table.core.windows.net"
   },
-  "Grafana": {
+  "Deployments": {
     "TableUri": "https://dotnetengstatusstaging.table.core.windows.net"
   },
   "Kusto": {

src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json

@@ -300,10 +300,10 @@
   "AzureTableTokenStore": {
     "TableName": "tokens"
   },
+  "Deployments": {
+    "TableName": "deployments"
+  },
   "Grafana": {
-    "BaseUrl": "https://dotnet-eng-grafana-staging.westus2.cloudapp.azure.com",
-    "ApiToken": "[vault(grafana-api-token)]",
-    "TableName": "deployments",
     "WebhookSecret": "[vault(grafana-webhook-secret)]"
   },
   "WebHooks": {

src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AnnotationsController.cs

@@ -28,12 +28,12 @@ public class AnnotationsController : ControllerBase
 {
     private const int _maximumServerCount = 10; // Safety limit on query complexity
     private readonly ILogger<AnnotationsController> _logger;
-    private readonly IOptionsMonitor<GrafanaOptions> _options;
+    private readonly IOptionsMonitor<DeploymentTableOptions> _options;
     private readonly IHostEnvironment _env;
 
     public AnnotationsController(
         ILogger<AnnotationsController> logger,
-        IOptionsMonitor<GrafanaOptions> options,
+        IOptionsMonitor<DeploymentTableOptions> options,
         IHostEnvironment env)
     {
         _logger = logger;

src/DotNet.Status.Web/DotNet.Status.Web/Controllers/DeploymentController.cs

@@ -4,25 +4,15 @@
 
 using Azure;
 using System;
-using System.Collections.Immutable;
 using System.ComponentModel.DataAnnotations;
-using System.IO;
-using System.Net.Http;
-using System.Net.Http.Headers;
-using System.Text;
-using System.Threading;
 using System.Threading.Tasks;
 using Azure.Data.Tables;
 using DotNet.Status.Web.Options;
 using Microsoft.AspNetCore.Mvc;
-using Microsoft.DotNet.Services.Utility;
 using Microsoft.Extensions.Hosting;
 using Microsoft.Extensions.Logging;
 using Microsoft.Extensions.Options;
-using Newtonsoft.Json;
-using Newtonsoft.Json.Serialization;
 using StatusWebAnnotationEntity = DotNet.Status.Web.Models.AnnotationEntity;
-using Microsoft.WindowsAzure.Storage.Table;
 using Azure.Identity;
 
 namespace DotNet.Status.Web.Controllers;
@@ -32,18 +22,15 @@ namespace DotNet.Status.Web.Controllers;
 public class DeploymentController : ControllerBase
 {
     private readonly IHostEnvironment _env;
-    private readonly ExponentialRetry _retry;
-    private readonly IOptionsMonitor<GrafanaOptions> _grafanaOptions;
+    private readonly IOptionsMonitor<DeploymentTableOptions> _deploymentTableOptions;
     private readonly ILogger<DeploymentController> _logger;
 
     public DeploymentController(
-        ExponentialRetry retry,
-        IOptionsMonitor<GrafanaOptions> grafanaOptions,
+        IOptionsMonitor<DeploymentTableOptions> deploymentTableOptions,
         ILogger<DeploymentController> logger,
         IHostEnvironment env)
     {
-        _retry = retry;
-        _grafanaOptions = grafanaOptions;
+        _deploymentTableOptions = deploymentTableOptions;
         _logger = logger;
         _env = env;
     }
@@ -52,99 +39,58 @@ public DeploymentController(
     public async Task<IActionResult> MarkStart([Required] string service, [Required] string id)
     {
         _logger.LogInformation("Recording start of deployment of '{service}' with id '{id}'", service, id);
-        NewGrafanaAnnotationRequest content = new NewGrafanaAnnotationRequest
+
+        // Recording a deployment annotation is best-effort telemetry and must never fail the calling
+        // deployment pipeline. The annotation is written to Azure Table storage, which Grafana reads
+        // directly; any failure is logged and swallowed so the endpoint returns success instead of a 500.
+        try
         {
-            Text = $"Deployment of {service}",
-            Tags = new[] {"deploy", $"deploy-{service}", service},
-            Time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-        };
-
-        NewGrafanaAnnotationResponse annotation;
-        using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }))
+            TableClient table = await GetCloudTable();
+            await table.UpsertEntityAsync(new StatusWebAnnotationEntity(service, id)
+            {
+                Started = DateTimeOffset.UtcNow,
+                ETag = new Azure.ETag("*")
+            });
+        }
+        catch (Exception ex)
         {
-            annotation = await _retry.RetryAsync(async () =>
-                {
-                    GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue;
-                    _logger.LogInformation("Creating annotation to {url}", grafanaOptions.BaseUrl);
-                    using (var request = new HttpRequestMessage(HttpMethod.Post,
-                               $"{grafanaOptions.BaseUrl}/api/annotations"))
-                    {
-                        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
-                        request.Headers.Authorization =
-                            new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken);
-                        request.Content = CreateObjectContent(content);
-
-                        using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None))
-                        {
-                            _logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase);
-                            response.EnsureSuccessStatusCode();
-                            return await ReadObjectContent<NewGrafanaAnnotationResponse>(response.Content);
-                        }
-                    }
-                },
-                e => _logger.LogWarning(e, "Failed to send new annotation"),
-                e => true
-            );
+            _logger.LogError(ex, "Failed to record start of deployment of '{service}' with id '{id}'", service, id);
         }
-        _logger.LogInformation("Created annotation {annotationId}, inserting into table", annotation.Id);
 
-        TableClient table = await GetCloudTable();
-        await table.UpsertEntityAsync(new DotNet.Status.Web.Models.AnnotationEntity(service, id, annotation.Id)
-        {
-            ETag = new Azure.ETag("*")
-        });
         return NoContent();
     }
 
     [HttpPost("{service}/{id}/end")]
     public async Task<IActionResult> MarkEnd([Required] string service, [Required] string id)
     {
         _logger.LogInformation("Recording end of deployment of '{service}' with id '{id}'", service, id);
-        TableClient table = await GetCloudTable();
-        _logger.LogInformation("Looking for existing deployment");
-        var getResult = await table.GetEntityIfExistsAsync<StatusWebAnnotationEntity>(service, id);
-        _logger.LogTrace("Table response code {responseCode}", getResult.GetRawResponse().Status);
-        if (!getResult.HasValue)
-        {
-            return NotFound();
-        }
-
-        var annotation = getResult.Value;
 
-        _logger.LogTrace("Updating end time of deployment...");
-        annotation.Ended = DateTimeOffset.UtcNow;
-
-        var updateResult = await table.UpdateEntityAsync(annotation, annotation.ETag, TableUpdateMode.Replace);
-        _logger.LogInformation("Update response code {responseCode}", updateResult.Status);
-
-        var content = new NewGrafanaAnnotationRequest
+        // As with MarkStart, recording the end of a deployment is best-effort telemetry written to
+        // Azure Table storage and must never fail the calling deployment pipeline. Any failure is
+        // logged and swallowed.
+        try
         {
-            TimeEnd = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(),
-        };
-
-        using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true }))
+            TableClient table = await GetCloudTable();
+            _logger.LogInformation("Looking for existing deployment");
+            var getResult = await table.GetEntityIfExistsAsync<StatusWebAnnotationEntity>(service, id);
+            _logger.LogTrace("Table response code {responseCode}", getResult.GetRawResponse().Status);
+            if (!getResult.HasValue)
+            {
+                _logger.LogWarning("No deployment start record found for '{service}' with id '{id}'; nothing to mark as ended", service, id);
+                return NoContent();
+            }
+
+            var annotation = getResult.Value;
+
+            _logger.LogTrace("Updating end time of deployment...");
+            annotation.Ended = DateTimeOffset.UtcNow;
+
+            var updateResult = await table.UpdateEntityAsync(annotation, annotation.ETag, TableUpdateMode.Replace);
+            _logger.LogInformation("Update response code {responseCode}", updateResult.Status);
+        }
+        catch (Exception ex)
         {
-            await _retry.RetryAsync(async () =>
-                {
-                    GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue;
-                    _logger.LogInformation("Updating annotation {annotationId} to {url}", annotation.GrafanaAnnotationId, grafanaOptions.BaseUrl);
-                    using (var request = new HttpRequestMessage(HttpMethod.Patch,
-                               $"{grafanaOptions.BaseUrl}/api/annotations/{annotation.GrafanaAnnotationId}"))
-                    {
-                        request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json"));
-                        request.Headers.Authorization =
-                            new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken);
-                        request.Content = CreateObjectContent(content);
-                        using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None))
-                        {
-                            _logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase);
-                            response.EnsureSuccessStatusCode();
-                        }
-                    }
-                },
-                e => _logger.LogWarning(e, "Failed to send new annotation"),
-                e => true
-            );
+            _logger.LogError(ex, "Failed to record end of deployment of '{service}' with id '{id}'", service, id);
         }
 
         return NoContent();
@@ -153,7 +99,7 @@ await _retry.RetryAsync(async () =>
     private async Task<TableClient> GetCloudTable()
     {
         TableClient table;
-        GrafanaOptions options = _grafanaOptions.CurrentValue;
+        DeploymentTableOptions options = _deploymentTableOptions.CurrentValue;
         if (_env.IsDevelopment())
         {
             table = new TableClient("UseDevelopmentStorage=true", options.TableName);
@@ -165,48 +111,4 @@ private async Task<TableClient> GetCloudTable()
         }
         return table;
     }
-
-    private static StringContent CreateObjectContent(object content)
-    {
-        StringWriter writer = new StringWriter();
-        s_grafanaSerializer.Serialize(writer, content);
-        return new StringContent(writer.ToString(), Encoding.UTF8, "application/json");
-    }
-
-    private static async Task<T> ReadObjectContent<T>(HttpContent content)
-    {
-        using (var streamReader = new StreamReader(await content.ReadAsStreamAsync()))
-        using (var jsonReader = new JsonTextReader(streamReader))
-        {
-            return s_grafanaSerializer.Deserialize<T>(jsonReader);
-        }
-    }
-
-    private static readonly JsonSerializer s_grafanaSerializer = new JsonSerializer
-    {
-        ContractResolver = new CamelCasePropertyNamesContractResolver(),
-        Formatting = Formatting.None,
-    };
-}
-
-public class DeploymentStartRequest
-{
-    [Required]
-    public string Service { get; set; }
-}
-
-public class NewGrafanaAnnotationRequest
-{
-    public int? DashboardId { get; set; }
-    public int? PanelId { get; set; }
-    public long? Time { get; set; }
-    public long? TimeEnd { get; set; }
-    public string[] Tags { get; set; }
-    public string Text { get; set; }
-}
-
-public class NewGrafanaAnnotationResponse
-{
-    public string Message { get; set; }
-    public int Id { get; set; }
 }

src/DotNet.Status.Web/DotNet.Status.Web/Options/DeploymentTableOptions.cs

@@ -0,0 +1,14 @@
+// Licensed to the .NET Foundation under one or more agreements.
+// The .NET Foundation licenses this file to you under the MIT license.
+// See the LICENSE file in the project root for more information.
+
+namespace DotNet.Status.Web.Options;
+
+// Storage coordinates for the deployment annotations table. The DeploymentController writes
+// deployment start/end records here and the AnnotationsController reads them back to serve
+// Grafana's annotation queries.
+public class DeploymentTableOptions
+{
+    public string TableUri { get; set; }
+    public string TableName { get; set; }
+}

src/DotNet.Status.Web/DotNet.Status.Web/Options/GrafanaOptions.cs

@@ -6,9 +6,5 @@ namespace DotNet.Status.Web.Options;
 
 public class GrafanaOptions
 {
-    public string BaseUrl { get; set; }
-    public string ApiToken { get; set; }
-    public string TableUri { get; set; }
-    public string TableName { get; set; }
     public string WebhookSecret { get; set; }
 }

src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs

@@ -92,6 +92,7 @@ private void ConfigureConfiguration(IServiceCollection services)
         services.Configure<TeamMentionForwardingOptions>(Configuration.GetSection("IssueMentionForwarding"));
         services.Configure<GitHubConnectionOptions>(Configuration.GetSection("GitHub"));
         services.Configure<GrafanaOptions>(Configuration.GetSection("Grafana"));
+        services.Configure<DeploymentTableOptions>(Configuration.GetSection("Deployments"));
         services.Configure<AzureDevOpsAlertOptions>(Configuration.GetSection("AzureDevOpsAlert"));
         services.Configure<AnnotationsOptions>(Configuration.GetSection("Annotations"));
         services.Configure<GitHubTokenProviderOptions>(Configuration.GetSection("GitHubAppAuth"));