diff --git a/.vault-config/dotneteng-status-local.yaml b/.vault-config/dotneteng-status-local.yaml index f4b90fde8..047010235 100644 --- a/.vault-config/dotneteng-status-local.yaml +++ b/.vault-config/dotneteng-status-local.yaml @@ -19,11 +19,6 @@ secrets: hasWebhookSecret: true hasOAuthSecret: true - grafana-api-token: - type: azure-managed-grafana-api-key - parameters: - environment: dotnet-eng-grafana-staging.westus2.cloudapp.azure.com - # Shared secret for Grafana webhook contact point Basic Auth grafana-webhook-secret: type: text diff --git a/.vault-config/dotneteng-status-prod.yaml b/.vault-config/dotneteng-status-prod.yaml index 2b96ee2bb..f9906c969 100644 --- a/.vault-config/dotneteng-status-prod.yaml +++ b/.vault-config/dotneteng-status-prod.yaml @@ -19,12 +19,6 @@ keys: importSecretsFrom: shared/dotneteng-status-secrets.yaml secrets: - # Grafana API key with admin privileges - grafana-api-token: - type: azure-managed-grafana-api-key - parameters: - environment: dotnet-eng-grafana.westus2.cloudapp.azure.com - # Shared secret for Grafana webhook contact point Basic Auth grafana-webhook-secret: type: text diff --git a/.vault-config/dotneteng-status-staging.yaml b/.vault-config/dotneteng-status-staging.yaml index 832e82520..c719205c4 100644 --- a/.vault-config/dotneteng-status-staging.yaml +++ b/.vault-config/dotneteng-status-staging.yaml @@ -19,12 +19,6 @@ keys: importSecretsFrom: shared/dotneteng-status-secrets.yaml secrets: - # Grafana API key with admin privileges - grafana-api-token: - type: azure-managed-grafana-api-key - parameters: - environment: dotnet-eng-grafana-staging.westus2.cloudapp.azure.com - # Shared secret for Grafana webhook contact point Basic Auth grafana-webhook-secret: type: text diff --git a/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AnnotationsControllerTests.cs b/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AnnotationsControllerTests.cs index 5835029ec..0ee4a9b66 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AnnotationsControllerTests.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web.Tests/AnnotationsControllerTests.cs @@ -122,7 +122,7 @@ public TestData() services.AddControllers() .AddApplicationPart(typeof(AnnotationsController).Assembly); - services.Configure(options => + services.Configure(options => { options.TableUri = "https://127.0.0.1:10002/devstoreaccount1/deployments"; }); diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Development.json b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Development.json index 2ff4b522a..83e0e3961 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Development.json +++ b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Development.json @@ -49,7 +49,7 @@ "StorageAccountConnectionString": "", "KeyIdentifier": "" }, - "Grafana": { + "Deployments": { "TableUri": "" }, "Kusto": { diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json index 984e23195..44e3dd132 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json +++ b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Production.json @@ -23,8 +23,7 @@ "AzureTableTokenStore": { "TableUri": "https://dotnetengstatusprod.table.core.windows.net" }, - "Grafana": { - "BaseUrl": "https://dotnet-eng-grafana.westus2.cloudapp.azure.com", + "Deployments": { "TableUri": "https://dotnetengstatusprod.table.core.windows.net" }, "Kusto": { diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json index 132c73bee..bcd736aa9 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json +++ b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.Staging.json @@ -25,7 +25,7 @@ "AzureTableTokenStore": { "TableUri": "https://dotnetengstatusstaging.table.core.windows.net" }, - "Grafana": { + "Deployments": { "TableUri": "https://dotnetengstatusstaging.table.core.windows.net" }, "Kusto": { diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json index f242641c8..94dc1bab5 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json +++ b/src/DotNet.Status.Web/DotNet.Status.Web/.config/settings.json @@ -300,10 +300,10 @@ "AzureTableTokenStore": { "TableName": "tokens" }, + "Deployments": { + "TableName": "deployments" + }, "Grafana": { - "BaseUrl": "https://dotnet-eng-grafana-staging.westus2.cloudapp.azure.com", - "ApiToken": "[vault(grafana-api-token)]", - "TableName": "deployments", "WebhookSecret": "[vault(grafana-webhook-secret)]" }, "WebHooks": { diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AnnotationsController.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AnnotationsController.cs index 329bfec26..96fd50d0d 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AnnotationsController.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/AnnotationsController.cs @@ -28,12 +28,12 @@ public class AnnotationsController : ControllerBase { private const int _maximumServerCount = 10; // Safety limit on query complexity private readonly ILogger _logger; - private readonly IOptionsMonitor _options; + private readonly IOptionsMonitor _options; private readonly IHostEnvironment _env; public AnnotationsController( ILogger logger, - IOptionsMonitor options, + IOptionsMonitor options, IHostEnvironment env) { _logger = logger; diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/DeploymentController.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/DeploymentController.cs index fd6df5bc8..7f459492d 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/DeploymentController.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Controllers/DeploymentController.cs @@ -4,25 +4,15 @@ using Azure; using System; -using System.Collections.Immutable; using System.ComponentModel.DataAnnotations; -using System.IO; -using System.Net.Http; -using System.Net.Http.Headers; -using System.Text; -using System.Threading; using System.Threading.Tasks; using Azure.Data.Tables; using DotNet.Status.Web.Options; using Microsoft.AspNetCore.Mvc; -using Microsoft.DotNet.Services.Utility; using Microsoft.Extensions.Hosting; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using Newtonsoft.Json; -using Newtonsoft.Json.Serialization; using StatusWebAnnotationEntity = DotNet.Status.Web.Models.AnnotationEntity; -using Microsoft.WindowsAzure.Storage.Table; using Azure.Identity; namespace DotNet.Status.Web.Controllers; @@ -32,18 +22,15 @@ namespace DotNet.Status.Web.Controllers; public class DeploymentController : ControllerBase { private readonly IHostEnvironment _env; - private readonly ExponentialRetry _retry; - private readonly IOptionsMonitor _grafanaOptions; + private readonly IOptionsMonitor _deploymentTableOptions; private readonly ILogger _logger; public DeploymentController( - ExponentialRetry retry, - IOptionsMonitor grafanaOptions, + IOptionsMonitor deploymentTableOptions, ILogger logger, IHostEnvironment env) { - _retry = retry; - _grafanaOptions = grafanaOptions; + _deploymentTableOptions = deploymentTableOptions; _logger = logger; _env = env; } @@ -52,47 +39,24 @@ public DeploymentController( public async Task MarkStart([Required] string service, [Required] string id) { _logger.LogInformation("Recording start of deployment of '{service}' with id '{id}'", service, id); - NewGrafanaAnnotationRequest content = new NewGrafanaAnnotationRequest + + // Recording a deployment annotation is best-effort telemetry and must never fail the calling + // deployment pipeline. The annotation is written to Azure Table storage, which Grafana reads + // directly; any failure is logged and swallowed so the endpoint returns success instead of a 500. + try { - Text = $"Deployment of {service}", - Tags = new[] {"deploy", $"deploy-{service}", service}, - Time = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), - }; - - NewGrafanaAnnotationResponse annotation; - using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true })) + TableClient table = await GetCloudTable(); + await table.UpsertEntityAsync(new StatusWebAnnotationEntity(service, id) + { + Started = DateTimeOffset.UtcNow, + ETag = new Azure.ETag("*") + }); + } + catch (Exception ex) { - annotation = await _retry.RetryAsync(async () => - { - GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue; - _logger.LogInformation("Creating annotation to {url}", grafanaOptions.BaseUrl); - using (var request = new HttpRequestMessage(HttpMethod.Post, - $"{grafanaOptions.BaseUrl}/api/annotations")) - { - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - request.Headers.Authorization = - new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken); - request.Content = CreateObjectContent(content); - - using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None)) - { - _logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase); - response.EnsureSuccessStatusCode(); - return await ReadObjectContent(response.Content); - } - } - }, - e => _logger.LogWarning(e, "Failed to send new annotation"), - e => true - ); + _logger.LogError(ex, "Failed to record start of deployment of '{service}' with id '{id}'", service, id); } - _logger.LogInformation("Created annotation {annotationId}, inserting into table", annotation.Id); - TableClient table = await GetCloudTable(); - await table.UpsertEntityAsync(new DotNet.Status.Web.Models.AnnotationEntity(service, id, annotation.Id) - { - ETag = new Azure.ETag("*") - }); return NoContent(); } @@ -100,51 +64,33 @@ await table.UpsertEntityAsync(new DotNet.Status.Web.Models.AnnotationEntity(serv public async Task MarkEnd([Required] string service, [Required] string id) { _logger.LogInformation("Recording end of deployment of '{service}' with id '{id}'", service, id); - TableClient table = await GetCloudTable(); - _logger.LogInformation("Looking for existing deployment"); - var getResult = await table.GetEntityIfExistsAsync(service, id); - _logger.LogTrace("Table response code {responseCode}", getResult.GetRawResponse().Status); - if (!getResult.HasValue) - { - return NotFound(); - } - - var annotation = getResult.Value; - _logger.LogTrace("Updating end time of deployment..."); - annotation.Ended = DateTimeOffset.UtcNow; - - var updateResult = await table.UpdateEntityAsync(annotation, annotation.ETag, TableUpdateMode.Replace); - _logger.LogInformation("Update response code {responseCode}", updateResult.Status); - - var content = new NewGrafanaAnnotationRequest + // As with MarkStart, recording the end of a deployment is best-effort telemetry written to + // Azure Table storage and must never fail the calling deployment pipeline. Any failure is + // logged and swallowed. + try { - TimeEnd = DateTimeOffset.UtcNow.ToUnixTimeMilliseconds(), - }; - - using (var client = new HttpClient(new HttpClientHandler() { CheckCertificateRevocationList = true })) + TableClient table = await GetCloudTable(); + _logger.LogInformation("Looking for existing deployment"); + var getResult = await table.GetEntityIfExistsAsync(service, id); + _logger.LogTrace("Table response code {responseCode}", getResult.GetRawResponse().Status); + if (!getResult.HasValue) + { + _logger.LogWarning("No deployment start record found for '{service}' with id '{id}'; nothing to mark as ended", service, id); + return NoContent(); + } + + var annotation = getResult.Value; + + _logger.LogTrace("Updating end time of deployment..."); + annotation.Ended = DateTimeOffset.UtcNow; + + var updateResult = await table.UpdateEntityAsync(annotation, annotation.ETag, TableUpdateMode.Replace); + _logger.LogInformation("Update response code {responseCode}", updateResult.Status); + } + catch (Exception ex) { - await _retry.RetryAsync(async () => - { - GrafanaOptions grafanaOptions = _grafanaOptions.CurrentValue; - _logger.LogInformation("Updating annotation {annotationId} to {url}", annotation.GrafanaAnnotationId, grafanaOptions.BaseUrl); - using (var request = new HttpRequestMessage(HttpMethod.Patch, - $"{grafanaOptions.BaseUrl}/api/annotations/{annotation.GrafanaAnnotationId}")) - { - request.Headers.Accept.Add(new MediaTypeWithQualityHeaderValue("application/json")); - request.Headers.Authorization = - new AuthenticationHeaderValue("Bearer", grafanaOptions.ApiToken); - request.Content = CreateObjectContent(content); - using (HttpResponseMessage response = await client.SendAsync(request, CancellationToken.None)) - { - _logger.LogTrace("Response from grafana {responseCode} {reason}", response.StatusCode, response.ReasonPhrase); - response.EnsureSuccessStatusCode(); - } - } - }, - e => _logger.LogWarning(e, "Failed to send new annotation"), - e => true - ); + _logger.LogError(ex, "Failed to record end of deployment of '{service}' with id '{id}'", service, id); } return NoContent(); @@ -153,7 +99,7 @@ await _retry.RetryAsync(async () => private async Task GetCloudTable() { TableClient table; - GrafanaOptions options = _grafanaOptions.CurrentValue; + DeploymentTableOptions options = _deploymentTableOptions.CurrentValue; if (_env.IsDevelopment()) { table = new TableClient("UseDevelopmentStorage=true", options.TableName); @@ -165,48 +111,4 @@ private async Task GetCloudTable() } return table; } - - private static StringContent CreateObjectContent(object content) - { - StringWriter writer = new StringWriter(); - s_grafanaSerializer.Serialize(writer, content); - return new StringContent(writer.ToString(), Encoding.UTF8, "application/json"); - } - - private static async Task ReadObjectContent(HttpContent content) - { - using (var streamReader = new StreamReader(await content.ReadAsStreamAsync())) - using (var jsonReader = new JsonTextReader(streamReader)) - { - return s_grafanaSerializer.Deserialize(jsonReader); - } - } - - private static readonly JsonSerializer s_grafanaSerializer = new JsonSerializer - { - ContractResolver = new CamelCasePropertyNamesContractResolver(), - Formatting = Formatting.None, - }; -} - -public class DeploymentStartRequest -{ - [Required] - public string Service { get; set; } -} - -public class NewGrafanaAnnotationRequest -{ - public int? DashboardId { get; set; } - public int? PanelId { get; set; } - public long? Time { get; set; } - public long? TimeEnd { get; set; } - public string[] Tags { get; set; } - public string Text { get; set; } -} - -public class NewGrafanaAnnotationResponse -{ - public string Message { get; set; } - public int Id { get; set; } } diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Options/DeploymentTableOptions.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Options/DeploymentTableOptions.cs new file mode 100644 index 000000000..365e669a6 --- /dev/null +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Options/DeploymentTableOptions.cs @@ -0,0 +1,14 @@ +// Licensed to the .NET Foundation under one or more agreements. +// The .NET Foundation licenses this file to you under the MIT license. +// See the LICENSE file in the project root for more information. + +namespace DotNet.Status.Web.Options; + +// Storage coordinates for the deployment annotations table. The DeploymentController writes +// deployment start/end records here and the AnnotationsController reads them back to serve +// Grafana's annotation queries. +public class DeploymentTableOptions +{ + public string TableUri { get; set; } + public string TableName { get; set; } +} diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Options/GrafanaOptions.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Options/GrafanaOptions.cs index ea333031e..61d4349aa 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/Options/GrafanaOptions.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Options/GrafanaOptions.cs @@ -6,9 +6,5 @@ namespace DotNet.Status.Web.Options; public class GrafanaOptions { - public string BaseUrl { get; set; } - public string ApiToken { get; set; } - public string TableUri { get; set; } - public string TableName { get; set; } public string WebhookSecret { get; set; } } diff --git a/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs b/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs index c89df642c..15fd57495 100644 --- a/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs +++ b/src/DotNet.Status.Web/DotNet.Status.Web/Startup.cs @@ -92,6 +92,7 @@ private void ConfigureConfiguration(IServiceCollection services) services.Configure(Configuration.GetSection("IssueMentionForwarding")); services.Configure(Configuration.GetSection("GitHub")); services.Configure(Configuration.GetSection("Grafana")); + services.Configure(Configuration.GetSection("Deployments")); services.Configure(Configuration.GetSection("AzureDevOpsAlert")); services.Configure(Configuration.GetSection("Annotations")); services.Configure(Configuration.GetSection("GitHubAppAuth"));