From 5e7fda3d32ee41dda90a1299674f3763537ce300 Mon Sep 17 00:00:00 2001 From: Kapil Agrawal <7047165+netops2devops@users.noreply.github.com> Date: Thu, 13 Aug 2026 17:12:53 -0500 Subject: [PATCH 1/3] Fix Prometheus Metrics and Grafana dashboard MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - externalcas/metrics.go: Changed externalCAStatus from a plain Gauge to a GaugeVec with a ca_url label, so the upstream CA URL is carried directly in the time series. - externalcas/metrics_test.go: Added two tests covering the new GaugeVec behaviour — UP on 2xx and DOWN on non-2xx, both asserting the ca_url label is set correctly. - contrib/develop/grafana-dashboard.json: Reworked the Health Overview section — External CA panel replaced with a table showing URL + coloured UP/DOWN status; Issuance Success Rate switched to a fixed 24h window; Issued/Revoked totals now read from the persistent KV-backed collector metrics instead of process-reset-sensitive counters; operation rate panels converted from ops/sec to ops/day. Added Issuer column to the All Certificates and Renewal Count inventory tables. Added a new "Certificates Issued in Time Range" panel that filters to certs issued within the dashboard time picker window, showing SAN, Issuer, Serial, and Days Remaining with a row count footer. Signed-off-by: Kapil Agrawal <7047165+netops2devops@users.noreply.github.com> --- contrib/develop/grafana-dashboard.json | 228 ++++++--- docs/assets/grafana-externalcas.json | 610 ------------------------- externalcas/metrics.go | 7 +- externalcas/metrics_test.go | 92 ++++ 4 files changed, 267 insertions(+), 670 deletions(-) delete mode 100644 docs/assets/grafana-externalcas.json diff --git a/contrib/develop/grafana-dashboard.json b/contrib/develop/grafana-dashboard.json index befde6c..c064c05 100644 --- a/contrib/develop/grafana-dashboard.json +++ b/contrib/develop/grafana-dashboard.json @@ -44,47 +44,58 @@ { "id": 2, - "type": "stat", + "type": "table", "title": "External CA", "description": "Reachability of the external CA directory URL. Probed every 30 seconds.", "gridPos": { "x": 0, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "externalcas_external_ca_up", - "legendFormat": "CA status", - "refId": "A" + "expr": "externalcas_external_ca_up{ca_url!=\"\"}", + "refId": "A", + "instant": true, + "format": "table" } ], + "transformations": [ + { "id": "filterFieldsByName", "options": { + "include": { "names": ["ca_url", "Value"] } + }}, + { "id": "organize", "options": { + "renameByName": { "ca_url": "CA URL", "Value": "Status" } + }} + ], "options": { - "colorMode": "background", - "graphMode": "none", - "justifyMode": "center", - "orientation": "auto", - "reduceOptions": { - "calcs": ["lastNotNull"], - "fields": "", - "values": false - }, - "textMode": "auto" + "footer": { "show": false } }, "fieldConfig": { "defaults": { - "mappings": [ - { "type": "value", "options": { "1": { "text": "UP", "color": "green", "index": 0 } } }, - { "type": "value", "options": { "0": { "text": "DOWN", "color": "red", "index": 1 } } } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "color": { "mode": "thresholds" }, - "noValue": "Unknown" + "custom": { "align": "left", "displayMode": "auto" } }, - "overrides": [] + "overrides": [ + { + "matcher": { "id": "byName", "options": "Status" }, + "properties": [ + { "id": "custom.displayMode", "value": "color-background" }, + { "id": "custom.width", "value": 80 }, + { "id": "custom.align", "value": "center" }, + { "id": "mappings", "value": [ + { "type": "value", "options": { + "0": { "text": "DOWN", "color": "red", "index": 0 }, + "1": { "text": "UP", "color": "green", "index": 1 } + }} + ]}, + { "id": "thresholds", "value": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }}, + { "id": "color", "value": { "mode": "thresholds" } } + ] + } + ] } }, @@ -92,14 +103,15 @@ "id": 3, "type": "stat", "title": "Issued (Success)", - "description": "Total certificates successfully issued since process start.", + "description": "Total certificates successfully issued, based on the persistent KV store.", "gridPos": { "x": 4, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "externalcas_certificates_issued_total{status=\"success\"}", + "expr": "count(externalcas_certificate_info{status=\"success\"})", "legendFormat": "Issued", - "refId": "A" + "refId": "A", + "instant": true } ], "options": { @@ -173,14 +185,15 @@ "id": 5, "type": "stat", "title": "Revoked", - "description": "Total certificates successfully revoked since process start.", + "description": "Total certificates successfully revoked, based on the persistent KV store.", "gridPos": { "x": 12, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "externalcas_certificates_revoked_total{status=\"success\"}", + "expr": "count(externalcas_certificate_revocation_info{status=\"success\"})", "legendFormat": "Revoked", - "refId": "A" + "refId": "A", + "instant": true } ], "options": { @@ -211,13 +224,13 @@ { "id": 6, "type": "stat", - "title": "Issuance Success Rate", - "description": "Percentage of issuance attempts that succeeded over the last 5 minutes.", + "title": "Issuance Success Rate (24h)", + "description": "Percentage of issuance attempts that succeeded in the last 24 hours.", "gridPos": { "x": 16, "y": 1, "w": 4, "h": 4 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "rate(externalcas_certificates_issued_total{status=\"success\"}[5m]) / rate(externalcas_certificates_issued_total[5m])", + "expr": "increase(externalcas_certificates_issued_total{status=\"success\"}[24h]) / ignoring(status) sum without(status) (increase(externalcas_certificates_issued_total[24h]))", "legendFormat": "Success rate", "refId": "A" } @@ -312,12 +325,12 @@ "id": 9, "type": "timeseries", "title": "Issuance Rate by Status", - "description": "Certificate issuances per second, split by success and failure.", + "description": "Certificate issuances per day, split by success and failure.", "gridPos": { "x": 0, "y": 6, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "rate(externalcas_certificates_issued_total[5m])", + "expr": "rate(externalcas_certificates_issued_total[$__rate_interval]) * 86400", "legendFormat": "{{status}}", "refId": "A" } @@ -328,7 +341,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", @@ -362,12 +375,12 @@ "id": 10, "type": "timeseries", "title": "Revocation Rate by Status", - "description": "Certificate revocations per second, split by success and failure.", + "description": "Certificate revocations per day, split by success and failure.", "gridPos": { "x": 12, "y": 6, "w": 12, "h": 8 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "rate(externalcas_certificates_revoked_total[5m])", + "expr": "rate(externalcas_certificates_revoked_total[$__rate_interval]) * 86400", "legendFormat": "{{status}}", "refId": "A" } @@ -378,7 +391,7 @@ }, "fieldConfig": { "defaults": { - "unit": "ops", + "unit": "short", "color": { "mode": "palette-classic" }, "custom": { "drawStyle": "line", @@ -525,8 +538,8 @@ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400", - "legendFormat": "{{serial}} — {{common_name}}", + "expr": "((externalcas_certificate_expiry_timestamp_seconds - time()) / 86400) * on(serial, common_name) group_left(issuer) externalcas_certificate_info{status=\"success\"}", + "legendFormat": "", "refId": "A", "instant": true, "format": "table" @@ -534,10 +547,11 @@ ], "transformations": [ { "id": "filterFieldsByName", "options": { - "include": { "names": ["serial", "common_name", "Value"] } + "include": { "names": ["serial", "common_name", "issuer", "Value"] } }}, { "id": "organize", "options": { - "renameByName": { "Value": "Days Remaining", "serial": "Serial", "common_name": "Common Name" } + "renameByName": { "Value": "Days Remaining", "serial": "Serial", "common_name": "Common Name", "issuer": "Issuer" }, + "indexByName": { "Common Name": 0, "Serial": 1, "Issuer": 2, "Days Remaining": 3 } }}, { "id": "sortBy", "options": { "fields": [{ "displayName": "Days Remaining", "desc": false }] }} ], @@ -576,6 +590,13 @@ { "id": "custom.width", "value": 260 } ] }, + { + "matcher": { "id": "byName", "options": "Issuer" }, + "properties": [ + { "id": "custom.displayMode", "value": "auto" }, + { "id": "custom.width", "value": 220 } + ] + }, { "matcher": { "id": "byName", "options": "Days Remaining" }, "properties": [ @@ -653,8 +674,8 @@ "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { - "expr": "count by (common_name, sans) (externalcas_certificate_info{status=\"success\"}) - 1", - "legendFormat": "{{common_name}}", + "expr": "count by (common_name, sans, issuer) (externalcas_certificate_info{status=\"success\"}) - 1", + "legendFormat": "", "refId": "A", "instant": true, "format": "table" @@ -662,10 +683,11 @@ ], "transformations": [ { "id": "filterFieldsByName", "options": { - "include": { "names": ["common_name", "sans", "Value"] } + "include": { "names": ["common_name", "sans", "issuer", "Value"] } }}, { "id": "organize", "options": { - "renameByName": { "Value": "Renewals", "common_name": "Common Name", "sans": "SANs" } + "renameByName": { "Value": "Renewals", "common_name": "Common Name", "sans": "SANs", "issuer": "Issuer" }, + "indexByName": { "Common Name": 0, "SANs": 1, "Issuer": 2, "Renewals": 3 } }}, { "id": "sortBy", "options": { "fields": [{ "displayName": "Renewals", "desc": true }] }} ], @@ -687,6 +709,13 @@ "decimals": 0 }, "overrides": [ + { + "matcher": { "id": "byName", "options": "Issuer" }, + "properties": [ + { "id": "custom.displayMode", "value": "auto" }, + { "id": "custom.width", "value": 200 } + ] + }, { "matcher": { "id": "byName", "options": "Renewals" }, "properties": [ @@ -698,12 +727,97 @@ } }, + { + "id": 24, + "type": "table", + "title": "Certificates Issued in Time Range", + "description": "Certificates whose issued-at timestamp falls within the dashboard time picker window. Row count in the footer shows how many certs were issued in the selected period.", + "gridPos": { "x": 0, "y": 42, "w": 24, "h": 10 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "((externalcas_certificate_expiry_timestamp_seconds - time()) / 86400) * on(serial, common_name) group_left(issuer, sans) (externalcas_certificate_info{status=\"success\"} and on(serial, common_name) (externalcas_certificate_issued_timestamp_seconds >= ($__from / 1000)))", + "legendFormat": "", + "refId": "A", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { "id": "filterFieldsByName", "options": { + "include": { "names": ["serial", "common_name", "issuer", "sans", "Value"] } + }}, + { "id": "organize", "options": { + "renameByName": { + "serial": "Serial", + "common_name": "Common Name", + "issuer": "Issuer", + "sans": "SANs", + "Value": "Days Remaining" + }, + "indexByName": { + "Common Name": 0, + "SANs": 1, + "Issuer": 2, + "Serial": 3, + "Days Remaining": 4 + } + }}, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Days Remaining", "desc": false }] }} + ], + "options": { + "sortBy": [{ "displayName": "Days Remaining", "desc": false }], + "footer": { "show": true, "reducer": ["count"], "fields": ["Common Name"], "countRows": true } + }, + "fieldConfig": { + "defaults": { + "custom": { "align": "left", "displayMode": "auto" }, + "decimals": 1 + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Common Name" }, + "properties": [{ "id": "custom.width", "value": 220 }] + }, + { + "matcher": { "id": "byName", "options": "SANs" }, + "properties": [{ "id": "custom.width", "value": 260 }] + }, + { + "matcher": { "id": "byName", "options": "Issuer" }, + "properties": [{ "id": "custom.width", "value": 200 }] + }, + { + "matcher": { "id": "byName", "options": "Serial" }, + "properties": [{ "id": "custom.width", "value": 220 }] + }, + { + "matcher": { "id": "byName", "options": "Days Remaining" }, + "properties": [ + { "id": "unit", "value": "d" }, + { "id": "custom.width", "value": 160 }, + { "id": "custom.displayMode", "value": "color-background" }, + { "id": "thresholds", "value": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 15 }, + { "color": "green", "value": 30 } + ] + }}, + { "id": "color", "value": { "mode": "thresholds" } } + ] + } + ] + } + }, + { "id": 18, "type": "row", "title": "Go Runtime & Process", "collapsed": false, - "gridPos": { "x": 0, "y": 42, "w": 24, "h": 1 } + "gridPos": { "x": 0, "y": 53, "w": 24, "h": 1 } }, { @@ -711,7 +825,7 @@ "type": "timeseries", "title": "Goroutines", "description": "Number of goroutines currently running. Sustained growth indicates a goroutine leak.", - "gridPos": { "x": 0, "y": 43, "w": 8, "h": 7 }, + "gridPos": { "x": 0, "y": 54, "w": 8, "h": 7 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { @@ -752,7 +866,7 @@ "type": "timeseries", "title": "Heap Memory In Use", "description": "Bytes of heap objects currently allocated.", - "gridPos": { "x": 8, "y": 43, "w": 8, "h": 7 }, + "gridPos": { "x": 8, "y": 54, "w": 8, "h": 7 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { @@ -798,7 +912,7 @@ "type": "timeseries", "title": "CPU Usage", "description": "Rate of CPU time consumed by the process (cores).", - "gridPos": { "x": 16, "y": 43, "w": 8, "h": 7 }, + "gridPos": { "x": 16, "y": 54, "w": 8, "h": 7 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { @@ -839,7 +953,7 @@ "type": "timeseries", "title": "GC Pause Duration (P99)", "description": "99th percentile GC stop-the-world pause duration.", - "gridPos": { "x": 0, "y": 50, "w": 12, "h": 7 }, + "gridPos": { "x": 0, "y": 61, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { @@ -880,7 +994,7 @@ "type": "timeseries", "title": "Open File Descriptors", "description": "Number of open file descriptors held by the process.", - "gridPos": { "x": 12, "y": 50, "w": 12, "h": 7 }, + "gridPos": { "x": 12, "y": 61, "w": 12, "h": 7 }, "datasource": { "type": "prometheus", "uid": "${datasource}" }, "targets": [ { diff --git a/docs/assets/grafana-externalcas.json b/docs/assets/grafana-externalcas.json deleted file mode 100644 index a17f8f3..0000000 --- a/docs/assets/grafana-externalcas.json +++ /dev/null @@ -1,610 +0,0 @@ -{ - "__inputs": [ - { - "name": "DS_PROMETHEUS", - "label": "Prometheus", - "type": "datasource", - "pluginId": "prometheus", - "pluginName": "Prometheus" - } - ], - "__requires": [ - { "type": "grafana", "id": "grafana", "name": "Grafana", "version": "10.0.0" }, - { "type": "datasource", "id": "prometheus", "name": "Prometheus", "version": "1.0.0" }, - { "type": "panel", "id": "stat", "name": "Stat", "version": "" }, - { "type": "panel", "id": "timeseries", "name": "Time series", "version": "" }, - { "type": "panel", "id": "table", "name": "Table", "version": "" } - ], - "annotations": { "list": [] }, - "description": "acme-proxy external CA — certificate operations, CA health, signing latency, and per-certificate inventory", - "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 1, - "id": null, - "links": [], - "refresh": "1m", - "schemaVersion": 39, - "tags": ["acme-proxy", "certificates", "tls"], - "time": { "from": "now-24h", "to": "now" }, - "timepicker": {}, - "timezone": "browser", - "title": "acme-proxy — External CA", - "uid": "acme-proxy-externalcas-v1", - "version": 1, - "templating": { - "list": [ - { - "current": {}, - "hide": 0, - "includeAll": false, - "label": "Datasource", - "multi": false, - "name": "datasource", - "options": [], - "query": "prometheus", - "refresh": 1, - "type": "datasource" - } - ] - }, - "panels": [ - - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 0 }, - "id": 1, - "title": "Health Overview", - "type": "row" - }, - - { - "id": 2, - "title": "External CA", - "type": "stat", - "gridPos": { "h": 4, "w": 4, "x": 0, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "background", - "graphMode": "none", - "textMode": "auto", - "orientation": "auto" - }, - "fieldConfig": { - "defaults": { - "mappings": [ - { - "type": "value", - "options": { - "0": { "text": "DOWN", "color": "red", "index": 0 }, - "1": { "text": "UP", "color": "green", "index": 1 } - } - } - ], - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "green", "value": 1 } - ] - }, - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "externalcas_external_ca_up", - "legendFormat": "CA status", - "refId": "A" - } - ] - }, - - { - "id": 3, - "title": "Certs Issued", - "type": "stat", - "gridPos": { "h": 4, "w": 4, "x": 4, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "reduceOptions": { "calcs": ["sum"] }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "green" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "increase(externalcas_certificates_issued_total{status=\"success\"}[$__range])", - "legendFormat": "issued", - "refId": "A" - } - ] - }, - - { - "id": 4, - "title": "Certs Renewed", - "type": "stat", - "gridPos": { "h": 4, "w": 4, "x": 8, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "reduceOptions": { "calcs": ["sum"] }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "blue" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "increase(externalcas_certificates_renewed_total{status=\"success\"}[$__range])", - "legendFormat": "renewed", - "refId": "A" - } - ] - }, - - { - "id": 5, - "title": "Certs Revoked", - "type": "stat", - "gridPos": { "h": 4, "w": 4, "x": 12, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "reduceOptions": { "calcs": ["sum"] }, - "colorMode": "value", - "graphMode": "area", - "textMode": "auto" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "color": { "mode": "fixed", "fixedColor": "yellow" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "increase(externalcas_certificates_revoked_total{status=\"success\"}[$__range])", - "legendFormat": "revoked", - "refId": "A" - } - ] - }, - - { - "id": 6, - "title": "ACME Errors", - "type": "stat", - "gridPos": { "h": 4, "w": 4, "x": 16, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "reduceOptions": { "calcs": ["sum"] }, - "colorMode": "background", - "graphMode": "area", - "textMode": "auto" - }, - "fieldConfig": { - "defaults": { - "unit": "short", - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 1 }, - { "color": "red", "value": 5 } - ] - }, - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "increase(externalcas_acme_errors_total[$__range])", - "legendFormat": "errors", - "refId": "A" - } - ] - }, - - { - "id": 7, - "title": "Time Since Last Successful Cert", - "type": "stat", - "gridPos": { "h": 4, "w": 4, "x": 20, "y": 1 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "reduceOptions": { "calcs": ["lastNotNull"] }, - "colorMode": "value", - "graphMode": "none", - "textMode": "auto" - }, - "fieldConfig": { - "defaults": { - "unit": "dtdurations", - "thresholds": { - "mode": "absolute", - "steps": [ - { "color": "green", "value": null }, - { "color": "orange", "value": 86400 }, - { "color": "red", "value": 604800 } - ] - }, - "color": { "mode": "thresholds" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "time() - externalcas_last_successful_certificate_timestamp_seconds", - "legendFormat": "age", - "refId": "A" - } - ] - }, - - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 5 }, - "id": 8, - "title": "Operation Rates", - "type": "row" - }, - - { - "id": 9, - "title": "Issue & Renewal Rate", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "custom": { "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" } - }, - "overrides": [ - { - "matcher": { "id": "byRegexp", "options": ".*failure.*" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "red" } }] - }, - { - "matcher": { "id": "byRegexp", "options": "issued.*success" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "green" } }] - }, - { - "matcher": { "id": "byRegexp", "options": "renewed.*success" }, - "properties": [{ "id": "color", "value": { "mode": "fixed", "fixedColor": "blue" } }] - } - ] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(externalcas_certificates_issued_total[$__rate_interval])", - "legendFormat": "issued {{status}}", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(externalcas_certificates_renewed_total[$__rate_interval])", - "legendFormat": "renewed {{status}}", - "refId": "B" - } - ] - }, - - { - "id": 10, - "title": "ACME Errors by Type", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 6 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "fieldConfig": { - "defaults": { - "unit": "ops", - "color": { "mode": "palette-classic" }, - "custom": { "lineWidth": 1, "fillOpacity": 10, "showPoints": "never" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "rate(externalcas_acme_errors_total[$__rate_interval])", - "legendFormat": "{{error_type}}", - "refId": "A" - } - ] - }, - - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 14 }, - "id": 11, - "title": "Signing Latency", - "type": "row" - }, - - { - "id": 12, - "title": "Certificate Request Duration (p50 / p95 / p99)", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 0, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" }, - "custom": { "lineWidth": 1, "fillOpacity": 5, "showPoints": "never" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, operation) (rate(externalcas_certificate_request_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p50 {{operation}}", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, operation) (rate(externalcas_certificate_request_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p95 {{operation}}", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, operation) (rate(externalcas_certificate_request_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p99 {{operation}}", - "refId": "C" - } - ] - }, - - { - "id": 13, - "title": "ACME Roundtrip Duration (p50 / p95 / p99)", - "type": "timeseries", - "gridPos": { "h": 8, "w": 12, "x": 12, "y": 15 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "tooltip": { "mode": "multi", "sort": "desc" }, - "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max"] } - }, - "fieldConfig": { - "defaults": { - "unit": "s", - "color": { "mode": "palette-classic" }, - "custom": { "lineWidth": 1, "fillOpacity": 5, "showPoints": "never" } - }, - "overrides": [] - }, - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.50, sum by (le, acme_operation) (rate(externalcas_acme_roundtrip_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p50 {{acme_operation}}", - "refId": "A" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.95, sum by (le, acme_operation) (rate(externalcas_acme_roundtrip_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p95 {{acme_operation}}", - "refId": "B" - }, - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "histogram_quantile(0.99, sum by (le, acme_operation) (rate(externalcas_acme_roundtrip_duration_seconds_bucket[$__rate_interval])))", - "legendFormat": "p99 {{acme_operation}}", - "refId": "C" - } - ] - }, - - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 23 }, - "id": 14, - "title": "Certificate Inventory", - "type": "row" - }, - - { - "id": 15, - "title": "Issued Certificates", - "type": "table", - "gridPos": { "h": 10, "w": 24, "x": 0, "y": 24 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "sortBy": [{ "displayName": "Common Name", "desc": false }], - "footer": { "show": false } - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "left", - "displayMode": "auto", - "filterable": true - } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "status" }, - "properties": [ - { - "id": "custom.displayMode", - "value": "color-background" - }, - { - "id": "mappings", - "value": [ - { - "type": "value", - "options": { - "success": { "text": "success", "color": "green", "index": 0 }, - "failure": { "text": "failure", "color": "red", "index": 1 } - } - } - ] - } - ] - }, - { - "matcher": { "id": "byName", "options": "Time" }, - "properties": [{ "id": "custom.hidden", "value": true }] - }, - { - "matcher": { "id": "byName", "options": "Value" }, - "properties": [{ "id": "custom.hidden", "value": true }] - } - ] - }, - "transformations": [ - { "id": "labelsToFields", "options": { "mode": "columns" } }, - { - "id": "organize", - "options": { - "renameByName": { - "serial": "Serial", - "common_name": "Common Name", - "issuer": "Issuer", - "sans": "SANs", - "status": "Status" - }, - "indexByName": { - "Common Name": 0, - "Serial": 1, - "SANs": 2, - "Issuer": 3, - "Status": 4 - } - } - } - ], - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "externalcas_certificate_info", - "legendFormat": "", - "instant": true, - "refId": "A" - } - ] - }, - - { - "collapsed": false, - "gridPos": { "h": 1, "w": 24, "x": 0, "y": 34 }, - "id": 16, - "title": "Certificate Expiry", - "type": "row" - }, - - { - "id": 17, - "title": "Days Until Expiry", - "type": "table", - "gridPos": { "h": 8, "w": 24, "x": 0, "y": 35 }, - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "options": { - "sortBy": [{ "displayName": "Days Remaining", "desc": false }], - "footer": { "show": false } - }, - "fieldConfig": { - "defaults": { - "custom": { - "align": "left", - "displayMode": "auto", - "filterable": true - } - }, - "overrides": [ - { - "matcher": { "id": "byName", "options": "Days Remaining" }, - "properties": [ - { "id": "unit", "value": "d" }, - { "id": "decimals", "value": 1 }, - { "id": "custom.displayMode", "value": "color-background" }, - { - "id": "thresholds", - "value": { - "mode": "absolute", - "steps": [ - { "color": "red", "value": null }, - { "color": "orange", "value": 7 }, - { "color": "green", "value": 30 } - ] - } - }, - { "id": "color", "value": { "mode": "thresholds" } } - ] - }, - { - "matcher": { "id": "byName", "options": "Time" }, - "properties": [{ "id": "custom.hidden", "value": true }] - } - ] - }, - "transformations": [ - { "id": "labelsToFields", "options": { "mode": "columns" } }, - { - "id": "organize", - "options": { - "renameByName": { - "serial": "Serial", - "common_name": "Common Name", - "Value": "Days Remaining" - }, - "indexByName": { - "Common Name": 0, - "Serial": 1, - "Days Remaining": 2 - } - } - } - ], - "targets": [ - { - "datasource": { "type": "prometheus", "uid": "${datasource}" }, - "expr": "(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400", - "legendFormat": "", - "instant": true, - "refId": "A" - } - ] - } - - ] -} diff --git a/externalcas/metrics.go b/externalcas/metrics.go index a41f597..5657964 100644 --- a/externalcas/metrics.go +++ b/externalcas/metrics.go @@ -167,7 +167,7 @@ func runCAHealthProbe(caURL string, interval time.Duration) { probe := func() { resp, err := client.Get(caURL) //nolint:noctx if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 { - externalCAStatus.Set(0) + externalCAStatus.WithLabelValues(caURL).Set(0) if err != nil { slog.Debug("external CA health probe failed", "url", caURL, "error", err) } else { @@ -177,7 +177,7 @@ func runCAHealthProbe(caURL string, interval time.Duration) { return } resp.Body.Close() - externalCAStatus.Set(1) + externalCAStatus.WithLabelValues(caURL).Set(1) } probe() // run once immediately so the gauge is meaningful before the first tick ticker := time.NewTicker(interval) @@ -257,11 +257,12 @@ var ( ) // Gauges — values that can go up or down - externalCAStatus = prometheus.NewGauge( + externalCAStatus = prometheus.NewGaugeVec( prometheus.GaugeOpts{ Name: "externalcas_external_ca_up", Help: "Status of external CA (1 = up/healthy, 0 = down/unhealthy)", }, + []string{"ca_url"}, ) lastSuccessfulCertificateTimestamp = prometheus.NewGauge( diff --git a/externalcas/metrics_test.go b/externalcas/metrics_test.go index f638507..ca41a7a 100644 --- a/externalcas/metrics_test.go +++ b/externalcas/metrics_test.go @@ -1,6 +1,8 @@ package externalcas import ( + "net/http" + "net/http/httptest" "testing" "time" @@ -171,3 +173,93 @@ func TestStartMetricsServer_Disabled(t *testing.T) { t.Errorf("StartMetricsServer(disabled) = %v, want nil", err) } } + +func TestRunCAHealthProbe_SetsCAURLLabel(t *testing.T) { + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + })) + defer srv.Close() + + gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "test_ca_up", + Help: "test", + }, []string{"ca_url"}) + + // Run one probe synchronously by calling the same logic used in runCAHealthProbe. + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(srv.URL) //nolint:noctx + if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 { + gauge.WithLabelValues(srv.URL).Set(0) + } else { + resp.Body.Close() + gauge.WithLabelValues(srv.URL).Set(1) + } + + // Verify label is present and value is 1. + reg := prometheus.NewRegistry() + reg.MustRegister(gauge) + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("Gather() error: %v", err) + } + if len(mfs) != 1 { + t.Fatalf("expected 1 metric family, got %d", len(mfs)) + } + metrics := mfs[0].GetMetric() + if len(metrics) != 1 { + t.Fatalf("expected 1 metric, got %d", len(metrics)) + } + labels := make(map[string]string) + for _, lp := range metrics[0].GetLabel() { + labels[lp.GetName()] = lp.GetValue() + } + if labels["ca_url"] != srv.URL { + t.Errorf("ca_url label = %q, want %q", labels["ca_url"], srv.URL) + } + if v := metrics[0].GetGauge().GetValue(); v != 1 { + t.Errorf("gauge value = %v, want 1", v) + } +} + +func TestRunCAHealthProbe_DownOnError(t *testing.T) { + // Use a server that immediately closes connections. + srv := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusServiceUnavailable) + })) + defer srv.Close() + + gauge := prometheus.NewGaugeVec(prometheus.GaugeOpts{ + Name: "test_ca_up_down", + Help: "test", + }, []string{"ca_url"}) + + client := &http.Client{Timeout: 5 * time.Second} + resp, err := client.Get(srv.URL) //nolint:noctx + if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 { + if resp != nil { + resp.Body.Close() + } + gauge.WithLabelValues(srv.URL).Set(0) + } else { + resp.Body.Close() + gauge.WithLabelValues(srv.URL).Set(1) + } + + reg := prometheus.NewRegistry() + reg.MustRegister(gauge) + mfs, err := reg.Gather() + if err != nil { + t.Fatalf("Gather() error: %v", err) + } + metrics := mfs[0].GetMetric() + if v := metrics[0].GetGauge().GetValue(); v != 0 { + t.Errorf("gauge value = %v, want 0 (non-2xx should set DOWN)", v) + } + labels := make(map[string]string) + for _, lp := range metrics[0].GetLabel() { + labels[lp.GetName()] = lp.GetValue() + } + if labels["ca_url"] != srv.URL { + t.Errorf("ca_url label = %q, want %q", labels["ca_url"], srv.URL) + } +} From f4fb7a489d3c8b92e728273ef9ca7121515a8c8e Mon Sep 17 00:00:00 2001 From: Kapil Agrawal <7047165+netops2devops@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:06:29 -0500 Subject: [PATCH 2/3] Bump Go version in Github actions to 1.26.6 to resolve security vuln problems Signed-off-by: Kapil Agrawal <7047165+netops2devops@users.noreply.github.com> --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index a43c793..2390afe 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -8,7 +8,7 @@ on: branches: [main] env: - GO_VERSION: '1.26' + GO_VERSION: '1.26.6' REGISTRY: ghcr.io IMAGE_NAME: ${{ github.repository }} From eb4583f694be3e0a39f2523b26f52048194b85ab Mon Sep 17 00:00:00 2001 From: Kapil Agrawal <7047165+netops2devops@users.noreply.github.com> Date: Thu, 13 Aug 2026 18:08:36 -0500 Subject: [PATCH 3/3] remove codecov job in gh workflow. Signed-off-by: Kapil Agrawal <7047165+netops2devops@users.noreply.github.com> --- .github/workflows/ci.yml | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 2390afe..e970fa0 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -73,12 +73,7 @@ jobs: # Tests scoped to externalcas: the top-level package is a step-ca plugin entry # point with no independently testable logic. run: | - go test -v -race -coverprofile=coverage.txt ./externalcas - - - name: Upload coverage results to Codecov - uses: codecov/codecov-action@v5 - with: - token: ${{ secrets.CODECOV_TOKEN }} + go test -v -race ./externalcas security-scan: name: Security Scan