diff --git a/contrib/develop/grafana-dashboard.json b/contrib/develop/grafana-dashboard.json new file mode 100644 index 0000000..befde6c --- /dev/null +++ b/contrib/develop/grafana-dashboard.json @@ -0,0 +1,920 @@ +{ + "id": null, + "uid": "acme-proxy-metrics", + "title": "acme-proxy", + "tags": ["acme-proxy", "certificates", "pki"], + "timezone": "browser", + "editable": true, + "graphTooltip": 1, + "schemaVersion": 39, + "version": 1, + "refresh": "30s", + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "templating": { + "list": [ + { + "name": "datasource", + "type": "datasource", + "pluginId": "prometheus", + "label": "Data source", + "hide": 0, + "refresh": 1, + "options": [], + "current": {} + } + ] + }, + "annotations": { + "list": [] + }, + "links": [], + "panels": [ + + { + "id": 1, + "type": "row", + "title": "Health Overview", + "collapsed": false, + "gridPos": { "x": 0, "y": 0, "w": 24, "h": 1 } + }, + + { + "id": 2, + "type": "stat", + "title": "External CA", + "description": "Reachability of the external CA directory URL. Probed every 30 seconds.", + "gridPos": { "x": 0, "y": 1, "w": 4, "h": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "externalcas_external_ca_up", + "legendFormat": "CA status", + "refId": "A" + } + ], + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "mappings": [ + { "type": "value", "options": { "1": { "text": "UP", "color": "green", "index": 0 } } }, + { "type": "value", "options": { "0": { "text": "DOWN", "color": "red", "index": 1 } } } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "green", "value": 1 } + ] + }, + "color": { "mode": "thresholds" }, + "noValue": "Unknown" + }, + "overrides": [] + } + }, + + { + "id": 3, + "type": "stat", + "title": "Issued (Success)", + "description": "Total certificates successfully issued since process start.", + "gridPos": { "x": 4, "y": 1, "w": 4, "h": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "externalcas_certificates_issued_total{status=\"success\"}", + "legendFormat": "Issued", + "refId": "A" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "green", "mode": "fixed" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 4, + "type": "stat", + "title": "Issued (Failure)", + "description": "Total failed certificate issuances since process start.", + "gridPos": { "x": 8, "y": 1, "w": 4, "h": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "externalcas_certificates_issued_total{status=\"failure\"}", + "legendFormat": "Failures", + "refId": "A" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "red", "value": 1 } + ] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 5, + "type": "stat", + "title": "Revoked", + "description": "Total certificates successfully revoked since process start.", + "gridPos": { "x": 12, "y": 1, "w": 4, "h": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "externalcas_certificates_revoked_total{status=\"success\"}", + "legendFormat": "Revoked", + "refId": "A" + } + ], + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "color": { "fixedColor": "orange", "mode": "fixed" }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "orange", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 6, + "type": "stat", + "title": "Issuance Success Rate", + "description": "Percentage of issuance attempts that succeeded over the last 5 minutes.", + "gridPos": { "x": 16, "y": 1, "w": 4, "h": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "rate(externalcas_certificates_issued_total{status=\"success\"}[5m]) / rate(externalcas_certificates_issued_total[5m])", + "legendFormat": "Success rate", + "refId": "A" + } + ], + "options": { + "colorMode": "background", + "graphMode": "area", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 0.9 }, + { "color": "green", "value": 0.99 } + ] + }, + "mappings": [], + "noValue": "No data" + }, + "overrides": [] + } + }, + + { + "id": 7, + "type": "stat", + "title": "Time Since Last Successful Cert", + "description": "Minutes elapsed since the most recently successfully issued certificate.", + "gridPos": { "x": 20, "y": 1, "w": 4, "h": 4 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "(time() - externalcas_last_successful_certificate_timestamp_seconds) / 60", + "legendFormat": "Minutes ago", + "refId": "A" + } + ], + "options": { + "colorMode": "background", + "graphMode": "none", + "justifyMode": "center", + "orientation": "auto", + "reduceOptions": { + "calcs": ["lastNotNull"], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "fieldConfig": { + "defaults": { + "unit": "m", + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "yellow", "value": 60 }, + { "color": "red", "value": 1440 } + ] + }, + "mappings": [ + { "type": "value", "options": { "0": { "text": "Never", "index": 0 } } } + ], + "noValue": "Never" + }, + "overrides": [] + } + }, + + { + "id": 8, + "type": "row", + "title": "Certificate Operations", + "collapsed": false, + "gridPos": { "x": 0, "y": 5, "w": 24, "h": 1 } + }, + + { + "id": 9, + "type": "timeseries", + "title": "Issuance Rate by Status", + "description": "Certificate issuances per second, split by success and failure.", + "gridPos": { "x": 0, "y": 6, "w": 12, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "rate(externalcas_certificates_issued_total[5m])", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "options": { + "tooltip": { "mode": "multi", "sort": "none" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "failure" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "success" }, + "properties": [{ "id": "color", "value": { "fixedColor": "green", "mode": "fixed" } }] + } + ] + } + }, + + { + "id": 10, + "type": "timeseries", + "title": "Revocation Rate by Status", + "description": "Certificate revocations per second, split by success and failure.", + "gridPos": { "x": 12, "y": 6, "w": 12, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "rate(externalcas_certificates_revoked_total[5m])", + "legendFormat": "{{status}}", + "refId": "A" + } + ], + "options": { + "tooltip": { "mode": "multi", "sort": "none" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "fieldConfig": { + "defaults": { + "unit": "ops", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "failure" }, + "properties": [{ "id": "color", "value": { "fixedColor": "red", "mode": "fixed" } }] + }, + { + "matcher": { "id": "byName", "options": "success" }, + "properties": [{ "id": "color", "value": { "fixedColor": "orange", "mode": "fixed" } }] + } + ] + } + }, + + { + "id": 11, + "type": "row", + "title": "Latency", + "collapsed": false, + "gridPos": { "x": 0, "y": 14, "w": 24, "h": 1 } + }, + + { + "id": 12, + "type": "timeseries", + "title": "End-to-End Request Latency", + "description": "P50, P95, and P99 end-to-end time from receiving the request to getting a response from the external CA, by operation (issue/revoke).", + "gridPos": { "x": 0, "y": 15, "w": 12, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "histogram_quantile(0.50, rate(externalcas_certificate_request_duration_seconds_bucket[5m]))", + "legendFormat": "{{operation}} P50", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.95, rate(externalcas_certificate_request_duration_seconds_bucket[5m]))", + "legendFormat": "{{operation}} P95", + "refId": "B" + }, + { + "expr": "histogram_quantile(0.99, rate(externalcas_certificate_request_duration_seconds_bucket[5m]))", + "legendFormat": "{{operation}} P99", + "refId": "C" + } + ], + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 13, + "type": "timeseries", + "title": "ACME Roundtrip Latency by Operation (P95)", + "description": "P95 time for individual ACME protocol calls: account registration, certificate obtain, and revoke. Useful for isolating which step is slow.", + "gridPos": { "x": 12, "y": 15, "w": 12, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "histogram_quantile(0.95, rate(externalcas_acme_roundtrip_duration_seconds_bucket[5m]))", + "legendFormat": "{{acme_operation}} P95", + "refId": "A" + } + ], + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "table", "placement": "bottom", "calcs": ["mean", "max", "last"] } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 14, + "type": "row", + "title": "Certificate Inventory", + "collapsed": false, + "gridPos": { "x": 0, "y": 23, "w": 24, "h": 1 } + }, + + { + "id": 15, + "type": "table", + "title": "All Certificates — Days Remaining", + "description": "One row per issued certificate. Sorted by days remaining ascending so certs closest to expiry appear first.", + "gridPos": { "x": 0, "y": 24, "w": 24, "h": 10 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400", + "legendFormat": "{{serial}} — {{common_name}}", + "refId": "A", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { "id": "filterFieldsByName", "options": { + "include": { "names": ["serial", "common_name", "Value"] } + }}, + { "id": "organize", "options": { + "renameByName": { "Value": "Days Remaining", "serial": "Serial", "common_name": "Common Name" } + }}, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Days Remaining", "desc": false }] }} + ], + "options": { + "sortBy": [{ "displayName": "Days Remaining", "desc": false }], + "footer": { "show": false } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 15 }, + { "color": "green", "value": 30 } + ] + }, + "custom": { + "displayMode": "color-background" + }, + "decimals": 1 + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Serial" }, + "properties": [ + { "id": "custom.displayMode", "value": "auto" }, + { "id": "custom.width", "value": 220 } + ] + }, + { + "matcher": { "id": "byName", "options": "Common Name" }, + "properties": [ + { "id": "custom.displayMode", "value": "auto" }, + { "id": "custom.width", "value": 260 } + ] + }, + { + "matcher": { "id": "byName", "options": "Days Remaining" }, + "properties": [ + { "id": "unit", "value": "d" }, + { "id": "custom.width", "value": 160 } + ] + } + ] + } + }, + + { + "id": 16, + "type": "table", + "title": "Certificates Expiring Within 30 Days", + "description": "Only shows certs where days remaining is below 30. Empty means no certs are approaching expiry.", + "gridPos": { "x": 0, "y": 34, "w": 14, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400 < 30", + "legendFormat": "{{serial}} — {{common_name}}", + "refId": "A", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { "id": "filterFieldsByName", "options": { + "include": { "names": ["serial", "common_name", "Value"] } + }}, + { "id": "organize", "options": { + "renameByName": { "Value": "Days Remaining", "serial": "Serial", "common_name": "Common Name" } + }}, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Days Remaining", "desc": false }] }} + ], + "options": { + "sortBy": [{ "displayName": "Days Remaining", "desc": false }], + "footer": { "show": false }, + "noValue": "No certificates expiring within 30 days" + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "red", "value": null }, + { "color": "yellow", "value": 15 } + ] + }, + "custom": { "displayMode": "color-background" }, + "decimals": 1, + "unit": "d" + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Serial" }, + "properties": [{ "id": "custom.displayMode", "value": "auto" }] + }, + { + "matcher": { "id": "byName", "options": "Common Name" }, + "properties": [{ "id": "custom.displayMode", "value": "auto" }] + } + ] + } + }, + + { + "id": 17, + "type": "table", + "title": "Renewal Count per CN/SAN", + "description": "Counts how many times each CN/SAN pair has been renewed. 0 = issued once, never renewed. Detected by counting distinct serials for the same CN/SAN.", + "gridPos": { "x": 14, "y": 34, "w": 10, "h": 8 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "count by (common_name, sans) (externalcas_certificate_info{status=\"success\"}) - 1", + "legendFormat": "{{common_name}}", + "refId": "A", + "instant": true, + "format": "table" + } + ], + "transformations": [ + { "id": "filterFieldsByName", "options": { + "include": { "names": ["common_name", "sans", "Value"] } + }}, + { "id": "organize", "options": { + "renameByName": { "Value": "Renewals", "common_name": "Common Name", "sans": "SANs" } + }}, + { "id": "sortBy", "options": { "fields": [{ "displayName": "Renewals", "desc": true }] }} + ], + "options": { + "sortBy": [{ "displayName": "Renewals", "desc": true }], + "footer": { "show": false } + }, + "fieldConfig": { + "defaults": { + "color": { "mode": "thresholds" }, + "thresholds": { + "mode": "absolute", + "steps": [ + { "color": "green", "value": null }, + { "color": "blue", "value": 1 } + ] + }, + "custom": {}, + "decimals": 0 + }, + "overrides": [ + { + "matcher": { "id": "byName", "options": "Renewals" }, + "properties": [ + { "id": "custom.displayMode", "value": "color-background" }, + { "id": "custom.width", "value": 100 } + ] + } + ] + } + }, + + { + "id": 18, + "type": "row", + "title": "Go Runtime & Process", + "collapsed": false, + "gridPos": { "x": 0, "y": 42, "w": 24, "h": 1 } + }, + + { + "id": 19, + "type": "timeseries", + "title": "Goroutines", + "description": "Number of goroutines currently running. Sustained growth indicates a goroutine leak.", + "gridPos": { "x": 0, "y": 43, "w": 8, "h": 7 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "go_goroutines", + "legendFormat": "goroutines", + "refId": "A" + } + ], + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "fixedColor": "blue", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 20, + "type": "timeseries", + "title": "Heap Memory In Use", + "description": "Bytes of heap objects currently allocated.", + "gridPos": { "x": 8, "y": 43, "w": 8, "h": 7 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "go_memstats_heap_inuse_bytes", + "legendFormat": "heap in use", + "refId": "A" + }, + { + "expr": "go_memstats_alloc_bytes", + "legendFormat": "alloc", + "refId": "B" + } + ], + "options": { + "tooltip": { "mode": "multi", "sort": "desc" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "bytes", + "color": { "mode": "palette-classic" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 21, + "type": "timeseries", + "title": "CPU Usage", + "description": "Rate of CPU time consumed by the process (cores).", + "gridPos": { "x": 16, "y": 43, "w": 8, "h": 7 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "rate(process_cpu_seconds_total[1m])", + "legendFormat": "cpu", + "refId": "A" + } + ], + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "percentunit", + "color": { "fixedColor": "purple", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 22, + "type": "timeseries", + "title": "GC Pause Duration (P99)", + "description": "99th percentile GC stop-the-world pause duration.", + "gridPos": { "x": 0, "y": 50, "w": 12, "h": 7 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "histogram_quantile(0.99, rate(go_gc_duration_seconds_bucket[5m]))", + "legendFormat": "GC pause P99", + "refId": "A" + } + ], + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "s", + "color": { "fixedColor": "yellow", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + }, + + { + "id": 23, + "type": "timeseries", + "title": "Open File Descriptors", + "description": "Number of open file descriptors held by the process.", + "gridPos": { "x": 12, "y": 50, "w": 12, "h": 7 }, + "datasource": { "type": "prometheus", "uid": "${datasource}" }, + "targets": [ + { + "expr": "process_open_fds", + "legendFormat": "open fds", + "refId": "A" + } + ], + "options": { + "tooltip": { "mode": "single", "sort": "none" }, + "legend": { "displayMode": "list", "placement": "bottom" } + }, + "fieldConfig": { + "defaults": { + "unit": "short", + "color": { "fixedColor": "orange", "mode": "fixed" }, + "custom": { + "drawStyle": "line", + "lineInterpolation": "linear", + "lineWidth": 2, + "fillOpacity": 10, + "showPoints": "never", + "spanNulls": false, + "stacking": { "mode": "none", "group": "A" } + }, + "thresholds": { + "mode": "absolute", + "steps": [{ "color": "green", "value": null }] + }, + "mappings": [] + }, + "overrides": [] + } + } + + ] +} diff --git a/contrib/develop/monitoring.md b/contrib/develop/monitoring.md new file mode 100644 index 0000000..b57ed2b --- /dev/null +++ b/contrib/develop/monitoring.md @@ -0,0 +1,307 @@ +# Monitoring acme-proxy with Prometheus and Grafana + +This guide covers setting up a local Prometheus and Grafana stack that scrapes metrics from an acme-proxy container running on the `acme-proxy-testbridge` Docker network. + +## Prerequisites + +- `acme-proxy-testbridge` network exists and acme-proxy is running on it (see `test-infra.md`) +- `ca.json` has `metrics.port` and `metrics.dataSource` configured +- A `prometheus.yml` scrape config + +## Start Prometheus + +```sh +docker run -d --name prometheus \ + --network acme-proxy-testbridge \ + -v "$(pwd)/contrib/develop/prometheus.yml":/etc/prometheus/prometheus.yml:ro \ + -p 9090:9090 \ + prom/prometheus:latest +``` + +The `-p 9090:9090` publish is optional. It exposes the Prometheus UI on your host at `http://localhost:9090` but is not required for Grafana — Grafana reaches Prometheus over the bridge by container name. + +## Start Grafana + +```sh +docker run -d --name grafana \ + --network acme-proxy-testbridge \ + -p 3000:3000 \ + grafana/grafana:latest +``` + +## Add Prometheus as a Grafana data source + +1. Open `http://localhost:3000` (default credentials: `admin` / `admin`) +2. Go to **Connections → Data sources → Add data source → Prometheus** +3. Set the URL to `http://prometheus:9090` — use the container name, not `localhost` +4. Click **Save & test** + +--- + +## Metrics reference + +acme-proxy exposes three categories of metrics on the `/metrics` endpoint. + +### Per-certificate metrics (custom collector) + +These are emitted on every Prometheus scrape from the in-memory sidecar store. One time series is emitted per certificate record — they are not rate-based and do not reset on process restart. + +| Metric | Labels | Description | +|---|---|---| +| `externalcas_certificate_info` | `serial`, `common_name`, `issuer`, `sans`, `status` | Metadata for each issued certificate. Value is always 1. Status is `success` or `failure`. Failed issuances have empty serial and issuer. | +| `externalcas_certificate_issued_timestamp_seconds` | `serial`, `common_name` | Unix timestamp of cert `NotBefore`. Only emitted for `status="success"`. | +| `externalcas_certificate_expiry_timestamp_seconds` | `serial`, `common_name` | Unix timestamp of cert `NotAfter`. Only emitted for `status="success"`. | +| `externalcas_certificate_signing_duration_seconds` | `serial`, `common_name`, `status` | Time in seconds the external CA took to sign this specific certificate. Emitted for both success and failure. | +| `externalcas_certificate_revocation_info` | `serial`, `common_name`, `issuer`, `sans`, `status` | Metadata for each revoked certificate. Value is always 1. | +| `externalcas_certificate_revocation_duration_seconds` | `serial`, `common_name`, `status` | Time in seconds the external CA took to process the revocation. | + +### Aggregate operation metrics (in-process) + +These are standard Prometheus counters, histograms, and gauges updated at operation time. They reset to zero if the process restarts. + +| Metric | Type | Labels | Description | +|---|---|---|---| +| `externalcas_certificates_issued_total` | counter | `status` (`success`, `failure`) | Total certificate issuances since process start. | +| `externalcas_certificates_revoked_total` | counter | `status` (`success`, `failure`) | Total certificate revocations since process start. | +| `externalcas_certificate_request_duration_seconds` | histogram | `operation` (`issue`, `revoke`) | End-to-end time from receiving the request to getting a response from the external CA. Buckets: 1s, 2.5s, 5s, 10s, 30s, 60s, 120s. | +| `externalcas_acme_roundtrip_duration_seconds` | histogram | `acme_operation` (`register`, `obtain`, `revoke`) | Time for individual ACME protocol calls. Useful for isolating which step is slow. Buckets: 100ms, 250ms, 500ms, 1s, 2.5s, 5s, 10s. | +| `externalcas_certificate_expiration_seconds` | histogram | `status` (`issued`) | Distribution of certificate lifetimes (NotAfter − NotBefore in seconds). Buckets: 1d, 7d, 30d, 60d, 90d, 365d. | +| `externalcas_external_ca_up` | gauge | — | Reachability of the external CA directory URL. 1 = last probe returned 2xx, 0 = error or non-2xx. Probed every 30 seconds. | +| `externalcas_last_successful_certificate_timestamp_seconds` | gauge | — | Unix timestamp of the most recently successfully issued certificate. | + +### Go runtime and process metrics + +Collected automatically from the Go runtime and OS process. A subset of the most useful ones: + +| Metric | Description | +|---|---| +| `go_goroutines` | Number of goroutines currently running. | +| `go_gc_duration_seconds` | Distribution of GC stop-the-world pause durations. | +| `go_memstats_alloc_bytes` | Bytes of heap objects currently allocated. | +| `go_memstats_heap_inuse_bytes` | Bytes in heap spans currently in use. | +| `go_memstats_sys_bytes` | Total bytes of memory obtained from the OS. | +| `process_resident_memory_bytes` | RSS — bytes of physical memory in use. | +| `process_cpu_seconds_total` | Total CPU time consumed by the process. | +| `process_open_fds` | Number of open file descriptors. | + +--- + +## Grafana dashboard PromQL + +The queries below are organized by recommended Grafana panel type. The suggested dashboard layout has five rows. + +### Row 1 — Health overview (Stat panels) + +These give an at-a-glance summary of system health at the top of the dashboard. + +**External CA status** +Use a Stat panel with value mappings: 1 → "UP" (green), 0 → "DOWN" (red). +```promql +externalcas_external_ca_up +``` + +**Total certificates issued (success)** +```promql +externalcas_certificates_issued_total{status="success"} +``` + +**Total certificates issued (failure)** +```promql +externalcas_certificates_issued_total{status="failure"} +``` + +**Total certificates revoked** +```promql +externalcas_certificates_revoked_total{status="success"} +``` + +**Issuance success rate** +Percentage of issuance attempts that succeeded over the last 5 minutes. Set unit to `percent (0-1)`. +```promql +rate(externalcas_certificates_issued_total{status="success"}[5m]) +/ +rate(externalcas_certificates_issued_total[5m]) +``` + +**Time since last successful certificate (minutes)** +Useful for alerting if no cert has been issued recently when one was expected. +```promql +(time() - externalcas_last_successful_certificate_timestamp_seconds) / 60 +``` + +--- + +### Row 2 — Certificate operations (Time series panels) + +**Issuance rate by status** +Two series on one panel: successful and failed issuances per second. +```promql +rate(externalcas_certificates_issued_total[5m]) +``` + +**Revocation rate by status** +```promql +rate(externalcas_certificates_revoked_total[5m]) +``` + +**Cumulative issuances over time** +Shows growth rate visually. Use `increase()` over your dashboard time range for a windowed view. +```promql +increase(externalcas_certificates_issued_total{status="success"}[$__range]) +``` + +--- + +### Row 3 — Latency (Time series and Heatmap panels) + +**End-to-end request latency percentiles (Time series)** +Three queries on one panel for P50, P95, P99. Set legend to `{{operation}} p{{quantile}}`. +```promql +histogram_quantile(0.50, rate(externalcas_certificate_request_duration_seconds_bucket[5m])) +``` +```promql +histogram_quantile(0.95, rate(externalcas_certificate_request_duration_seconds_bucket[5m])) +``` +```promql +histogram_quantile(0.99, rate(externalcas_certificate_request_duration_seconds_bucket[5m])) +``` + +**ACME roundtrip latency by operation (Time series)** +Shows how long each individual ACME protocol step takes: account registration, certificate obtain, revoke. +```promql +histogram_quantile(0.95, rate(externalcas_acme_roundtrip_duration_seconds_bucket[5m])) +``` + +**Certificate lifetime distribution (Bar gauge)** +Shows what proportion of issued certs fall into each lifetime bucket (30-day, 90-day, etc.). +```promql +rate(externalcas_certificate_expiration_seconds_bucket[1h]) +``` + +--- + +### Row 4 — Certificate inventory (Table panels) + +**All active certificates with days remaining** +One row per cert, sorted by days remaining ascending. Set column `Days Remaining` with threshold coloring: red < 15, yellow < 30, green otherwise. +```promql +(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400 +``` + +**Certificates expiring within 30 days** +Filters to only certs approaching expiry. Use as the basis for an alert rule. +```promql +(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400 < 30 +``` + +**Days remaining on the current active cert per CN/SAN pair** +When a cert has been renewed, multiple serials exist for the same CN/SAN. `max` selects the newest cert's expiry without needing to know the serial. +```promql +( + max by (common_name, sans) (externalcas_certificate_expiry_timestamp_seconds) + - time() +) / 86400 +``` + +**Renewal count per CN/SAN pair** +Any CN/SAN pair with more than one successful issuance has been renewed. Value of 0 = never renewed, 1 = renewed once, etc. +```promql +count by (common_name, sans) (externalcas_certificate_info{status="success"}) - 1 +``` + +**Full certificate inventory (Table)** +Displays all cert metadata. In Grafana, use a Table panel and enable the label columns `serial`, `common_name`, `issuer`, `sans`, `status`. +```promql +externalcas_certificate_info +``` + +**Revoked certificate inventory (Table)** +```promql +externalcas_certificate_revocation_info +``` + +--- + +### Row 5 — Go runtime and process (Time series panels) + +**Goroutine count** +Sustained growth here indicates a goroutine leak. +```promql +go_goroutines +``` + +**Heap memory in use** +```promql +go_memstats_heap_inuse_bytes +``` + +**GC pause duration (P99)** +```promql +histogram_quantile(0.99, rate(go_gc_duration_seconds_bucket[5m])) +``` + +**CPU usage (rate)** +```promql +rate(process_cpu_seconds_total[1m]) +``` + +**Open file descriptors** +```promql +process_open_fds +``` + +--- + +## Alert rules + +The following PromQL expressions evaluate to a non-empty result when the condition is true, which is the standard Prometheus alerting pattern. + +**External CA unreachable** +```promql +externalcas_external_ca_up == 0 +``` + +**Certificate expiring within 30 days** +```promql +(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400 < 30 +``` + +**Certificate expiring within 7 days** +```promql +(externalcas_certificate_expiry_timestamp_seconds - time()) / 86400 < 7 +``` + +**Issuance failure rate above 10% over last 5 minutes** +```promql +rate(externalcas_certificates_issued_total{status="failure"}[5m]) +/ +rate(externalcas_certificates_issued_total[5m]) +> 0.1 +``` + +**No successful issuance in the last hour** +Fires only if at least one issuance has ever occurred (gauge is non-zero) but nothing recent has succeeded. +```promql +( + externalcas_last_successful_certificate_timestamp_seconds > 0 + and + (time() - externalcas_last_successful_certificate_timestamp_seconds) > 3600 +) +``` + +--- + +## Notes on renewal detection + +acme-proxy does not explicitly distinguish a first issuance from a renewal at the metric level — both go through `CreateCertificate` and increment `externalcas_certificates_issued_total{status="success"}`. Renewal detection is done at query time: when an ACME client renews a cert it receives a new serial number for the same set of SANs. The `externalcas_certificate_info` metric emits one series per serial, so two series sharing the same `common_name` and `sans` labels but with different `serial` labels indicates a renewal occurred. + +The `max` aggregation in several queries above exploits the fact that the newer cert always has a later `NotBefore` and `NotAfter`, so it naturally selects the active cert without needing to inspect the serial directly. + +--- + +## Teardown + +```sh +docker kill prometheus grafana +docker rm prometheus grafana +``` diff --git a/contrib/develop/prometheus.yml b/contrib/develop/prometheus.yml new file mode 100644 index 0000000..bf33f54 --- /dev/null +++ b/contrib/develop/prometheus.yml @@ -0,0 +1,7 @@ +global: + scrape_interval: 5s + +scrape_configs: + - job_name: acme-proxy + static_configs: + - targets: ['acme-proxy.duckdns.org:9234'] diff --git a/externalcas/config.go b/externalcas/config.go index 39ac28f..0b44abc 100644 --- a/externalcas/config.go +++ b/externalcas/config.go @@ -41,9 +41,8 @@ type acmeProxyConfig struct { Metrics metrics `json:"metrics"` // derived during Validate(); not marshaled - useEAB bool - useDNS01 bool - metricsEnabled bool + useEAB bool + useDNS01 bool } // Validate checks if the values provided in ca.json file contain required fields @@ -68,7 +67,7 @@ func (c *acmeProxyConfig) Validate() error { // Consider Metrics enabled only when port & datasource both are defined if c.Metrics.Port > 0 && c.Metrics.DataSource != "" { - c.metricsEnabled = true + c.Metrics.Enabled = true } if (c.Metrics.Port > 0 && c.Metrics.DataSource == "") || (c.Metrics.Port == 0 && c.Metrics.DataSource != "") { diff --git a/externalcas/config_test.go b/externalcas/config_test.go index b8345ee..b7b16d6 100644 --- a/externalcas/config_test.go +++ b/externalcas/config_test.go @@ -115,7 +115,7 @@ func TestAcmeProxyConfig_Validate(t *testing.T) { } func TestAcmeProxyConfig_Validate_MetricsEnabled(t *testing.T) { - t.Run("metricsEnabled set when port and datasource both present", func(t *testing.T) { + t.Run("Metrics.Enabled set when port and datasource both present", func(t *testing.T) { cfg := acmeProxyConfig{ CaURL: "https://acme.example.com", Kid: "test-kid", @@ -125,12 +125,12 @@ func TestAcmeProxyConfig_Validate_MetricsEnabled(t *testing.T) { if err := cfg.Validate(); err != nil { t.Fatalf("Validate() unexpected error: %v", err) } - if !cfg.metricsEnabled { - t.Error("metricsEnabled = false, want true when port and datasource are both set") + if !cfg.Metrics.Enabled { + t.Error("Metrics.Enabled = false, want true when port and datasource are both set") } }) - t.Run("metricsEnabled false when metrics not configured", func(t *testing.T) { + t.Run("Metrics.Enabled false when metrics not configured", func(t *testing.T) { cfg := acmeProxyConfig{ CaURL: "https://acme.example.com", Kid: "test-kid", @@ -139,12 +139,12 @@ func TestAcmeProxyConfig_Validate_MetricsEnabled(t *testing.T) { if err := cfg.Validate(); err != nil { t.Fatalf("Validate() unexpected error: %v", err) } - if cfg.metricsEnabled { - t.Error("metricsEnabled = true, want false when metrics are not configured") + if cfg.Metrics.Enabled { + t.Error("Metrics.Enabled = true, want false when metrics are not configured") } }) - t.Run("metricsEnabled false when only port set (invalid)", func(t *testing.T) { + t.Run("Metrics.Enabled false when only port set (invalid)", func(t *testing.T) { cfg := acmeProxyConfig{ CaURL: "https://acme.example.com", Kid: "test-kid", @@ -155,8 +155,8 @@ func TestAcmeProxyConfig_Validate_MetricsEnabled(t *testing.T) { if err == nil { t.Fatal("Validate() expected error for partial metrics config, got nil") } - if cfg.metricsEnabled { - t.Error("metricsEnabled must remain false when Validate() returns an error") + if cfg.Metrics.Enabled { + t.Error("Metrics.Enabled must remain false when Validate() returns an error") } }) } @@ -284,8 +284,8 @@ func TestParseConfig_MetricsFieldValues(t *testing.T) { if cfg.Metrics.DataSource != "/opt/acme-proxy/db/metrics" { t.Errorf("Metrics.DataSource = %q, want %q", cfg.Metrics.DataSource, "/opt/acme-proxy/db/metrics") } - if !cfg.metricsEnabled { - t.Error("metricsEnabled = false, want true after parseConfig with port and dataSource set") + if !cfg.Metrics.Enabled { + t.Error("Metrics.Enabled = false, want true after parseConfig with port and dataSource set") } } diff --git a/externalcas/external.go b/externalcas/external.go index b98fedb..a48ffaa 100644 --- a/externalcas/external.go +++ b/externalcas/external.go @@ -47,7 +47,7 @@ func New(ctx context.Context, opts apiv1.Options) (*ExternalCAS, error) { } cas.dnsProvider = provider } - if err := StartMetricsServer(cfg.Metrics); err != nil { + if err := StartMetricsServer(cfg.Metrics, cfg.CaURL); err != nil { return nil, err } return cas, nil @@ -130,19 +130,27 @@ func (c *ExternalCAS) createLegoClient(cfg *acmeProxyConfig) (ACMEClient, error) // Account registration — EAB takes precedence when configured if cfg.useEAB { + regStart := time.Now() reg, err := client.Registration.RegisterWithExternalAccountBinding(registration.RegisterEABOptions{ TermsOfServiceAgreed: true, Kid: cfg.Kid, HmacEncoded: cfg.HmacKey, }) + if metricsEnabled { + acmeRoundtripDuration.WithLabelValues("register").Observe(time.Since(regStart).Seconds()) + } if err != nil { return nil, fmt.Errorf("lego acme client registration failed with CA: %w", err) } user.Registration = reg } else { + regStart := time.Now() reg, err := client.Registration.Register(registration.RegisterOptions{ TermsOfServiceAgreed: true, }) + if metricsEnabled { + acmeRoundtripDuration.WithLabelValues("register").Observe(time.Since(regStart).Seconds()) + } if err != nil { return nil, fmt.Errorf("lego acme client registration failed with CA: %w", err) } @@ -229,6 +237,8 @@ func (c *ExternalCAS) CreateCertificate(req *apiv1.CreateCertificateRequest) (*a if result.err != nil { if metricsEnabled { certificatesIssuedTotal.WithLabelValues("failure").Inc() + certificateRequestDuration.WithLabelValues("issue").Observe(result.duration.Seconds()) + acmeRoundtripDuration.WithLabelValues("obtain").Observe(result.duration.Seconds()) if req.CSR != nil { if err := globalStore.recordIssued(CertRecord{ CommonName: req.CSR.Subject.CommonName, @@ -245,7 +255,12 @@ func (c *ExternalCAS) CreateCertificate(req *apiv1.CreateCertificateRequest) (*a slog.Info("obtained certificate from external CA", "domains", req.CSR.DNSNames) if metricsEnabled { certificatesIssuedTotal.WithLabelValues("success").Inc() + certificateRequestDuration.WithLabelValues("issue").Observe(result.duration.Seconds()) + acmeRoundtripDuration.WithLabelValues("obtain").Observe(result.duration.Seconds()) cert := result.response.Certificate + lifetime := cert.NotAfter.Sub(cert.NotBefore).Seconds() + certificateExpirationTime.WithLabelValues("issued").Observe(lifetime) + lastSuccessfulCertificateTimestamp.SetToCurrentTime() if err := globalStore.recordIssued(CertRecord{ Serial: cert.SerialNumber.Text(16), CommonName: cert.Subject.CommonName, @@ -320,6 +335,8 @@ func (c *ExternalCAS) RevokeCertificate(req *apiv1.RevokeCertificateRequest) (*a ) if metricsEnabled { certificatesRevokedTotal.WithLabelValues("failure").Inc() + certificateRequestDuration.WithLabelValues("revoke").Observe(revokeDuration.Seconds()) + acmeRoundtripDuration.WithLabelValues("revoke").Observe(revokeDuration.Seconds()) cert := req.Certificate if err := globalStore.recordRevoked(CertRecord{ Serial: cert.SerialNumber.Text(16), @@ -343,6 +360,8 @@ func (c *ExternalCAS) RevokeCertificate(req *apiv1.RevokeCertificateRequest) (*a ) if metricsEnabled { certificatesRevokedTotal.WithLabelValues("success").Inc() + certificateRequestDuration.WithLabelValues("revoke").Observe(revokeDuration.Seconds()) + acmeRoundtripDuration.WithLabelValues("revoke").Observe(revokeDuration.Seconds()) cert := req.Certificate if err := globalStore.recordRevoked(CertRecord{ Serial: cert.SerialNumber.Text(16), diff --git a/externalcas/metrics.go b/externalcas/metrics.go index dc5367e..4c9f002 100644 --- a/externalcas/metrics.go +++ b/externalcas/metrics.go @@ -1,10 +1,3 @@ -// Total certs issued, renewed, revoked -// client metadata (hostname, src_ip, acme_client used) -// SAN(s), isssue/renew status (success, failed) -// cert expiraiton date -// external CA acme endpoint status (up/down) -// measure time it takes to get certs signed from external CA - package externalcas import ( @@ -16,7 +9,7 @@ import ( "time" "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/client_golang/prometheus/promauto" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" ) @@ -119,7 +112,7 @@ func (c *certMetaCollector) Collect(ch chan<- prometheus.Metric) { // StartMetricsServer starts the Prometheus metrics HTTP server once. // DataSource is guaranteed non-empty by AcmeProxyConfig.Validate() when enabled. // Returns an error if the cert store cannot be opened — this fails server startup. -func StartMetricsServer(m metrics) error { +func StartMetricsServer(m metrics, caURL string) error { if !m.Enabled { return nil } @@ -131,11 +124,15 @@ func StartMetricsServer(m metrics) error { return } globalStore = s + if err := registry.Register(newCertMetaCollector(s)); err != nil { startErr = fmt.Errorf("failed to register cert meta collector: %w", err) return } + metricsEnabled = true + go runCAHealthProbe(caURL, 30*time.Second) + port := m.Port if port == 0 { port = 9123 @@ -158,6 +155,33 @@ func StartMetricsServer(m metrics) error { return startErr } +// runCAHealthProbe periodically GETs caURL and updates externalCAStatus. +// A 2xx response sets the gauge to 1 (up); any error or non-2xx sets it to 0 (down). +func runCAHealthProbe(caURL string, interval time.Duration) { + client := &http.Client{Timeout: 10 * time.Second} + probe := func() { + resp, err := client.Get(caURL) //nolint:noctx + if err != nil || resp.StatusCode < 200 || resp.StatusCode >= 300 { + externalCAStatus.Set(0) + if err != nil { + slog.Debug("external CA health probe failed", "url", caURL, "error", err) + } else { + resp.Body.Close() + slog.Debug("external CA health probe non-2xx", "url", caURL, "status", resp.StatusCode) + } + return + } + resp.Body.Close() + externalCAStatus.Set(1) + } + probe() // run once immediately so the gauge is meaningful before the first tick + ticker := time.NewTicker(interval) + defer ticker.Stop() + for range ticker.C { + probe() + } +} + var ( // metricsServerOnce ensures the metrics HTTP server starts exactly once metricsServerOnce sync.Once @@ -168,11 +192,19 @@ var ( // globalStore is the sidecar cert store; nil when DataSource is not configured globalStore *certStore - // Prometheus registry for all externalcas metrics - registry *prometheus.Registry + // Prometheus registry for all externalcas metrics; isolated from the default registry + // so Go/process collectors added here don't collide with any host-level defaults. + registry = func() *prometheus.Registry { + r := prometheus.NewRegistry() + r.MustRegister( + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + return r + }() - // Counters - monotonically increasing values - certificatesIssuedTotal = promauto.NewCounterVec( + // Counters — monotonically increasing + certificatesIssuedTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "externalcas_certificates_issued_total", Help: "Total number of certificates issued from external CA, labeled by status (success/failure)", @@ -180,15 +212,7 @@ var ( []string{"status"}, ) - certificatesRenewedTotal = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "externalcas_certificates_renewed_total", - Help: "Total number of certificates renewed from external CA, labeled by status (success/failure)", - }, - []string{"status"}, - ) - - certificatesRevokedTotal = promauto.NewCounterVec( + certificatesRevokedTotal = prometheus.NewCounterVec( prometheus.CounterOpts{ Name: "externalcas_certificates_revoked_total", Help: "Total number of certificates revoked at external CA, labeled by status (success/failure)", @@ -196,26 +220,18 @@ var ( []string{"status"}, ) - acmeErrorsTotal = promauto.NewCounterVec( - prometheus.CounterOpts{ - Name: "externalcas_acme_errors_total", - Help: "Total number of ACME protocol errors encountered, labeled by error type", - }, - []string{"error_type"}, - ) - - // Histograms - distribution of observed values (request durations) - certificateRequestDuration = promauto.NewHistogramVec( + // Histograms — distribution of observed values + certificateRequestDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "externalcas_certificate_request_duration_seconds", - Help: "Time taken to obtain certificate from external CA (in seconds)", + Help: "Time taken to obtain or revoke a certificate from the external CA (in seconds)", // Buckets: 1s, 2.5s, 5s, 10s, 30s, 60s, 120s Buckets: []float64{1, 2.5, 5, 10, 30, 60, 120}, }, []string{"operation"}, // operation: issue, revoke ) - acmeRoundtripDuration = promauto.NewHistogramVec( + acmeRoundtripDuration = prometheus.NewHistogramVec( prometheus.HistogramOpts{ Name: "externalcas_acme_roundtrip_duration_seconds", Help: "Time taken for individual ACME API calls (in seconds)", @@ -225,49 +241,40 @@ var ( []string{"acme_operation"}, // acme_operation: register, obtain, revoke ) - // Gauges - values that can go up or down - externalCAStatus = promauto.NewGauge( + certificateExpirationTime = prometheus.NewHistogramVec( + prometheus.HistogramOpts{ + Name: "externalcas_certificate_expiration_seconds", + Help: "Distribution of certificate lifetimes (NotAfter - NotBefore, in seconds)", + // Buckets: 1 day, 7 days, 30 days, 60 days, 90 days, 365 days + Buckets: []float64{86400, 604800, 2592000, 5184000, 7776000, 31536000}, + }, + []string{"status"}, // status: issued, renewed + ) + + // Gauges — values that can go up or down + externalCAStatus = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "externalcas_external_ca_up", Help: "Status of external CA (1 = up/healthy, 0 = down/unhealthy)", }, ) - lastSuccessfulCertificateTimestamp = promauto.NewGauge( + lastSuccessfulCertificateTimestamp = prometheus.NewGauge( prometheus.GaugeOpts{ Name: "externalcas_last_successful_certificate_timestamp_seconds", Help: "Unix timestamp of the last successfully issued certificate", }, ) - - certificateExpirationTime = promauto.NewHistogramVec( - prometheus.HistogramOpts{ - Name: "externalcas_certificate_expiration_seconds", - Help: "Distribution of certificate expiration times (lifetime in seconds)", - // Buckets: 1 day, 7 days, 30 days, 60 days, 90 days, 365 days - Buckets: []float64{86400, 604800, 2592000, 5184000, 7776000, 31536000}, - }, - []string{"status"}, // status: issued, renewed - ) ) func init() { - // Create a dedicated Prometheus registry for externalcas metrics - // This allows isolation from other metrics that might exist in the application - registry = prometheus.NewRegistry() - - // Register all metrics with the custom registry - registry.MustRegister(certificatesIssuedTotal) - registry.MustRegister(certificatesRenewedTotal) - registry.MustRegister(certificatesRevokedTotal) - registry.MustRegister(acmeErrorsTotal) - registry.MustRegister(certificateRequestDuration) - registry.MustRegister(acmeRoundtripDuration) - registry.MustRegister(externalCAStatus) - registry.MustRegister(lastSuccessfulCertificateTimestamp) - registry.MustRegister(certificateExpirationTime) - - // Initialize external CA status to unknown (0) - // Will be set to 1 on first successful operation - externalCAStatus.Set(0) + registry.MustRegister( + certificatesIssuedTotal, + certificatesRevokedTotal, + certificateRequestDuration, + acmeRoundtripDuration, + certificateExpirationTime, + externalCAStatus, + lastSuccessfulCertificateTimestamp, + ) } diff --git a/externalcas/metrics_test.go b/externalcas/metrics_test.go index 8409ebd..65154c2 100644 --- a/externalcas/metrics_test.go +++ b/externalcas/metrics_test.go @@ -146,7 +146,7 @@ func TestCertMetaCollector_Collect_LabelValues(t *testing.T) { } func TestStartMetricsServer_Disabled(t *testing.T) { - if err := StartMetricsServer(metrics{Enabled: false}); err != nil { + if err := StartMetricsServer(metrics{Enabled: false}, ""); err != nil { t.Errorf("StartMetricsServer(disabled) = %v, want nil", err) } }