diff --git a/controller/cmd/main.go b/controller/cmd/main.go index 4123faed1..767a7973a 100644 --- a/controller/cmd/main.go +++ b/controller/cmd/main.go @@ -48,6 +48,7 @@ import ( "github.com/jumpstarter-dev/jumpstarter/controller/internal/authorization" "github.com/jumpstarter-dev/jumpstarter/controller/internal/config" "github.com/jumpstarter-dev/jumpstarter/controller/internal/controller" + jmpmetrics "github.com/jumpstarter-dev/jumpstarter/controller/internal/metrics" "github.com/jumpstarter-dev/jumpstarter/controller/internal/oidc" "github.com/jumpstarter-dev/jumpstarter/controller/internal/service" "github.com/jumpstarter-dev/jumpstarter/controller/internal/service/login" @@ -201,6 +202,8 @@ func main() { os.Exit(1) } + jmpmetrics.RegisterDefaults() + oidcCert, err := service.NewSelfSignedCertificate("jumpstarter oidc", []string{"localhost"}, []net.IP{}) if err != nil { setupLog.Error(err, "unable to generate certificate for internal oidc provider") diff --git a/controller/deploy/operator/internal/controller/jumpstarter/controller_metrics_bind_test.go b/controller/deploy/operator/internal/controller/jumpstarter/controller_metrics_bind_test.go new file mode 100644 index 000000000..64b5985e2 --- /dev/null +++ b/controller/deploy/operator/internal/controller/jumpstarter/controller_metrics_bind_test.go @@ -0,0 +1,66 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package jumpstarter + +import ( + operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/deploy/operator/api/v1alpha1" + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +var _ = Describe("createControllerDeployment metrics bind", func() { + var r *JumpstarterReconciler + var js *operatorv1alpha1.Jumpstarter + + BeforeEach(func() { + r = &JumpstarterReconciler{} + js = &operatorv1alpha1.Jumpstarter{ + ObjectMeta: metav1.ObjectMeta{ + Name: "jumpstarter", + Namespace: "jumpstarter-lab", + }, + Spec: operatorv1alpha1.JumpstarterSpec{ + Controller: operatorv1alpha1.ControllerConfig{ + Image: "example.com/controller:test", + ImagePullPolicy: corev1.PullIfNotPresent, + Replicas: 1, + }, + }, + } + }) + + It("exposes -metrics-bind-address=:8080 and metrics port 8080", func() { + dep := r.createControllerDeployment(js, "testhash") + Expect(dep).NotTo(BeNil()) + Expect(dep.Spec.Template.Spec.Containers).NotTo(BeEmpty()) + + c := dep.Spec.Template.Spec.Containers[0] + Expect(c.Args).To(ContainElement("-metrics-bind-address=:8080")) + + var metricsPort *corev1.ContainerPort + for i := range c.Ports { + if c.Ports[i].Name == "metrics" { + metricsPort = &c.Ports[i] + break + } + } + Expect(metricsPort).NotTo(BeNil(), "expected container port named metrics") + Expect(metricsPort.ContainerPort).To(Equal(int32(8080))) + }) +}) diff --git a/controller/internal/controller/lease_acquisition_metrics_test.go b/controller/internal/controller/lease_acquisition_metrics_test.go new file mode 100644 index 000000000..a6b2376b8 --- /dev/null +++ b/controller/internal/controller/lease_acquisition_metrics_test.go @@ -0,0 +1,75 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package controller + +import ( + "testing" + + jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" + jmpmetrics "github.com/jumpstarter-dev/jumpstarter/controller/internal/metrics" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestLeaseAcquisitionTransitionResult_SuccessOnce(t *testing.T) { + lease := &jumpstarterdevv1alpha1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: "lease-1"}, + Status: jumpstarterdevv1alpha1.LeaseStatus{ + ExporterRef: &corev1.LocalObjectReference{Name: "exporter-a"}, + }, + } + + result, ok := leaseAcquisitionTransitionResult(nil, false, lease) + if !ok || result != jmpmetrics.ResultSuccess { + t.Fatalf("first transition: got (%q, %v), want (%q, true)", result, ok, jmpmetrics.ResultSuccess) + } + + // Reconcile retry after status persisted: priorExporterRef is set. + result, ok = leaseAcquisitionTransitionResult(lease.Status.ExporterRef, false, lease) + if ok { + t.Fatalf("second observation after persist must not record, got %q", result) + } +} + +func TestLeaseAcquisitionTransitionResult_FailureOnce(t *testing.T) { + lease := &jumpstarterdevv1alpha1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: "lease-2"}, + } + lease.SetStatusUnsatisfiable("NoAccess", "no exporters approved") + + result, ok := leaseAcquisitionTransitionResult(nil, false, lease) + if !ok || result != jmpmetrics.ResultFailure { + t.Fatalf("first unsatisfiable transition: got (%q, %v), want (%q, true)", result, ok, jmpmetrics.ResultFailure) + } + + // Reconcile retry after Unsatisfiable was persisted. + result, ok = leaseAcquisitionTransitionResult(nil, true, lease) + if ok { + t.Fatalf("second unsatisfiable observation after persist must not record, got %q", result) + } +} + +func TestLeaseAcquisitionTransitionResult_NoTransition(t *testing.T) { + lease := &jumpstarterdevv1alpha1.Lease{ + ObjectMeta: metav1.ObjectMeta{Name: "lease-3"}, + } + + result, ok := leaseAcquisitionTransitionResult(nil, false, lease) + if ok { + t.Fatalf("pending lease with no exporter/unsatisfiable must not record, got %q", result) + } +} diff --git a/controller/internal/controller/lease_controller.go b/controller/internal/controller/lease_controller.go index 42fb71ae1..5d78c1339 100755 --- a/controller/internal/controller/lease_controller.go +++ b/controller/internal/controller/lease_controller.go @@ -24,6 +24,7 @@ import ( "time" jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1" + jmpmetrics "github.com/jumpstarter-dev/jumpstarter/controller/internal/metrics" corev1 "k8s.io/api/core/v1" k8serrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/api/meta" @@ -93,6 +94,12 @@ func (r *LeaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl ctx = ctrl.LoggerInto(ctx, logger) var result ctrl.Result + priorExporterRef := lease.Status.ExporterRef + priorUnsatisfiable := meta.IsStatusConditionTrue( + lease.Status.Conditions, + string(jumpstarterdevv1alpha1.LeaseConditionTypeUnsatisfiable), + ) + if err := r.reconcileStatusExporterRef(ctx, &result, &lease); err != nil { return result, err } @@ -109,6 +116,10 @@ func (r *LeaseReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl return RequeueConflict(logger, result, err) } + // Record acquisition only after status is persisted so a failed Update + // cannot double-count on requeue (success or failure). + recordLeaseAcquisitionTransition(ctx, &lease, priorExporterRef, priorUnsatisfiable) + if lease.Labels == nil { lease.Labels = make(map[string]string) } @@ -395,6 +406,55 @@ func (r *LeaseReconciler) reconcileStatusExporterRef( return nil } +// leaseAcquisitionTransitionResult returns the metric result for a persisted +// status transition, or ("", false) when no acquisition should be recorded. +// Recording is based on the pre-reconcile persisted snapshot so requeues after +// a successful Status().Update do not double-count. +func leaseAcquisitionTransitionResult( + priorExporterRef *corev1.LocalObjectReference, + priorUnsatisfiable bool, + lease *jumpstarterdevv1alpha1.Lease, +) (string, bool) { + if lease == nil { + return "", false + } + if priorExporterRef == nil && lease.Status.ExporterRef != nil { + return jmpmetrics.ResultSuccess, true + } + nowUnsatisfiable := meta.IsStatusConditionTrue( + lease.Status.Conditions, + string(jumpstarterdevv1alpha1.LeaseConditionTypeUnsatisfiable), + ) + if !priorUnsatisfiable && nowUnsatisfiable { + return jmpmetrics.ResultFailure, true + } + return "", false +} + +func recordLeaseAcquisitionTransition( + ctx context.Context, + lease *jumpstarterdevv1alpha1.Lease, + priorExporterRef *corev1.LocalObjectReference, + priorUnsatisfiable bool, +) { + result, ok := leaseAcquisitionTransitionResult(priorExporterRef, priorUnsatisfiable, lease) + if !ok { + return + } + recordLeaseAcquisition(ctx, lease, result) +} + +func recordLeaseAcquisition(ctx context.Context, lease *jumpstarterdevv1alpha1.Lease, result string) { + exemplars := map[string]string{} + if lease != nil { + exemplars["lease_id"] = lease.Name + if lease.Spec.ClientRef.Name != "" { + exemplars["client"] = lease.Spec.ClientRef.Name + } + } + jmpmetrics.Default.RecordAcquisition(ctx, result, exemplars) +} + // attachMatchingPolicies attaches the matching policies to the list of online exporters // if the exporter matches the policy and the client matches the policy's client selector // the exporter is approved for leasing diff --git a/controller/internal/metrics/lease.go b/controller/internal/metrics/lease.go new file mode 100644 index 000000000..91b1f03bd --- /dev/null +++ b/controller/internal/metrics/lease.go @@ -0,0 +1,208 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package metrics provides Jumpstarter controller Prometheus metrics (JEP-0013 Phase 2). +package metrics + +import ( + "context" + "unicode/utf8" + + jlog "github.com/jumpstarter-dev/jumpstarter/controller/internal/log" + "github.com/prometheus/client_golang/prometheus" + "sigs.k8s.io/controller-runtime/pkg/log" + ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics" +) + +const ( + // LeaseAcquisitionsTotal is the JEP-0013 counter for lease acquire attempts. + LeaseAcquisitionsTotal = "jumpstarter_lease_acquisitions_total" + + ResultSuccess = "success" + ResultFailure = "failure" + + // maxExemplarRunes is the OpenMetrics 1.0 combined label name+value rune limit. + // Exceeding it makes prometheus.ExemplarAdder.AddWithExemplar panic. + maxExemplarRunes = 128 +) + +// DefaultExemplarKeys are the JEP-0013 default exemplar allowlist keys. +var DefaultExemplarKeys = []string{"client", "lease_id"} + +// LeaseMetrics holds lease-related Prometheus collectors. +type LeaseMetrics struct { + acquisitions *prometheus.CounterVec +} + +// NewLeaseMetrics constructs lease metrics. Call Register before RecordAcquisition. +func NewLeaseMetrics() *LeaseMetrics { + return &LeaseMetrics{ + acquisitions: prometheus.NewCounterVec( + prometheus.CounterOpts{ + Name: LeaseAcquisitionsTotal, + Help: "Lease acquire attempts on the Jumpstarter controller.", + }, + []string{"result"}, + ), + } +} + +// Register registers collectors with the given registerer and pre-creates +// the success/failure series at zero so scrapes are never missing samples. +func (m *LeaseMetrics) Register(r prometheus.Registerer) error { + if err := r.Register(m.acquisitions); err != nil { + return err + } + m.initializeSeries() + return nil +} + +// MustRegister registers collectors and panics on error. +func (m *LeaseMetrics) MustRegister(r prometheus.Registerer) { + r.MustRegister(m.acquisitions) + m.initializeSeries() +} + +// MustRegisterWithControllerRuntime registers on the controller-runtime metrics registry. +func (m *LeaseMetrics) MustRegisterWithControllerRuntime() { + m.MustRegister(ctrlmetrics.Registry) +} + +// initializeSeries ensures result=success and result=failure time series exist at 0. +// CounterVec is lazily created; without this, /metrics shows HELP/TYPE but no samples +// until the first observation (https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics). +func (m *LeaseMetrics) initializeSeries() { + if m == nil || m.acquisitions == nil { + return + } + _, _ = m.acquisitions.GetMetricWithLabelValues(ResultSuccess) + _, _ = m.acquisitions.GetMetricWithLabelValues(ResultFailure) +} + +// RecordAcquisition increments jumpstarter_lease_acquisitions_total for result +// and attaches exemplar labels from the allowlist (client, lease_id when present). +// Invalid or over-budget exemplar values are truncated or dropped; the counter +// always increments. Budget truncation/drops are logged at Warning (logr V(1)). +func (m *LeaseMetrics) RecordAcquisition(ctx context.Context, result string, exemplars map[string]string) { + if m == nil || m.acquisitions == nil { + return + } + metric, err := m.acquisitions.GetMetricWithLabelValues(result) + if err != nil { + return + } + filtered := filterExemplarLabels(exemplars) + labels, constrained := constrainExemplarLabels(filtered) + if constrained { + jlog.Warning(log.FromContext(ctx), + "lease acquisition exemplar exceeded OpenMetrics budget; truncated or dropped labels", + "budget_runes", maxExemplarRunes, + "result", result, + "original", filtered, + "constrained", labels, + ) + } + if len(labels) > 0 { + if adder, ok := metric.(prometheus.ExemplarAdder); ok { + adder.AddWithExemplar(1, labels) + return + } + } + metric.Inc() +} + +func filterExemplarLabels(in map[string]string) prometheus.Labels { + if len(in) == 0 { + return nil + } + out := prometheus.Labels{} + for _, key := range DefaultExemplarKeys { + if v, ok := in[key]; ok && v != "" { + out[key] = v + } + } + if len(out) == 0 { + return nil + } + return out +} + +// constrainExemplarLabels validates UTF-8 and fits labels into the OpenMetrics +// 128-rune exemplar budget in DefaultExemplarKeys order. Values are truncated +// when needed; keys that cannot fit (even truncated) are dropped. +// The bool is true when any label was truncated or dropped for the budget. +func constrainExemplarLabels(in prometheus.Labels) (prometheus.Labels, bool) { + if len(in) == 0 { + return nil, false + } + out := prometheus.Labels{} + constrained := false + remaining := maxExemplarRunes + for _, key := range DefaultExemplarKeys { + v, ok := in[key] + if !ok || v == "" { + continue + } + if !utf8.ValidString(key) || !utf8.ValidString(v) { + constrained = true + continue + } + keyRunes := utf8.RuneCountInString(key) + if keyRunes >= remaining { + constrained = true + continue + } + valueBudget := remaining - keyRunes + valueRunes := utf8.RuneCountInString(v) + if valueRunes > valueBudget { + constrained = true + v = truncateRunes(v, valueBudget) + if v == "" { + continue + } + valueRunes = utf8.RuneCountInString(v) + } + out[key] = v + remaining -= keyRunes + valueRunes + } + if len(out) == 0 { + return nil, constrained + } + return out, constrained +} + +func truncateRunes(s string, maxRunes int) string { + if maxRunes <= 0 { + return "" + } + if utf8.RuneCountInString(s) <= maxRunes { + return s + } + i := 0 + for n := 0; n < maxRunes; n++ { + _, size := utf8.DecodeRuneInString(s[i:]) + i += size + } + return s[:i] +} + +// Default is the process-wide lease metrics instance used by the controller. +var Default = NewLeaseMetrics() + +// RegisterDefaults registers Default with the controller-runtime registry. +func RegisterDefaults() { + Default.MustRegisterWithControllerRuntime() +} diff --git a/controller/internal/metrics/lease_test.go b/controller/internal/metrics/lease_test.go new file mode 100644 index 000000000..e40a1316f --- /dev/null +++ b/controller/internal/metrics/lease_test.go @@ -0,0 +1,295 @@ +/* +Copyright 2026. The Jumpstarter Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package metrics + +import ( + "context" + "strings" + "testing" + "unicode/utf8" + + "github.com/go-logr/logr/funcr" + jlog "github.com/jumpstarter-dev/jumpstarter/controller/internal/log" + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "sigs.k8s.io/controller-runtime/pkg/log" +) + +func TestLeaseAcquisitionsMetricName(t *testing.T) { + if LeaseAcquisitionsTotal != "jumpstarter_lease_acquisitions_total" { + t.Fatalf("unexpected metric name %q", LeaseAcquisitionsTotal) + } +} + +func TestRegisterInitializesZeroSeries(t *testing.T) { + reg := prometheus.NewRegistry() + m := NewLeaseMetrics() + if err := m.Register(reg); err != nil { + t.Fatalf("register: %v", err) + } + + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + var found *dto.MetricFamily + for _, f := range families { + if f.GetName() == LeaseAcquisitionsTotal { + found = f + break + } + } + if found == nil { + t.Fatalf("metric %s not found after register", LeaseAcquisitionsTotal) + } + success := sampleValue(found, map[string]string{"result": ResultSuccess}) + failure := sampleValue(found, map[string]string{"result": ResultFailure}) + if success != 0 { + t.Fatalf("success count = %v, want 0 before observations", success) + } + if failure != 0 { + t.Fatalf("failure count = %v, want 0 before observations", failure) + } +} + +func TestRecordLeaseAcquisitionIncrementsCounter(t *testing.T) { + reg := prometheus.NewRegistry() + m := NewLeaseMetrics() + if err := m.Register(reg); err != nil { + t.Fatalf("register: %v", err) + } + + ctx := context.Background() + m.RecordAcquisition(ctx, ResultSuccess, map[string]string{ + "client": "p16v", + "lease_id": "lease-abc", + }) + m.RecordAcquisition(ctx, ResultFailure, map[string]string{ + "client": "p16v", + "lease_id": "lease-def", + }) + + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + + var found *dto.MetricFamily + for _, f := range families { + if f.GetName() == LeaseAcquisitionsTotal { + found = f + break + } + } + if found == nil { + t.Fatalf("metric %s not found in registry output", LeaseAcquisitionsTotal) + } + if found.GetType() != dto.MetricType_COUNTER { + t.Fatalf("expected COUNTER, got %v", found.GetType()) + } + + success := sampleValue(found, map[string]string{"result": ResultSuccess}) + failure := sampleValue(found, map[string]string{"result": ResultFailure}) + if success != 1 { + t.Fatalf("success count = %v, want 1", success) + } + if failure != 1 { + t.Fatalf("failure count = %v, want 1", failure) + } +} + +func TestRecordLeaseAcquisitionAttachesExemplars(t *testing.T) { + reg := prometheus.NewRegistry() + m := NewLeaseMetrics() + if err := m.Register(reg); err != nil { + t.Fatalf("register: %v", err) + } + + m.RecordAcquisition(context.Background(), ResultSuccess, map[string]string{ + "client": "ci-bot", + "lease_id": "lease-xyz", + "ignored": "should-not-appear", + }) + + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + + var metric *dto.Metric + for _, f := range families { + if f.GetName() != LeaseAcquisitionsTotal { + continue + } + for _, met := range f.Metric { + if labelValue(met, "result") == ResultSuccess { + metric = met + break + } + } + } + if metric == nil { + t.Fatal("success sample not found") + } + if metric.GetCounter() == nil || metric.GetCounter().GetExemplar() == nil { + t.Fatal("expected exemplar on success observation") + } + + ex := exemplarMap(metric.GetCounter().GetExemplar()) + if ex["client"] != "ci-bot" { + t.Fatalf("exemplar client = %q, want ci-bot", ex["client"]) + } + if ex["lease_id"] != "lease-xyz" { + t.Fatalf("exemplar lease_id = %q, want lease-xyz", ex["lease_id"]) + } + if _, ok := ex["ignored"]; ok { + t.Fatal("non-allowlisted exemplar key should not be present") + } +} + +func TestPrometheusTextContainsSeries(t *testing.T) { + reg := prometheus.NewRegistry() + m := NewLeaseMetrics() + if err := m.Register(reg); err != nil { + t.Fatalf("register: %v", err) + } + m.RecordAcquisition(context.Background(), ResultSuccess, map[string]string{"client": "c", "lease_id": "l"}) + + var b strings.Builder + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + for _, f := range families { + if _, err := expfmt.MetricFamilyToText(&b, f); err != nil { + t.Fatalf("encode: %v", err) + } + } + out := b.String() + if !strings.Contains(out, LeaseAcquisitionsTotal) { + t.Fatalf("Prometheus text missing %s:\n%s", LeaseAcquisitionsTotal, out) + } + if !strings.Contains(out, `result="success"`) { + t.Fatalf("Prometheus text missing result label:\n%s", out) + } +} + +func TestRecordAcquisitionTruncatesOverBudgetExemplars(t *testing.T) { + reg := prometheus.NewRegistry() + m := NewLeaseMetrics() + if err := m.Register(reg); err != nil { + t.Fatalf("register: %v", err) + } + + var logged []string + logger := funcr.New(func(_, args string) { + logged = append(logged, args) + }, funcr.Options{Verbosity: jlog.LevelWarning}) + ctx := log.IntoContext(context.Background(), logger) + + // Combined key+value runes exceed OpenMetrics' 128 limit; must not panic, + // must still increment, and must attach a constrained exemplar. + long := strings.Repeat("a", 200) + m.RecordAcquisition(ctx, ResultSuccess, map[string]string{ + "client": long, + "lease_id": long, + }) + + if len(logged) == 0 { + t.Fatal("expected warning log when exemplar budget is exceeded") + } + if !strings.Contains(logged[0], "OpenMetrics budget") { + t.Fatalf("unexpected log message %q", logged[0]) + } + + families, err := reg.Gather() + if err != nil { + t.Fatalf("gather: %v", err) + } + var found *dto.MetricFamily + for _, f := range families { + if f.GetName() == LeaseAcquisitionsTotal { + found = f + break + } + } + if found == nil { + t.Fatalf("metric %s not found", LeaseAcquisitionsTotal) + } + if sampleValue(found, map[string]string{"result": ResultSuccess}) != 1 { + t.Fatal("counter must increment even when exemplars are truncated") + } + + var metric *dto.Metric + for _, met := range found.Metric { + if labelValue(met, "result") == ResultSuccess { + metric = met + break + } + } + if metric == nil || metric.GetCounter() == nil || metric.GetCounter().GetExemplar() == nil { + t.Fatal("expected constrained exemplar on success observation") + } + ex := exemplarMap(metric.GetCounter().GetExemplar()) + total := 0 + for k, v := range ex { + total += utf8.RuneCountInString(k) + utf8.RuneCountInString(v) + } + if total > maxExemplarRunes { + t.Fatalf("exemplar rune budget exceeded: %d > %d (%v)", total, maxExemplarRunes, ex) + } + if ex["client"] == "" { + t.Fatal("expected truncated client exemplar to be retained") + } + if utf8.RuneCountInString(ex["client"]) > maxExemplarRunes-utf8.RuneCountInString("client") { + t.Fatalf("client exemplar value too long: %q", ex["client"]) + } +} + +func sampleValue(f *dto.MetricFamily, want map[string]string) float64 { + for _, m := range f.Metric { + ok := true + for k, v := range want { + if labelValue(m, k) != v { + ok = false + break + } + } + if ok { + return m.GetCounter().GetValue() + } + } + return 0 +} + +func labelValue(m *dto.Metric, name string) string { + for _, l := range m.Label { + if l.GetName() == name { + return l.GetValue() + } + } + return "" +} + +func exemplarMap(e *dto.Exemplar) map[string]string { + out := make(map[string]string) + for _, l := range e.Label { + out[l.GetName()] = l.GetValue() + } + return out +}