Skip to content
Open
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions controller/cmd/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -48,6 +48,7 @@ import (
"github.com/jumpstarter-dev/jumpstarter/controller/internal/authorization"
"github.com/jumpstarter-dev/jumpstarter/controller/internal/config"
"github.com/jumpstarter-dev/jumpstarter/controller/internal/controller"
jmpmetrics "github.com/jumpstarter-dev/jumpstarter/controller/internal/metrics"
"github.com/jumpstarter-dev/jumpstarter/controller/internal/oidc"
"github.com/jumpstarter-dev/jumpstarter/controller/internal/service"
"github.com/jumpstarter-dev/jumpstarter/controller/internal/service/login"
Expand Down Expand Up @@ -201,6 +202,8 @@ func main() {
os.Exit(1)
}

jmpmetrics.RegisterDefaults()

oidcCert, err := service.NewSelfSignedCertificate("jumpstarter oidc", []string{"localhost"}, []net.IP{})
if err != nil {
setupLog.Error(err, "unable to generate certificate for internal oidc provider")
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,77 @@
/*
Copyright 2026. The Jumpstarter Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package jumpstarter

import (
"testing"

operatorv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/deploy/operator/api/v1alpha1"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)

// Stdlib unit test (no envtest): asserts Controller Deployment metrics bind for JEP-0013 Phase 2.
func TestControllerDeploymentMetricsBind(t *testing.T) {
r := &JumpstarterReconciler{}
js := &operatorv1alpha1.Jumpstarter{
ObjectMeta: metav1.ObjectMeta{
Name: "jumpstarter",
Namespace: "jumpstarter-lab",
},
Spec: operatorv1alpha1.JumpstarterSpec{
Controller: operatorv1alpha1.ControllerConfig{
Image: "example.com/controller:test",
ImagePullPolicy: corev1.PullIfNotPresent,
Replicas: 1,
},
},
}

dep := r.createControllerDeployment(js, "testhash")
if dep == nil {
t.Fatal("expected non-nil deployment")
}
if len(dep.Spec.Template.Spec.Containers) == 0 {
t.Fatal("expected at least one container")
}

c := dep.Spec.Template.Spec.Containers[0]
foundArg := false
for _, arg := range c.Args {
if arg == "-metrics-bind-address=:8080" {
foundArg = true
break
}
}
if !foundArg {
t.Fatalf("expected -metrics-bind-address=:8080 in args, got %#v", c.Args)
}

foundPort := false
for _, p := range c.Ports {
if p.Name == "metrics" {
foundPort = true
if p.ContainerPort != 8080 {
t.Fatalf("metrics port = %d, want 8080", p.ContainerPort)
}
break
}
}
if !foundPort {
t.Fatal("expected container port named metrics")
}
}
Comment thread
coderabbitai[bot] marked this conversation as resolved.
Outdated
38 changes: 31 additions & 7 deletions controller/internal/controller/lease_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"time"

jumpstarterdevv1alpha1 "github.com/jumpstarter-dev/jumpstarter/controller/api/v1alpha1"
jmpmetrics "github.com/jumpstarter-dev/jumpstarter/controller/internal/metrics"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/meta"
Expand Down Expand Up @@ -242,7 +243,7 @@ func (r *LeaseReconciler) reconcileStatusExporterRef(
Name: lease.Spec.ExporterRef.Name,
}, &exporter); err != nil {
if k8serrors.IsNotFound(err) {
lease.SetStatusUnsatisfiable(
setUnsatisfiableAndRecord(lease,
"ExporterNotFound",
"Requested exporter %s was not found",
lease.Spec.ExporterRef.Name,
Expand All @@ -252,7 +253,7 @@ func (r *LeaseReconciler) reconcileStatusExporterRef(
return fmt.Errorf("reconcileStatusExporterRef: failed to get requested exporter: %w", err)
}
if !selector.Empty() && !selector.Matches(labels.Set(exporter.Labels)) {
lease.SetStatusUnsatisfiable(
setUnsatisfiableAndRecord(lease,
"SelectorMismatch",
"Requested exporter %s does not match selector %s",
exporter.Name,
Expand All @@ -262,7 +263,7 @@ func (r *LeaseReconciler) reconcileStatusExporterRef(
}
// Check if the explicitly requested exporter is disabled
if !exporter.IsEnabled() && !lease.Spec.AllowDisabled {
lease.SetStatusUnsatisfiable(
setUnsatisfiableAndRecord(lease,
"ExporterDisabled",
"Requested exporter %s is disabled. "+
"To lease a disabled exporter, set spec.allowDisabled: true on the Lease, "+
Expand All @@ -281,7 +282,7 @@ func (r *LeaseReconciler) reconcileStatusExporterRef(
// Filter out disabled exporters from selector-based listing
matchingExporters = filterOutDisabledExporters(listed.Items)
if len(matchingExporters) == 0 && len(listed.Items) > 0 {
lease.SetStatusUnsatisfiable(
setUnsatisfiableAndRecord(lease,
"AllDisabled",
"All %d exporters matching the selector are disabled",
len(listed.Items),
Expand All @@ -301,12 +302,12 @@ func (r *LeaseReconciler) reconcileStatusExporterRef(
if len(desc) > 4096 {
desc = desc[:4096] + "..."
}
lease.SetStatusUnsatisfiable("NoAccess",
setUnsatisfiableAndRecord(lease, "NoAccess",
"While there are %d exporters matching the selector, none of them are approved by any policy for your client. Matching policies: %s",
len(matchingExporters), desc,
)
} else {
lease.SetStatusUnsatisfiable("NoAccess",
setUnsatisfiableAndRecord(lease, "NoAccess",
"While there are %d exporters matching the selector, none of them are approved by any policy for your client",
len(matchingExporters),
)
Expand Down Expand Up @@ -336,7 +337,7 @@ func (r *LeaseReconciler) reconcileStatusExporterRef(
orderedExporters := orderApprovedExporters(onlineApprovedExporters)

if len(orderedExporters) > 0 && orderedExporters[0].Policy.SpotAccess {
lease.SetStatusUnsatisfiable("SpotAccess",
setUnsatisfiableAndRecord(lease, "SpotAccess",
"The only possible exporters are under spot access (i.e. %s), but spot access is still not implemented",
orderedExporters[0].Exporter.Name)
return nil
Expand Down Expand Up @@ -389,12 +390,35 @@ func (r *LeaseReconciler) reconcileStatusExporterRef(
lease.Status.ExporterRef = &corev1.LocalObjectReference{
Name: selected.Exporter.Name,
}
recordLeaseAcquisition(lease, jmpmetrics.ResultSuccess)

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

The success metric is recorded here before status is persisted to the API server.

return nil
}

return nil
}

func recordLeaseAcquisition(lease *jumpstarterdevv1alpha1.Lease, result string) {
exemplars := map[string]string{}
if lease != nil {
exemplars["lease_id"] = lease.Name

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

from the pov of exemplars and useful data, each lease_id is different, so it would not provide much statistical value.

What do you think about recording the assigned exporter name instead?

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Having only really run this in development in my own homelab I requested an evaluation of the implications of having lease_id both with and without and additional exporter as the initial take was 'The JEP has a specific design and the lease_id should not be removed'. The results sound reasonable, but as one of the authors of the JEP-0013 you might have a better idea than me if it simply "sounds reasonable" or is actually reasonable:

H5 in detail: lease_id vs exporter name on controller exemplars

mangelajo’s point is right for aggregation: each lease_id is unique, so it does not help “which hardware fails often?” style rates. Exemplars are not for that. JEP-0013 puts unbounded IDs in exemplars for drill-down, and keeps bounded identity like exporter on Prom/Loki labels where series exist for that purpose.

This PR’s series is only:

jumpstarter_lease_acquisitions_total{result="success|failure"}

with default exemplars client + lease_id. There is no exporter label on this counter.


1. With only client + lease_id (today / JEP defaults)

Operator flow when a dashboard shows a spike in failures (or a success blip):

  1. Grafana shows an exemplar dot on jumpstarter_lease_acquisitions_total{result="failure"} (last-seen exemplar per series).
  2. Click → see e.g. {client="ci-bot", lease_id="lease-abc"}.
  3. Use lease_id as the join key into Loki, e.g.
    {component="controller"} | json | lease_id="lease-abc"
    (and the same filter on exporter/cli streams once Phase 1/3 correlation fields are on the lines).
  4. From those log lines you recover the rest: assigned exporter (if any), selector, unsatisfiable reason, spec.context, etc.

So without an exporter exemplar key, lease_id is the primary bridge from “this metric sample” → “the one lease’s logs / CR”. client answers “which client,” not “which DUT.”

Limits of this path:

  • You need a working Loki (or API) lookup; the exemplar panel alone does not name the exporter.
  • Failure acquires often have no exporter assigned (ExporterNotFound, NoAccess, …), so an exporter key would be empty anyway.
  • Last-exemplar-wins: on a busy result="failure" series you only see the most recent lease, not a histogram of exporters (that still comes from logs or from exporter-labeled metrics elsewhere).

2. If exporter is also on the exemplar (in addition to lease_id)

On success (and only when Status.ExporterRef is set), the exemplar might look like:

{client="ci-bot", lease_id="lease-abc", exporter="sidekick-t450s"}

Operator flow:

  1. Same spike / click on the exemplar.
  2. Immediately see which exporter was assigned — useful for “was this the flaky board?” without opening Loki first.
  3. Still use lease_id for precise correlation: one exporter has many leases over time; lease_id picks the exact run and joins to that lease’s log cluster / spec.context.
  4. Optionally pivot: Loki by exporter="sidekick-t450s" for hardware trends, or by lease_id for one CI run.

What this does not replace:

  • Rates per exporter still belong on metrics that already have an exporter label (e.g. planned jumpstarter_operations_total{exporter=...}), not on unique exemplar values.
  • Replacing lease_id with exporter alone would weaken the join: many samples share one exporter name; you lose one-click identity of the lease that caused this observation.

Budget note: OpenMetrics caps exemplars at 128 runes. Adding exporter plus key name costs ~15–40+ characters and can force truncation of lease_id/client (you already log when that happens).


Side-by-side

Goal lease_id only lease_id + exporter
Join metric sample → one lease’s logs Direct Direct (same)
See DUT name in the exemplar popup Indirect (via Loki/API) Direct on success
Failure with no assignment Fine (lease_id still set) exporter omitted/empty
“Which exporter fails often?” Wrong tool (use labeled series / LogQL) Still wrong tool for rates; only a hint on last sample
Cardinality of the Prom series Unchanged Unchanged

Verdict for #932

Keep JEP defaults: client + lease_id. Do not replace lease_id with exporter.

Optional later (or a short PR reply to mangelajo): on success only, also attach exporter when Status.ExporterRef is set, still under the allowlist / 128-rune budget — additive drill-down, not a substitute for lease_id.

That matches the JEP table: lease_id is exemplar-for-drill-down; exporter is primarily a bounded Prom/Loki label on exporter-scoped metrics, not the lease-acquire join key.


Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

@kirkbrauer @raballew As others who reviewed the original JEP-0013 PR #631 will likely be afk for a few days at least maybe you have an educated opinion to chime in with?

if lease.Spec.ClientRef.Name != "" {
exemplars["client"] = lease.Spec.ClientRef.Name
}
}
jmpmetrics.Default.RecordAcquisition(result, exemplars)
}

func setUnsatisfiableAndRecord(lease *jumpstarterdevv1alpha1.Lease, reason, messageFormat string, a ...any) {
already := meta.IsStatusConditionTrue(
lease.Status.Conditions,
string(jumpstarterdevv1alpha1.LeaseConditionTypeUnsatisfiable),
)
lease.SetStatusUnsatisfiable(reason, messageFormat, a...)
if !already {
recordLeaseAcquisition(lease, jmpmetrics.ResultFailure)
}

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Some test for the check that prevents double-counting on re-reconciliation would be nice.

}

// attachMatchingPolicies attaches the matching policies to the list of online exporters
// if the exporter matches the policy and the client matches the policy's client selector
// the exporter is approved for leasing
Expand Down
111 changes: 111 additions & 0 deletions controller/internal/metrics/lease.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,111 @@
/*
Copyright 2026. The Jumpstarter Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

// Package metrics provides Jumpstarter controller Prometheus metrics (JEP-0013 Phase 2).
package metrics

import (
"github.com/prometheus/client_golang/prometheus"
ctrlmetrics "sigs.k8s.io/controller-runtime/pkg/metrics"
)

const (
// LeaseAcquisitionsTotal is the JEP-0013 counter for lease acquire attempts.
LeaseAcquisitionsTotal = "jumpstarter_lease_acquisitions_total"

ResultSuccess = "success"
ResultFailure = "failure"
)

// DefaultExemplarKeys are the JEP-0013 default exemplar allowlist keys.
var DefaultExemplarKeys = []string{"client", "lease_id"}

// LeaseMetrics holds lease-related Prometheus collectors.
type LeaseMetrics struct {
acquisitions *prometheus.CounterVec
}

// NewLeaseMetrics constructs lease metrics. Call Register before RecordAcquisition.
func NewLeaseMetrics() *LeaseMetrics {
return &LeaseMetrics{
acquisitions: prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: LeaseAcquisitionsTotal,
Help: "Lease acquire attempts on the Jumpstarter controller.",
},
[]string{"result"},
),
}
}
Comment on lines +51 to +61

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Counter time series not pre-initialized with zero values. So, before any lease acquisition, scraping /metrics produces TYPE/HELP metadata but no data lines.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Do you need to pre-initialize to 0 ? I assumed that was the default in go.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Zero values in go - " Variables declared without an explicit initial value are given their zero value. "

@raballew raballew Aug 3, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

https://prometheus.io/docs/practices/instrumentation/#avoid-missing-metrics - its lazily initialized hence if you do do not prepopulate no graph will show.


// Register registers collectors with the given registerer.
func (m *LeaseMetrics) Register(r prometheus.Registerer) error {
return r.Register(m.acquisitions)
}

// MustRegister registers collectors and panics on error.
func (m *LeaseMetrics) MustRegister(r prometheus.Registerer) {
r.MustRegister(m.acquisitions)
}

// MustRegisterWithControllerRuntime registers on the controller-runtime metrics registry.
func (m *LeaseMetrics) MustRegisterWithControllerRuntime() {
m.MustRegister(ctrlmetrics.Registry)
}

// RecordAcquisition increments jumpstarter_lease_acquisitions_total for result
// and attaches exemplar labels from the allowlist (client, lease_id when present).
func (m *LeaseMetrics) RecordAcquisition(result string, exemplars map[string]string) {
if m == nil || m.acquisitions == nil {
return
}
metric, err := m.acquisitions.GetMetricWithLabelValues(result)
if err != nil {
return
}
labels := filterExemplarLabels(exemplars)
if len(labels) > 0 {
if adder, ok := metric.(prometheus.ExemplarAdder); ok {
adder.AddWithExemplar(1, labels)
return
Comment thread
coderabbitai[bot] marked this conversation as resolved.
}
}
metric.Inc()
}

func filterExemplarLabels(in map[string]string) prometheus.Labels {
if len(in) == 0 {
return nil
}
out := prometheus.Labels{}
for _, key := range DefaultExemplarKeys {
if v, ok := in[key]; ok && v != "" {
out[key] = v
}
}
if len(out) == 0 {
return nil
}
return out
}

// Default is the process-wide lease metrics instance used by the controller.
var Default = NewLeaseMetrics()

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

All controller tests share the global counter, breaking test isolation.

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Don't those tests run in serial?, is it worth complicating the code in favor of testability? could be, just sharing the question here, I am not familiar with the prometheus metrics framework to be able to figure out if it'd get too complex, or if it really won't.

Copy link
Copy Markdown
Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Evaluated and deferred as not required right now, although may be in the future.

H4. Global Default breaks test isolation
lease.go (var Default)
Claim: All controller tests share one process-wide counter.

Verdict: Soft accept — improve if cheap; don’t over-engineer

Metrics package tests already use NewLeaseMetrics() + private registries. Pain appears only if controller tests assert Default values (or run in parallel against it).

mangelajo’s question is fair: serial Ginkgo reduces flake risk; full DI may not be worth it for Phase 2.

Recommended change: Prefer injecting *LeaseMetrics on LeaseReconciler (defaulting to Default in production). Avoid a large framework change. If injection is deferred, at least don’t assert absolute global counter values in controller tests.


// RegisterDefaults registers Default with the controller-runtime registry.
func RegisterDefaults() {
Default.MustRegisterWithControllerRuntime()
}
Loading
Loading