diff --git a/.gitignore b/.gitignore index 36f673b28..8bcdae200 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,11 @@ __pycache__/ test/e2e/**/output test/statistics-analysis/output +*.test + +# Extracted sysfs fixture directories used by pkg/sysfs, pkg/resmgr/lib/memory, +# and pkg/topology tests. These are unpacked from tarballs at test time and +# should not be tracked. +pkg/sysfs/testdata/ +pkg/resmgr/lib/memory/testdata/ +pkg/topology/testdata/ diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 4bf4c71fd..d0058cfc1 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -254,6 +254,12 @@ func (p *balloons) Start() error { return nil } +// Stop shuts down this policy. The balloons policy holds no resources +// to release. +func (p *balloons) Stop() error { + return nil +} + // Sync synchronizes the active policy state. func (p *balloons) Sync(add []cache.Container, del []cache.Container) error { irq.BlockWrites() diff --git a/cmd/plugins/template/policy/template-policy.go b/cmd/plugins/template/policy/template-policy.go index 443be4d5a..6c04c295d 100644 --- a/cmd/plugins/template/policy/template-policy.go +++ b/cmd/plugins/template/policy/template-policy.go @@ -75,6 +75,12 @@ func (p *policy) Start() error { return nil } +// Stop shuts down this policy. The template policy holds no resources +// to release. +func (p *policy) Stop() error { + return nil +} + // Reconfigure this policy. func (p *policy) Reconfigure(newCfg any) error { cfg, ok := newCfg.(*cfgapi.Config) diff --git a/cmd/plugins/topology-aware/policy/cache_test.go b/cmd/plugins/topology-aware/policy/cache_test.go index 7ce1a7b80..57ba09e7c 100644 --- a/cmd/plugins/topology-aware/policy/cache_test.go +++ b/cmd/plugins/topology-aware/policy/cache_test.go @@ -88,11 +88,11 @@ func TestAllocationMarshalling(t *testing.T) { }{ { name: "non-zero Exclusive", - data: []byte(`{"key1":{"PrettyName":"","Exclusive":"1","Part":1,"CPUType":0,"Container":"1","Pool":"testnode","MemoryPool":0,"MemType":"DRAM,PMEM,HBM","MemSize":0,"ColdStart":0}}`), + data: []byte(`{"key1":{"PrettyName":"","Exclusive":"1","Part":1,"CPUType":0,"CPUClass":"","Container":"1","Pool":"testnode","MemoryPool":0,"MemType":"DRAM,PMEM,HBM","MemSize":0,"ColdStart":0,"Irqs":null}}`), }, { name: "zero Exclusive", - data: []byte(`{"key1":{"PrettyName":"","Exclusive":"","Part":1,"CPUType":0,"Container":"1","Pool":"testnode","MemoryPool":0,"MemType":"DRAM,PMEM,HBM","MemSize":0,"ColdStart":0}}`), + data: []byte(`{"key1":{"PrettyName":"","Exclusive":"","Part":1,"CPUType":0,"CPUClass":"","Container":"1","Pool":"testnode","MemoryPool":0,"MemType":"DRAM,PMEM,HBM","MemSize":0,"ColdStart":0,"Irqs":null}}`), }, } for _, tc := range tcases { diff --git a/cmd/plugins/topology-aware/policy/dra.go b/cmd/plugins/topology-aware/policy/dra.go new file mode 100644 index 000000000..23d960106 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra.go @@ -0,0 +1,123 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" + + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// buildDRAPlugin constructs p.draPlugin from the current policy +// configuration (p.cfg, p.cpuClasses, p.cache) and the given backend +// options. Called once from Setup() when cfg.DRAEnabled() is true, and +// never from Reconfigure() (which refuses any change to DRAEnabled outright +// rather than tearing down/rebuilding p.draPlugin). +// +// A missing kube client or node name is treated as "DRA not ready yet" +// rather than a hard Setup() failure: this logs a warning and leaves +// p.draPlugin nil. Every other Backend lifecycle method already nil-checks +// p.draPlugin, so this degrades to "DRA disabled" rather than crashing. +// +// An empty (nil) p.cpuClasses is different: the plugin is still built, just +// with an empty device set (every *cpuclass.Handler method the adapter +// calls is nil-receiver-safe, see policyDRAAdapter's doc comment). Since +// buildDRAPlugin only ever runs once, leaving p.draPlugin nil here would +// prevent a later Reconfigure() that adds cpuClasses from ever enabling DRA. +// +// Genuine construction failures (CDI writer setup, dra.New's own +// dependency validation) are returned as errors, since those indicate a +// real misconfiguration rather than a timing issue. +func (p *policy) buildDRAPlugin(opts *policyapi.BackendOptions) error { + if opts.KubeClientFn == nil { + log.Warnf("dra: no KubeClientFn provided, DRA plugin not started") + return nil + } + kubeClient := opts.KubeClientFn() + if kubeClient == nil { + log.Warnf("dra: no kube client available yet, DRA plugin not started") + return nil + } + if opts.NodeName == "" { + log.Warnf("dra: node name not known yet, DRA plugin not started") + return nil + } + + adapter := &policyDRAAdapter{p: p} + + cdiWriter, err := dra.NewCDIWriter(DRADriverName, p.cdiDir) + if err != nil { + return policyError("failed to create DRA CDI writer: %w", err) + } + + deps := dra.Deps{ + KubeClient: kubeClient, + NodeName: opts.NodeName, + // ValidateClasses captures p, not a config snapshot, so it observes + // whatever p.cfg is live when called, including after Reconfigure() + // swaps it. DRASharedCounters() is the nil-safe getter — without it, + // a Reconfigure that removes the dra: section would panic on + // p.cfg.DRA.SharedCounters here. + ValidateClasses: func() error { + return cpuclass.ValidateCPUClassesForDRA(p.cfg.CPUClasses, p.cfg.DRASharedCounters()) + }, + // ValidateCPUsInPool mirrors the check allocateClaim performs via + // poolForCPUs at container-creation time, so a claim whose picked + // CPUs straddle more than one leaf pool (e.g. a punit spanning an + // entire package while leaf pools are NUMA or L3 nodes) is rejected + // at Prepare time instead of being persisted and always failing + // later when its container is created. + ValidateCPUsInPool: func(cpus cpuset.CPUSet) error { + _, err := p.poolForCPUs(cpus) + return err + }, + DeviceLister: adapter, + ClaimAllocator: adapter, + CDIWriter: cdiWriter, + ClaimStore: dra.NewCacheClaimStore(p.cache), + ClaimUnprepare: p.unprepareDRAClaim, + WithLock: opts.WithLock, + Logger: log, + } + + plugin, err := dra.New(DRADriverName, deps) + if err != nil { + return policyError("failed to create DRA plugin: %w", err) + } + + p.draPlugin = plugin + + return nil +} + +// triggerDRARepublish enqueues a DRA ResourceSlice republication when +// PCT-based HP capacity is active. Called (deferred) from AllocateResources, +// ReleaseResources, and UpdateResources so that any change to non-DRA HP CPU +// usage (hpUsed) caused by those NRI-path allocations is reflected in the +// next ResourceSlice. +// The enqueue is non-blocking and safe to call while holding the resmgr lock; +// the actual publish runs in the DRA plugin's republisherLoop goroutine, which +// acquires the lock itself after the NRI handler releases it. +func (p *policy) triggerDRARepublish() { + if p.draPlugin == nil { + return + } + if p.cpuClasses == nil || !p.cpuClasses.PctActive() { + return + } + p.draPlugin.TriggerRepublish() +} diff --git a/cmd/plugins/topology-aware/policy/dra_adapter.go b/cmd/plugins/topology-aware/policy/dra_adapter.go new file mode 100644 index 000000000..d4ed7e639 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra_adapter.go @@ -0,0 +1,75 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + resourceapi "k8s.io/api/resource/v1" + + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// DRADriverName is the DRA driver name used to publish and identify CPU +// devices for the topology-aware policy. Defined once here; referenced +// everywhere else DRA devices for this policy are published or identified. +const DRADriverName = "nri.topology-aware.cpu" + +// policyDRAAdapter implements dra.ClaimAllocator and dra.DeviceLister by +// forwarding every call to the *current* p.cpuClasses at call time. +// +// A field holding *cpuclass.Handler directly (captured once, at +// construction) would go stale: initialize() sets p.cpuClasses = nil and +// installs a brand new *cpuclass.Handler on every Reconfigure. Because this +// adapter only ever holds the *policy back-pointer and dereferences +// p.cpuClasses fresh on each call, it always observes the handler that is +// live at call time. +// +// All *cpuclass.Handler methods used here are nil-receiver-safe (they check +// h == nil internally and return zero-values/errors), so no additional nil +// guard is required in the adapter for a nil p.cpuClasses. +type policyDRAAdapter struct { + p *policy +} + +// Make sure policyDRAAdapter implements the interfaces the DRA plugin needs. +var ( + _ dra.ClaimAllocator = &policyDRAAdapter{} + _ dra.DeviceLister = &policyDRAAdapter{} +) + +// PickHpCpus routes to the current p.cpuClasses handler's PickHpCpus. +func (a *policyDRAAdapter) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { + return a.p.cpuClasses.PickHpCpus(pkgID, punitID, n, held) +} + +// ReleaseHpCpus routes to the current p.cpuClasses handler's ReleaseHpCpus. +func (a *policyDRAAdapter) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { + a.p.cpuClasses.ReleaseHpCpus(pkgID, punitID, cpus) +} + +// AccountHpCpus routes to the current p.cpuClasses handler's AccountHpCpus. +func (a *policyDRAAdapter) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { + return a.p.cpuClasses.AccountHpCpus(pkgID, punitID, cpus) +} + +// IsHPClass routes to the current p.cpuClasses handler's IsHPClass. +func (a *policyDRAAdapter) IsHPClass(className string) bool { + return a.p.cpuClasses.IsHPClass(className) +} + +// DRADevices routes to the current p.cpuClasses handler's DRADevices. +func (a *policyDRAAdapter) DRADevices(driverName string) ([]resourceapi.Device, error) { + return a.p.cpuClasses.DRADevices(driverName) +} diff --git a/cmd/plugins/topology-aware/policy/dra_adapter_test.go b/cmd/plugins/topology-aware/policy/dra_adapter_test.go new file mode 100644 index 000000000..8d1803971 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra_adapter_test.go @@ -0,0 +1,162 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "testing" + + idset "github.com/intel/goresctrl/pkg/utils" + + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// adapterTestSys is a minimal sysfs.System implementation sufficient for +// cpuclass.New()/Configure(). Only CPUIDs is overridden; every other method +// is delegated to the embedded nil interface, which panics if called (and is +// never called by cpuclass.New/Configure in practice). +type adapterTestSys struct { + sysfs.System +} + +func (s *adapterTestSys) CPUIDs() []idset.ID { return nil } + +// newActiveClassHandler builds a *cpuclass.Handler with an active managed +// PCT allocator (via the goresctrl SST in-memory mock), one HP class named +// "hp". Mirrors cpuclass.newConfiguredHandler (internal, unexported), kept +// in sync intentionally — this package cannot import that internal test +// helper. +func newActiveClassHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", `{"supported":true,"clos_count":4,"packages":[{"id":0,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"punits":[{"id":0,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":4}]}]}`) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + h, err := cpuclass.New(&adapterTestSys{}) + if err != nil { + t.Fatalf("cpuclass.New() failed: %v", err) + } + if err := h.Configure(cpuclass.ConfigSpec{ + Classes: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }); err != nil { + t.Fatalf("Configure() failed: %v", err) + } + return h +} + +// newInactiveClassHandler builds a *cpuclass.Handler without SST support, so +// PCT stays disabled ("inactive"): all DRA-facing methods report their +// nil/zero-value defaults. +func newInactiveClassHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", "") + h, err := cpuclass.New(&adapterTestSys{}) + if err != nil { + t.Fatalf("cpuclass.New() failed: %v", err) + } + _ = h.Configure(cpuclass.ConfigSpec{ + Classes: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }) + return h +} + +// TestPolicyDRAAdapterRoutesToCurrentHandler verifies the adapter forwards +// to whatever *cpuclass.Handler is installed in p.cpuClasses *at call time*, +// not to a handler captured once at construction. This matters because +// initialize() sets p.cpuClasses = nil and installs a brand new Handler on +// every Reconfigure — a cached pointer would go stale. +func TestPolicyDRAAdapterRoutesToCurrentHandler(t *testing.T) { + p := &policy{} + a := &policyDRAAdapter{p: p} + + // 1. cpuClasses nil ("no cpuClass config yet"): all methods must return + // safe zero-values, no panic. + p.cpuClasses = nil + if a.IsHPClass("hp") { + t.Error("IsHPClass with nil cpuClasses: got true, want false") + } + if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus with nil cpuClasses: got nil error, want error") + } + if devs, err := a.DRADevices(DRADriverName); err != nil || len(devs) != 0 { + t.Errorf("DRADevices with nil cpuClasses: got (%v, %v), want (empty, nil)", devs, err) + } + a.ReleaseHpCpus(0, 0, cpuset.New()) // must not panic + if err := a.AccountHpCpus(0, 0, cpuset.New()); err == nil { + t.Error("AccountHpCpus with nil cpuClasses: got nil error, want error") + } + + // 2. Swap in an inactive handler (PCT unsupported): still all + // zero-value defaults, but now backed by a real (non-nil) Handler. + p.cpuClasses = newInactiveClassHandler(t) + if a.IsHPClass("hp") { + t.Error("IsHPClass with inactive handler: got true, want false") + } + if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus with inactive handler: got nil error, want error") + } + + // 3. Swap in an active handler with class "hp": the adapter must + // immediately observe the new handler's behavior — proving it reads + // p.cpuClasses fresh on every call rather than a cached pointer. + p.cpuClasses = newActiveClassHandler(t) + if !a.IsHPClass("hp") { + t.Error("IsHPClass with active handler: got false, want true") + } + cpus, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus with active handler: %v", err) + } + if cpus.Size() != 2 { + t.Errorf("PickHpCpus with active handler: got %d CPUs, want 2", cpus.Size()) + } + devs, err := a.DRADevices(DRADriverName) + if err != nil { + t.Fatalf("DRADevices with active handler: %v", err) + } + if len(devs) == 0 { + t.Error("DRADevices with active handler: got 0 devices, want > 0") + } + a.ReleaseHpCpus(0, 0, cpus) + if err := a.AccountHpCpus(0, 0, cpus); err != nil { + t.Errorf("AccountHpCpus with active handler: %v", err) + } + + // 4. Swap back to nil: must return to safe zero-values, proving the + // forwarding is not sticky/cached from step 3. + p.cpuClasses = nil + if a.IsHPClass("hp") { + t.Error("IsHPClass after swap back to nil: got true, want false") + } +} + +func TestPolicyDRAAdapterNilCpuClassesNoPanic(t *testing.T) { + p := &policy{cpuClasses: nil} + a := &policyDRAAdapter{p: p} + + defer func() { + if r := recover(); r != nil { + t.Fatalf("adapter method panicked with nil cpuClasses: %v", r) + } + }() + + _, _ = a.PickHpCpus(0, 0, 1, cpuset.New()) + a.ReleaseHpCpus(0, 0, cpuset.New()) + _ = a.AccountHpCpus(0, 0, cpuset.New()) + _ = a.IsHPClass("hp") + _, _ = a.DRADevices(DRADriverName) +} diff --git a/cmd/plugins/topology-aware/policy/dra_test.go b/cmd/plugins/topology-aware/policy/dra_test.go new file mode 100644 index 000000000..7b97e3084 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra_test.go @@ -0,0 +1,511 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "context" + "os" + "path" + "strings" + "testing" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" + system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/testutils" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// setupDRATestPolicy builds a real *policy from the "server" sysfs test +// data (same fixture as newDRATestPolicy in resources_test.go) and runs it +// through Setup(). mutateCfg and mutateOpts (both optional) let each test +// case tweak the Config / BackendOptions before Setup() runs; preSetup +// (optional) runs on the freshly constructed, not-yet-Setup *policy — used +// to inject p.cdiDir before buildDRAPlugin would otherwise fall back to the +// real /var/run/cdi default. +func setupDRATestPolicy( + t *testing.T, + mutateCfg func(*cfgapi.Config), + mutateOpts func(*policyapi.BackendOptions), + preSetup func(*policy), +) (*policy, error) { + t.Helper() + + dir, err := os.MkdirTemp("", "nri-resource-policy-test-sysfs-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + t.Cleanup(func() { removeAll(t, dir) }) + + if err := testutils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { + t.Fatalf("failed to uncompress test sysfs data: %v", err) + } + + sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + if err != nil { + t.Fatalf("failed to discover test system: %v", err) + } + + cfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{ + cfgapi.CPU: "750m", + }, + } + if mutateCfg != nil { + mutateCfg(cfg) + } + + opts := &policyapi.BackendOptions{ + Cache: &mockCache{}, + System: sys, + Config: cfg, + } + if mutateOpts != nil { + mutateOpts(opts) + } + + p := New().(*policy) + if preSetup != nil { + preSetup(p) + } + + return p, p.Setup(opts) +} + +// withOneHPClass sets a single valid HP cpuClass, sufficient to make +// initialize() install a non-nil p.cpuClasses handler. +func withOneHPClass(cfg *cfgapi.Config) { + cfg.CPUClasses = []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}} +} + +// withDRAEnabled sets cfg.DRA.Enabled = true (SharedCounters left false). +func withDRAEnabled(cfg *cfgapi.Config) { + cfg.DRA = &cfgapi.TopologyAwareDRA{Enabled: true} +} + +// TestSetupDRADisabledLeavesPluginNil verifies that when DRA is disabled +// (the default zero-value Config.DRA == nil), Setup() never calls +// buildDRAPlugin, leaving p.draPlugin nil, and that the DRA-adjacent +// lifecycle methods (Start, Stop) remain no-ops/no-panics against that nil +// state — i.e. behavior is unchanged from before Step 8. +func TestSetupDRADisabledLeavesPluginNil(t *testing.T) { + p, err := setupDRATestPolicy(t, withOneHPClass, nil, nil) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin != nil { + t.Error("draPlugin: got non-nil, want nil (DRA disabled)") + } + + if err := p.Start(); err != nil { + t.Fatalf("Start() with DRA disabled: got %v, want nil", err) + } + if err := p.Stop(); err != nil { + t.Fatalf("Stop() with DRA disabled: got %v, want nil", err) + } +} + +// TestSetupDRAEnabledNilKubeClientLeavesPluginNil verifies that when DRA is +// enabled but opts.KubeClientFn() returns a nil kubernetes.Interface (e.g. +// local-config mode, or too early during agent startup), Setup() logs a +// warning and leaves p.draPlugin nil rather than failing. +func TestSetupDRAEnabledNilKubeClientLeavesPluginNil(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return nil } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + nil, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin != nil { + t.Error("draPlugin: got non-nil, want nil (no kube client)") + } +} + +// TestSetupDRAEnabledEmptyNodeNameLeavesPluginNil verifies that when DRA is +// enabled, a kube client is available, but opts.NodeName is empty (also +// possible in local-config mode), Setup() logs a warning and leaves +// p.draPlugin nil rather than failing or calling dra.New with an empty +// NodeName (which would itself error). +func TestSetupDRAEnabledEmptyNodeNameLeavesPluginNil(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "" + opts.WithLock = func(f func()) { f() } + }, + nil, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin != nil { + t.Error("draPlugin: got non-nil, want nil (empty node name)") + } +} + +// TestSetupDRAEnabledNoCPUClassesBuildsPluginWithNoDevices verifies that +// when DRA is enabled but no cpuClasses are configured (p.cpuClasses stays +// nil after initialize()), Setup() still builds a non-nil p.draPlugin, +// publishing zero devices — rather than leaving it permanently nil, which +// would prevent a later Reconfigure() that adds cpuClasses from ever +// recovering (buildDRAPlugin only ever runs once, from Setup()). +func TestSetupDRAEnabledNoCPUClassesBuildsPluginWithNoDevices(t *testing.T) { + p, err := setupDRATestPolicy(t, + withDRAEnabled, // no withOneHPClass: CPUClasses stays empty + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.cpuClasses != nil { + t.Fatal("test setup invariant violated: p.cpuClasses is non-nil, want nil") + } + if p.draPlugin == nil { + t.Fatal("draPlugin: got nil, want non-nil (empty cpuClasses must not prevent construction)") + } + + // *dra.Plugin doesn't expose DeviceLister directly; go through the same + // adapter path buildDRAPlugin wired in, to confirm the nil-cpuClasses + // case really does yield an empty (not erroring) device list rather + // than merely a non-nil plugin. + adapter := &policyDRAAdapter{p: p} + devs, err := adapter.DRADevices(DRADriverName) + if err != nil { + t.Fatalf("DRADevices() unexpected error: %v", err) + } + if len(devs) != 0 { + t.Errorf("DRADevices() = %v, want empty for nil p.cpuClasses", devs) + } +} + +// TestSetupDRAEnabledValidDepsBuildsPlugin verifies that when DRA is +// enabled and every dependency (kube client, node name, cpuClass +// configuration) is available, Setup() builds a non-nil p.draPlugin. +func TestSetupDRAEnabledValidDepsBuildsPlugin(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin == nil { + t.Fatal("draPlugin: got nil, want non-nil") + } +} + +// TestSetupDRAEnabledCDIWriterFailureReturnsError verifies buildDRAPlugin's +// genuine-hard-failure path: unlike the four "not ready yet" guards above +// (nil kube client, empty node name, no cpuClasses — all warn-and-nil), a +// real construction failure in one of its own dependencies (here, +// dra.NewCDIWriter failing because p.cdiDir cannot be created) must be +// returned as an error from Setup(), not swallowed. +func TestSetupDRAEnabledCDIWriterFailureReturnsError(t *testing.T) { + // A regular file can't be MkdirAll'd into: NewCDIWriter's os.MkdirAll on + // p.cdiDir (or a path beneath it) will fail with ENOTDIR. + tmp := t.TempDir() + blocker := path.Join(tmp, "not-a-directory") + if err := os.WriteFile(blocker, []byte("x"), 0644); err != nil { + t.Fatalf("failed to create blocking file: %v", err) + } + cdiDir := path.Join(blocker, "cdi") + + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = cdiDir }, + ) + if err == nil { + t.Fatalf("Setup() with an unusable cdiDir: got nil error, want a descriptive error") + } + if p.draPlugin != nil { + t.Errorf("draPlugin: got non-nil after a failed Setup(), want nil") + } +} + +// TestStopCancelsContextAndStopsDRAPlugin verifies that Stop() cancels the +// context draCtxCancel was set with and calls draPlugin.Stop(), and that +// calling Stop() a second time is safe (both context.CancelFunc and +// dra.Plugin.Stop are documented as idempotent). +func TestStopCancelsContextAndStopsDRAPlugin(t *testing.T) { + p := &policy{} + p.draPlugin = newTestDRAPlugin(t, cpuset.New(0), "dev0") + + ctx, cancel := context.WithCancel(context.Background()) + p.draCtxCancel = cancel + + if err := p.Stop(); err != nil { + t.Fatalf("Stop() = %v, want nil", err) + } + if ctx.Err() == nil { + t.Error("Stop() did not cancel the context") + } + + if err := p.Stop(); err != nil { + t.Fatalf("second Stop() = %v, want nil", err) + } +} + +// newConflictingTierClassHandler builds a *cpuclass.Handler configured with +// two managed PCT classes at the same tier (both PctPriority: "high"), the +// exact shape ValidateCPUClassesForDRA rejects when sharedCounters is +// false. SST support is left disabled (PCT inactive) since +// ValidateCPUClassesForDRA inspects the class list itself, not device +// activity — mirrors newInactiveClassHandler in dra_adapter_test.go. +func newConflictingTierClassHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", "") + h, err := cpuclass.New(&adapterTestSys{}) + if err != nil { + t.Fatalf("cpuclass.New() failed: %v", err) + } + classes := []*cfgapi.CPUClass{ + {Name: "hp1", PctPriority: "high"}, + {Name: "hp2", PctPriority: "high"}, + } + if err := h.Configure(cpuclass.ConfigSpec{ + Classes: classes, + Allowed: cpuset.MustParse("0-7"), + }); err != nil { + t.Fatalf("Configure() failed: %v", err) + } + return h +} + +// TestBuildDRAPluginValidateClassesUsesLiveConfig verifies that the +// ValidateClasses closure buildDRAPlugin hands to the DRA plugin reads +// p.cfg live (via the nil-safe DRASharedCounters() getter) rather than a +// config snapshot taken at buildDRAPlugin call time — required so that a +// later Reconfigure() (which swaps p.cfg for a new *Config) is observed +// without rebuilding the plugin. The second phase resolves the tier +// conflict by dropping to a single published class, not by setting +// SharedCounters: true — that option is rejected outright regardless of +// conflicts, since Model C (KEP-5941) isn't implemented. +// +// PublishResources is used as the probe: it runs ValidateClasses before +// checking whether Start() has been called, so the distinction between "a +// tier-conflict error" (ValidateClasses failed) and "called before Start" +// (ValidateClasses passed) is directly observable without ever calling +// Start() (which would require real kubelet registration directories). +func TestBuildDRAPluginValidateClassesUsesLiveConfig(t *testing.T) { + classes := []*cfgapi.CPUClass{ + {Name: "hp1", PctPriority: "high"}, + {Name: "hp2", PctPriority: "high"}, + } + p := &policy{ + cache: &mockCache{}, + cpuClasses: newConflictingTierClassHandler(t), + cfg: &cfgapi.Config{ + CPUClasses: classes, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true, SharedCounters: false}, + }, + cdiDir: t.TempDir(), + } + + opts := &policyapi.BackendOptions{ + KubeClientFn: func() kubernetes.Interface { return fake.NewClientset() }, + NodeName: "test-node", + WithLock: func(f func()) { f() }, + } + + if err := p.buildDRAPlugin(opts); err != nil { + t.Fatalf("buildDRAPlugin() = %v, want nil", err) + } + if p.draPlugin == nil { + t.Fatal("draPlugin: got nil, want non-nil") + } + + // SharedCounters is false and both classes are at the same PCT tier: + // ValidateClasses must fail with a tier-conflict error. + if err := p.draPlugin.PublishResources(context.Background()); err == nil || !strings.Contains(err.Error(), "tier") { + t.Fatalf("PublishResources() before simulated Reconfigure = %v, want tier-conflict error", err) + } + + // Simulate a Reconfigure that swaps p.cfg for a new *Config whose + // CPUClasses no longer conflict (down to a single HP class). + // buildDRAPlugin's ValidateClasses closure captured p, not cfg, so it + // must observe this change immediately — with no need to rebuild + // p.draPlugin. (SharedCounters stays false: it is rejected outright by + // ValidateCPUClassesForDRA regardless of tier conflicts, since Model C + // isn't implemented — it can no longer be used to "fix" a conflict.) + p.cfg = &cfgapi.Config{ + CPUClasses: []*cfgapi.CPUClass{classes[0]}, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true, SharedCounters: false}, + } + + err := p.draPlugin.PublishResources(context.Background()) + if err == nil || !strings.Contains(err.Error(), "called before Start") { + t.Fatalf("PublishResources() after simulated Reconfigure = %v, want \"called before Start\" error (proves ValidateClasses passed)", err) + } +} + +// ---- Reconfigure refusal: DRA config changes require a restart. ---- + +// TestReconfigureRefusesAnyChangeWhileDRAEnabled verifies that Reconfigure() +// refuses a config change unrelated to DRA itself (here: ReservedResources) +// once DRA is enabled -- any config change while DRA is (or was) enabled now +// requires a restart, not just a dra.enabled flip or a live-claim conflict. +func TestReconfigureRefusesAnyChangeWhileDRAEnabled(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin == nil { + t.Fatal("test setup error: draPlugin unexpectedly nil") + } + + oldCfg := p.cfg + oldDRAPlugin := p.draPlugin + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "1000m"}, + CPUClasses: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true}, + } + + err = p.Reconfigure(newCfg) + if err == nil || !strings.Contains(err.Error(), "restart") { + t.Fatalf("Reconfigure() = %v, want an error naming the restart requirement", err) + } + if p.cfg != oldCfg { + t.Error("p.cfg was replaced despite the refused Reconfigure") + } + if opt != oldCfg { + t.Error("opt was replaced despite the refused Reconfigure") + } + if p.draPlugin != oldDRAPlugin { + t.Error("draPlugin was replaced despite the refused Reconfigure") + } +} + +// TestReconfigureRefusesDRAEnabledFlipToTrue verifies that Reconfigure() +// refuses a config change that turns DRA on when it was off at Setup() time. +func TestReconfigureRefusesDRAEnabledFlipToTrue(t *testing.T) { + p := newDRATestPolicy(t) // DRA disabled by default (cfg.DRA == nil) + if p.draPlugin != nil { + t.Fatal("test setup error: draPlugin unexpectedly non-nil") + } + + oldCfg := p.cfg + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "750m"}, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true}, + } + + err := p.Reconfigure(newCfg) + if err == nil { + t.Fatal("Reconfigure() = nil, want an error (DRAEnabled() flip false -> true)") + } + if opt != oldCfg { + t.Error("opt not restored to the pre-Reconfigure config after the refused DRAEnabled flip") + } + if p.draPlugin != nil { + t.Error("draPlugin unexpectedly built by a refused Reconfigure") + } +} + +// TestReconfigureRefusesDRAEnabledFlipToFalse mirrors +// TestReconfigureRefusesDRAEnabledFlipToTrue for the opposite direction: DRA +// was enabled (and successfully built) at Setup() time, and a later +// Reconfigure() tries to turn it off. +func TestReconfigureRefusesDRAEnabledFlipToFalse(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + oldCfg := p.cfg + oldDRAPlugin := p.draPlugin + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "750m"}, + CPUClasses: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + // DRA left nil: DRAEnabled() == false. + } + + err = p.Reconfigure(newCfg) + if err == nil { + t.Fatal("Reconfigure() = nil, want an error (DRAEnabled() flip true -> false)") + } + if opt != oldCfg { + t.Error("opt not restored to the pre-Reconfigure config after the refused DRAEnabled flip") + } + if p.draPlugin != oldDRAPlugin { + t.Error("draPlugin was replaced/cleared despite the refused Reconfigure") + } +} + +// TestReconfigureUnaffectedWhenDRANeverEnabled verifies that Reconfigure() +// still succeeds for an ordinary config change when DRA was never enabled +// (nil in both the old and new config) -- guards against the DRA restart +// guard accidentally widening to non-DRA reconfigures. +func TestReconfigureUnaffectedWhenDRANeverEnabled(t *testing.T) { + p := newDRATestPolicy(t) + if p.cfg.DRAEnabled() { + t.Fatal("test setup error: DRA unexpectedly enabled") + } + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "1000m"}, + } + + if err := p.Reconfigure(newCfg); err != nil { + t.Fatalf("Reconfigure() = %v, want nil (DRA never enabled)", err) + } + if p.cfg != newCfg { + t.Error("p.cfg was not updated by a successful Reconfigure") + } +} diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 28b074dbf..91f26197d 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -367,6 +367,9 @@ type mockContainer struct { returnValueForGetID string returnValueForQOSClass v1.PodQOSClass pod cache.Pod + cdiDeviceNames []string + cpusetCpus string + setCpusetCpusCalls []string } func (m *mockContainer) GetPod() (cache.Pod, bool) { @@ -428,6 +431,9 @@ func (m *mockContainer) GetMounts() []*cache.Mount { func (m *mockContainer) GetDevices() []*cache.Device { panic("unimplemented") } +func (m *mockContainer) GetCDIDeviceNames() []string { + return m.cdiDeviceNames +} func (m *mockContainer) PrettyName() string { return m.name } @@ -490,7 +496,9 @@ func (m *mockContainer) SetCPUPeriod(int64) { func (m *mockContainer) SetCPUQuota(int64) { panic("unimplemented") } -func (m *mockContainer) SetCpusetCpus(string) { +func (m *mockContainer) SetCpusetCpus(cpus string) { + m.setCpusetCpusCalls = append(m.setCpusetCpusCalls, cpus) + m.cpusetCpus = cpus } func (m *mockContainer) SetCpusetMems(string) { } @@ -585,7 +593,7 @@ func (m *mockContainer) GetCPUPeriod() int64 { panic("unimplemented") } func (m *mockContainer) GetCpusetCpus() string { - panic("unimplemented") + return m.cpusetCpus } func (m *mockContainer) GetCpusetMems() string { panic("unimplemented") @@ -719,6 +727,7 @@ type mockCache struct { returnValueForGetPolicyEntry bool returnValue1ForLookupContainer cache.Container returnValue2ForLookupContainer bool + containers []cache.Container } func (m *mockCache) InsertPod(*nri.PodSandbox, <-chan *podresapi.PodResources) cache.Pod { @@ -749,7 +758,7 @@ func (m *mockCache) GetPods() []cache.Pod { panic("unimplemented") } func (m *mockCache) GetContainers() []cache.Container { - panic("unimplemented") + return m.containers } func (m *mockCache) GetContainerIds() []string { panic("unimplemented") diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index 2eb1a7c4c..00567da83 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -18,15 +18,19 @@ import ( "fmt" "math" "sort" + "strings" "github.com/containers/nri-plugins/pkg/utils/cpuset" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/dra" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" system "github.com/containers/nri-plugins/pkg/sysfs" idset "github.com/intel/goresctrl/pkg/utils" + "tags.cncf.io/container-device-interface/pkg/parser" ) // buildPoolsByTopology builds a hierarchical tree of pools based on HW topology. @@ -257,7 +261,7 @@ func (p *policy) getCpuSupply(node Node, cpus cpuset.CPUSet) (Supply, Supply) { log.Infof(" %s CPU: %s", node.Name(), s.DumpCapacity()) - return s, s.Clone() + return s, newSupply(node, isolated, reserved, sharable, 0, 0) } func (p *policy) getMemSupply(node Node, cpus cpuset.CPUSet) (dram, pmem, hbm idset.IDSet) { @@ -513,8 +517,9 @@ func (p *policy) allocatePool(container cache.Container, poolHint string) (Grant // setPreferredCpusetCpus pins container's CPUs according to what has been // allocated for it, taking into account if the container should run -// with hyperthreads hidden. -func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated cpuset.CPUSet, info string) { +// with hyperthreads hidden. CPUs in preserve are always included in the +// final cpuset regardless of hide-hyperthreads filtering (e.g. DRA claimed CPUs). +func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated, preserve cpuset.CPUSet, info string) { allow := allocated hidingInfo := "" pod, ok := container.GetPod() @@ -526,6 +531,7 @@ func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated cpu hidingInfo = " (no hyperthreads to hide)" } } + allow = allow.Union(preserve) log.Infof("%s%s", info, hidingInfo) container.SetCpusetCpus(allow.String()) } @@ -568,6 +574,13 @@ func (p *policy) applyGrant(grant Grant) { return } + // Collect CPUs this container holds via a live DRA claim. These are + // removed from pool free supply by allocateClaim, but that never + // touches the container's cgroup cpuset — this is the only place that + // pins them. They are passed as preserve to setPreferredCpusetCpus so + // that hide-hyperthreads filtering on the normal grant cannot drop them. + claimed := p.claimedCPUsByContainer[container.GetID()] + mems := libmem.NodeMask(0) if opt.PinMemory { mems = grant.GetMemoryZone() @@ -575,10 +588,20 @@ func (p *policy) applyGrant(grant Grant) { if opt.PinCPU { if cpuType == cpuPreserve { + if !claimed.IsEmpty() { + preserved, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + log.Errorf(" => failed to parse %s cpuset %q while adding DRA claim %s: %v", + container.PrettyName(), container.GetCpusetCpus(), claimed, err) + } else { + preserved = preserved.Union(claimed) + container.SetCpusetCpus(preserved.String()) + } + } log.Infof(" => preserving %s cpuset %s", container.PrettyName(), container.GetCpusetCpus()) } else { - if cpus.Size() > 0 { - p.setPreferredCpusetCpus(container, cpus, + if cpus.Size() > 0 || !claimed.IsEmpty() { + p.setPreferredCpusetCpus(container, cpus, claimed, fmt.Sprintf(" => pinning %s to (%s) cpuset %s", container.PrettyName(), kind, cpus)) } else { @@ -701,12 +724,17 @@ func (p *policy) updateSharedAllocations(grant *Grant) { if opt.PinCPU { shared := other.GetCPUNode().FreeSupply().SharableCPUs() exclusive := other.ExclusiveCPUs() + // Pass claimed CPUs as preserve so hide-hyperthreads filtering + // on the normal grant cpuset cannot drop them. + claimed := p.claimedCPUsByContainer[other.GetContainer().GetID()] if exclusive.IsEmpty() { - p.setPreferredCpusetCpus(other.GetContainer(), shared, + cpus := shared + p.setPreferredCpusetCpus(other.GetContainer(), cpus, claimed, fmt.Sprintf(" => updating %s with shared CPUs of %s: %s...", - other, other.GetCPUNode().Name(), shared.String())) + other, other.GetCPUNode().Name(), cpus.String())) } else { - p.setPreferredCpusetCpus(other.GetContainer(), exclusive.Union(shared), + cpus := exclusive.Union(shared) + p.setPreferredCpusetCpus(other.GetContainer(), cpus, claimed, fmt.Sprintf(" => updating %s with exclusive+shared CPUs of %s: %s+%s...", other, other.GetCPUNode().Name(), exclusive.String(), shared.String())) } @@ -1339,3 +1367,491 @@ func combineHintScores(scores map[string]float64) (float64, float64) { } return combined, filtered } + +// +// DRA claim identification and pool accounting. +// + +// cdiClaimDeviceNamePrefix is the fixed prefix of every qualified CDI device +// name the DRA plugin (pkg/resmgr/dra) generates for this driver: the CDI +// "device" class is hardcoded there (cdi.go: cdiClass = "device"), and every +// device name it builds starts with "claim-" (cdi.go: cdiDeviceName). This is +// duplicated here (rather than imported) because cdiClass and cdiDeviceName +// are unexported in package dra. +const cdiClaimDeviceNamePrefix = DRADriverName + "/device=claim-" + +// parseCDIClaimUID extracts the DRA ResourceClaim UID from a qualified CDI +// device name of the form "nri.topology-aware.cpu/device=claim----" +// (see pkg/resmgr/dra/cdi.go's cdiDeviceName), by trimming the trailing +// "---" three '-'-separated tokens and returning what's +// left. +// +// This is a best-effort fast path, not a lossless inverse of cdiDeviceName: +// and are themselves sanitized names that may contain +// '-' (see sanitizeCDIName), so when they do, the split boundary can land in +// the wrong place and the returned string will include extra trailing +// tokens that actually belong to /. It is exact whenever +// and are single tokens (the common case), which is all +// this function alone can guarantee. claimCPUsFromContainer compensates for +// the ambiguous case by falling back to matching against the caller's known +// set of live claim UIDs, which this function has no access to. +func parseCDIClaimUID(deviceName string) (string, bool) { + rest, ok := strings.CutPrefix(deviceName, cdiClaimDeviceNamePrefix) + if !ok { + return "", false + } + + tokens := strings.Split(rest, "-") + if len(tokens) < 4 { + return "", false + } + + uid := strings.Join(tokens[:len(tokens)-3], "-") + if uid == "" { + return "", false + } + + return uid, true +} + +// claimLister is the minimal slice of *dra.Plugin's API that +// claimCPUsFromContainer needs. Declaring it locally (instead of taking a +// *dra.Plugin directly) lets tests exercise the CDI-name-to-claim lookup +// logic with a lightweight fake instead of standing up a full dra.Plugin +// (kubelet registration, CDI writer, claim store, etc.); *dra.Plugin +// satisfies this interface, so production call sites are unaffected. +// +// Callers must nil-check the concrete *dra.Plugin *before* passing it in +// here: a nil *dra.Plugin wrapped in a non-nil claimLister interface value +// would panic inside LiveClaimsLocked (typed-nil trap). +type claimLister interface { + LiveClaimsLocked() map[types.UID][]dra.ResultAlloc + DriverName() string +} + +// classifyClaimCPUs parses every alloc's CPUs field and returns the union of +// all of them plus a grouping of the same CPUs by cpuClass name. +// +// A prepared claim's DeviceRequestAllocationResults are constrained by +// pkg/resmgr/dra to a single punit (PrepareResourceClaims rejects any claim +// whose results span more than one), but that punit can still publish more +// than one HP class: cpuclass.ValidateCPUClassesForDRA allows at most one +// published class per tier, not one class overall, so a punit with several +// tiers publishes a device per (class, tier) pair. Grouping by class here +// lets callers apply each class only to the CPUs that actually belong to it +// (see allocateClaim/remarkClaimInSupply), so a multi-class claim never has +// the wrong physical class silently applied to part of its CPUs. +// +// Allocs whose CPUs field fails to parse are logged and skipped; they +// contribute to neither the returned union nor the per-class grouping. +func classifyClaimCPUs(uid types.UID, allocs []dra.ResultAlloc) (cpuset.CPUSet, map[string]cpuset.CPUSet) { + cpus := cpuset.New() + classCPUs := map[string]cpuset.CPUSet{} + + for _, a := range allocs { + parsed, err := cpuset.Parse(a.CPUs) + if err != nil { + log.Warnf("dra: claim %s: failed to parse allocated CPUs %q: %v", uid, a.CPUs, err) + continue + } + cpus = cpus.Union(parsed) + if existing, ok := classCPUs[a.ClassName]; ok { + classCPUs[a.ClassName] = existing.Union(parsed) + } else { + classCPUs[a.ClassName] = parsed + } + } + + return cpus, classCPUs +} + +// containerClaim describes one live DRA claim's contribution to a +// container: its UID, the union of its allocated CPUs, and those CPUs +// grouped by cpuClass name (see classifyClaimCPUs — a single claim can span +// more than one class). +type containerClaim struct { + UID types.UID + CPUs cpuset.CPUSet + ClassCPUs map[string]cpuset.CPUSet +} + +// claimCPUsFromContainer looks for CDI device names on c that identify live +// DRA claims and returns one containerClaim per distinct live claim UID +// found. Returns an empty slice if c carries no recognizable claim device +// name, or if every embedded UID has no corresponding entry in +// plugin.LiveClaimsLocked() (e.g. a foreign/stale CDI device, or a claim +// that has already been unprepared). +// +// A container is most commonly backed by a single live TA CPU ResourceClaim, +// but Kubernetes' general pod.spec.resourceClaims plumbing does not forbid a +// container from referencing more than one distinct ResourceClaim for this +// driver (this is orthogonal to a single ResourceClaim with +// AllowMultipleAllocations fanning out to *many containers*, which is +// handled by the claimContainerRefs refcount in allocateClaim/releaseClaim). +// Callers must loop over every entry in the returned slice, not just the +// first, to avoid leaving a distinct claim's CPUs unaccounted for (and thus +// double-bookable by a subsequent, unrelated allocation). +func claimCPUsFromContainer(c cache.Container, plugin claimLister) []containerClaim { + if plugin == nil { + return nil + } + + live := plugin.LiveClaimsLocked() + + var result []containerClaim + for _, name := range c.GetCDIDeviceNames() { + for uid, allocs := range live { + for i, alloc := range allocs { + expected := parser.QualifiedName(plugin.DriverName(), "device", + dra.CDIDeviceName(uid, alloc.Request, alloc.Device, i)) + if name != expected { + continue + } + claimCPUs, claimClassCPUs := classifyClaimCPUs(uid, []dra.ResultAlloc{alloc}) + if claimCPUs.IsEmpty() { + continue + } + for j := range result { + if result[j].UID != uid { + continue + } + result[j].CPUs = result[j].CPUs.Union(claimCPUs) + for class, cpus := range claimClassCPUs { + result[j].ClassCPUs[class] = result[j].ClassCPUs[class].Union(cpus) + } + goto nextDevice + } + result = append(result, containerClaim{UID: uid, CPUs: claimCPUs, ClassCPUs: claimClassCPUs}) + goto nextDevice + } + } + nextDevice: + } + + return result +} + +// poolForCPUs returns the tightest (deepest) *leaf* pool whose statically +// assigned CPU range (GetSupply(), which — unlike FreeSupply() — never +// changes with allocation or claim accounting) fully contains cpus. Returns +// an error if no single pool's range is a superset of cpus: either cpus lies +// (at least partly) outside p.allowed altogether, or it straddles more than +// one leaf pool (which a legitimate single-punit DRA CPU pick never does). +// +// Candidates are restricted to leaf pools (Node.IsLeafNode()) deliberately: +// every non-leaf ancestor's static range is, by construction, the union of +// its descendants' ranges, so it is always a superset of any cpus subset of +// p.allowed — including a cpus set that straddles two *different* leaf +// pools. Without this restriction, such a straddling cpus set would +// incorrectly resolve to the lowest common ancestor (worst case, root) +// instead of being rejected: Supply.ClaimCPUs only walks *up* the +// node.Parent() chain from whatever pool it's called on, never down into +// children, so marking the CPUs claimed at the ancestor would fail to +// exclude them from either leaf's own FreeSupply() — a double-booking gap +// this restriction closes. +func (p *policy) poolForCPUs(cpus cpuset.CPUSet) (Node, error) { + var ( + best Node + bestDepth = -1 + ) + + for _, n := range p.pools { + if !n.IsLeafNode() { + continue + } + full := n.GetSupply() + total := full.IsolatedCPUs().Union(full.ReservedCPUs()).Union(full.SharableCPUs()) + if !cpus.IsSubsetOf(total) { + continue + } + if n.RootDistance() > bestDepth { + best = n + bestDepth = n.RootDistance() + } + } + + if best == nil { + return nil, policyError("no single pool contains CPUs %s (outside allowed CPUs, or spanning more than one pool)", cpus) + } + + return best, nil +} + +// applyClassCPUs applies each (className, subset) pair in classCPUs via +// cpuClasses.UseClass, one call per class. This is a no-op if p.cpuClasses +// is nil. Empty class names (an alloc whose device carried no nri/cpuClass +// attribute — should not happen given upstream validation, but defensively +// skipped rather than trusted) and empty subsets are skipped. verb is used +// only for logging ("apply" vs "re-apply"). +// +// classCPUs groups a single DRA claim's CPUs by cpuClass (see +// classifyClaimCPUs): a claim's DeviceRequestAllocationResults can resolve +// to devices of different classes published for the same punit, so the +// physical class must be applied per subset — applying one class to the +// claim's entire unioned CPU set would silently mis-apply it to part of the +// claim. +func (p *policy) applyClassCPUs(verb string, uid types.UID, classCPUs map[string]cpuset.CPUSet) error { + if p.cpuClasses == nil { + return nil + } + for className, subset := range classCPUs { + if className == "" || subset.IsEmpty() { + continue + } + if err := p.cpuClasses.UseClass(className, subset); err != nil { + return fmt.Errorf("dra: failed to %s CPU class %q to claim %s CPUs %s: %w", + verb, className, uid, subset, err) + } + } + return nil +} + +// allocateClaim marks cpus as claimed by DRA ResourceClaim uid in the +// tightest pool that fully contains them, evicting and requeueing for +// reallocation any exclusive grant that overlaps those CPUs. +// +// Safe to call more than once for the same uid: a ResourceClaim with +// AllowMultipleAllocations can back more than one container, and +// AllocateResources calls this once per container. The pool marking itself +// (and any eviction it triggers) is only performed for the first container; +// subsequent calls just bump the per-claim container refcount so that +// releaseClaim knows to keep the CPUs marked until the last referencing +// container is released. +func (p *policy) allocateClaim(uid types.UID, cpus cpuset.CPUSet, classCPUs map[string]cpuset.CPUSet) error { + if cpus.IsEmpty() { + return policyError("cannot allocate DRA claim %s: empty CPU set", uid) + } + + if p.claimContainerRefs == nil { + p.claimContainerRefs = make(map[types.UID]int) + } + + if p.claimContainerRefs[uid] == 0 { + pool, err := p.poolForCPUs(cpus) + if err != nil { + return policyError("cannot allocate DRA claim %s (CPUs %s): %v", uid, cpus, err) + } + + evicted, evictedCpusets := p.evictOverlappingGrants(cpus, fmt.Sprintf("claim %s", uid)) + + pool.FreeSupply().ClaimCPUs(uid, cpus) + + // Apply the physical cpuClass (SST-CP CLOS association, EPP, + // governor, ...) to the claimed CPUs. Without this, pool accounting + // excludes the CPUs from regular grants but the hardware is left in + // whatever class it was in before — the whole point of associating a + // DRA claim with a cpuClass would otherwise have no physical effect. + // classCPUs groups the claimed CPUs by class (see classifyClaimCPUs): + // a single claim can span more than one class (e.g. requests + // resolving to devices of different classes published for the same + // punit), so UseClass is applied once per (class, subset) pair + // rather than once for the whole unioned cpus with a single, + // possibly-wrong class. + if err := p.applyClassCPUs("apply", uid, classCPUs); err != nil { + // Roll back the supply mark so pool accounting stays consistent. + pool.FreeSupply().UnclaimCPUs(uid) + p.resetCpuClass(fmt.Sprintf("dra: rollback claim %s", uid), cpus) + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + log.Errorf("dra: claim %s: failed to restore evicted grants during CPU class rollback: %v", uid, reallocErr) + } + return policyError("dra: claim %s: failed to apply CPU class: %v", uid, err) + } + + // The claimed CPUs may have been subtracted from the sharable pool + // (not just isolated/exclusive), in which case containers already + // pinned (via applyGrant) to the pool's previous, wider sharable + // cpuset need to be re-pinned to the now-reduced set — otherwise they + // keep running on CPUs the pool considers exclusively owned by this + // DRA claim. Run this unconditionally: it is a no-op for containers + // whose cpuset does not include any of the reserved-CPU-type grants + // affected here (see updateSharedAllocations). + p.updateSharedAllocations(nil) + + if err := p.reallocateEvicted(evicted, evictedCpusets, cpus, uid); err != nil { + pool.FreeSupply().UnclaimCPUs(uid) + p.resetCpuClass(fmt.Sprintf("dra: rollback claim %s", uid), cpus) + p.updateSharedAllocations(nil) + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + log.Errorf("dra: claim %s: failed to restore evicted grants during rollback: %v", uid, reallocErr) + } + return policyError("dra: claim %s: evicted %d container(s) to free CPUs %s but failed "+ + "to fully reallocate them: %v", uid, len(evicted), cpus, err) + } + } + + p.claimContainerRefs[uid]++ + + return nil +} + +// evictOverlappingGrants releases the exclusive grant of every container +// whose ExclusiveCPUs() overlaps cpus, returning the evicted containers and +// a snapshot of their cgroup cpuset.cpus (as it was right before eviction) — +// the latter is needed by reallocateEvicted's safety net if reallocation +// later fails for one of them. reason is used only for logging. +func (p *policy) evictOverlappingGrants(cpus cpuset.CPUSet, reason string) ([]cache.Container, map[string]string) { + var evicted []cache.Container + evictedCpusets := map[string]string{} + for _, g := range p.allocations.grants { + if g.ExclusiveCPUs().Intersection(cpus).IsEmpty() { + continue + } + c := g.GetContainer() + evicted = append(evicted, c) + evictedCpusets[c.GetID()] = c.GetCpusetCpus() + } + + for _, c := range evicted { + log.Infof("dra: evicting %s to free CPUs %s for %s", c.PrettyName(), cpus, reason) + p.releasePool(c) + } + + return evicted, evictedCpusets +} + +// reallocateEvicted attempts to reallocate the containers evicted (by +// evictOverlappingGrants) to free cpus for DRA claim uid. If reallocation +// fails for one or more of them, it forcibly strips cpus out of their +// last-known cgroup cpuset (evictedCpusets, as captured before eviction) as +// a safety net: an evicted container that ends up with no grant must not be +// left pinned to a cpuset that overlaps the CPUs a DRA claim now exclusively +// owns — that would let two workloads run on the same physical CPUs +// simultaneously. Returns the (possibly partial-reallocation) error from +// reallocateResources, or nil if evicted is empty or reallocation succeeded. +func (p *policy) reallocateEvicted(evicted []cache.Container, evictedCpusets map[string]string, cpus cpuset.CPUSet, uid types.UID) error { + if len(evicted) == 0 { + return nil + } + + if err := p.reallocateResources(evicted, nil); err != nil { + log.Errorf("dra: failed to fully reallocate %d container(s) evicted for claim %s: %v", + len(evicted), uid, err) + + for _, c := range evicted { + if _, ok := p.allocations.getGrant(c.GetID()); ok { + continue + } + prev, perr := cpuset.Parse(evictedCpusets[c.GetID()]) + if perr != nil { + log.Errorf("dra: claim %s: cannot safely re-pin %s off claimed CPUs %s: %v", + uid, c.PrettyName(), cpus, perr) + continue + } + safe := prev.Difference(cpus) + if safe.IsEmpty() { + // The victim's entire previous cpuset is within the claimed CPUs. + // An empty cpuset string is treated by NRI as "no restriction", so + // we must not call SetCpusetCpus with it — doing so would leave the + // container running on the DRA-claimed CPUs. Try to find at least + // one non-claimed CPU from the system-wide sharable pool as a + // fallback to pin the container to. + fallback := p.root.FreeSupply().SharableCPUs() + if fallback.IsEmpty() { + log.Errorf("dra: claim %s: cannot safely re-pin %s off claimed CPUs %s: "+ + "previous cpuset %s is entirely claimed and no sharable fallback CPU exists", + uid, c.PrettyName(), cpus, prev) + continue + } + safe = cpuset.New(fallback.List()[0]) + } + log.Warnf("dra: claim %s: %s could not be reallocated after eviction; "+ + "forcing cpuset from %s to %s to avoid overlap with claimed CPUs %s", + uid, c.PrettyName(), prev, safe, cpus) + c.SetCpusetCpus(safe.String()) + } + + return err + } + + return nil +} + +// releaseClaim decrements the container refcount for DRA claim uid and, +// once the last referencing container has been released, restores cpus to +// the pool that had them marked as claimed. A no-op (not an error) for a uid +// that allocateClaim was never called for, or that has already been fully +// released — ReleaseResources may run for containers the policy never saw +// AllocateResources for (e.g. across a restart). +func (p *policy) releaseClaim(uid types.UID, cpus cpuset.CPUSet) error { + if p.claimContainerRefs == nil || p.claimContainerRefs[uid] == 0 { + return nil + } + + p.claimContainerRefs[uid]-- + if p.claimContainerRefs[uid] > 0 { + return nil + } + + delete(p.claimContainerRefs, uid) + + // If unprepareDRAClaim ran first (while we still had refs), it tombstoned + // the allocs rather than releasing the HP hold. Do that now. + if allocs, ok := p.tombstonedDRAClaims[uid]; ok { + delete(p.tombstonedDRAClaims, uid) + p.releaseHpCPUsForAllocs(uid, allocs) + } + + pool, err := p.poolForCPUs(cpus) + if err != nil { + return policyError("cannot release DRA claim %s (CPUs %s): %v", uid, cpus, err) + } + + pool.FreeSupply().UnclaimCPUs(uid) + + // Mirror releasePool's resetCpuClass: allocateClaim applied a physical + // cpuClass (SST-CP CLOS, EPP, governor, ...) to these CPUs via + // cpuClasses.UseClass. Without resetting it here, the CPUs go back into + // the pool's regular exclusive/shared rotation still carrying whatever + // class the claim used — silently affecting the next, unrelated + // container the pool hands them to, until the next full + // Reconfigure/restart resets every allowed CPU's class from scratch. + p.resetCpuClass(fmt.Sprintf("dra: release claim %s", uid), cpus) + + // The released CPUs may have been restored to the sharable pool (not + // just isolated/exclusive), widening the cpuset available to containers + // already running in that pool's shared allocation. Let them reclaim it + // — mirrors the same call in allocateClaim/reapplyDRAClaims. + p.updateSharedAllocations(nil) + + return nil +} + +func (p *policy) unprepareDRAClaim(uid types.UID, allocs []dra.ResultAlloc) { + if p.claimContainerRefs != nil && p.claimContainerRefs[uid] > 0 { + // Containers still using this claim's CPUs. Defer the HP-CPU and pool + // release until the last container drops (releaseClaim will drain it). + if p.tombstonedDRAClaims == nil { + p.tombstonedDRAClaims = make(map[types.UID][]dra.ResultAlloc) + } + p.tombstonedDRAClaims[uid] = allocs + return + } + + cpus, _ := classifyClaimCPUs(uid, allocs) + if cpus.IsEmpty() { + return + } + pool, err := p.poolForCPUs(cpus) + if err != nil { + log.Errorf("dra: cannot unclaim prepared claim %s (CPUs %s): %v", uid, cpus, err) + return + } + p.releaseHpCPUsForAllocs(uid, allocs) + pool.FreeSupply().UnclaimCPUs(uid) + p.resetCpuClass(fmt.Sprintf("dra: unprepare claim %s", uid), cpus) + p.updateSharedAllocations(nil) +} + +// releaseHpCPUsForAllocs calls cpuClasses.ReleaseHpCpus for every allocation +// in allocs, parsing CPUs from the stored string. Parse errors are logged and +// skipped (matching UnprepareResourceClaims' own parse-error policy). +func (p *policy) releaseHpCPUsForAllocs(uid types.UID, allocs []dra.ResultAlloc) { + for _, alloc := range allocs { + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + log.Warnf("dra: release claim %s device %s: parse CPUs %q: %v (skipping HP release)", uid, alloc.Device, alloc.CPUs, err) + continue + } + p.cpuClasses.ReleaseHpCpus(alloc.PkgID, alloc.PunitID, cpus) + } +} diff --git a/cmd/plugins/topology-aware/policy/pools_test.go b/cmd/plugins/topology-aware/policy/pools_test.go index e11b978f5..46d191540 100644 --- a/cmd/plugins/topology-aware/policy/pools_test.go +++ b/cmd/plugins/topology-aware/policy/pools_test.go @@ -18,13 +18,19 @@ import ( "fmt" "os" "path" + "strings" "testing" + "k8s.io/apimachinery/pkg/types" + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/dra" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/testutils" + "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func findNodeWithName(name string, nodes []Node) Node { @@ -556,3 +562,613 @@ func TestAffinities(t *testing.T) { }) } } + +// +// DRA claim identification and pool accounting (Step 8, Task 6). +// + +func TestParseCDIClaimUID(t *testing.T) { + tcases := []struct { + name string + deviceName string + wantUID string + wantOK bool + }{ + { + name: "simple uid", + deviceName: "nri.topology-aware.cpu/device=claim-abc123-req-dev-0", + wantUID: "abc123", + wantOK: true, + }, + { + // A uid containing '-' (the real Kubernetes UID shape) is still + // recovered exactly, as long as and are each + // single tokens: the split only has to strip a fixed 3 trailing + // tokens, so it doesn't matter how many tokens the leading uid + // part itself has. + name: "uid with embedded dashes, single-token request/device", + deviceName: "nri.topology-aware.cpu/device=claim-13f7db45-eb2a-4dd1-b0cb-1234567890ab-myreq-dev0-0", + wantUID: "13f7db45-eb2a-4dd1-b0cb-1234567890ab", + wantOK: true, + }, + { + // Documents the known best-effort limitation: when is + // itself multi-token after sanitization (e.g. "punit-0-0"), the + // fixed trailing-3-token strip lands in the wrong place and + // "swallows" part of into the returned uid. + // claimCPUsFromContainer compensates for this by iterating over + // its known-live UIDs and doing exact device-name construction + // instead of relying on parseCDIClaimUID alone. + name: "documents limitation: multi-token device swallows the request", + deviceName: "nri.topology-aware.cpu/device=claim-13f7db45-myreq-punit-0-0-0", + wantUID: "13f7db45-myreq-punit", + wantOK: true, + }, + { + name: "wrong driver prefix", + deviceName: "other.driver/device=claim-abc123-req-dev-0", + wantOK: false, + }, + { + name: "not a CDI qualified name at all", + deviceName: "not-a-cdi-device-name", + wantOK: false, + }, + { + name: "right prefix, too few remaining tokens", + deviceName: "nri.topology-aware.cpu/device=claim-dev-0", + wantOK: false, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + uid, ok := parseCDIClaimUID(tc.deviceName) + if ok != tc.wantOK { + t.Fatalf("parseCDIClaimUID(%q) ok = %v, want %v", tc.deviceName, ok, tc.wantOK) + } + if ok && uid != tc.wantUID { + t.Errorf("parseCDIClaimUID(%q) = %q, want %q", tc.deviceName, uid, tc.wantUID) + } + }) + } +} + +// fakeClaimLister is a minimal claimLister for tests, avoiding the need to +// stand up a full *dra.Plugin (kubelet registration, CDI writer, claim +// store, etc.) just to exercise claimCPUsFromContainer's lookup logic. +type fakeClaimLister struct { + claims map[types.UID][]dra.ResultAlloc +} + +func (f *fakeClaimLister) LiveClaimsLocked() map[types.UID][]dra.ResultAlloc { + return f.claims +} + +func (*fakeClaimLister) DriverName() string { + return DRADriverName +} + +func cdiClaimDeviceName(uid types.UID, request, device string, idx int) string { + return DRADriverName + "/device=" + dra.CDIDeviceName(uid, request, device, idx) +} + +func TestClaimCPUsFromContainer(t *testing.T) { + uid := types.UID("claim-uid-1") + name := cdiClaimDeviceName(uid, "myreq", "dev0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: {{Request: "myreq", Device: "dev0", ClassName: "gold", CPUs: "0-1"}}, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if got.UID != uid { + t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) + } + if want := cpuset.MustParse("0-1"); !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) + } + if len(got.ClassCPUs) != 1 { + t.Errorf("claimCPUsFromContainer() classCPUs = %v, want exactly one class", got.ClassCPUs) + } + if want := cpuset.MustParse("0-1"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s", got.CPUs, want) + } +} + +func TestClaimCPUsFromContainerDashedUIDAndDashedDeviceName(t *testing.T) { + // A real k8s UID (embedded dashes) combined with a device name that + // is itself multi-token after sanitization (e.g. "punit-0-0") is exactly + // the case parseCDIClaimUID's fast path alone mis-splits (see + // TestParseCDIClaimUID's "documents limitation" case). claimCPUsFromContainer + // must still resolve it correctly via exact device-name construction. + uid := types.UID("13f7db45-eb2a-4dd1-b0cb-1234567890ab") + name := cdiClaimDeviceName(uid, "myreq", "punit-0-0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: {{Request: "myreq", Device: "punit-0-0", ClassName: "gold", CPUs: "4-5"}}, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if got.UID != uid { + t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) + } + if want := cpuset.MustParse("4-5"); !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) + } + if want := cpuset.MustParse("4-5"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s", got.CPUs, want) + } +} + +func TestClaimCPUsFromContainerUsesOnlyConsumedAllocs(t *testing.T) { + // A single claim can back more than one DeviceRequestAllocationResult + // (e.g. multiple requests in one claim); the union of their CPUs must be + // returned, and (since every alloc here shares the same class) a single + // classCPUs entry covering that whole union. + uid := types.UID("claim-uid-multi") + name0 := cdiClaimDeviceName(uid, "req0", "dev0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name0}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: { + {Request: "req0", Device: "dev0", ClassName: "gold", CPUs: "0-1"}, + {Request: "req1", Device: "dev1", ClassName: "gold", CPUs: "2-3"}, + }, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if want := cpuset.MustParse("0-1"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s (only the consumed allocation)", got.CPUs, want) + } + if want := cpuset.MustParse("0-1"); len(got.ClassCPUs) != 1 || !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs = %v, want {gold: %s}", got.ClassCPUs, want) + } +} + +// TestClaimCPUsFromContainerMultipleClasses covers the MAJOR finding this +// fix addresses: a single ResourceClaim can legitimately contain +// DeviceRequestAllocationResults resolving to devices of *different* +// cpuClasses (the per-punit multi-class-overcommit validation only guards a +// single punit's classes against each other, it does not forbid a claim's +// requests from spanning more than one punit/class). claimCPUsFromContainer +// must group the claimed CPUs by class rather than collapsing to a single +// className, so callers can apply each class only to the CPUs that actually +// belong to it. +func TestClaimCPUsFromContainerMultipleClasses(t *testing.T) { + uid := types.UID("claim-uid-multiclass") + name0 := cdiClaimDeviceName(uid, "req0", "dev0", 0) + name1 := cdiClaimDeviceName(uid, "req1", "dev1", 1) + + c := &mockContainer{cdiDeviceNames: []string{name0, name1}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: { + {Request: "req0", Device: "dev0", ClassName: "gold", CPUs: "0-1"}, + {Request: "req1", Device: "dev1", ClassName: "silver", CPUs: "2-3"}, + }, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if got.UID != uid { + t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) + } + if want := cpuset.MustParse("0-3"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s (union across classes)", got.CPUs, want) + } + if len(got.ClassCPUs) != 2 { + t.Fatalf("claimCPUsFromContainer() classCPUs = %v, want two distinct classes", got.ClassCPUs) + } + if want := cpuset.MustParse("0-1"); !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) + } + if want := cpuset.MustParse("2-3"); !got.ClassCPUs["silver"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[silver] = %s, want %s", got.ClassCPUs["silver"], want) + } +} + +func TestClaimCPUsFromContainerNoCDIDevices(t *testing.T) { + c := &mockContainer{} + lister := &fakeClaimLister{claims: map[types.UID][]dra.ResultAlloc{}} + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 0 { + t.Errorf("claimCPUsFromContainer() = %v, want empty for a container with no CDI devices", claims) + } +} + +func TestClaimCPUsFromContainerUnknownClaim(t *testing.T) { + // The container carries a well-formed claim device name, but the parsed + // UID has no entry in LiveClaimsLocked() (e.g. the claim was already + // unprepared, or the device belongs to some other driver's namespace). + uid := types.UID("claim-uid-gone") + name := cdiClaimDeviceName(uid, "myreq", "dev0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name}} + lister := &fakeClaimLister{claims: map[types.UID][]dra.ResultAlloc{}} + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 0 { + t.Errorf("claimCPUsFromContainer() = %v, want empty for an unknown/stale claim UID", claims) + } +} + +// TestClaimCPUsFromContainerMultipleDistinctClaims verifies that a +// container whose CDI devices resolve to two distinct live claim UIDs must +// have both claims' CPUs accounted for, not just the first one encountered. +func TestClaimCPUsFromContainerMultipleDistinctClaims(t *testing.T) { + uid1 := types.UID("claim-uid-first") + uid2 := types.UID("claim-uid-second") + name1 := cdiClaimDeviceName(uid1, "req0", "dev0", 0) + name2 := cdiClaimDeviceName(uid2, "req0", "dev1", 0) + + c := &mockContainer{cdiDeviceNames: []string{name1, name2}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid1: {{Request: "req0", Device: "dev0", ClassName: "gold", CPUs: "0-1"}}, + uid2: {{Request: "req0", Device: "dev1", ClassName: "silver", CPUs: "2-3"}}, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 2 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 2 (both distinct live claims must be accounted for)", len(claims)) + } + + byUID := map[types.UID]containerClaim{} + for _, cl := range claims { + byUID[cl.UID] = cl + } + if cl, ok := byUID[uid1]; !ok || !cl.CPUs.Equals(cpuset.MustParse("0-1")) { + t.Errorf("claim %s CPUs = %v, want 0-1", uid1, cl.CPUs) + } + if cl, ok := byUID[uid2]; !ok || !cl.CPUs.Equals(cpuset.MustParse("2-3")) { + t.Errorf("claim %s CPUs = %v, want 2-3", uid2, cl.CPUs) + } +} + +// goldClassCPUs wraps cpus as the single-class classCPUs allocateClaim/ +// remarkClaimInSupply expect, for tests that don't care about the +// multi-class case (see TestAllocateClaimAppliesPerAllocClass for that). +func goldClassCPUs(cpus cpuset.CPUSet) map[string]cpuset.CPUSet { + return map[string]cpuset.CPUSet{"gold": cpus} +} + +// addTestGrant hands out an exclusive grant for container from pool's +// FreeSupply, mirroring (at the level of supply-state mutation) what +// supply.AllocateCPU does for the granting node itself, then records the +// grant in p.allocations so p.releasePool/p.reallocateResources can find it. +// This lets eviction tests exercise the real p.releasePool/grant.Release() +// code path without going through the full container-annotation-driven +// request/offer pipeline (which coldstart_test.go notes is impractical to +// mock with a bare container). +func addTestGrant(t *testing.T, p *policy, pool Node, container cache.Container, exclusive cpuset.CPUSet) Grant { + t.Helper() + + g := newGrant(pool, container, cpuNormal, "", exclusive, 0, memoryDRAM, nil, 0) + + s, ok := pool.FreeSupply().(*supply) + if !ok { + t.Fatalf("pool %q FreeSupply() is not a *supply", pool.Name()) + } + s.isolated = s.isolated.Difference(exclusive) + s.sharable = s.sharable.Difference(exclusive) + g.AccountAllocateCPU() + + p.allocations.addGrant(g) + + return g +} + +func TestAllocateClaimMarksTightestPool(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + cpus := cpuset.New(sharable[0], sharable[1]) + + uid := types.UID("claim-mark") + if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("claimed CPUs %s still present in %q sharable set: %s", cpus, leaf.Name(), got) + } + + // A regular exclusive-CPU request for another container must not be + // able to pick the claimed CPUs. + free := leaf.FreeSupply().AllocatableSharedCPU() + full := free / 1000 + if full < 1 { + t.Fatalf("expected at least 1 full CPU still allocatable on %q, got %dm", leaf.Name(), free) + } + req := &request{full: full, container: &mockContainer{}} + offer, err := leaf.FreeSupply().GetCPUOffer(req) + if err != nil { + t.Fatalf("GetCPUOffer for %d full CPUs failed unexpectedly: %v", full, err) + } + if offer.Intersection(cpus).Size() != 0 { + t.Errorf("CPU offer %s for another container includes claimed CPUs %s", offer, cpus) + } +} + +func TestAllocateClaimOutsideAllowedReturnsError(t *testing.T) { + p := newDRATestPolicy(t) + + // CPU 99999 does not exist on the test system at all, so it can't be a + // subset of any pool's (including root's) statically assigned range. + cpus := cpuset.New(99999) + + if err := p.allocateClaim(types.UID("claim-outside"), cpus, goldClassCPUs(cpus)); err == nil { + t.Fatalf("allocateClaim() with CPUs outside the allowed set: got nil error, want a descriptive error") + } +} + +// TestAllocateClaimSpanningNoPoolReturnsError covers the other poolForCPUs +// failure mode from TestAllocateClaimOutsideAllowedReturnsError: CPUs that +// are individually within p.allowed (each belongs to some pool), but +// straddle two sibling pools so that no single pool's static range is a +// superset of the whole set. A legitimate single-punit DRA CPU pick never +// does this; allocateClaim must still reject it with a descriptive error +// rather than, say, silently marking one of the two pools. +func TestAllocateClaimSpanningNoPoolReturnsError(t *testing.T) { + p := newDRATestPolicy(t) + + // "NUMA node #0" and "NUMA node #2" are siblings under "socket #0" (see + // TestSupplyClaimCPUsAncestorNotDoubleSubtracted): no pool below "root" + // (or "socket #0") is a strict subset spanning both, so a cpuset with + // one CPU from each cannot be contained by any single pool. + leafA := findPoolNode(t, p, "NUMA node #0") + leafB := findPoolNode(t, p, "NUMA node #2") + + cpuA := leafA.GetSupply().SharableCPUs().List() + cpuB := leafB.GetSupply().SharableCPUs().List() + if len(cpuA) < 1 || len(cpuB) < 1 { + t.Fatalf("expected at least 1 CPU on both %q and %q", leafA.Name(), leafB.Name()) + } + spanning := cpuset.New(cpuA[0], cpuB[0]) + + err := p.allocateClaim(types.UID("claim-spanning"), spanning, goldClassCPUs(spanning)) + if err == nil { + t.Fatalf("allocateClaim() with CPUs spanning two pools: got nil error, want a descriptive error") + } + if _, exists := p.claimContainerRefs[types.UID("claim-spanning")]; exists { + t.Errorf("claimContainerRefs unexpectedly populated for a claim that failed to allocate") + } +} + +func TestAllocateClaimRefcountsMultipleContainers(t *testing.T) { + // A ResourceClaim with AllowMultipleAllocations backs more than one + // container; allocateClaim is called once per container sharing it. + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + cpus := cpuset.New(sharable[0]) + uid := types.UID("claim-shared") + + if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { + t.Fatalf("first allocateClaim() failed: %v", err) + } + afterFirst := leaf.FreeSupply().SharableCPUs() + + if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { + t.Fatalf("second allocateClaim() (second container, same claim) failed: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(afterFirst) { + t.Errorf("second allocateClaim() for the same uid changed pool supply: got %s, want unchanged %s", got, afterFirst) + } + if got := p.claimContainerRefs[uid]; got != 2 { + t.Errorf("claimContainerRefs[%s] = %d, want 2 after two containers", uid, got) + } + + // Releasing once (one of the two containers) must not restore the CPUs yet. + if err := p.releaseClaim(uid, cpus); err != nil { + t.Fatalf("first releaseClaim() failed: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("CPUs %s restored after releasing only one of two referencing containers: sharable=%s", cpus, got) + } + + // Releasing the second (last) container must restore the CPUs. + if err := p.releaseClaim(uid, cpus); err != nil { + t.Fatalf("second releaseClaim() failed: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != cpus.Size() { + t.Errorf("CPUs %s not restored after releasing the last referencing container: sharable=%s", cpus, got) + } + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs[%s] still present after refcount reached zero", uid) + } +} + +func TestReleaseClaimUnknownUIDNoop(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + before := leaf.FreeSupply().SharableCPUs() + + if err := p.releaseClaim(types.UID("never-claimed"), cpuset.New(before.List()[0])); err != nil { + t.Errorf("releaseClaim() for an unknown uid: got error %v, want nil (idempotent)", err) + } + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(before) { + t.Errorf("releaseClaim() for an unknown uid changed pool supply: got %s, want unchanged %s", got, before) + } +} + +// TestReleaseClaimResetsCpuClass verifies that releaseClaim is symmetric +// with allocateClaim's cpuClasses.UseClass call: releasing a claim must +// reset the physical cpuClass on the unclaimed CPUs back to the shared-pool +// baseline (mirroring releasePool's resetCpuClass call), not leave them +// stuck in the claim's class for whatever unrelated container the pool +// hands them to next. +func TestReleaseClaimResetsCpuClass(t *testing.T) { + p := newDRATestPolicyWithCPUClasses(t, "shared", "gold") + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + cpu := sharable[0] + claimed := cpuset.New(cpu) + + // A sibling CPU never touched by the claim: its class reflects whatever + // initialize() applied via resetCpuClass("initialize", p.allowed) — the + // shared-pool baseline every allowed CPU starts in. + baselineCPU := sharable[1] + baseline := p.cpuClasses.ClassForCPU(baselineCPU) + + uid := types.UID("claim-cpuclass-reset") + if err := p.allocateClaim(uid, claimed, goldClassCPUs(claimed)); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + if got := p.cpuClasses.ClassForCPU(cpu); got == baseline { + t.Fatalf("test setup error: claimed CPU %d class unchanged (%q) after allocateClaim with class %q", + cpu, got, "gold") + } + + if err := p.releaseClaim(uid, claimed); err != nil { + t.Fatalf("releaseClaim() failed: %v", err) + } + if got := p.cpuClasses.ClassForCPU(cpu); got != baseline { + t.Errorf("claimed CPU %d class = %q after releaseClaim(), want reset back to shared-pool baseline %q", + cpu, got, baseline) + } +} + +// TestAllocateClaimAppliesPerAllocClass covers the MAJOR finding this fix +// addresses: a single DRA claim can resolve to more than one cpuClass +// across its ResultAllocs (see classifyClaimCPUs), and allocateClaim must +// apply each class only to the CPU subset that actually belongs to it, +// rather than applying whichever class happened to come first to the +// claim's entire (unioned) CPU set. +func TestAllocateClaimAppliesPerAllocClass(t *testing.T) { + p := newDRATestPolicyWithCPUClasses(t, "shared", "gold", "silver") + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + goldCPU := cpuset.New(sharable[0]) + silverCPU := cpuset.New(sharable[1]) + claimed := goldCPU.Union(silverCPU) + + uid := types.UID("claim-multiclass") + classCPUs := map[string]cpuset.CPUSet{ + "gold": goldCPU, + "silver": silverCPU, + } + if err := p.allocateClaim(uid, claimed, classCPUs); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + // ClassForCPU may return a synthetic, decorated class name (e.g. with a + // per-die suffix) rather than the literal config name — see + // TestReleaseClaimResetsCpuClass, which compares against a baseline + // instead of a literal string for the same reason. What matters here is + // that the gold- and silver-alloc CPUs end up in *different*, class + // specific buckets: proof that allocateClaim applied each alloc's own + // class to its own CPU subset, instead of the pre-fix behavior of + // picking one class (from the first alloc) and applying it to the whole + // unioned CPU set. + goldGot := p.cpuClasses.ClassForCPU(sharable[0]) + silverGot := p.cpuClasses.ClassForCPU(sharable[1]) + if !strings.HasPrefix(goldGot, "gold") { + t.Errorf("gold-alloc CPU %d class = %q, want a class derived from %q", sharable[0], goldGot, "gold") + } + if !strings.HasPrefix(silverGot, "silver") { + t.Errorf("silver-alloc CPU %d class = %q, want a class derived from %q", sharable[1], silverGot, "silver") + } + if goldGot == silverGot { + t.Errorf("gold-alloc and silver-alloc CPUs ended up in the same class %q; "+ + "per-alloc class application did not take effect", goldGot) + } +} + +func TestAllocateClaimEvictsOverlappingExclusiveGrant(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + victim := &mockContainer{returnValueForGetID: "victim"} + addTestGrant(t, p, leaf, victim, claimed) + + if _, ok := p.allocations.getGrant("victim"); !ok { + t.Fatalf("test setup error: victim grant not present before allocateClaim") + } + + if err := p.allocateClaim(types.UID("evict-claim"), claimed, goldClassCPUs(claimed)); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + // No grant anywhere may still exclusively hold the claimed CPU: either + // the victim's original grant was released outright, or it was + // reallocated to CPUs that no longer overlap the claim. + for _, g := range p.allocations.grants { + if g.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still exclusively granted to %s after eviction", + claimed, g.GetContainer().PrettyName()) + } + } + + // And the claimed CPU must not be free for a new regular allocation. + free := leaf.FreeSupply() + if free.SharableCPUs().Union(free.IsolatedCPUs()).Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still free in %q supply after allocateClaim", claimed, leaf.Name()) + } + + // allocateClaim returned nil, so the evicted victim must actually have + // been reallocated a new grant (not just released and forgotten) — there + // is ample capacity left on this test system for reallocatePool to + // succeed. The new grant's exact shape (exclusive vs. shared/fractional) + // depends on the request reallocatePool derives from the container's own + // declared resource requirements — zero for the bare mockContainer used + // here as "victim" — so only its existence and non-overlap with the + // claimed CPU are asserted, not its exact size/type. + newGrant, ok := p.allocations.getGrant("victim") + if !ok { + t.Fatalf("victim has no grant after allocateClaim() succeeded; eviction must reallocate, not just release") + } + if newGrant.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("victim's new grant %s still overlaps claimed CPU %s", newGrant.ExclusiveCPUs(), claimed) + } +} diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 6307d743e..4be1b32cc 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -20,6 +20,8 @@ import ( "strconv" "time" + "k8s.io/apimachinery/pkg/types" + "github.com/containers/nri-plugins/pkg/agent/podresapi" "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/topology" @@ -77,6 +79,14 @@ type Supply interface { AccountAllocateCPU(Grant) // AccountReleaseCPU accounts for (reinserts) released exclusive capacity into the supply. AccountReleaseCPU(Grant) + // ClaimCPUs marks cpus as claimed by a DRA ResourceClaim (uid), subtracting + // them from isolated/sharable capacity in this supply and, tree-wide, in + // every ancestor supply. Idempotent per uid: a second call for the same + // uid replaces (does not stack on top of) the previous marking. + ClaimCPUs(uid types.UID, cpus cpuset.CPUSet) + // UnclaimCPUs reverses a previous ClaimCPUs marking for uid, restoring the + // claimed CPUs tree-wide. A no-op if uid is unknown. + UnclaimCPUs(uid types.UID) // GetScore calculates how well this supply fits/fulfills the given request. GetScore(Request) Score // AllocatableSharedCPU calculates the allocatable amount of shared CPU of this supply. @@ -236,6 +246,16 @@ type supply struct { sharable cpuset.CPUSet // sharable CPUs at this node grantedReserved int // amount of reserved CPUs allocated grantedShared int // amount of shareable CPUs allocated + + // claimRefs tracks, per DRA claim UID, the CPUs subtracted from isolated/ + // sharable capacity of this supply on behalf of that claim. Marking is + // tree-wide: the same UID is (re)marked in every ancestor supply too (see + // ClaimCPUs/UnclaimCPUs). + claimRefs map[types.UID]cpuset.CPUSet + + // cloned indicates that this supply is a Clone() copy and should not + // propagate ClaimCPUs/UnclaimCPUs to real ancestor nodes. + cloned bool } var _ Supply = &supply{} @@ -322,7 +342,15 @@ func (cs *supply) GetNode() Node { // Clone clones the given CPU supply. func (cs *supply) Clone() Supply { - return newSupply(cs.node, cs.isolated, cs.reserved, cs.sharable, cs.grantedReserved, cs.grantedShared) + clone := newSupply(cs.node, cs.isolated, cs.reserved, cs.sharable, cs.grantedReserved, cs.grantedShared).(*supply) + if len(cs.claimRefs) > 0 { + clone.claimRefs = make(map[types.UID]cpuset.CPUSet, len(cs.claimRefs)) + for uid, cpus := range cs.claimRefs { + clone.claimRefs[uid] = cpus.Clone() + } + } + clone.cloned = true + return clone } // IsolatedCpus returns the isolated CPUSet of this supply. @@ -387,6 +415,69 @@ func (cs *supply) AccountReleaseCPU(g Grant) { cs.sharable = cs.sharable.Union(sharable) } +// ClaimCPUs marks cpus as claimed by a DRA ResourceClaim (uid), subtracting +// them from isolated/reserved/sharable capacity of this supply, then +// propagating the same marking up the node.Parent() chain so every ancestor +// supply also excludes these CPUs from its own isolated/reserved/sharable +// capacity (tree-wide accounting, mirroring AccountAllocateCPU/ +// AccountReleaseCPU). +// +// Reserved CPUs are included because poolForCPUs (pools.go) matches a +// pool's static range as isolated+reserved+sharable: p.allowed (which is +// what the DRA CPU-pick allocator's own "allowed" domain is configured +// from) is not required to exclude p.reserved, so a legitimate DRA pick can +// land on a CPU that is also part of a pool's reserved partition. Without +// subtracting it here too, AllocatableReservedCPU would keep advertising +// that CPU's fractional capacity to ordinary reserved-type grants even +// though a DRA claim already exclusively owns it — a double-booking gap on +// the reserved partition mirroring the one this method already closes for +// isolated/sharable. (Re-pinning any reserved-type container *already* +// running on that CPU is a separate, pre-existing gap: updateSharedAllocations +// explicitly skips cpuType == cpuReserved grants; not addressed here.) +// +// A second ClaimCPUs call for the same uid replaces (rather than stacks on +// top of) the previous marking at each level: the old cpuset for uid is first +// restored, then the new one is subtracted. This makes re-applying claim +// marks after a policy rebuild (Reconfigure/restart) idempotent. +func (cs *supply) ClaimCPUs(uid types.UID, cpus cpuset.CPUSet) { + if old, ok := cs.claimRefs[uid]; ok { + full := cs.node.GetSupply() + cs.isolated = cs.isolated.Union(old.Intersection(full.IsolatedCPUs())) + cs.reserved = cs.reserved.Union(old.Intersection(full.ReservedCPUs())) + cs.sharable = cs.sharable.Union(old.Intersection(full.SharableCPUs())) + } + + if cs.claimRefs == nil { + cs.claimRefs = make(map[types.UID]cpuset.CPUSet) + } + cs.claimRefs[uid] = cpus.Clone() + cs.isolated = cs.isolated.Difference(cpus) + cs.reserved = cs.reserved.Difference(cpus) + cs.sharable = cs.sharable.Difference(cpus) + + if parent := cs.node.Parent(); !cs.cloned && !parent.IsNil() { + parent.FreeSupply().ClaimCPUs(uid, cpus) + } +} + +// UnclaimCPUs reverses a previous ClaimCPUs marking for uid in this supply, +// restoring the claimed CPUs to isolated/reserved/sharable capacity, then +// propagates the same reversal up the node.Parent() chain. A no-op (at every +// level it reaches) for a uid that was never claimed. +func (cs *supply) UnclaimCPUs(uid types.UID) { + if cpus, ok := cs.claimRefs[uid]; ok { + delete(cs.claimRefs, uid) + full := cs.node.GetSupply() + cs.isolated = cs.isolated.Union(cpus.Intersection(full.IsolatedCPUs())) + cs.reserved = cs.reserved.Union(cpus.Intersection(full.ReservedCPUs())) + cs.sharable = cs.sharable.Union(cpus.Intersection(full.SharableCPUs())) + } + + if parent := cs.node.Parent(); !cs.cloned && !parent.IsNil() { + parent.FreeSupply().UnclaimCPUs(uid) + } +} + // Allocate allocates a grant from the supply. func (cs *supply) Allocate(r Request, o *libmem.Offer) (Grant, map[string]libmem.NodeMask, error) { if o == nil { diff --git a/cmd/plugins/topology-aware/policy/resources_test.go b/cmd/plugins/topology-aware/policy/resources_test.go new file mode 100644 index 000000000..ff1035fc7 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/resources_test.go @@ -0,0 +1,462 @@ +// Copyright 2026 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "os" + "path" + "testing" + + "k8s.io/apimachinery/pkg/types" + + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" + system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/testutils" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// newDRATestPolicy builds a real policy (with a real, multi-level node tree: +// root -> socket -> NUMA node) from the "server" sysfs test data. This gives +// us actual Parent()/Children() wiring, which is what ClaimCPUs/UnclaimCPUs +// tree-wide propagation depends on. +func newDRATestPolicy(t *testing.T) *policy { + t.Helper() + // WithLock must be non-nil: Start() runs reapplyDRAClaims() under + // p.options.WithLock whenever p.draPlugin != nil (tests below set + // p.draPlugin directly), mirroring the real resmgr write lock. + return newDRATestPolicyWithLock(t, func(f func()) { f() }) +} + +// newDRATestPolicyWithLock is newDRATestPolicy with an injectable WithLock, +// letting lock-contract tests (see topology-aware-policy_test.go) observe +// exactly when Start()/Reconfigure() hold the resmgr write lock. +func newDRATestPolicyWithLock(t *testing.T, withLock func(func())) *policy { + t.Helper() + + dir, err := os.MkdirTemp("", "nri-resource-policy-test-sysfs-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + t.Cleanup(func() { removeAll(t, dir) }) + + if err := testutils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { + t.Fatalf("failed to uncompress test sysfs data: %v", err) + } + + sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + if err != nil { + t.Fatalf("failed to discover test system: %v", err) + } + + policyOptions := &policyapi.BackendOptions{ + Cache: &mockCache{}, + System: sys, + Config: &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{ + cfgapi.CPU: "750m", + }, + }, + WithLock: withLock, + } + + p := New().(*policy) + if err := p.Setup(policyOptions); err != nil { + t.Fatalf("failed to set up test policy: %v", err) + } + + return p +} + +// newDRATestPolicyWithCPUClasses is newDRATestPolicy plus a real +// *cpuclass.Handler (p.cpuClasses), configured with sharedClass (used as +// SharedPoolCpuClass, i.e. the class initialize()'s resetCpuClass and +// releaseClaim's resetCpuClass reapply to CPUs no longer exclusively held) +// plus one or more claimClasses (e.g. distinct classes different +// ResultAllocs within one DRA claim can resolve to). Lets tests observe +// cpuClass.UseClass side effects via Handler.ClassForCPU without needing a +// live SST/PCT backend. +func newDRATestPolicyWithCPUClasses(t *testing.T, sharedClass string, claimClasses ...string) *policy { + t.Helper() + + dir, err := os.MkdirTemp("", "nri-resource-policy-test-sysfs-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + t.Cleanup(func() { removeAll(t, dir) }) + + if err := testutils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { + t.Fatalf("failed to uncompress test sysfs data: %v", err) + } + + sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + if err != nil { + t.Fatalf("failed to discover test system: %v", err) + } + + cpuClasses := []*cfgapi.CPUClass{{Name: sharedClass}} + for _, c := range claimClasses { + cpuClasses = append(cpuClasses, &cfgapi.CPUClass{Name: c}) + } + + policyOptions := &policyapi.BackendOptions{ + Cache: &mockCache{}, + System: sys, + Config: &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{ + cfgapi.CPU: "750m", + }, + CPUClasses: cpuClasses, + SharedPoolCpuClass: sharedClass, + }, + // See newDRATestPolicy: Start() requires a non-nil WithLock whenever + // p.draPlugin is set. + WithLock: func(f func()) { f() }, + } + + p := New().(*policy) + if err := p.Setup(policyOptions); err != nil { + t.Fatalf("failed to set up test policy: %v", err) + } + if p.cpuClasses == nil { + t.Fatalf("test setup error: p.cpuClasses is nil despite non-empty CPUClasses config") + } + + return p +} + +// findPoolNode returns the pool Node with the given name, failing the test if +// it isn't found. +func findPoolNode(t *testing.T, p *policy, name string) Node { + t.Helper() + for _, n := range p.pools { + if n.Name() == name { + return n + } + } + t.Fatalf("no pool node named %q found", name) + return nil +} + +// ancestorsOf returns node's ancestor chain, closest first, by walking +// Parent(). +func ancestorsOf(n Node) []Node { + var ancestors []Node + for parent := n.Parent(); !parent.IsNil(); parent = parent.Parent() { + ancestors = append(ancestors, parent) + } + return ancestors +} + +func TestSupplyClaimCPUsTreeWide(t *testing.T) { + p := newDRATestPolicy(t) + + // "NUMA node #0" is a leaf under "socket #0", which is a child of "root". + leaf := findPoolNode(t, p, "NUMA node #0") + ancestors := ancestorsOf(leaf) + if len(ancestors) == 0 { + t.Fatalf("expected %q to have ancestors, got none", leaf.Name()) + } + + // Pick a couple of sharable CPUs known to belong to this leaf's supply. + claimed := leaf.FreeSupply().SharableCPUs() + if claimed.Size() < 2 { + t.Fatalf("expected leaf %q to have at least 2 sharable CPUs, got %s", leaf.Name(), claimed) + } + // Take exactly two CPUs from the leaf's sharable set. + claimedList := claimed.List() + cpus := cpuset.New(claimedList[0], claimedList[1]) + + uid := types.UID("claim-uid-1") + + leafSharableBefore := leaf.FreeSupply().SharableCPUs() + leafAllocatableBefore := leaf.FreeSupply().AllocatableSharedCPU() + ancestorSharableBefore := make(map[string]cpuset.CPUSet, len(ancestors)) + for _, a := range ancestors { + ancestorSharableBefore[a.Name()] = a.FreeSupply().SharableCPUs() + } + + leaf.FreeSupply().ClaimCPUs(uid, cpus) + + // The leaf's own sharable set must have shrunk by exactly `cpus`. + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(leafSharableBefore.Difference(cpus)) { + t.Errorf("leaf %q sharable CPUs after claim = %s, want %s", + leaf.Name(), got, leafSharableBefore.Difference(cpus)) + } + + // Every ancestor's sharable set must also have shrunk by `cpus`. + for _, a := range ancestors { + want := ancestorSharableBefore[a.Name()].Difference(cpus) + if got := a.FreeSupply().SharableCPUs(); !got.Equals(want) { + t.Errorf("ancestor %q sharable CPUs after claim = %s, want %s", a.Name(), got, want) + } + } + + // The claimed CPUs must no longer be available for a regular allocation + // from the leaf: AllocatableSharedCPU (milli-CPU) must have dropped by + // exactly 2 full CPUs worth (2000m). + freeAfter := leaf.FreeSupply().AllocatableSharedCPU() + if want := leafAllocatableBefore - 2000; freeAfter != want { + t.Errorf("claimed leaf allocatable shared CPU = %dm, want %dm (before claim: %dm)", + freeAfter, want, leafAllocatableBefore) + } + + // A regular exclusive-CPU allocation request for another container must + // not be able to pick the claimed CPUs, no matter how many full CPUs it + // asks for out of what's still nominally available. + full := freeAfter / 1000 + if full < 1 { + t.Fatalf("expected at least 1 full CPU still allocatable on %q after claim, got %dm", leaf.Name(), freeAfter) + } + req := &request{ + full: full, + container: &mockContainer{}, + } + offer, err := leaf.FreeSupply().GetCPUOffer(req) + if err != nil { + t.Fatalf("GetCPUOffer for %d full CPUs failed unexpectedly: %v", full, err) + } + if offer.Intersection(cpus).Size() != 0 { + t.Errorf("CPU offer %s for another container includes claimed CPUs %s", offer, cpus) + } +} + +// TestSupplyClaimCPUsReservedPartition covers the reserved-CPU case +// poolForCPUs (pools.go) allows for but that ClaimCPUs/UnclaimCPUs used to +// ignore: poolForCPUs matches a pool's static range as +// isolated+reserved+sharable, so a DRA claim can legitimately land on a CPU +// that is part of a pool's reserved partition (the DRA CPU-pick allocator's +// own "allowed" domain, p.allowed, is not required to exclude p.reserved). +// ClaimCPUs must subtract from the reserved cpuset too, or +// AllocatableReservedCPU would keep advertising that CPU's capacity to +// ordinary reserved-type grants even though DRA already owns it exclusively +// (a double-booking gap on the reserved partition). +func TestSupplyClaimCPUsReservedPartition(t *testing.T) { + p := newDRATestPolicy(t) + + var reservedPool Node + for _, n := range p.pools { + if n.GetSupply().ReservedCPUs().Size() > 0 { + reservedPool = n + break + } + } + if reservedPool == nil { + t.Fatalf("test setup error: no pool in the test topology has a non-empty reserved partition") + } + + reservedSupply, ok := reservedPool.FreeSupply().(*supply) + if !ok { + t.Fatalf("pool %q FreeSupply() is not a *supply", reservedPool.Name()) + } + + reserved := reservedSupply.ReservedCPUs() + cpus := cpuset.New(reserved.List()[0]) + uid := types.UID("claim-uid-reserved") + + reservedAllocatableBefore := reservedSupply.AllocatableReservedCPU() + + reservedSupply.ClaimCPUs(uid, cpus) + + if got := reservedSupply.ReservedCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("claimed reserved CPU %s still present in %q reserved set after ClaimCPUs: %s", + cpus, reservedPool.Name(), got) + } + // AllocatableReservedCPU has a special sentinel: it returns -1 (not a + // proportional milliCPU amount) once the reserved cpuset becomes + // entirely empty, rather than when its granted capacity merely drops to + // zero. Claiming the last reserved CPU on this pool hits that sentinel; + // claiming one of several would not. + wantAfterClaim := reservedAllocatableBefore - 1000 + if reserved.Difference(cpus).IsEmpty() { + wantAfterClaim = -1 + } + if got := reservedSupply.AllocatableReservedCPU(); got != wantAfterClaim { + t.Errorf("%q allocatable reserved CPU after claiming 1 reserved CPU = %dm, want %dm", + reservedPool.Name(), got, wantAfterClaim) + } + + reservedSupply.UnclaimCPUs(uid) + + if got := reservedSupply.ReservedCPUs(); !got.Equals(reserved) { + t.Errorf("%q reserved set after UnclaimCPUs = %s, want restored %s", reservedPool.Name(), got, reserved) + } + if got := reservedSupply.AllocatableReservedCPU(); got != reservedAllocatableBefore { + t.Errorf("%q allocatable reserved CPU after UnclaimCPUs = %dm, want restored %dm", + reservedPool.Name(), got, reservedAllocatableBefore) + } +} + +func TestSupplyClaimCPUsIdempotentReplace(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + + uid := types.UID("claim-uid-replace") + cpusA := cpuset.New(sharable[0]) + cpusB := cpuset.New(sharable[1]) + + before := leaf.FreeSupply().SharableCPUs() + + leaf.FreeSupply().ClaimCPUs(uid, cpusA) + leaf.FreeSupply().ClaimCPUs(uid, cpusB) + + // Only cpusB should be subtracted; cpusA must have been restored when the + // second ClaimCPUs call replaced the mark for the same uid. + want := before.Difference(cpusB) + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(want) { + t.Errorf("sharable CPUs after replacing claim = %s, want %s (cpusA=%s should be restored, cpusB=%s subtracted)", + got, want, cpusA, cpusB) + } + + // And ancestors must reflect the same non-stacked (single) subtraction. + for _, a := range ancestorsOf(leaf) { + full := a.FreeSupply().SharableCPUs() + if full.Intersection(cpusA).Size() != cpusA.Size() { + t.Errorf("ancestor %q is missing cpusA=%s (claim replace should have restored it there too): sharable=%s", + a.Name(), cpusA, full) + } + if full.Intersection(cpusB).Size() != 0 { + t.Errorf("ancestor %q still contains cpusB=%s that should be claimed: sharable=%s", + a.Name(), cpusB, full) + } + } +} + +func TestSupplyUnclaimCPUsRestores(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + ancestors := ancestorsOf(leaf) + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + + uid := types.UID("claim-uid-unclaim") + cpus := cpuset.New(sharable[0]) + + leafBefore := leaf.FreeSupply().SharableCPUs() + ancestorBefore := make(map[string]cpuset.CPUSet, len(ancestors)) + for _, a := range ancestors { + ancestorBefore[a.Name()] = a.FreeSupply().SharableCPUs() + } + + leaf.FreeSupply().ClaimCPUs(uid, cpus) + leaf.FreeSupply().UnclaimCPUs(uid) + + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(leafBefore) { + t.Errorf("leaf %q sharable CPUs after unclaim = %s, want restored %s", leaf.Name(), got, leafBefore) + } + for _, a := range ancestors { + if got := a.FreeSupply().SharableCPUs(); !got.Equals(ancestorBefore[a.Name()]) { + t.Errorf("ancestor %q sharable CPUs after unclaim = %s, want restored %s", + a.Name(), got, ancestorBefore[a.Name()]) + } + } +} + +func TestSupplyUnclaimCPUsUnknownUIDNoop(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + before := leaf.FreeSupply().SharableCPUs() + + defer func() { + if r := recover(); r != nil { + t.Fatalf("UnclaimCPUs for an unknown UID must not panic, got: %v", r) + } + }() + + leaf.FreeSupply().UnclaimCPUs(types.UID("never-claimed")) + + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(before) { + t.Errorf("sharable CPUs changed after unclaiming an unknown UID: got %s, want unchanged %s", got, before) + } +} + +func TestSupplyCloneCarriesClaimRefs(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + + uid := types.UID("claim-uid-clone") + cpus := cpuset.New(sharable[0]) + + leaf.FreeSupply().ClaimCPUs(uid, cpus) + + clone := leaf.FreeSupply().Clone() + + // The clone must already reflect the subtraction (it was taken after the + // claim), and — because claimRefs travels with Clone() — unclaiming on + // the clone must be able to restore the CPU, proving the clone knows + // which cpus belong to uid. + beforeUnclaim := clone.SharableCPUs() + if beforeUnclaim.Intersection(cpus).Size() != 0 { + t.Fatalf("clone did not inherit the claimed subtraction: sharable=%s still contains %s", beforeUnclaim, cpus) + } + + clone.UnclaimCPUs(uid) + + afterUnclaim := clone.SharableCPUs() + if afterUnclaim.Intersection(cpus).Size() != cpus.Size() { + t.Errorf("clone.UnclaimCPUs(%s) did not restore %s: sharable=%s -- Clone() must carry claimRefs", + uid, cpus, afterUnclaim) + } + + // The original leaf supply must be unaffected by unclaiming on the clone. + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("unclaiming on the clone leaked back into the original supply: sharable=%s still missing %s", got, cpus) + } +} + +func TestSupplyClaimCPUsAncestorNotDoubleSubtracted(t *testing.T) { + p := newDRATestPolicy(t) + + // "NUMA node #0" and "NUMA node #2" are siblings under "socket #0". + leafA := findPoolNode(t, p, "NUMA node #0") + leafB := findPoolNode(t, p, "NUMA node #2") + ancestor := findPoolNode(t, p, "socket #0") + + if leafA.Parent().Name() != ancestor.Name() || leafB.Parent().Name() != ancestor.Name() { + t.Fatalf("expected %q and %q to share parent %q; got parents %q and %q", + leafA.Name(), leafB.Name(), ancestor.Name(), leafA.Parent().Name(), leafB.Parent().Name()) + } + + cpusA := cpuset.New(leafA.FreeSupply().SharableCPUs().List()[0]) + cpusB := cpuset.New(leafB.FreeSupply().SharableCPUs().List()[0]) + if cpusA.Intersection(cpusB).Size() != 0 { + t.Fatalf("test setup error: cpusA and cpusB must be disjoint, got %s and %s", cpusA, cpusB) + } + + ancestorBefore := ancestor.FreeSupply().SharableCPUs() + + leafA.FreeSupply().ClaimCPUs(types.UID("claim-a"), cpusA) + leafB.FreeSupply().ClaimCPUs(types.UID("claim-b"), cpusB) + + want := ancestorBefore.Difference(cpusA).Difference(cpusB) + if got := ancestor.FreeSupply().SharableCPUs(); !got.Equals(want) { + t.Errorf("ancestor %q sharable CPUs after two independent child claims = %s, want %s (exactly one subtraction per claim, no double-subtraction)", + ancestor.Name(), got, want) + } +} diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index e9dc71036..cf1489eb1 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -15,17 +15,20 @@ package topologyaware import ( + "context" "errors" "fmt" "github.com/containers/nri-plugins/pkg/irq" "github.com/containers/nri-plugins/pkg/utils/cpuset" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/cpuallocator" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/resmgr/dra" "github.com/containers/nri-plugins/pkg/resmgr/events" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" @@ -74,6 +77,60 @@ type policy struct { cpuClasses *cpuclass.Handler // CPU class handler (cpufreq, SST/PCT, etc.) metrics *TopologyAwareMetrics // metrics provided by this policy irqCnt int // last applied [allocations.]irqCnt + + // claimContainerRefs counts, per DRA ResourceClaim UID, how many live + // containers currently reference that claim's CPUs (allocateClaim + // increments, releaseClaim decrements). The pool supply is marked via + // Supply.ClaimCPUs on the first container and unmarked via + // Supply.UnclaimCPUs only once the last referencing container is + // released — this is what makes a multi-container ResourceClaim + // (AllowMultipleAllocations) safe. + claimContainerRefs map[types.UID]int + + // tombstonedDRAClaims holds the allocation results for claims that have + // been unprepared (UnprepareResourceClaims) while at least one container + // was still using them. unprepareDRAClaim defers both the pool unclaim + // and the HP-CPU release until the last container drops; releaseClaim + // drains this map when the refcount reaches zero. + tombstonedDRAClaims map[types.UID][]dra.ResultAlloc + + // claimedCPUsByContainer is, per container ID, the union of every live + // DRA claim's CPUs that container consumes (populated/cleared in + // AllocateResources/ReleaseResources alongside allocateClaim/ + // releaseClaim, and repopulated by reapplyDRAClaims after a restart/ + // Reconfigure). applyGrant/updateSharedAllocations union this in before + // pinning a container's cpuset, so a claim consumer's normal grant + // (computed independently, without regard to its claimed CPUs) doesn't + // end up excluding the CPUs its CDI-injected NRI_CPU env vars claim + // it has. + claimedCPUsByContainer map[string]cpuset.CPUSet + + // draClaimsByContainer keeps the claims consumed by each container + // independently of the plugin's live-claim map, which may lose a claim + // during UnprepareResourceClaims before the container is released. + draClaimsByContainer map[string][]containerClaim + + // draPlugin is the DRA kubelet plugin instance for this driver, or nil + // when DRA is disabled (cfg.DRAEnabled() == false) or Setup() could not + // build one (see buildDRAPlugin in dra.go: missing kube client or node + // name at Setup() time — an empty cpuClasses configuration no longer + // prevents construction; the plugin is built with an empty device set + // instead). AllocateResources + // and ReleaseResources nil-check this field before passing it anywhere a + // claimLister is expected: a nil *dra.Plugin handed to an interface + // parameter would produce a non-nil interface wrapping a nil pointer + // (the typed-nil trap), so callers must guard on p.draPlugin != nil + // themselves rather than relying on claimCPUsFromContainer's internal + // nil check. + draPlugin *dra.Plugin + // draCtxCancel cancels the context draPlugin.Start() was given. Called + // from Stop() to shut the DRA plugin's background goroutines down. nil + // when draPlugin is nil. + draCtxCancel context.CancelFunc + // cdiDir is the directory DRA CDI spec files are written to. Empty + // means the dra package's default (/var/run/cdi). Overridable so tests + // can inject a temporary directory. + cdiDir string } var opt = &cfgapi.Config{} @@ -127,6 +184,15 @@ func (p *policy) Setup(opts *policyapi.BackendOptions) error { return policyError("failed to initialize %s policy: %w", PolicyName, err) } + // Build the DRA plugin, if enabled, once at initial Setup() time. There + // is no DRAEnabled-flip check here — see buildDRAPlugin's doc comment + // (dra.go) for why that check belongs in Reconfigure() instead. + if p.cfg.DRAEnabled() { + if err := p.buildDRAPlugin(opts); err != nil { + return policyError("failed to initialize %s policy: %w", PolicyName, err) + } + } + log.Infof("***** default CPU priority is %s", defaultPrio) return nil @@ -144,6 +210,46 @@ func (p *policy) Description() string { // Start prepares this policy for accepting allocation/release requests. func (p *policy) Start() error { + if err := p.restoreCache(); err != nil { + return policyError("failed to start: %v", err) + } + + // Start the DRA plugin (if built by Setup()) before reapplyDRAClaims: + // draPlugin.Start(ctx) loads the persisted ClaimStore, which + // reapplyDRAClaims reads via LiveClaimsLocked() to re-mark pool + // supplies. Start/PublishResources are not made while holding the + // resmgr lock — they take their own internal WithLock only for the + // specific sections that touch shared Handler/claim state. However, + // once draPlugin.Start(ctx) returns, the kubelet plugin is registered + // and may immediately start serving PrepareResourceClaims/ + // UnprepareResourceClaims RPCs in background goroutines; those mutate + // p.claims under deps.WithLock (= the resmgr write lock). reapplyDRAClaims + // (via LiveClaimsLocked) reads p.claims and therefore must also run under + // that same lock to avoid a concurrent unsynchronized map access. + if p.draPlugin != nil { + ctx, cancel := context.WithCancel(context.Background()) + p.draCtxCancel = cancel + if err := p.draPlugin.Start(ctx); err != nil { + cancel() + return policyError("failed to start DRA plugin: %v", err) + } + if err := p.draPlugin.PublishResources(ctx); err != nil { + cancel() + p.draPlugin.Stop() + return policyError("failed to publish DRA resources: %v", err) + } + + // Re-mark any DRA-claimed CPUs in the freshly rebuilt pool + // supplies. Must run under the resmgr write lock (see above); + // p.options.WithLock is the same closure the DRA plugin itself + // uses (dra.Deps.WithLock), and reapplyDRAClaims/its helpers + // (evictOverlappingGrants, remarkClaimInSupply, reallocateEvicted) + // do not call WithLock themselves, so this cannot deadlock. + p.options.WithLock(func() { + p.reapplyDRAClaims() + }) + } + // Turn coldstart forcibly off if we have movable non-DRAM memory. // Note that although this can change dynamically we only check it // during startup and trust users to either not fiddle with memory @@ -163,6 +269,20 @@ func (p *policy) Start() error { return nil } +// Stop shuts down this policy: it cancels the DRA plugin's context and +// stops the DRA plugin (kubeletplugin registration, background helper +// goroutines), if one was built. A no-op when DRA is disabled +// (draPlugin == nil) — safe to call regardless of whether Start() ran. +func (p *policy) Stop() error { + if p.draCtxCancel != nil { + p.draCtxCancel() + } + if p.draPlugin != nil { + p.draPlugin.Stop() + } + return nil +} + // Sync synchronizes the state of this policy. func (p *policy) Sync(add []cache.Container, del []cache.Container) error { irq.BlockWrites() @@ -243,9 +363,30 @@ func (p *policy) AllocateResources(container cache.Container) error { defer p.commitCpuClasses(container.PrettyName()) defer p.applyIrqAffinity(container.PrettyName()) + defer p.triggerDRARepublish() + + var markedClaims []containerClaim + if p.draPlugin != nil { + for _, cl := range claimCPUsFromContainer(container, p.draPlugin) { + if err := p.allocateClaim(cl.UID, cl.CPUs, cl.ClassCPUs); err != nil { + p.rollbackClaimMarks(container, markedClaims) + return policyError("failed to allocate resources for %s: %v", + container.PrettyName(), err) + } + markedClaims = append(markedClaims, cl) + } + if len(markedClaims) > 0 { + p.setClaimedCPUs(container, markedClaims) + if p.draClaimsByContainer == nil { + p.draClaimsByContainer = make(map[string][]containerClaim) + } + p.draClaimsByContainer[container.GetID()] = markedClaims + } + } err := p.allocateResources(container, "") if err != nil { + p.rollbackClaimMarks(container, markedClaims) return err } @@ -257,6 +398,41 @@ func (p *policy) AllocateResources(container cache.Container) error { return nil } +// setClaimedCPUs unions every claim's CPUs in marked and records the result +// in p.claimedCPUsByContainer, keyed by container's ID. Used by +// AllocateResources after successfully marking every live claim a container +// carries, and by reapplyDRAClaims to repopulate the map after a restart/ +// Reconfigure (see its own comment for why that repopulation alone isn't +// sufficient there). +func (p *policy) setClaimedCPUs(container cache.Container, marked []containerClaim) { + union := cpuset.New() + for _, cl := range marked { + union = union.Union(cl.CPUs) + } + if p.claimedCPUsByContainer == nil { + p.claimedCPUsByContainer = map[string]cpuset.CPUSet{} + } + p.claimedCPUsByContainer[container.GetID()] = union +} + +// rollbackClaimMarks releases every claim mark accumulated so far in an +// AllocateResources call that failed partway through (whether from a later +// claim's own allocateClaim call or from the subsequent normal pool +// allocation), so a partial failure never leaks a claimContainerRefs entry +// (or a stale claimedCPUsByContainer union) for a container that ultimately +// never got its resources allocated. +func (p *policy) rollbackClaimMarks(container cache.Container, marked []containerClaim) { + for _, cl := range marked { + if err := p.releaseClaim(cl.UID, cl.CPUs); err != nil { + log.Errorf("dra: rollback: failed to release claim %s: %v", cl.UID, err) + } + } + if len(marked) > 0 { + delete(p.draClaimsByContainer, container.GetID()) + delete(p.claimedCPUsByContainer, container.GetID()) + } +} + func (p *policy) allocateResources(container cache.Container, poolHint string) error { grant, err := p.allocatePool(container, poolHint) if err != nil { @@ -278,6 +454,21 @@ func (p *policy) ReleaseResources(container cache.Container) error { defer p.commitCpuClasses(container.PrettyName()) defer p.applyIrqAffinity(container.PrettyName()) + defer p.triggerDRARepublish() + + if p.draPlugin != nil { + claims := p.draClaimsByContainer[container.GetID()] + if claims == nil { + claims = claimCPUsFromContainer(container, p.draPlugin) + } + for _, cl := range claims { + if err := p.releaseClaim(cl.UID, cl.CPUs); err != nil { + log.Errorf("failed to release DRA claim for %s: %v", container.PrettyName(), err) + } + } + delete(p.draClaimsByContainer, container.GetID()) + delete(p.claimedCPUsByContainer, container.GetID()) + } if grant, found := p.releasePool(container); found { p.updateSharedAllocations(&grant) @@ -300,6 +491,7 @@ func (p *policy) UpdateResources(container cache.Container) error { defer p.commitCpuClasses(container.PrettyName()) defer p.applyIrqAffinity(container.PrettyName()) + defer p.triggerDRARepublish() grant, found := p.releasePool(container) if !found { @@ -537,6 +729,18 @@ func (p *policy) Reconfigure(newCfg any) error { savedPolicy := *p allocations := savedPolicy.allocations.clone() + // DRA config changes cannot be applied live: buildDRAPlugin only ever + // runs once, from the initial Setup() (see its doc comment in dra.go + // for why this check cannot live there) — Reconfigure() never tears + // down or (re)builds p.draPlugin, so there is no way to safely apply a + // dra.enabled flip or a cpuClass attribute change (which could + // invalidate a live DRA claim) without a restart. Refuse any config + // change outright whenever DRA is (or was) enabled. + if cfg.DRAEnabled() || p.cfg.DRAEnabled() { + return policyError("failed to reconfigure: DRA config changes require a restart " + + "(dra.enabled, cpuClass changes, etc. cannot be applied via live reconfigure)") + } + opt = cfg p.cfg = cfg defaultPrio = cfg.DefaultCPUPriority.Value() @@ -546,6 +750,8 @@ func (p *policy) Reconfigure(newCfg any) error { if err := p.initialize(); err != nil { *p = savedPolicy + opt = p.cfg + defaultPrio = p.cfg.DefaultCPUPriority.Value() return policyError("failed to reconfigure: %v", err) } @@ -566,6 +772,7 @@ func (p *policy) Reconfigure(newCfg any) error { if err := p.restoreAllocations(&allocations); err != nil { *p = savedPolicy opt = p.cfg + defaultPrio = p.cfg.DefaultCPUPriority.Value() return policyError("failed to reconfigure: %v", err) } @@ -739,6 +946,172 @@ func (p *policy) findExistingTopologyLevel(level cfgapi.CPUTopologyLevel) cfgapi return cfgapi.CPUTopologyLevelPackage } +func (p *policy) restoreCache() error { + allocations := p.newAllocations() + if p.cache.GetPolicyEntry(keyAllocations, &allocations) { + if err := p.restoreAllocations(&allocations); err != nil { + return policyError("failed to restore allocations from cache: %v", err) + } + p.allocations.Dump(log.Infof, "restored ") + } + p.saveAllocations() + + return nil +} + +// remarkClaimInSupply marks cpus as claimed by DRA ResourceClaim uid in the +// tightest pool that fully contains them (tree-wide, via +// Supply.ClaimCPUs — see resources.go), without touching +// claimContainerRefs. This is the marking-only counterpart to allocateClaim +// (pools.go): reapplyDRAClaims uses it to restore Supply.claimRefs after +// Start()/Reconfigure() rebuild pool/supply state in initialize(), which +// discards any marks a prior allocateClaim call applied. +// +// Using allocateClaim here instead of this marking-only path would +// double-count in the Reconfigure() case: p.claimContainerRefs is an +// in-process map that Reconfigure() never resets, so containers backing a +// live claim are already reflected in it from the AllocateResources call +// that admitted them. +// +// That reasoning does NOT hold across a process restart: claimContainerRefs +// is a plain in-memory map with no persistence, so it is zero-valued right +// after Start(), even though containers backed by live claims are already +// running. The correct refcount is rebuilt indirectly: pkg/resmgr/nri.go's +// syncWithNRI/Synchronize forces every already-running container through +// ReleaseResources (a no-op here, since the refcount is already 0) followed +// by AllocateResources (which calls allocateClaim and increments the +// refcount) as part of the NRI resync that always follows agent Start(). +// reapplyDRAClaims only has to fix up Supply.claimRefs (pool CPU exclusion) +// for the window between Start() and that resync; claimContainerRefs catches +// up once the resync runs. If that syncWithNRI invariant ever changes (e.g. +// running containers stop being included in both the "allocated" and +// "released" lists), releaseClaim will silently no-op on the eventual real +// ReleaseResources (refcount already 0) and the claimed CPUs will leak out +// of pool capacity permanently, until the next restart. +// +// Also re-applies the physical cpuClass (className) to cpus: initialize() +// (called by both Start() and Reconfigure() before this runs) resets every +// allowed CPU's class back to the shared-pool default +// (resetCpuClass("initialize", p.allowed)), which would otherwise silently +// strip the SST-CP/EPP/governor settings a live DRA claim depends on. +func (p *policy) remarkClaimInSupply(uid types.UID, cpus cpuset.CPUSet, classCPUs map[string]cpuset.CPUSet) error { + if cpus.IsEmpty() { + return policyError("cannot remark DRA claim %s: empty CPU set", uid) + } + + pool, err := p.poolForCPUs(cpus) + if err != nil { + return policyError("cannot remark DRA claim %s (CPUs %s): %v", uid, cpus, err) + } + + pool.FreeSupply().ClaimCPUs(uid, cpus) + + // classCPUs groups cpus by cpuClass (see classifyClaimCPUs): applied per + // subset so a claim spanning more than one class re-applies each class + // only to the CPUs that actually belong to it. Best-effort on the + // restart/reconfigure reapply path: log but do not fail the remark. + if err := p.applyClassCPUs("re-apply", uid, classCPUs); err != nil { + log.Errorf("dra: %v", err) + } + + return nil +} + +// reapplyDRAClaims re-marks pool supplies for every currently live DRA +// claim, restoring the Supply.claimRefs bookkeeping that Start()'s +// restoreCache() and Reconfigure()'s restoreAllocations() lose whenever +// initialize() rebuilds the pool/supply tree from scratch. It is a no-op if +// DRA is disabled (draPlugin == nil). +// +// reapplyDRAClaims runs *after* restoreCache()/restoreAllocations() have +// already reinstated grants (see Start()/Reconfigure()), at a point where +// Supply.claimRefs has just been wiped by initialize() and not yet re-marked. +// reinstateGrants/reallocateResources are therefore unaware of live DRA +// claims while they run: if grant restoration (verbatim reinstatement, or +// its allocatePool-based fallback) happens to hand a regular container CPUs +// that a live claim already owns, that overlap is a real double-booking — +// two workloads pinned to the same physical CPUs. Before marking each +// claim's CPUs here, evict and requeue for reallocation any restored grant +// that overlaps them, exactly like allocateClaim's first-time eviction path +// (evictOverlappingGrants/reallocateEvicted, pools.go) — this does not touch +// claimContainerRefs, so it stays consistent with remarkClaimInSupply's +// marking-only contract. +// +// Caller must hold the resmgr write lock (LiveClaimsLocked's contract). +// Reconfigure() already runs under the caller's lock, so it calls this +// directly. Start() runs unlocked (see its comment), so it must establish +// the lock itself via p.options.WithLock(func() { p.reapplyDRAClaims() }) — +// do not call reapplyDRAClaims from inside a callback that is nested inside +// an *already held* WithLock/resmgr-lock scope, since the lock is not +// reentrant and a second acquisition would deadlock. +func (p *policy) reapplyDRAClaims() { + if p.draPlugin == nil { + return + } + + remarked := false + for uid, allocs := range p.draPlugin.LiveClaimsLocked() { + cpus, classCPUs := classifyClaimCPUs(uid, allocs) + + if cpus.IsEmpty() { + continue + } + + evicted, evictedCpusets := p.evictOverlappingGrants(cpus, fmt.Sprintf("reapplyDRAClaims: claim %s", uid)) + + if err := p.remarkClaimInSupply(uid, cpus, classCPUs); err != nil { + log.Errorf("dra: reapplyDRAClaims: %v", err) + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + log.Errorf("dra: reapplyDRAClaims: failed to restore grants after claim %s could not be remarked: %v", uid, reallocErr) + } + continue + } + remarked = true + + if err := p.reallocateEvicted(evicted, evictedCpusets, cpus, uid); err != nil { + log.Errorf("dra: reapplyDRAClaims: claim %s: evicted %d restored grant(s) overlapping "+ + "claimed CPUs %s but failed to fully reallocate them: %v", uid, len(evicted), cpus, err) + } + } + + // claimedCPUsByContainer is a plain in-memory map with no persistence, + // so it is empty here even though pool accounting above is now + // correct. A prepared claim with no consumer is explicitly unclaimed + // through the DRA plugin's ClaimUnprepare callback, rather than relying + // on a later syncWithNRI resync. restoreCache()'s/Reconfigure()'s own + // restoreAllocations() call already ran applyGrant for these + // containers *before* reapplyDRAClaims was ever reached, without the + // union -- their cpusets need an explicit re-pin here, now, not just + // the map. Re-resolve every live claim's consuming container(s) and + // repopulate the map, then re-pin unconditionally (not gated on + // remarked/updateSharedAllocations's shared-portion check below, which + // skips exactly the common plain-exclusive-CPU claim consumer case). + for _, c := range p.cache.GetContainers() { + claims := claimCPUsFromContainer(c, p.draPlugin) + if len(claims) == 0 { + continue + } + p.setClaimedCPUs(c, claims) + + if grant, ok := p.allocations.getGrant(c.GetID()); ok { + p.applyGrant(grant) + } else if opt.PinCPU { + union := p.claimedCPUsByContainer[c.GetID()] + p.setPreferredCpusetCpus(c, cpuset.New(), union, + fmt.Sprintf(" => re-pinning %s to claimed cpuset %s (no regular grant)", c.PrettyName(), union)) + } + } + + // Re-marking may have subtracted CPUs from one or more pools' sharable + // capacity; any container already pinned (via applyGrant, from before + // the rebuild) to the previous, wider sharable cpuset must be re-pinned + // to the now-reduced set so it cannot keep running on CPUs a live DRA + // claim exclusively owns. + if remarked { + p.updateSharedAllocations(nil) + } +} + func (p *policy) checkColdstartOff() { for _, id := range p.sys.NodeIDs() { node := p.sys.Node(id) diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go b/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go new file mode 100644 index 000000000..8305f91a9 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go @@ -0,0 +1,863 @@ +// Copyright 2026 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "context" + "fmt" + "sync" + "testing" + + corev1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// test helpers: a minimal real *dra.Plugin seeded with a live +// claim via PrepareResourceClaims. Building a real Plugin (rather than +// re-deriving the claimLister interface with a fake) is what lets these +// tests exercise the exact typed-nil-trap-prone p.draPlugin field the +// production AllocateResources/ReleaseResources call sites use. ---- + +// fakeDRADeviceLister is a dra.DeviceLister that always returns a fixed list +// of devices, regardless of driverName. +type fakeDRADeviceLister struct { + devices []resourceapi.Device +} + +func (f *fakeDRADeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { + return f.devices, nil +} + +// fakeDRAClaimAllocator is a dra.ClaimAllocator that always "picks" a fixed +// CPUSet and reports every class as HP-eligible. Good enough for +// PrepareResourceClaims to succeed; the pool-accounting side effects this +// package cares about (Supply.ClaimCPUs/UnclaimCPUs) are exercised +// separately via allocateClaim/releaseClaim, not via this allocator. +type fakeDRAClaimAllocator struct { + pick cpuset.CPUSet +} + +func (f *fakeDRAClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { + return f.pick, nil +} +func (f *fakeDRAClaimAllocator) ReleaseHpCpus(_, _ int, _ cpuset.CPUSet) {} +func (f *fakeDRAClaimAllocator) AccountHpCpus(_, _ int, _ cpuset.CPUSet) error { return nil } +func (f *fakeDRAClaimAllocator) IsHPClass(_ string) bool { return true } + +// fakeDRACDIWriter is a dra.CDIWriter that tracks per-UID "written" state +// in memory instead of touching disk. Stateful (rather than a fixed +// false/nil) so that ClaimSpecExists/ListClaims accurately reflect prior +// WriteClaim/RemoveClaim calls: dra.Plugin.Start()'s orphan-claim sweep +// calls ClaimSpecExists for every persisted claim and drops any claim it +// reports as missing. +type fakeDRACDIWriter struct { + written map[types.UID]bool +} + +func (w *fakeDRACDIWriter) WriteClaim(uid types.UID, _ []dra.CDIDevice) error { + if w.written == nil { + w.written = map[types.UID]bool{} + } + w.written[uid] = true + return nil +} +func (w *fakeDRACDIWriter) RemoveClaim(uid types.UID) error { + delete(w.written, uid) + return nil +} +func (w *fakeDRACDIWriter) ClaimSpecExists(uid types.UID) bool { return w.written[uid] } +func (w *fakeDRACDIWriter) ListClaims() ([]types.UID, error) { + uids := make([]types.UID, 0, len(w.written)) + for uid := range w.written { + uids = append(uids, uid) + } + return uids, nil +} + +// fakeDRAClaimStore is a dra.ClaimStore that succeeds without persisting +// anything. +type fakeDRAClaimStore struct{} + +func (*fakeDRAClaimStore) Save(map[types.UID]*dra.ClaimState) error { return nil } +func (*fakeDRAClaimStore) Load() (map[types.UID]*dra.ClaimState, error) { return nil, nil } + +// newTestDRAPlugin builds a real *dra.Plugin backed entirely by fakes, ready +// for PrepareResourceClaims calls. pick is the CPUSet the fake allocator +// hands out for every PickHpCpus call. +func newTestDRAPlugin(t *testing.T, pick cpuset.CPUSet, deviceName string) *dra.Plugin { + t.Helper() + return newTestDRAPluginWithLock(t, pick, deviceName, func(f func()) { f() }) +} + +// newTestDRAPluginWithLock is newTestDRAPlugin with an injectable WithLock, +// letting lock-contract tests share a single non-reentrant stub between the +// DRA plugin's deps.WithLock and the policy's options.WithLock (both are +// backed by the same resmgr write lock in production). +func newTestDRAPluginWithLock(t *testing.T, pick cpuset.CPUSet, deviceName string, withLock func(func())) *dra.Plugin { + t.Helper() + + className := "gold" + device := resourceapi.Device{ + Name: deviceName, + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/cpuClass": {StringValue: &className}, + "nri/packageID": {IntValue: new(int64)}, + "nri/punitID": {IntValue: new(int64)}, + }, + } + + deps := dra.Deps{ + KubeClient: fake.NewClientset(), + NodeName: "test-node", + RegistrarDir: t.TempDir(), + PluginDataDir: t.TempDir(), + ValidateClasses: func() error { return nil }, + ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + DeviceLister: &fakeDRADeviceLister{devices: []resourceapi.Device{device}}, + ClaimAllocator: &fakeDRAClaimAllocator{pick: pick}, + CDIWriter: &fakeDRACDIWriter{}, + ClaimStore: &fakeDRAClaimStore{}, + WithLock: withLock, + ClaimUnprepare: func(_ types.UID, _ []dra.ResultAlloc) {}, + Logger: log, + } + + p, err := dra.New(DRADriverName, deps) + if err != nil { + t.Fatalf("dra.New() failed: %v", err) + } + return p +} + +// seedLiveClaim runs PrepareResourceClaims for a single claim UID against +// plugin, returning the qualified CDI device name the runtime would see on +// the container's CDIDevices (the same string GetCDIDeviceNames() returns). +func seedLiveClaim(t *testing.T, plugin *dra.Plugin, uid types.UID, deviceName string, numCPUs int) string { + t.Helper() + + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + { + Driver: DRADriverName, + Pool: "pool0", + Device: deviceName, + Request: "req0", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{ + "nri/cpus": resource.MustParse(fmt.Sprintf("%d", numCPUs)), + }, + }, + }, + }, + }, + }, + } + + result, err := plugin.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil { + t.Fatalf("PrepareResourceClaims() unexpected error: %v", err) + } + r, ok := result[uid] + if !ok { + t.Fatalf("PrepareResourceClaims() result missing uid %s", uid) + } + if r.Err != nil { + t.Fatalf("PrepareResourceClaims() PrepareResult.Err = %v, want nil", r.Err) + } + if len(r.Devices) != 1 || len(r.Devices[0].CDIDeviceIDs) != 1 { + t.Fatalf("PrepareResourceClaims() result = %+v, want exactly one device with one CDI device ID", r) + } + + return r.Devices[0].CDIDeviceIDs[0] +} + +// TestAllocateResourcesWithTAClaimCallsAllocateClaim verifies that +// AllocateResources recognizes a container carrying a live TA DRA claim's +// CDI device, marks the claimed CPUs in the pool supply (so a subsequent +// regular allocation cannot pick them), and bumps claimContainerRefs. +func TestAllocateResourcesWithTAClaimCallsAllocateClaim(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-alloc-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + + p.draPlugin = plugin + + container := &mockContainer{returnValueForGetID: "c1", cdiDeviceNames: []string{cdiName}} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPUs %s still present in %q sharable set after AllocateResources: %s", claimed, leaf.Name(), got) + } + if got := p.claimContainerRefs[uid]; got != 1 { + t.Errorf("claimContainerRefs[%s] = %d after AllocateResources, want 1", uid, got) + } +} + +// TestReleaseResourcesWithTAClaimCallsReleaseClaim verifies that +// ReleaseResources restores CPUs claimed by allocateClaim once the last +// referencing container is released. +func TestReleaseResourcesWithTAClaimCallsReleaseClaim(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-release-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + + p.draPlugin = plugin + + container := &mockContainer{returnValueForGetID: "c1", cdiDeviceNames: []string{cdiName}} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Fatalf("test setup error: claimed CPUs %s not marked before ReleaseResources", claimed) + } + + if err := p.ReleaseResources(container); err != nil { + t.Fatalf("ReleaseResources() unexpected error: %v", err) + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != claimed.Size() { + t.Errorf("claimed CPUs %s not restored after ReleaseResources: sharable=%s", claimed, got) + } + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs[%s] still present after ReleaseResources released the last container", uid) + } +} + +// TestAllocateResourcesRollsBackClaimOnPoolAllocationFailure verifies that +// AllocateResources rolls back a successful claim-ref mark if the subsequent +// normal pool allocation fails, instead of leaking a claimContainerRefs +// entry (and the corresponding pool supply mark) for a container that never +// actually got its resources allocated. +func TestAllocateResourcesRollsBackClaimOnPoolAllocationFailure(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-rollback-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + + p.draPlugin = plugin + + // An absurdly large exclusive CPU request guarantees the subsequent + // normal pool allocation fails, regardless of topology. + container := &mockContainer{ + returnValueForGetID: "c1", + cdiDeviceNames: []string{cdiName}, + returnValueForGetResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + }, + } + + if err := p.AllocateResources(container); err == nil { + t.Fatal("AllocateResources() expected error from an unsatisfiable resource request, got nil") + } + + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs[%s] still present after AllocateResources failed; claim mark was not rolled back", uid) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != claimed.Size() { + t.Errorf("claimed CPUs %s not restored to pool supply after AllocateResources rollback: sharable=%s", claimed, got) + } +} + +// TestApplyGrantUnionsClaimedCPUsIntoContainerCpuset is the regression test +// for DRA-claimed CPUs missing from a consumer's cpuset: allocateClaim only +// marks pool supply (and, separately, physically paints the claim's +// cpuClass) — it has no way to touch the consuming container's own cgroup +// cpuset. applyGrant is what actually +// calls container.SetCpusetCpus, but until this fix it only ever pinned the +// *normal* grant's own CPUs (Y), silently excluding whatever CPUs (X) the +// container also holds via a live DRA claim — even though the CDI-injected +// NRI_CPU env vars tell the container it has X. This asserts the +// container's actual pinned cpuset is the union of X and Y, and that X was +// not counted towards Y's own (independently-computed) sizing. +func TestApplyGrantUnionsClaimedCPUsIntoContainerCpuset(t *testing.T) { + p := newDRATestPolicy(t) + // applyGrant only calls container.SetCpusetCpus at all when PinCPU is + // enabled; opt and p.cfg alias the same *cfgapi.Config after Setup(). + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimedCPU := sharable[0] + claimed := cpuset.New(claimedCPU) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-cpuset-union-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + container := &mockContainer{ + returnValueForGetID: "c1", + cdiDeviceNames: []string{cdiName}, + returnValueForGetResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + } + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + + gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) + } + if !gotCpus.Contains(claimedCPU) { + t.Errorf("container cpuset %s does not include claimed CPU %d (X) — only the normal grant's own CPUs (Y) were pinned", gotCpus, claimedCPU) + } + // The normal 1-CPU exclusive request (Y) must have been sized on its + // own, independent of the claimed CPU: exactly one CPU besides the + // claimed one, not zero (which would mean Y's sizing was inflated by + // counting the already-claimed CPU as if it were Y's own). + exclusiveOnly := gotCpus.Difference(claimed) + if exclusiveOnly.Size() != 1 { + t.Errorf("container cpuset %s minus claimed CPU %s = %s, want exactly 1 CPU for the normal grant (Y), got double-counting or a missing grant", + gotCpus, claimed, exclusiveOnly) + } +} + +// TestAllocateResourcesRollbackClearsClaimedCPUsByContainer verifies that +// rollbackClaimMarks also clears claimedCPUsByContainer's entry for a +// container whose AllocateResources call failed partway through, not just +// release the claims in pool/refcount terms -- otherwise a rolled-back call +// could leave a stale cpuset union in place for a container that no longer +// holds the claim. +func TestAllocateResourcesRollbackClearsClaimedCPUsByContainer(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-rollback-clear-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + container := &mockContainer{ + returnValueForGetID: "rollback-clear-c1", + cdiDeviceNames: []string{cdiName}, + returnValueForGetResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + }, + } + + if err := p.AllocateResources(container); err == nil { + t.Fatal("AllocateResources() expected error from an unsatisfiable resource request, got nil") + } + + if _, exists := p.claimedCPUsByContainer[container.GetID()]; exists { + t.Errorf("claimedCPUsByContainer[%s] still present after AllocateResources rollback", container.GetID()) + } +} + +// TestApplyGrantEmptyCpusUsesClaimedCPUsInsteadOfBlanking covers applyGrant's +// cpus.Size() == 0 exit (which otherwise calls container.SetCpusetCpus("")): +// a claim consumer whose *normal* grant happens to compute an empty cpuset +// must not lose access to its claimed CPUs entirely. Uses a +// cpuReserved grant on a leaf pool with no reserved CPUs of its own to +// exercise an empty `cpus` without needing to fully drain a pool's sharable +// capacity. +func TestApplyGrantEmptyCpusUsesClaimedCPUsInsteadOfBlanking(t *testing.T) { + p := newDRATestPolicy(t) + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #1") + if !leaf.GetSupply().ReservedCPUs().IsEmpty() { + t.Fatalf("test setup error: %q has reserved CPUs, want none", leaf.Name()) + } + + container := &mockContainer{returnValueForGetID: "empty-grant-claim-1"} + claimed := cpuset.New(leaf.FreeSupply().SharableCPUs().List()[0]) + p.claimedCPUsByContainer = map[string]cpuset.CPUSet{ + container.GetID(): claimed, + } + + g := newGrant(leaf, container, cpuReserved, "", cpuset.New(), 0, memoryDRAM, nil, 0) + p.applyGrant(g) + + gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) + } + if !gotCpus.Equals(claimed) { + t.Errorf("container cpuset = %s after applyGrant with an empty (reserved) grant, want claimed CPUs %s (must not be blanked to \"\")", gotCpus, claimed) + } +} + +// TestUpdateSharedAllocationsPreservesClaimedCPUsOnRepin covers the +// updateSharedAllocations re-pin path triggered from allocateClaim: +// container A already holds a live claim (claimACPU) plus a plain +// shared-portion grant (no exclusive CPUs). When an unrelated claim for +// container B is allocated, allocateClaim's own updateSharedAllocations(nil) +// call re-pins every other grant -- including A's -- to the pool's now- +// smaller sharable set. That re-pin must still include A's own claimed CPU, +// not just the (shrunk) shared set. +func TestUpdateSharedAllocationsPreservesClaimedCPUsOnRepin(t *testing.T) { + p := newDRATestPolicy(t) + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + + containerA := &mockContainer{returnValueForGetID: "shared-alloc-a"} + claimACPU := sharable[0] + p.claimedCPUsByContainer = map[string]cpuset.CPUSet{ + containerA.GetID(): cpuset.New(claimACPU), + } + grantA := newGrant(leaf, containerA, cpuNormal, "", cpuset.New(), 100, memoryDRAM, nil, 0) + p.allocations.addGrant(grantA) + + claimBCPU := sharable[1] + uidB := types.UID("claim-shared-repin-b") + if err := p.allocateClaim(uidB, cpuset.New(claimBCPU), goldClassCPUs(cpuset.New(claimBCPU))); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + gotCpus, err := cpuset.Parse(containerA.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container A's cpuset %q: %v", containerA.GetCpusetCpus(), err) + } + if !gotCpus.Contains(claimACPU) { + t.Errorf("container A's cpuset %s lost its own claimed CPU %d after an unrelated claim's updateSharedAllocations re-pin", gotCpus, claimACPU) + } +} + +// TestReapplyDRAClaimsRepinsContainerCpusetWithGrant is the restart-window +// regression test for reapplyDRAClaims's cpuset re-pin: a container +// restored with a regular grant (grantCPU, e.g. by restoreCache() before +// reapplyDRAClaims is ever reached) that also carries a live DRA claim +// (claimedCPU) must end up with both in its cpuset once reapplyDRAClaims +// runs -- not just wait for the next NRI resync's Release+AllocateResources +// to add the claimed CPU in. +func TestReapplyDRAClaimsRepinsContainerCpusetWithGrant(t *testing.T) { + p := newDRATestPolicy(t) + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimedCPU := sharable[0] + grantCPU := sharable[1] + claimed := cpuset.New(claimedCPU) + + container := &mockContainer{returnValueForGetID: "reapply-repin-1"} + addTestGrant(t, p, leaf, container, cpuset.New(grantCPU)) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-reapply-repin-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + container.cdiDeviceNames = []string{cdiName} + p.draPlugin = plugin + + mc, ok := p.cache.(*mockCache) + if !ok { + t.Fatalf("test setup error: p.cache is not *mockCache") + } + mc.containers = []cache.Container{container} + + p.reapplyDRAClaims() + + gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) + } + want := cpuset.New(claimedCPU, grantCPU) + if !gotCpus.Equals(want) { + t.Errorf("container cpuset after reapplyDRAClaims() = %s, want %s (claimed + grant, re-pinned without waiting for the next NRI resync)", gotCpus, want) + } +} + +// TestAllocateResourcesNoCDIDevicesUnaffected verifies that a container +// without any TA CDI device is unaffected by an active (non-nil) draPlugin: +// no claim bookkeeping happens, and regular allocation behaves exactly as it +// did before Step 8. +func TestAllocateResourcesNoCDIDevicesUnaffected(t *testing.T) { + p := newDRATestPolicy(t) + p.draPlugin = newTestDRAPlugin(t, cpuset.New(), "dev0") // no live claims seeded + + container := &mockContainer{returnValueForGetID: "c1"} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + if len(p.claimContainerRefs) != 0 { + t.Errorf("claimContainerRefs = %v after AllocateResources on a container with no CDI devices, want empty", p.claimContainerRefs) + } +} + +// TestAllocateResourcesNilDRAPluginNoCrash verifies that AllocateResources +// and ReleaseResources on a policy with DRA disabled (draPlugin == nil, the +// default) neither panic nor attempt any claim lookup. +func TestAllocateResourcesNilDRAPluginNoCrash(t *testing.T) { + p := newDRATestPolicy(t) + if p.draPlugin != nil { + t.Fatalf("test setup error: draPlugin unexpectedly non-nil") + } + + container := &mockContainer{returnValueForGetID: "c1"} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() with nil draPlugin: unexpected error: %v", err) + } + if err := p.ReleaseResources(container); err != nil { + t.Fatalf("ReleaseResources() with nil draPlugin: unexpected error: %v", err) + } +} + +// reapplyDRAClaims()/remarkClaimInSupply() and their +// Start()/Reconfigure() wiring. ---- + +// TestStartMarksLiveDRAClaimsInPoolSupply verifies that Start() re-marks +// pool supplies for every claim the (already-loaded) DRA plugin reports as +// live, so a subsequent regular allocation cannot pick those CPUs. This +// stands in for a full ClaimStore-backed reload: p.draPlugin is seeded with +// a live claim via PrepareResourceClaims before Start() runs. +func TestStartMarksLiveDRAClaimsInPoolSupply(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-start-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + if err := p.Start(); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + + free := leaf.FreeSupply() + if got := free.SharableCPUs().Union(free.IsolatedCPUs()).Intersection(claimed); got.Size() != 0 { + t.Errorf("claimed CPUs %s still free in %q supply after Start(): %s", claimed, leaf.Name(), got) + } +} + +// TestStartReappliesDRAClaimsAfterRestoreCache verifies the ordering that +// Start() must observe: restoreCache() alone (which only restores +// previously-cached container allocations, and knows nothing about DRA +// claims) must not mark any claimed CPUs; only the rest of Start() (which +// calls reapplyDRAClaims() after restoreCache() returns) does. +func TestStartReappliesDRAClaimsAfterRestoreCache(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-start-order-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + if err := p.restoreCache(); err != nil { + t.Fatalf("restoreCache() unexpected error: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != claimed.Size() { + t.Fatalf("test setup error: claimed CPUs %s unexpectedly marked by restoreCache() alone (want unmarked at this point): sharable=%s", + claimed, got) + } + + if err := p.Start(); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPUs %s not marked once Start() completed: sharable=%s", claimed, got) + } +} + +// lockContractStub is a WithLock stand-in that panics if invoked while +// already "held", i.e. re-entrantly. Mirrors pkg/resmgr/policy's +// lockContractStub: used here to assert that Start()'s draPlugin.Start(), +// draPlugin.PublishResources(), and reapplyDRAClaims() calls each acquire +// the (shared, non-reentrant) resmgr write lock in strict sequence, never +// nested — the exact bug class of the reapplyDRAClaims/LiveClaimsLocked +// unsynchronized-access race this test guards against. +type lockContractStub struct { + mu sync.Mutex + held bool +} + +func (s *lockContractStub) run(f func()) { + s.mu.Lock() + if s.held { + s.mu.Unlock() + panic("WithLock invoked re-entrantly") + } + s.held = true + s.mu.Unlock() + + defer func() { + s.mu.Lock() + s.held = false + s.mu.Unlock() + }() + + f() +} + +// TestStartReapplyDRAClaimsHoldsWriteLockNotReentrant verifies that Start() +// runs reapplyDRAClaims() (and therefore LiveClaimsLocked(), which reads +// p.claims with no internal synchronization) under the same non-reentrant +// WithLock the DRA plugin's own Start()/PublishResources() use — and never +// nests a second acquisition inside an already-held one, which would +// deadlock. A single lockContractStub is shared between the policy's +// options.WithLock and the DRA plugin's deps.WithLock, exactly as production +// wiring shares one resmgr write lock (m.withWriteLock) between both. +func TestStartReapplyDRAClaimsHoldsWriteLockNotReentrant(t *testing.T) { + stub := &lockContractStub{} + p := newDRATestPolicyWithLock(t, stub.run) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPluginWithLock(t, claimed, "dev0", stub.run) + uid := types.UID("claim-lock-contract-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + var startErr error + panicked := false + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + startErr = p.Start() + }() + + if panicked { + t.Fatalf("Start() panicked: WithLock was invoked re-entrantly") + } + if startErr != nil { + t.Fatalf("Start() unexpected error: %v", startErr) + } + if stub.held { + t.Errorf("resmgr write lock still held after Start() returned") + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPUs %s not marked once Start() completed: sharable=%s", claimed, got) + } +} + +// TestClaimContainerRefsRebuiltAfterStartResync verifies the mechanism +// documented on remarkClaimInSupply/reapplyDRAClaims for the restart case: +// p.claimContainerRefs is a plain in-memory map, so it is empty right after +// Start(), even though a container backed by a live DRA claim is already +// running. It is only rebuilt once pkg/resmgr/nri.go's syncWithNRI/ +// Synchronize forces that already-running container through +// ReleaseResources (a no-op, since the refcount is already 0) followed by +// AllocateResources (which increments it) — reproduced here directly via +// p.Sync(add, del) with the same container in both lists, exactly as +// syncWithNRI does for every container discovered in ContainerStateRunning/ +// ContainerStateCreated. +func TestClaimContainerRefsRebuiltAfterStartResync(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-restart-resync-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + if err := p.Start(); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + + // Right after Start(), the pool supply already excludes the claimed + // CPUs (reapplyDRAClaims), but claimContainerRefs knows nothing about + // the container yet — it hasn't gone through AllocateResources in this + // process. + if got := p.claimContainerRefs[uid]; got != 0 { + t.Fatalf("test setup error: claimContainerRefs[%s] = %d right after Start(), want 0", uid, got) + } + + container := &mockContainer{returnValueForGetID: "restart-c1", cdiDeviceNames: []string{cdiName}} + + // Mirror syncWithNRI: an already-running container is placed in both + // the "allocated" and "released" lists so Sync() releases (no-op) then + // re-allocates it. + if err := p.Sync([]cache.Container{container}, []cache.Container{container}); err != nil { + t.Fatalf("Sync() unexpected error: %v", err) + } + + if got := p.claimContainerRefs[uid]; got != 1 { + t.Errorf("claimContainerRefs[%s] = %d after Start()+resync Sync(), want 1", uid, got) + } +} + +// TestReapplyDRAClaimsNilDRAPluginNoop verifies that reapplyDRAClaims() is a +// no-op (no panic, no supply changes) when DRA is disabled (draPlugin == +// nil, the default). +func TestReapplyDRAClaimsNilDRAPluginNoop(t *testing.T) { + p := newDRATestPolicy(t) + if p.draPlugin != nil { + t.Fatalf("test setup error: draPlugin unexpectedly non-nil") + } + + leaf := findPoolNode(t, p, "NUMA node #0") + before := leaf.FreeSupply().SharableCPUs() + + p.reapplyDRAClaims() + + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(before) { + t.Errorf("reapplyDRAClaims() with nil draPlugin changed pool supply: got %s, want unchanged %s", got, before) + } +} + +// TestReapplyDRAClaimsEvictsOverlappingRestoredGrant covers the double- +// booking gap identified in the DRA step 8 review: reapplyDRAClaims() runs +// *after* restoreCache()'s/restoreAllocations()'s grant restoration, at a +// point where Supply.claimRefs has just been wiped by initialize() and not +// yet re-marked. If grant restoration (verbatim reinstatement, or its +// allocatePool-based fallback) handed a regular container CPUs that a live +// DRA claim already owns, that overlap must be detected and evicted here — +// otherwise two workloads end up pinned to the same physical CPUs until the +// next restart. addTestGrant stands in for "restoreCache() already +// reinstated this grant" without needing the full cache/offer machinery. +func TestReapplyDRAClaimsEvictsOverlappingRestoredGrant(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + // Simulate restoreCache()/restoreAllocations() having already reinstated + // (or freshly reallocated) a regular grant that happens to overlap the + // CPU a live DRA claim owns — the exact scenario reapplyDRAClaims must + // still be able to correct despite running after grant restoration. + victim := &mockContainer{returnValueForGetID: "reapply-victim"} + addTestGrant(t, p, leaf, victim, claimed) + if _, ok := p.allocations.getGrant("reapply-victim"); !ok { + t.Fatalf("test setup error: victim grant not present before reapplyDRAClaims") + } + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-reapply-evict-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + p.reapplyDRAClaims() + + // No grant anywhere may still exclusively hold the claimed CPU: either + // the victim's original grant was released outright, or it was + // reallocated to CPUs that no longer overlap the claim. + for _, g := range p.allocations.grants { + if g.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still exclusively granted to %s after reapplyDRAClaims", + claimed, g.GetContainer().PrettyName()) + } + } + + // And the claimed CPU must not be free for a new regular allocation. + free := leaf.FreeSupply() + if free.SharableCPUs().Union(free.IsolatedCPUs()).Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still free in %q supply after reapplyDRAClaims", claimed, leaf.Name()) + } + + // The evicted victim must actually have been reallocated a new grant + // (not just released and forgotten) — ample capacity remains on this + // test system for reallocatePool to succeed. + newGrant, ok := p.allocations.getGrant("reapply-victim") + if !ok { + t.Fatalf("victim has no grant after reapplyDRAClaims(); eviction must reallocate, not just release") + } + if newGrant.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("victim's new grant %s still overlaps claimed CPU %s", newGrant.ExclusiveCPUs(), claimed) + } + + // reapplyDRAClaims() must not touch claimContainerRefs (marking-only + // contract) — even though it evicted/reallocated a container here. + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs unexpectedly populated by reapplyDRAClaims (marking-only contract violated)") + } +} diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index b144e3cc1..30c7e7da0 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -781,6 +781,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 diff --git a/config/crd/bases/config.nri_topologyawarepolicies.yaml b/config/crd/bases/config.nri_topologyawarepolicies.yaml index 92c0de2b9..175e85fc1 100644 --- a/config/crd/bases/config.nri_topologyawarepolicies.yaml +++ b/config/crd/bases/config.nri_topologyawarepolicies.yaml @@ -482,6 +482,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 @@ -603,6 +615,25 @@ spec: assigned for excusive use to containers which are not annotated to use any CPU class. type: string + dra: + description: |- + DRA controls whether the policy runs a DRA (Dynamic Resource Allocation) + kubelet plugin publishing CPU devices derived from the configured + CPUClasses. If unset, DRA integration is disabled. + properties: + enabled: + default: false + description: |- + Enabled controls whether the DRA kubelet plugin is started and CPU + devices are published for the configured CPUClasses. + type: boolean + sharedCounters: + default: false + description: |- + SharedCounters is reserved for future use (KEP-5941 shared counters). + Setting this to true is currently rejected by validation; leave it unset or false. + type: boolean + type: object instrumentation: description: Config provides runtime configuration for instrumentation. properties: diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index b144e3cc1..30c7e7da0 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -781,6 +781,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 diff --git a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml index 92c0de2b9..175e85fc1 100644 --- a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml +++ b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml @@ -482,6 +482,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 @@ -603,6 +615,25 @@ spec: assigned for excusive use to containers which are not annotated to use any CPU class. type: string + dra: + description: |- + DRA controls whether the policy runs a DRA (Dynamic Resource Allocation) + kubelet plugin publishing CPU devices derived from the configured + CPUClasses. If unset, DRA integration is disabled. + properties: + enabled: + default: false + description: |- + Enabled controls whether the DRA kubelet plugin is started and CPU + devices are published for the configured CPUClasses. + type: boolean + sharedCounters: + default: false + description: |- + SharedCounters is reserved for future use (KEP-5941 shared counters). + Setting this to true is currently rejected by validation; leave it unset or false. + type: boolean + type: object instrumentation: description: Config provides runtime configuration for instrumentation. properties: diff --git a/deployment/helm/topology-aware/templates/clusterrole.yaml b/deployment/helm/topology-aware/templates/clusterrole.yaml index 3ea31273d..4951c300b 100644 --- a/deployment/helm/topology-aware/templates/clusterrole.yaml +++ b/deployment/helm/topology-aware/templates/clusterrole.yaml @@ -30,3 +30,21 @@ rules: - list - update - delete +{{- if (.Values.config.dra).enabled }} +- apiGroups: + - resource.k8s.io + resources: + - resourceslices + verbs: + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + verbs: + - get +{{- end }} diff --git a/deployment/helm/topology-aware/templates/daemonset.yaml b/deployment/helm/topology-aware/templates/daemonset.yaml index 0302c2bb2..ea5f39b5e 100644 --- a/deployment/helm/topology-aware/templates/daemonset.yaml +++ b/deployment/helm/topology-aware/templates/daemonset.yaml @@ -142,6 +142,14 @@ spec: - name: hostdev mountPath: /host/dev {{- end }} + {{- if (.Values.config.dra).enabled }} + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: kubelet-plugins-registry + mountPath: /var/lib/kubelet/plugins_registry + - name: cdi-spec-dir + mountPath: /var/run/cdi + {{- end }} {{- if .Values.podPriorityClassNodeCritical }} priorityClassName: system-node-critical @@ -176,6 +184,20 @@ spec: path: /dev type: Directory {{- end }} + {{- if (.Values.config.dra).enabled }} + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: kubelet-plugins-registry + hostPath: + path: /var/lib/kubelet/plugins_registry + type: DirectoryOrCreate + - name: cdi-spec-dir + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + {{- end }} {{- if .Values.nri.runtime.patchConfig }} - name: containerd-config hostPath: diff --git a/deployment/helm/topology-aware/templates/deviceclass.yaml b/deployment/helm/topology-aware/templates/deviceclass.yaml new file mode 100644 index 000000000..8b765192a --- /dev/null +++ b/deployment/helm/topology-aware/templates/deviceclass.yaml @@ -0,0 +1,12 @@ +{{- if (.Values.config.dra).enabled }} +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: nri.topology-aware.cpu + labels: + {{- include "nri-plugin.labels" . | nindent 4 }} +spec: + selectors: + - cel: + expression: device.driver == "nri.topology-aware.cpu" +{{- end }} diff --git a/deployment/helm/topology-aware/values.yaml b/deployment/helm/topology-aware/values.yaml index 81c5224ac..807b532c1 100644 --- a/deployment/helm/topology-aware/values.yaml +++ b/deployment/helm/topology-aware/values.yaml @@ -66,6 +66,14 @@ config: prometheusExport: false reportPeriod: 60s samplingRatePerMillion: 0 + # DRA (Dynamic Resource Allocation) kubelet plugin integration. See + # TopologyAwareDRA in + # pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go for what + # these fields control. Requires Kubernetes 1.34+ (resource.k8s.io/v1, + # used by the DeviceClass this chart installs when enabled=true). + dra: + enabled: false + sharedCounters: false # configGroupLabel: config.nri/group diff --git a/docs/resource-policy/policy/topology-aware.md b/docs/resource-policy/policy/topology-aware.md index 8ece885dd..7739c6ee6 100644 --- a/docs/resource-policy/policy/topology-aware.md +++ b/docs/resource-policy/policy/topology-aware.md @@ -733,6 +733,94 @@ both on startup and when they are returned to the shared pool from exclusive allocation. The CPU assigned for the reserved pool (`kube-system` namespace) will get configured according to the `reserved` CPU class. +### Dynamic Resource Allocation + +The topology-aware policy can optionally register a Kubernetes [Dynamic Resource +Allocation](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/) +(DRA) kubelet plugin driver that publishes `cpuClasses` (see [Class Based CPU +Tuning](#class-based-cpu-tuning)) as DRA devices. This lets pods request CPUs with a +specific `cpuClass` via a `ResourceClaim` instead of (or in addition to) the +`cpu-class.resource-policy.nri.io` annotation, and lets the scheduler see and account +for `cpuClass`-tagged capacity when fitting pods onto nodes. + +#### Prerequisites + +- Kubernetes 1.34+, with the [KEP-5075](https://github.com/kubernetes/enhancements/issues/5075) + (`DRAConsumableCapacity` / consumable capacity) alpha feature gate enabled on the + API server, kube-scheduler, kube-controller-manager, and kubelet. The `DeviceClass` object below uses the `resource.k8s.io/v1` + API, which requires Kubernetes 1.34+. +- Optionally, Kubernetes 1.37+, with the + [KEP-5517](https://github.com/kubernetes/enhancements/issues/5517) + (`DRANodeAllocatableResources` / node allocatable resources) alpha feature gate + enabled, so the scheduler accounts for `cpuClass`-tagged capacity in its own + in-memory node-allocatable bookkeeping and records the resolved allocation on + `pod.status.nodeAllocatableResourceClaimStatuses[]` at PreBind — **not** on + `node.status.allocatable`, which this feature gate never mutates. Without it, + DRA allocation still works but capacity mirroring is skipped. +- The container runtime must support NRI (as for the rest of this policy). + +#### Enabling + +DRA support is off by default. Enable it via the Helm chart's `config.dra.enabled` +value: + +```console +helm install topology-aware deployment/helm/topology-aware \ + --set config.dra.enabled=true +``` + +This does three things: +- Installs the extra RBAC rules the driver needs (`resourceslices` CRUD, + `resourceclaims: get`). +- Adds the host mounts the driver's kubelet plugin needs + (`/var/lib/kubelet/plugins`, `/var/lib/kubelet/plugins_registry`, `/var/run/cdi`). +- Installs the base `DeviceClass` named `nri.topology-aware.cpu`. + +It also sets `spec.dra.enabled: true` in the `TopologyAwarePolicy` CR, which turns on +the policy's own DRA plugin (kubelet-plugin registration and device publication). + +`config.dra.sharedCounters` (also under `config.dra`, default `false`) enables Model C +publication once the underlying [KEP-5941](https://github.com/kubernetes/enhancements/issues/5941) +shared-counters support is available; leave it `false` otherwise. + +#### Requesting a `cpuClass` via `ResourceClaim` + +Each defined `cpuClass` is published as a DRA device under the +`nri.topology-aware.cpu` `DeviceClass`, with the class's config attributes (such as +`nri/pctPriority`) exposed as device attributes. **Only high-priority PCT classes +are currently published this way** — non-HP classes are filtered out at +publication time, and the driver's `Prepare` rejects claims against them, so +ordinary configured classes without a high-priority tier will not appear as +devices. An eligible HP class can still be hidden from DRA by setting +`cpuClass.dra.publish: false`, which leaves it configurable via the normal +`cpu-class.resource-policy.nri.io` annotation but excludes it from device +publication; `dra.publish` defaults to `true` when unset. A `ResourceClaim` +can select on any of these attributes. For example, to request 2 CPUs from a +high-priority PCT class: + +```yaml +apiVersion: resource.k8s.io/v1 +kind: ResourceClaim +metadata: + name: hp-turbo-cpus +spec: + devices: + requests: + - name: cpus + exactly: + deviceClassName: nri.topology-aware.cpu + capacity: + requests: + nri/cpus: "2" + selectors: + - cel: + expression: | + device.attributes["nri"].pctPriority == "high" +``` + +A pod then references the claim via `resourceClaims` and consumes it in a container's +`resources.claims`, per the standard Kubernetes DRA pod API. + ### IRQ CPU Affinity Tuning Containers eligible for exclusive CPU allocation can be annotated with IRQ diff --git a/go.mod b/go.mod index eebdbe85b..28cffae5e 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/containers/nri-plugins/pkg/topology v0.0.0 github.com/coreos/go-systemd/v22 v22.7.0 github.com/fsnotify/fsnotify v1.10.1 + github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 github.com/intel/goresctrl v0.13.0 github.com/intel/memtierd v0.1.1 @@ -37,14 +38,17 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 golang.org/x/sys v0.47.0 google.golang.org/grpc v1.83.1 - k8s.io/api v0.34.11 - k8s.io/apimachinery v0.34.11 - k8s.io/client-go v0.34.11 + k8s.io/api v0.37.0 + k8s.io/apimachinery v0.37.0 + k8s.io/client-go v0.37.0 + k8s.io/dynamic-resource-allocation v0.37.0 k8s.io/klog/v2 v2.140.0 - k8s.io/kubelet v0.34.11 - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + k8s.io/kubelet v0.37.0 + k8s.io/utils v0.0.0-20260626114624-be93311217bd sigs.k8s.io/controller-runtime v0.22.5 sigs.k8s.io/yaml v1.6.0 + tags.cncf.io/container-device-interface v1.1.0 + tags.cncf.io/container-device-interface/specs-go v1.1.0 ) require ( @@ -54,13 +58,23 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect - github.com/go-logr/logr v1.4.4 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/go-openapi/jsonreference v1.0.0 // indirect + github.com/go-openapi/swag v0.27.1 // indirect + github.com/go-openapi/swag/cmdutils v0.27.1 // indirect + github.com/go-openapi/swag/conv v0.27.1 // indirect + github.com/go-openapi/swag/fileutils v0.27.1 // indirect + github.com/go-openapi/swag/jsonutils v0.27.1 // indirect + github.com/go-openapi/swag/loading v0.27.1 // indirect + github.com/go-openapi/swag/mangling v0.27.1 // indirect + github.com/go-openapi/swag/netutils v0.27.1 // indirect + github.com/go-openapi/swag/pools v0.27.1 // indirect + github.com/go-openapi/swag/stringutils v0.27.1 // indirect + github.com/go-openapi/swag/typeutils v0.27.1 // indirect + github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -68,26 +82,28 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knqyf263/go-plugin v0.9.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect + github.com/moby/sys/capability v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.3.0 // indirect - github.com/pkg/errors v0.9.1 // indirect + github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/tetratelabs/wazero v1.11.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/mod v0.37.0 // indirect @@ -96,25 +112,25 @@ require ( golang.org/x/sync v0.22.0 // indirect golang.org/x/term v0.45.0 // indirect golang.org/x/text v0.40.0 // indirect - golang.org/x/time v0.9.0 // indirect + golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.47.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/code-generator v0.34.11 // indirect - k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect + k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect ) replace ( github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology - github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 + github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 ) tool k8s.io/code-generator diff --git a/go.sum b/go.sum index 19e7836e8..0e4b23618 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/askervin/gofmbt v0.0.0-20260730061456-f663bfa65cdf h1:0AmJ+APMItCB+TS github.com/askervin/gofmbt v0.0.0-20260730061456-f663bfa65cdf/go.mod h1:1rWH2fCHPoGz1ApWyGyEV9YhZ2ZHeeCPaHcicW3b6uk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/brianvoe/gofakeit/v7 v7.12.1 h1:df1tiI4SL1dR5Ix4D/r6a3a+nXBJ/OBGU5jEKRBmmqg= github.com/brianvoe/gofakeit/v7 v7.12.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= @@ -20,17 +22,16 @@ github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uih github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= +github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -42,14 +43,40 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= +github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= +github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg= +github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE= +github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE= +github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU= +github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs= +github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM= +github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU= +github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= +github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY= +github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE= +github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA= +github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= +github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU= +github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E= +github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= +github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8= +github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= +github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc= +github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= +github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA= +github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= @@ -71,12 +98,14 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/intel/goresctrl v0.13.0 h1:5fhKjNq4V5MYDFHa//6M6x0jP6Iq5EXwZc6/eYxdEtQ= github.com/intel/goresctrl v0.13.0/go.mod h1:KFHS91JGOmeeuEog+nTQcsGjLC81nRqdsdhcqf69fjU= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -89,21 +118,18 @@ github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJn github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= +github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -118,10 +144,12 @@ github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 h1:tAKu3NkKWZYpqBSOJKwTxT1wIGueiF7gcmcNgr5pNTY= +github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116/go.mod h1:DKDEfzxvRkoQ6n9TGhxQgg2IM1lY4aM0eaQP4e3oElw= +github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -139,17 +167,12 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q= github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= @@ -164,8 +187,16 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/etcd/client/pkg/v3 v3.7.0 h1:sW9njJzS3vXKcAJjjLQ4nk+avNUJ12Bcijcx8ehUskE= +go.etcd.io/etcd/client/pkg/v3 v3.7.0/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 h1:oEl2Pw/i4OQwhAuda2pAHFAcOMivA+Xa+iTccBfab/g= @@ -208,6 +239,10 @@ go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6Tb go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= @@ -215,8 +250,8 @@ go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= @@ -245,8 +280,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -274,38 +309,43 @@ google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.34.11 h1:LkxGIHlj06urNrS5Gpgj2iFRRgja+0tUV0+ejZ6j/T8= -k8s.io/api v0.34.11/go.mod h1:b1wzM4EH4H4FtXVDK3zAnG1zG/NBQBwtPeahyyYYDIU= -k8s.io/apimachinery v0.34.11 h1:N29JKPe3cXeLmc8QhFILkKU7jYq47EWG1zAZNOIawnM= -k8s.io/apimachinery v0.34.11/go.mod h1:xfCr+Akw9yI3OXIqWDjOaQCklbC498VcPxtFJpRK+FI= -k8s.io/client-go v0.34.11 h1:ibO8IP6RMp6JImv47JIW6ZUcZky98zLnzaGIsD6yQWM= -k8s.io/client-go v0.34.11/go.mod h1:MxE93lNP2kK12JL4S74ddZctG0Pt6JVLiK8nLwlMxyY= +k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4= +k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk= +k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0= +k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE= +k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ= +k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0= k8s.io/code-generator v0.34.11 h1:o7jWDfqo27pQygEm7a+g8QytR9ZaVEHE3urIfNmFYhU= k8s.io/code-generator v0.34.11/go.mod h1:ACmWPlRob//4Azs/4/wIi+CFPmnGhYuixpJfKUUhyJs= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f h1:SLb+kxmzfA87x4E4brQzB33VBbT2+x7Zq9ROIHmGn9Q= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/dynamic-resource-allocation v0.37.0 h1:pfn3hsUQVvhqdDh0gA1DpV0w5njRzB68rjYT15AteT4= +k8s.io/dynamic-resource-allocation v0.37.0/go.mod h1:Nkono0X3H5tNnEf3IOK0Ml+I7HB4o0/UXaiiNCgB2Vw= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/kubelet v0.34.11 h1:jp81mYXdPnHRwEqtRWpaVGpdB3+rDSruVL1A0C90ahQ= -k8s.io/kubelet v0.34.11/go.mod h1:wu6sd9D6svx7r1PLzfDVX8hZ9M3C/4BIUhK87fJyHaY= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/kubelet v0.37.0 h1:VhaZanjlE5CkoAPAjKw0DH+Q0BYXVfYekZgzZDAJMMg= +k8s.io/kubelet v0.37.0/go.mod h1:PHXfQuVqsTzFVqOeP67UUNv5Ajri+dLGoeaEnW6gGjE= +k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= +k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/controller-runtime v0.22.5 h1:v3nfSUMowX/2WMp27J9slwGFyAt7IV0YwBxAkrUr0GE= sigs.k8s.io/controller-runtime v0.22.5/go.mod h1:pc5SoYWnWI6I+cBHYYdZ7B6YHZVY5xNfll88JB+vniI= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +tags.cncf.io/container-device-interface v1.1.0 h1:RnxNhxF1JOu6CJUVpetTYvrXHdxw9j9jFYgZpI+anSY= +tags.cncf.io/container-device-interface v1.1.0/go.mod h1:76Oj0Yqp9FwTx/pySDc8Bxjpg+VqXfDb50cKAXVJ34Q= +tags.cncf.io/container-device-interface/specs-go v1.1.0 h1:QRZVeAceQM+zTZe12eyfuJuuzp524EKYwhmvLd+h+yQ= +tags.cncf.io/container-device-interface/specs-go v1.1.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ= diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index f577e7b87..3c98c223d 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -29,13 +29,12 @@ import ( "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/clientcmd" nrtapi "github.com/containers/nri-plugins/pkg/agent/nrtapi" "github.com/containers/nri-plugins/pkg/agent/podresapi" - "github.com/containers/nri-plugins/pkg/agent/watch" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1" - k8sclient "k8s.io/client-go/kubernetes" + "github.com/containers/nri-plugins/pkg/kubernetes/client" + "github.com/containers/nri-plugins/pkg/kubernetes/watch" logger "github.com/containers/nri-plugins/pkg/log" ) @@ -129,12 +128,11 @@ type Agent struct { kubeConfig string // kubeconfig path configFile string // configuration file to use instead of custom resource - cfgIf ConfigInterface // custom resource access interface - httpCli *http.Client // shared HTTP client - k8sCli *k8sclient.Clientset // kubernetes client - nrtCli *nrtapi.Client // NRT custom resources client - nrtLock sync.Mutex // serialize NRT custom resource updates - podResCli *podresapi.Client // pod resources API client + cfgIf ConfigInterface // custom resource access interface + k8sCli *client.Client // wrapped kubernetes client + REST config + HTTP client + nrtCli *nrtapi.Client // NRT custom resources client + nrtLock sync.Mutex // serialize NRT custom resource updates + podResCli *podresapi.Client // pod resources API client notifyFn NotifyFn // config resource change notification callback nodeWatch watch.Interface // kubernetes node watch @@ -300,12 +298,11 @@ func (a *Agent) configure(newConfig metav1.Object) { switch { case cfg.NodeResourceTopology && a.nrtCli == nil: log.Infof("enabling NRT client") - cfg, err := a.getRESTConfig() - if err != nil { - log.Errorf("failed to setup NRT client: %v", err) + if a.k8sCli == nil { + log.Errorf("failed to setup NRT client: no kubernetes client") break } - cli, err := nrtapi.NewForConfigAndClient(cfg, a.httpCli) + cli, err := nrtapi.NewForConfigAndClient(a.k8sCli.RestConfig(), a.k8sCli.HttpClient()) if err != nil { log.Errorf("failed to setup NRT client: %v", err) break @@ -346,30 +343,17 @@ func (a *Agent) setupClients() error { return nil } - // Create HTTP/REST client and K8s client on initial startup. Any failure - // to create these is a failure start up. - if a.httpCli == nil { - log.Infof("setting up HTTP/REST client...") - restCfg, err := a.getRESTConfig() - if err != nil { - return err - } - - a.httpCli, err = rest.HTTPClientFor(restCfg) + // Create the kubernetes client on initial startup. Any failure is fatal. + if a.k8sCli == nil { + log.Infof("setting up kubernetes client...") + c, err := client.New(client.WithKubeOrInClusterConfig(a.kubeConfig)) if err != nil { - return fmt.Errorf("failed to setup kubernetes HTTP client: %w", err) - } - - log.Infof("setting up K8s client...") - a.k8sCli, err = k8sclient.NewForConfigAndClient(restCfg, a.httpCli) - if err != nil { - a.cleanupClients() return fmt.Errorf("failed to setup kubernetes client: %w", err) } + a.k8sCli = c - kubeCfg := *restCfg - err = a.cfgIf.SetKubeClient(a.httpCli, &kubeCfg) - if err != nil { + if err := a.cfgIf.SetKubeClient(a.k8sCli.HttpClient(), a.k8sCli.RestConfig()); err != nil { + a.cleanupClients() return fmt.Errorf("failed to setup kubernetes config resource client: %w", err) } } @@ -380,31 +364,35 @@ func (a *Agent) setupClients() error { } func (a *Agent) cleanupClients() { - if a.httpCli != nil { - a.httpCli.CloseIdleConnections() - } - a.httpCli = nil + a.k8sCli.Close() a.k8sCli = nil a.nrtCli = nil } -func (a *Agent) getRESTConfig() (*rest.Config, error) { - var ( - cfg *rest.Config - err error - ) +// NodeName returns the kubernetes node name this agent is running on. +func (a *Agent) NodeName() string { + return a.nodeName +} - if a.kubeConfig == "" { - cfg, err = rest.InClusterConfig() - } else { - cfg, err = clientcmd.BuildConfigFromFlags("", a.kubeConfig) - } +// KubeClient returns the shared kubernetes client wrapper. Returns nil +// before setupClients has run successfully. +func (a *Agent) KubeClient() *client.Client { + return a.k8sCli +} - if err != nil { - return nil, fmt.Errorf("failed to get kubernetes REST client config: %w", err) - } +// KubeConfig returns the kubeconfig file path this agent was configured +// with. Returns the empty string when running with in-cluster credentials. +func (a *Agent) KubeConfig() string { + return a.kubeConfig +} - return cfg, err +// RestConfig returns a copy of the REST config used by the shared +// kubernetes client, or nil before setupClients has run successfully. +func (a *Agent) RestConfig() *rest.Config { + if a.k8sCli == nil { + return nil + } + return a.k8sCli.RestConfig() } func (a *Agent) setupNodeWatch() error { diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go index 3e4258fda..df6ad8e22 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go @@ -19,6 +19,16 @@ import ( "fmt" ) +// CPUClassDRA holds per-cpuClass DRA publication options. +// +k8s:deepcopy-gen=true +type CPUClassDRA struct { + // Publish controls whether this cpuClass is exposed as a DRA device. + // Defaults to true when unset. Set to false to hide this class from + // DRA while keeping it usable via the normal nri-plugins config path. + // +optional + Publish *bool `json:"publish,omitempty"` +} + // CPUClass specifies CPU frequency, C-state, and turbo attributes // for a CPU class. // +k8s:deepcopy-gen=true @@ -99,6 +109,19 @@ type CPUClass struct { // on a single node. Has effect only when the class also carries // PctPriority or SstClosID. Experimental. PublishExtendedResource bool `json:"publishExtendedResource,omitempty"` + // DRA holds DRA publication options for this class. Optional; defaults + // to publish=true (the class is visible to the DRA driver). + // +optional + DRA *CPUClassDRA `json:"dra,omitempty"` +} + +// DRAPublish reports whether this cpuClass should be published as a DRA device. +// Returns true if DRA is nil or DRA.Publish is nil. +func (cc *CPUClass) DRAPublish() bool { + if cc.DRA == nil || cc.DRA.Publish == nil { + return true + } + return *cc.DRA.Publish } func (cc *CPUClass) Validate() error { diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass_test.go b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass_test.go new file mode 100644 index 000000000..87e33a12f --- /dev/null +++ b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass_test.go @@ -0,0 +1,82 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "testing" +) + +// ptr returns a pointer to v; helper for bool/int literals in tests. +func ptr[T any](v T) *T { return &v } + +func TestCPUClassDRAPublish(t *testing.T) { + tests := []struct { + name string + cc *CPUClass + want bool + }{ + { + name: "nil DRA field defaults to true", + cc: &CPUClass{}, + want: true, + }, + { + name: "DRA set but Publish nil defaults to true", + cc: &CPUClass{DRA: &CPUClassDRA{}}, + want: true, + }, + { + name: "explicit false", + cc: &CPUClass{DRA: &CPUClassDRA{Publish: ptr(false)}}, + want: false, + }, + { + name: "explicit true", + cc: &CPUClass{DRA: &CPUClassDRA{Publish: ptr(true)}}, + want: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := tc.cc.DRAPublish() + if got != tc.want { + t.Errorf("DRAPublish() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestCPUClassDRADeepCopy(t *testing.T) { + orig := &CPUClass{ + Name: "hp", + DRA: &CPUClassDRA{Publish: ptr(false)}, + } + copy := orig.DeepCopy() + if copy == orig { + t.Fatal("DeepCopy returned same pointer") + } + if copy.DRA == orig.DRA { + t.Fatal("DRA pointer not deep-copied") + } + if copy.DRA.Publish == orig.DRA.Publish { + t.Fatal("DRA.Publish pointer not deep-copied (shared storage)") + } + + // Mutating the copy must not affect the original. + *copy.DRA.Publish = true + if *orig.DRA.Publish != false { + t.Error("mutating copy's DRA.Publish affected the original") + } +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go index 7be7b1153..8c9a0faff 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go @@ -197,6 +197,27 @@ type Config struct { // +optional // +kubebuilder:default={"*"} ControllableInterrupts []string `json:"controllableInterrupts,omitempty"` + // DRA controls whether the policy runs a DRA (Dynamic Resource Allocation) + // kubelet plugin publishing CPU devices derived from the configured + // CPUClasses. If unset, DRA integration is disabled. + // +optional + DRA *TopologyAwareDRA `json:"dra,omitempty"` +} + +// TopologyAwareDRA controls the DRA (Dynamic Resource Allocation) kubelet +// plugin integration for the topology-aware policy. +// +kubebuilder:object:generate=true +type TopologyAwareDRA struct { + // Enabled controls whether the DRA kubelet plugin is started and CPU + // devices are published for the configured CPUClasses. + // +kubebuilder:default=false + // +optional + Enabled bool `json:"enabled,omitempty"` + // SharedCounters is reserved for future use (KEP-5941 shared counters). + // Setting this to true is currently rejected by validation; leave it unset or false. + // +kubebuilder:default=false + // +optional + SharedCounters bool `json:"sharedCounters,omitempty"` } var ( @@ -277,6 +298,25 @@ func (c *Config) Validate() error { return errors.Join(errs...) } +// DRAEnabled returns whether the DRA kubelet plugin integration is enabled. +// It is nil-safe: a nil Config.DRA (the default) means disabled. +func (c *Config) DRAEnabled() bool { + if c == nil || c.DRA == nil { + return false + } + return c.DRA.Enabled +} + +// DRASharedCounters returns whether published DRA CPU devices should use +// shared per-class counters. It is nil-safe: a nil Config.DRA (the default) +// means false. +func (c *Config) DRASharedCounters() bool { + if c == nil || c.DRA == nil { + return false + } + return c.DRA.SharedCounters +} + // GetSchedulingClass returns the named class or nil if it is not defined. func (c *Config) GetSchedulingClass(name string) *SchedulingClass { for _, sc := range c.SchedulingClasses { diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config_test.go b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config_test.go new file mode 100644 index 000000000..fcd6274e9 --- /dev/null +++ b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config_test.go @@ -0,0 +1,45 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDRAEnabledZeroConfig(t *testing.T) { + c := &Config{} + require.False(t, c.DRAEnabled(), "zero Config (nil DRA) should report DRAEnabled() == false") + require.False(t, c.DRASharedCounters(), "zero Config (nil DRA) should report DRASharedCounters() == false") +} + +func TestDRAEnabledTrue(t *testing.T) { + c := &Config{ + ReservedResources: Constraints{CPU: "750m"}, + DRA: &TopologyAwareDRA{Enabled: true}, + } + require.True(t, c.DRAEnabled()) + require.NoError(t, c.Validate()) +} + +func TestDRASharedCountersTrue(t *testing.T) { + c := &Config{ + ReservedResources: Constraints{CPU: "750m"}, + DRA: &TopologyAwareDRA{SharedCounters: true}, + } + require.True(t, c.DRASharedCounters()) + require.NoError(t, c.Validate()) +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go index 73ad92456..c2b2ada52 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go @@ -93,6 +93,11 @@ func (in *Config) DeepCopyInto(out *Config) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.DRA != nil { + in, out := &in.DRA, &out.DRA + *out = new(TopologyAwareDRA) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. @@ -104,3 +109,18 @@ func (in *Config) DeepCopy() *Config { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TopologyAwareDRA) DeepCopyInto(out *TopologyAwareDRA) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologyAwareDRA. +func (in *TopologyAwareDRA) DeepCopy() *TopologyAwareDRA { + if in == nil { + return nil + } + out := new(TopologyAwareDRA) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go index ebb0af11a..21f947c1f 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go @@ -33,6 +33,11 @@ func (in *CPUClass) DeepCopyInto(out *CPUClass) { *out = new(int) **out = **in } + if in.DRA != nil { + in, out := &in.DRA, &out.DRA + *out = new(CPUClassDRA) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPUClass. @@ -45,6 +50,26 @@ func (in *CPUClass) DeepCopy() *CPUClass { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CPUClassDRA) DeepCopyInto(out *CPUClassDRA) { + *out = *in + if in.Publish != nil { + in, out := &in.Publish, &out.Publish + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPUClassDRA. +func (in *CPUClassDRA) DeepCopy() *CPUClassDRA { + if in == nil { + return nil + } + out := new(CPUClassDRA) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchedulingClass) DeepCopyInto(out *SchedulingClass) { *out = *in diff --git a/pkg/kubernetes/client/client.go b/pkg/kubernetes/client/client.go new file mode 100644 index 000000000..27c65f239 --- /dev/null +++ b/pkg/kubernetes/client/client.go @@ -0,0 +1,219 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package client builds a *kubernetes.Clientset from a kubeconfig file or +// in-cluster credentials, and exposes the REST config and HTTP client it +// was built from so callers can share one client. +package client + +import ( + "errors" + "net/http" + "strings" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" +) + +// Wire content types accepted by the Kubernetes API server. +const ( + ContentTypeJSON = "application/json" + ContentTypeProtobuf = "application/vnd.kubernetes.protobuf" +) + +// Client wraps a Kubernetes clientset together with the REST config and +// HTTP client it was built from. Use the embedded Clientset directly for +// API calls, or HttpClient()/RestConfig() to build other clients sharing +// the same transport. +type Client struct { + cfg *rest.Config + http *http.Client + *kubernetes.Clientset +} + +// Option configures a Client during construction via New. Options apply +// in order; config-dependent options (WithContentType, +// WithAcceptContentTypes) require a config-source option (WithKubeConfig, +// WithInClusterConfig, or WithRestConfig) earlier in the list. +type Option func(*Client) error + +// errNoConfigSet is returned by options that require the REST config +// to be present but are called before any config-source option. +var errNoConfigSet = errors.New("option requires REST config; pass a config-source option (WithKubeConfig, WithInClusterConfig, or WithRestConfig) before this option") + +// GetConfigForFile returns a REST configuration parsed from the given +// kubeconfig file path. Thin wrapper over clientcmd.BuildConfigFromFlags +// exposed for callers that need a config but not a full Client. +func GetConfigForFile(kubeConfig string) (*rest.Config, error) { + return clientcmd.BuildConfigFromFlags("", kubeConfig) +} + +// InClusterConfig returns the REST configuration for the pod's service +// account, if the process is running inside a Kubernetes cluster. +// Returns rest.ErrNotInCluster (wrapped) when not in a cluster. +func InClusterConfig() (*rest.Config, error) { + return rest.InClusterConfig() +} + +// New constructs a Client by applying the given options in order, +// defaulting to WithInClusterConfig() if none set a REST config. +func New(options ...Option) (*Client, error) { + c := &Client{} + + for _, o := range options { + if err := o(c); err != nil { + return nil, err + } + } + + if c.cfg == nil { + if err := WithInClusterConfig()(c); err != nil { + return nil, err + } + } + + if c.http == nil { + hc, err := rest.HTTPClientFor(c.cfg) + if err != nil { + return nil, err + } + c.http = hc + } + + cs, err := kubernetes.NewForConfigAndClient(c.cfg, c.http) + if err != nil { + return nil, err + } + c.Clientset = cs + + return c, nil +} + +// WithKubeConfig returns an Option that resolves the REST config from +// the given kubeconfig file. +func WithKubeConfig(file string) Option { + return func(c *Client) error { + cfg, err := GetConfigForFile(file) + if err != nil { + return err + } + return WithRestConfig(cfg)(c) + } +} + +// WithInClusterConfig returns an Option that resolves the REST config +// from the pod's service-account credentials. +func WithInClusterConfig() Option { + return func(c *Client) error { + cfg, err := InClusterConfig() + if err != nil { + return err + } + return WithRestConfig(cfg)(c) + } +} + +// WithKubeOrInClusterConfig resolves the REST config from the given +// kubeconfig file if non-empty, or from in-cluster credentials otherwise. +func WithKubeOrInClusterConfig(file string) Option { + if file == "" { + return WithInClusterConfig() + } + return WithKubeConfig(file) +} + +// WithRestConfig uses a deep copy (via rest.CopyConfig) of the given REST +// config, so the caller keeps ownership of the original. +func WithRestConfig(cfg *rest.Config) Option { + return func(c *Client) error { + if cfg == nil { + return errors.New("rest config must not be nil") + } + c.cfg = rest.CopyConfig(cfg) + return nil + } +} + +// WithHttpClient returns an Option that uses the given pre-built HTTP +// client. Useful when multiple components should share one client +// (and therefore its connection pool). +func WithHttpClient(hc *http.Client) Option { + return func(c *Client) error { + c.http = hc + return nil + } +} + +// WithAcceptContentTypes sets the Accept content types to negotiate with +// the API server, joined with commas. Requires a config-source option +// earlier in the list. +func WithAcceptContentTypes(contentTypes ...string) Option { + return func(c *Client) error { + if c.cfg == nil { + return errNoConfigSet + } + c.cfg.AcceptContentTypes = strings.Join(contentTypes, ",") + return nil + } +} + +// WithContentType sets the wire content type used for requests. Requires +// a config-source option earlier in the list. +func WithContentType(contentType string) Option { + return func(c *Client) error { + if c.cfg == nil { + return errNoConfigSet + } + c.cfg.ContentType = contentType + return nil + } +} + +// RestConfig returns a copy of the Client's REST config. Top-level and +// value-typed nested fields may be freely overwritten, but nested +// maps/slices (e.g. TLSClientConfig.CAData) share storage with the +// Client's internal config and must not be mutated. +func (c *Client) RestConfig() *rest.Config { + return rest.CopyConfig(c.cfg) +} + +// HttpClient returns the Client's underlying HTTP client, e.g. for +// constructing other clients that share the same transport. +func (c *Client) HttpClient() *http.Client { + return c.http +} + +// K8sClient returns the Client's underlying *kubernetes.Clientset. +// Callers may alternatively use the embedded Clientset directly on +// the Client value. +func (c *Client) K8sClient() *kubernetes.Clientset { + return c.Clientset +} + +// Close releases resources held by the Client. Safe to call on a nil or +// already-closed Client. +func (c *Client) Close() { + if c == nil { + return + } + if c.http != nil { + c.http.CloseIdleConnections() + } + c.cfg = nil + c.http = nil + c.Clientset = nil +} diff --git a/pkg/kubernetes/client/client_test.go b/pkg/kubernetes/client/client_test.go new file mode 100644 index 000000000..3990af7f3 --- /dev/null +++ b/pkg/kubernetes/client/client_test.go @@ -0,0 +1,370 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package client + +import ( + "errors" + "net/http" + "os" + "path/filepath" + "testing" + + "k8s.io/client-go/rest" +) + +// fixtureKubeconfig is the path to a minimal valid kubeconfig used by tests +// that need a config source without contacting a real API server. +const fixtureKubeconfig = "testdata/kubeconfig-example.yaml" + +// skipIfInCluster fails-fast for tests that assert not-in-cluster behavior +// when they happen to run inside a Pod (e.g. e2e CI). +func skipIfInCluster(t *testing.T) { + t.Helper() + if os.Getenv("KUBERNETES_SERVICE_HOST") != "" { + t.Skip("running inside a Kubernetes Pod; skipping not-in-cluster test") + } +} + +func TestGetConfigForFile_Success(t *testing.T) { + cfg, err := GetConfigForFile(fixtureKubeconfig) + if err != nil { + t.Fatalf("GetConfigForFile(%q) returned error: %v", fixtureKubeconfig, err) + } + if cfg == nil { + t.Fatal("GetConfigForFile returned nil config with no error") + } + if cfg.Host == "" { + t.Errorf("returned config has empty Host; expected value from fixture") + } +} + +func TestGetConfigForFile_MissingFile(t *testing.T) { + missing := filepath.Join(t.TempDir(), "nonexistent-kubeconfig.yaml") + cfg, err := GetConfigForFile(missing) + if err == nil { + t.Fatalf("expected error for missing file, got config: %+v", cfg) + } + if cfg != nil { + t.Errorf("expected nil config on error, got %+v", cfg) + } +} + +func TestGetConfigForFile_MalformedFile(t *testing.T) { + malformed := filepath.Join(t.TempDir(), "malformed-kubeconfig.yaml") + if err := os.WriteFile(malformed, []byte("this: is: not: valid: yaml: {[}\n"), 0o600); err != nil { + t.Fatalf("failed to write malformed fixture: %v", err) + } + cfg, err := GetConfigForFile(malformed) + if err == nil { + t.Fatalf("expected error for malformed file, got config: %+v", cfg) + } + if cfg != nil { + t.Errorf("expected nil config on error, got %+v", cfg) + } +} + +func TestInClusterConfig_NotInCluster(t *testing.T) { + skipIfInCluster(t) + cfg, err := InClusterConfig() + if err == nil { + t.Fatalf("expected error outside a cluster, got config: %+v", cfg) + } + if !errors.Is(err, rest.ErrNotInCluster) { + t.Errorf("expected rest.ErrNotInCluster, got: %v", err) + } + if cfg != nil { + t.Errorf("expected nil config on error, got %+v", cfg) + } +} + +// TestNew_NoOptions verifies that New() with no options falls back to +// WithInClusterConfig — which fails when the test runs outside a Pod. +// Only exercises the fallback path; the success path requires an in- +// cluster environment which is not usable from a unit test. +func TestNew_NoOptions(t *testing.T) { + skipIfInCluster(t) + c, err := New() + if err == nil { + t.Fatalf("expected error from New() outside a cluster, got: %+v", c) + } + if !errors.Is(err, rest.ErrNotInCluster) { + t.Errorf("expected rest.ErrNotInCluster, got: %v", err) + } + if c != nil { + t.Errorf("expected nil client on error, got %+v", c) + } +} + +func TestNew_WithKubeConfig_Success(t *testing.T) { + c, err := New(WithKubeConfig(fixtureKubeconfig)) + if err != nil { + t.Fatalf("New(WithKubeConfig) returned error: %v", err) + } + if c == nil { + t.Fatal("New returned nil client with no error") + } + if c.K8sClient() == nil { + t.Error("Client.K8sClient() is nil") + } + if c.RestConfig() == nil { + t.Error("Client.RestConfig() is nil") + } + if c.HttpClient() == nil { + t.Error("Client.HttpClient() is nil") + } +} + +func TestNew_WithKubeConfig_MissingFile(t *testing.T) { + missing := filepath.Join(t.TempDir(), "nope.yaml") + c, err := New(WithKubeConfig(missing)) + if err == nil { + t.Fatalf("expected error for missing kubeconfig, got: %+v", c) + } + if c != nil { + t.Errorf("expected nil client on error, got %+v", c) + } +} + +func TestNew_WithInClusterConfig(t *testing.T) { + skipIfInCluster(t) + c, err := New(WithInClusterConfig()) + if err == nil { + t.Fatalf("expected error outside a cluster, got: %+v", c) + } + if !errors.Is(err, rest.ErrNotInCluster) { + t.Errorf("expected rest.ErrNotInCluster, got: %v", err) + } + if c != nil { + t.Errorf("expected nil client on error, got %+v", c) + } +} + +func TestNew_WithKubeOrInClusterConfig_EmptyFallsBack(t *testing.T) { + skipIfInCluster(t) + // Empty file path should fall back to in-cluster, which fails outside + // a cluster; that's how we know the fallback path was taken. + _, err := New(WithKubeOrInClusterConfig("")) + if !errors.Is(err, rest.ErrNotInCluster) { + t.Errorf("expected rest.ErrNotInCluster from empty-file fallback, got: %v", err) + } +} + +func TestNew_WithKubeOrInClusterConfig_FileWins(t *testing.T) { + c, err := New(WithKubeOrInClusterConfig(fixtureKubeconfig)) + if err != nil { + t.Fatalf("New(WithKubeOrInClusterConfig(file)) returned error: %v", err) + } + if c == nil || c.K8sClient() == nil { + t.Fatalf("expected non-nil client, got %+v", c) + } +} + +func TestNew_WithRestConfig(t *testing.T) { + // Build a config via the file helper first; use it as input to WithRestConfig + // to skip the file/in-cluster resolvers entirely. + cfg, err := GetConfigForFile(fixtureKubeconfig) + if err != nil { + t.Fatalf("GetConfigForFile fixture failed: %v", err) + } + c, err := New(WithRestConfig(cfg)) + if err != nil { + t.Fatalf("New(WithRestConfig) returned error: %v", err) + } + if c == nil || c.K8sClient() == nil { + t.Fatalf("expected non-nil client, got %+v", c) + } +} + +func TestNew_WithRestConfig_NilConfig(t *testing.T) { + // A nil *rest.Config must produce a normal error, not a panic inside + // rest.CopyConfig. + _, err := New(WithRestConfig(nil)) + if err == nil { + t.Fatal("New(WithRestConfig(nil)) returned nil error, want an error") + } +} + +func TestNew_WithHttpClient(t *testing.T) { + // Provide a pre-built HTTP client, verify HttpClient() returns the same pointer. + hc := &http.Client{} + c, err := New(WithHttpClient(hc), WithKubeConfig(fixtureKubeconfig)) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + if c.HttpClient() != hc { + t.Errorf("HttpClient() returned different pointer than provided: got %p, want %p", c.HttpClient(), hc) + } +} + +// newTestConfig returns a fresh rest.Config. Built inline (not via the +// fixture) so tests fully control every field. Uses Insecure=true and no +// CAData so rest.HTTPClientFor inside New() does not try to parse CAData +// as a PEM block. +func newTestConfig() *rest.Config { + return &rest.Config{ + Host: "https://example.com:6443", + TLSClientConfig: rest.TLSClientConfig{ + Insecure: true, + }, + UserAgent: "test-user-agent", + } +} + +// TestClient_RestConfig_CopySemantics verifies RestConfig() returns a copy +// with rest.CopyConfig semantics: top-level and value-struct fields are +// safely overwritable on the returned value without affecting subsequent +// RestConfig() calls. Nested map/slice contents are NOT tested here — +// they share storage per rest.CopyConfig's contract. +func TestClient_RestConfig_CopySemantics(t *testing.T) { + c, err := New(WithRestConfig(newTestConfig())) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + first := c.RestConfig() + first.Host = "https://mutated.example.com" + first.UserAgent = "mutated-user-agent" + + second := c.RestConfig() + if second.Host == first.Host { + t.Errorf("RestConfig Host mutation leaked: got %q, want unchanged", second.Host) + } + if second.UserAgent == first.UserAgent { + t.Errorf("RestConfig UserAgent mutation leaked: got %q, want unchanged", second.UserAgent) + } +} + +// TestClient_WithRestConfig_CopySemanticsOnInput verifies that WithRestConfig +// takes a rest.CopyConfig copy of its input — post-New mutations of the +// original config's top-level fields do not leak into the client. +func TestClient_WithRestConfig_CopySemanticsOnInput(t *testing.T) { + cfg := newTestConfig() + + c, err := New(WithRestConfig(cfg)) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + + cfg.Host = "https://mutated-input.example.com" + cfg.UserAgent = "mutated-input-user-agent" + + rc := c.RestConfig() + if rc.Host == cfg.Host { + t.Errorf("input-side Host mutation leaked to client: got %q", rc.Host) + } + if rc.UserAgent == cfg.UserAgent { + t.Errorf("input-side UserAgent mutation leaked to client: got %q", rc.UserAgent) + } +} + +func TestWithAcceptContentTypes(t *testing.T) { + c, err := New( + WithKubeConfig(fixtureKubeconfig), + WithAcceptContentTypes(ContentTypeProtobuf, ContentTypeJSON), + ) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + got := c.RestConfig().AcceptContentTypes + want := ContentTypeProtobuf + "," + ContentTypeJSON + if got != want { + t.Errorf("AcceptContentTypes: got %q, want %q", got, want) + } +} + +func TestWithContentType(t *testing.T) { + c, err := New( + WithKubeConfig(fixtureKubeconfig), + WithContentType(ContentTypeProtobuf), + ) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + if got := c.RestConfig().ContentType; got != ContentTypeProtobuf { + t.Errorf("ContentType: got %q, want %q", got, ContentTypeProtobuf) + } +} + +// TestContentType_OrderDependence verifies that config-dependent options +// (WithContentType, WithAcceptContentTypes) must appear after the config- +// source option; placing them before the config source returns an error. +func TestContentType_OrderDependence(t *testing.T) { + // Case A: config-source first, content-type second — works correctly. + a, err := New( + WithKubeConfig(fixtureKubeconfig), + WithContentType(ContentTypeProtobuf), + WithAcceptContentTypes(ContentTypeProtobuf, ContentTypeJSON), + ) + if err != nil { + t.Fatalf("case A New returned error: %v", err) + } + if a.RestConfig().ContentType != ContentTypeProtobuf { + t.Errorf("ContentType: got %q, want %q", a.RestConfig().ContentType, ContentTypeProtobuf) + } + + // Case B: content-type before config-source — must return an error. + _, err = New( + WithContentType(ContentTypeProtobuf), + WithKubeConfig(fixtureKubeconfig), + ) + if err == nil { + t.Fatal("case B New should return an error when content-type precedes config source, got nil") + } +} + +// TestContentType_OnlyNoConfigSource verifies that a content-type option +// alone (no config-source option) returns an error because the REST config +// is not yet set when the option is applied. +func TestContentType_OnlyNoConfigSource(t *testing.T) { + _, err := New(WithContentType(ContentTypeProtobuf)) + if err == nil { + t.Fatal("New(WithContentType) without a config source should return an error, got nil") + } +} + +// TestContentType_MultipleOptions verifies that when multiple content-type +// options are passed after the config source, all of them apply in the +// order they were passed (last write wins). +func TestContentType_MultipleOptions(t *testing.T) { + c, err := New( + WithKubeConfig(fixtureKubeconfig), // provides the config + WithAcceptContentTypes(ContentTypeProtobuf), // overridden below + WithAcceptContentTypes(ContentTypeJSON), // last one wins + WithContentType(ContentTypeProtobuf), + ) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + // Last WithAcceptContentTypes wins. + if got := c.RestConfig().AcceptContentTypes; got != ContentTypeJSON { + t.Errorf("last-applied AcceptContentTypes should win: got %q, want %q", got, ContentTypeJSON) + } + if got := c.RestConfig().ContentType; got != ContentTypeProtobuf { + t.Errorf("ContentType: got %q, want %q", got, ContentTypeProtobuf) + } +} + +func TestClient_Close_Idempotent(t *testing.T) { + c, err := New(WithKubeConfig(fixtureKubeconfig)) + if err != nil { + t.Fatalf("New returned error: %v", err) + } + // First call — must not panic. + c.Close() + // Second call — must also not panic. + c.Close() +} diff --git a/pkg/kubernetes/client/testdata/kubeconfig-example.yaml b/pkg/kubernetes/client/testdata/kubeconfig-example.yaml new file mode 100644 index 000000000..350cfe160 --- /dev/null +++ b/pkg/kubernetes/client/testdata/kubeconfig-example.yaml @@ -0,0 +1,17 @@ +apiVersion: v1 +kind: Config +current-context: nri-plugins-test +clusters: +- name: nri-plugins-test-cluster + cluster: + server: https://example.com:6443 + insecure-skip-tls-verify: true +contexts: +- name: nri-plugins-test + context: + cluster: nri-plugins-test-cluster + user: nri-plugins-test-user +users: +- name: nri-plugins-test-user + user: + token: dummy-token diff --git a/pkg/agent/watch/file.go b/pkg/kubernetes/watch/file.go similarity index 100% rename from pkg/agent/watch/file.go rename to pkg/kubernetes/watch/file.go diff --git a/pkg/agent/watch/object.go b/pkg/kubernetes/watch/object.go similarity index 100% rename from pkg/agent/watch/object.go rename to pkg/kubernetes/watch/object.go diff --git a/pkg/agent/watch/watch.go b/pkg/kubernetes/watch/watch.go similarity index 97% rename from pkg/agent/watch/watch.go rename to pkg/kubernetes/watch/watch.go index 66735213f..473725c7d 100644 --- a/pkg/agent/watch/watch.go +++ b/pkg/kubernetes/watch/watch.go @@ -35,5 +35,5 @@ const ( ) var ( - log = logger.Get("agent") + log = logger.Get("watch") ) diff --git a/pkg/kubernetes/watch/watch_test.go b/pkg/kubernetes/watch/watch_test.go new file mode 100644 index 000000000..0cdad4631 --- /dev/null +++ b/pkg/kubernetes/watch/watch_test.go @@ -0,0 +1,196 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package watch + +import ( + "context" + "os" + "path/filepath" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + k8swatch "k8s.io/apimachinery/pkg/watch" +) + +// Compile-time assertions: the re-exported aliases really are the same +// types and values as the upstream ones. These lines would fail to +// compile if the aliases drifted. +var ( + _ Interface = k8swatch.Interface(nil) + _ EventType = k8swatch.EventType("") + _ Event = k8swatch.Event{} +) + +func TestEventTypeConstants(t *testing.T) { + tests := []struct { + name string + local EventType + want k8swatch.EventType + }{ + {"Added", Added, k8swatch.Added}, + {"Modified", Modified, k8swatch.Modified}, + {"Deleted", Deleted, k8swatch.Deleted}, + {"Bookmark", Bookmark, k8swatch.Bookmark}, + {"Error", Error, k8swatch.Error}, + } + for _, tc := range tests { + if tc.local != tc.want { + t.Errorf("%s: got %q, want %q", tc.name, tc.local, tc.want) + } + } +} + +// TestObject_HappyPath confirms events sent to the fake Interface +// returned by CreateFn make it out through Object's ResultChan. +func TestObject_HappyPath(t *testing.T) { + fake := k8swatch.NewFake() + create := func(ctx context.Context, ns, name string) (Interface, error) { + return fake, nil + } + + ctx, cancel := context.WithCancel(t.Context()) + defer cancel() + + ow, err := Object(ctx, "default", "cm-x", create) + if err != nil { + t.Fatalf("Object returned error: %v", err) + } + defer ow.Stop() + + // Push an Added and a Modified event through the fake; expect both. + go func() { + fake.Add(&metav1.Status{Message: "added"}) + fake.Modify(&metav1.Status{Message: "modified"}) + }() + + got := drainEvents(t, ow.ResultChan(), 2, 2*time.Second) + if len(got) < 2 { + t.Fatalf("expected at least 2 events, got %d: %+v", len(got), got) + } + if got[0].Type != Added { + t.Errorf("event[0].Type = %q, want %q", got[0].Type, Added) + } + if got[1].Type != Modified { + t.Errorf("event[1].Type = %q, want %q", got[1].Type, Modified) + } +} + +// TestObject_StopIdempotent verifies Stop() can be called more than +// once without panicking. The existing implementation uses sync.Once +// on the internal stop path; this is a defensive guard. +func TestObject_StopIdempotent(t *testing.T) { + fake := k8swatch.NewFake() + create := func(ctx context.Context, ns, name string) (Interface, error) { + return fake, nil + } + + ow, err := Object(t.Context(), "default", "cm-x", create) + if err != nil { + t.Fatalf("Object returned error: %v", err) + } + ow.Stop() + ow.Stop() // must not panic +} + +// TestFile_CreateVsWriteEventTypes is a defensive guard against +// regressing the Create-emits-Added / Write-emits-Modified distinction. +// PR #536's parallel implementation ships a copy-paste bug where both +// cases emit Added; we don't have that bug today, and this test +// ensures we don't accidentally introduce it in a future edit. +func TestFile_CreateVsWriteEventTypes(t *testing.T) { + dir := t.TempDir() + file := filepath.Join(dir, "watched.yaml") + + unmarshal := func(data []byte, name string) (runtime.Object, error) { + return &metav1.Status{Message: string(data)}, nil + } + + fw, err := File(file, unmarshal) + if err != nil { + t.Fatalf("File returned error: %v", err) + } + defer fw.Stop() + + // Step 1: create the file — expect an Added event from the fsnotify + // Create. + if err := os.WriteFile(file, []byte("v1"), 0o600); err != nil { + t.Fatalf("initial write: %v", err) + } + + // Step 2: modify the file using O_WRONLY|O_APPEND so fsnotify + // emits only Write (not Create). os.WriteFile uses O_CREATE|O_TRUNC + // which fires a Create event even for an existing file — that would + // give us a second Added instead of the Modified we're testing for. + time.Sleep(150 * time.Millisecond) + f, err := os.OpenFile(file, os.O_WRONLY|os.O_APPEND, 0) + if err != nil { + t.Fatalf("open for append: %v", err) + } + if _, err := f.Write([]byte("+v2")); err != nil { + _ = f.Close() + t.Fatalf("append write: %v", err) + } + if err := f.Close(); err != nil { + t.Fatalf("close after append: %v", err) + } + + // Drain up to 4 events (initial-Added-from-run + Create-Added + Write-Modified + slack). + got := drainEvents(t, fw.ResultChan(), 4, 3*time.Second) + sawAdded, sawModified := false, false + for _, ev := range got { + if ev.Type == Added { + sawAdded = true + } + if ev.Type == Modified { + sawModified = true + } + } + if !sawAdded { + t.Errorf("expected at least one Added event; got types: %v", eventTypes(got)) + } + if !sawModified { + t.Errorf("expected at least one Modified event (write path); got types: %v", eventTypes(got)) + } +} + +// drainEvents receives up to `want` events from ch or times out. +// Returns whatever it received. +func drainEvents(t *testing.T, ch <-chan Event, want int, timeout time.Duration) []Event { + t.Helper() + got := make([]Event, 0, want) + deadline := time.After(timeout) + for len(got) < want { + select { + case ev, ok := <-ch: + if !ok { + return got + } + got = append(got, ev) + case <-deadline: + return got + } + } + return got +} + +func eventTypes(evs []Event) []EventType { + out := make([]EventType, len(evs)) + for i, ev := range evs { + out[i] = ev.Type + } + return out +} diff --git a/pkg/resmgr/cache/cache.go b/pkg/resmgr/cache/cache.go index f5091999d..677c1e67d 100644 --- a/pkg/resmgr/cache/cache.go +++ b/pkg/resmgr/cache/cache.go @@ -237,6 +237,9 @@ type Container interface { GetMounts() []*Mount // GetDevices returns all the linux devices of the container. GetDevices() []*Device + // GetCDIDeviceNames returns the qualified names of all CDI devices + // injected into this container (nil/empty if there are none). + GetCDIDeviceNames() []string // PrettyName returns the user-friendly $namespace/$pod/$container for the container. PrettyName() string diff --git a/pkg/resmgr/cache/container.go b/pkg/resmgr/cache/container.go index 62b18a28d..5eba1ebe2 100644 --- a/pkg/resmgr/cache/container.go +++ b/pkg/resmgr/cache/container.go @@ -513,6 +513,20 @@ func (c *container) GetDevices() []*Device { return devices } +func (c *container) GetCDIDeviceNames() []string { + cdiDevices := c.Ctr.GetCDIDevices() + if len(cdiDevices) == 0 { + return nil + } + + names := make([]string, 0, len(cdiDevices)) + for _, d := range cdiDevices { + names = append(names, d.GetName()) + } + + return names +} + func (c *container) GetResmgrLabel(key string) (string, bool) { value, ok := c.GetLabel(kubernetes.ResmgrKey(key)) return value, ok diff --git a/pkg/resmgr/cache/container_test.go b/pkg/resmgr/cache/container_test.go index 7331f50af..445d4962f 100644 --- a/pkg/resmgr/cache/container_test.go +++ b/pkg/resmgr/cache/container_test.go @@ -396,6 +396,45 @@ var _ = Describe("Container", func() { Expect(ctrs[0].GetDevices()).To(Equal(devices)) }) + It("can return its CDI device names", func() { + var ( + names = []string{ + "nri.topology-aware.cpu/device=claim-abc-req-dev-0", + "nri.topology-aware.cpu/device=claim-abc-req-dev-1", + } + nriPods = []*nri.PodSandbox{ + makePod(), + } + nriCtrs = []*nri.Container{ + makeCtr( + WithCtrPodID(nriPods[0].GetId()), + WithCtrCDIDevices(names), + ), + } + ) + + _, _, ctrs := makePopulatedCache(nriPods, nriCtrs) + + Expect(ctrs[0].GetCDIDeviceNames()).To(Equal(names)) + }) + + It("returns no CDI device names when there are none", func() { + var ( + nriPods = []*nri.PodSandbox{ + makePod(), + } + nriCtrs = []*nri.Container{ + makeCtr( + WithCtrPodID(nriPods[0].GetId()), + ), + } + ) + + _, _, ctrs := makePopulatedCache(nriPods, nriCtrs) + + Expect(ctrs[0].GetCDIDeviceNames()).To(BeEmpty()) + }) + }) var _ = Describe("Container", func() { @@ -948,6 +987,22 @@ func WithCtrDevices(devices []*nri.LinuxDevice) CtrOption { } } +func WithCtrCDIDevices(names []string) CtrOption { + return func(nriCtr *nri.Container) error { + if names == nil { + nriCtr.CDIDevices = nil + return nil + } + nriCtr.CDIDevices = make([]*nri.CDIDevice, len(names)) + for i, n := range names { + nriCtr.CDIDevices[i] = &nri.CDIDevice{ + Name: n, + } + } + return nil + } +} + func WithCtrLabels(labels map[string]string) CtrOption { return func(nriCtr *nri.Container) error { if labels == nil { diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index 34548cea5..f5dad786f 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -25,6 +25,7 @@ package cpuclass import ( + "errors" "fmt" "sort" @@ -41,6 +42,12 @@ import ( var log = logger.NewLogger("cpuclass") +// ErrAllocatorInactive is returned by AccountHpCpus when the PCT allocator +// has not been configured yet. Callers that tolerate a deferred Configure() +// (e.g. plugin.Start with an inactive allocator) should treat this as a +// warn-and-continue condition rather than a fatal error. +var ErrAllocatorInactive = errors.New("cpuclass: pct allocator not active") + // AllocationIntent describes an upcoming CPU allocation for which // the caller wants placement preferences. type AllocationIntent = types.AllocationIntent @@ -78,6 +85,11 @@ type Handler struct { cpufreq *cpufreq.Allocator pct *pct.Allocator + // classes is the last-applied cpuClass list. Updated on every + // Configure() call. Used by DRADevices to enumerate published classes. + // Caller-owned slice; not deep-copied (consistent with pct.Configure behavior). + classes []*policyapi.CPUClass + // defs maps synthetic class name -> resolved class definition. // Populated by SetClassDef calls from the cpufreq allocator. defs map[string]types.ClassDef @@ -138,6 +150,80 @@ func (h *Handler) PctActive() bool { return h != nil && h.pct != nil && h.pct.Active() } +// PickHpCpus selects n HP-eligible CPUs from the punit identified by +// (pkgID, punitID), excluding CPUs in held and those already tracked +// in hpUsed or hpDRAUsed. Delegates to the PCT allocator. Returns an +// error when the handler or its PCT allocator is nil, or when the +// underlying pick fails (inactive allocator, punit not found, or +// insufficient HP capacity). +func (h *Handler) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { + if h == nil || h.pct == nil { + return cpuset.New(), fmt.Errorf("cpuclass: PickHpCpus: pct allocator not initialized") + } + return h.pct.PickHpCpus(pkgID, punitID, n, held) +} + +// ReleaseHpCpus removes cpus from DRA HP accounting on the punit +// identified by (pkgID, punitID). Delegates to the PCT allocator. +// No-op when the handler or its PCT allocator is nil, or when the +// punit is unknown (idempotent). +func (h *Handler) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { + if h == nil || h.pct == nil { + return + } + h.pct.ReleaseHpCpus(pkgID, punitID, cpus) +} + +// AccountHpCpus records cpus as DRA HP-held on the punit identified +// by (pkgID, punitID). Used during restart reconciliation to rebuild +// HP accounting from persisted claim state without re-allocating CPUs. +// Delegates to the PCT allocator. Returns an error when the handler +// or its PCT allocator is nil, or when accounting fails (inactive +// allocator, punit not found, or HP-ineligible punit). +func (h *Handler) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { + if h == nil || h.pct == nil { + return fmt.Errorf("cpuclass: AccountHpCpus: pct allocator not initialized: %w", ErrAllocatorInactive) + } + if !h.pct.Active() { + return fmt.Errorf("cpuclass: AccountHpCpus: pct allocator not active: %w", ErrAllocatorInactive) + } + return h.pct.AccountHpCpus(pkgID, punitID, cpus) +} + +// IsHPClass reports whether className is currently classified as PCT +// high priority. Delegates to the PCT allocator. Returns false when +// the handler or its PCT allocator is nil, or when the allocator is +// inactive. +func (h *Handler) IsHPClass(className string) bool { + if h == nil || h.pct == nil { + return false + } + return h.pct.IsHPClass(className) +} + +// ClassForCPU returns the synthetic class name currently assigned to cpu by +// the most recent UseClass/AssignCPUs call (as tracked in h.cpuClass), or "" +// if cpu is unmanaged or explicitly assigned to no class. Primarily useful +// for tests that need to verify a UseClass call actually changed (or +// restored) a CPU's class, since Handler otherwise exposes no per-CPU class +// query. Nil-safe. +// +// This is a test-only accessor kept on the exported Handler API rather than +// behind an export_test.go shim: its only callers are cmd/plugins/topology- +// aware/policy's tests, a different package, and Go test files are never +// compiled into a package's importable surface across package boundaries — +// an export_test.go in this package would be invisible there. Duplicating +// equivalent instrumentation on the caller side (e.g. inspecting SST mock +// state directly) would require exposing Handler's internal CLOS/class +// mapping some other way, which is a larger change than this narrow, +// clearly-documented read accessor. +func (h *Handler) ClassForCPU(cpu int) string { + if h == nil { + return "" + } + return h.cpuClass[cpu] +} + // Configure (re)applies a configuration spec. Idempotent: may be // called repeatedly with changed classes, turbo-domain mode, or // allowed set. @@ -158,6 +244,10 @@ func (h *Handler) Configure(spec ConfigSpec) error { return fmt.Errorf("cpuclass: pct configure: %w", err) } + // h.classes is set after all fallible operations so that a partial + // Configure failure leaves h.classes consistent with the previously + // committed state (not a half-applied new config). + h.classes = spec.Classes h.classNames = map[string]struct{}{} for _, cls := range spec.Classes { h.classNames[cls.Name] = struct{}{} diff --git a/pkg/resmgr/cpuclass/cpuclass_dra_test.go b/pkg/resmgr/cpuclass/cpuclass_dra_test.go new file mode 100644 index 000000000..aad99d883 --- /dev/null +++ b/pkg/resmgr/cpuclass/cpuclass_dra_test.go @@ -0,0 +1,244 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for the DRA ClaimAllocator pass-through methods on Handler. + +package cpuclass_test + +import ( + "testing" + + idset "github.com/intel/goresctrl/pkg/utils" + + policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// Compile-time assertion: *Handler must satisfy dra.ClaimAllocator. +var _ dra.ClaimAllocator = (*cpuclass.Handler)(nil) + +// draTestSys is a minimal sysfs.System implementation for DRA pass-through +// tests. Only CPUIDs is overridden; all other methods are delegated to the +// embedded nil interface, which panics if called. In practice, only CPUIDs +// is invoked during Handler.New() (by cpufreq platform discovery). +type draTestSys struct { + sysfs.System +} + +func (s *draTestSys) CPUIDs() []idset.ID { return nil } + +// newConfiguredHandler creates a Handler with an active managed PCT +// allocator using the SST in-memory mock (OVERRIDE_SST). The mock is +// seeded with one package (ID 0), one punit (ID 0), CPUs 0-7, and +// GuaranteedHpCpus=4. t.Setenv restores the env after the test. +func newConfiguredHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", `{"supported":true,"clos_count":4,"packages":[{"id":0,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"punits":[{"id":0,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":4}]}]}`) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + h, err := cpuclass.New(&draTestSys{}) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + if err := h.Configure(cpuclass.ConfigSpec{ + Classes: []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }); err != nil { + t.Fatalf("Configure() failed: %v", err) + } + return h +} + +// newInactiveHandler creates a Handler without OVERRIDE_SST so that +// SST is reported as unsupported and PCT stays in disabled mode. +// Configure is called with an HP class to exercise the "SST not +// supported → PCT disabled" path. +func newInactiveHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + // Ensure OVERRIDE_SST is unset (t.Setenv restores original value). + t.Setenv("OVERRIDE_SST", "") + h, err := cpuclass.New(&draTestSys{}) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + // Configure with a PCT class; SST unsupported → pct stays disabled. + _ = h.Configure(cpuclass.ConfigSpec{ + Classes: []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }) + return h +} + +// --------------------------------------------------------------------------- +// PickHpCpus +// --------------------------------------------------------------------------- + +func TestHandlerPickHpCpus_NilHandler(t *testing.T) { + var h *cpuclass.Handler + _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + if err == nil { + t.Fatal("expected error from nil handler, got nil") + } +} + +func TestHandlerPickHpCpus_NilPct(t *testing.T) { + h := &cpuclass.Handler{} // pct is nil + _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + if err == nil { + t.Fatal("expected error from nil pct, got nil") + } +} + +func TestHandlerPickHpCpus_InactivePct(t *testing.T) { + h := newInactiveHandler(t) + _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + if err == nil { + t.Fatal("expected error from inactive PCT, got nil") + } +} + +func TestHandlerPickHpCpus_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + got, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus(0,0,2) = %v, want nil error", err) + } + if got.Size() != 2 { + t.Errorf("PickHpCpus(0,0,2) returned %d CPUs, want 2", got.Size()) + } + // Requesting more than GuaranteedHpCpus (4) must error. + if _, err := h.PickHpCpus(0, 0, 5, cpuset.New()); err == nil { + t.Error("PickHpCpus(0,0,5) with capacity=4: expected error, got nil") + } +} + +// --------------------------------------------------------------------------- +// ReleaseHpCpus +// --------------------------------------------------------------------------- + +func TestHandlerReleaseHpCpus_NilHandler(t *testing.T) { + var h *cpuclass.Handler + // must not panic + h.ReleaseHpCpus(0, 0, cpuset.New()) +} + +func TestHandlerReleaseHpCpus_NilPct(t *testing.T) { + h := &cpuclass.Handler{} + // must not panic + h.ReleaseHpCpus(0, 0, cpuset.New()) +} + +func TestHandlerReleaseHpCpus_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + // Pick 2 CPUs, then release them. + cpus, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus setup: %v", err) + } + h.ReleaseHpCpus(0, 0, cpus) + // After release, the full 4-CPU capacity must be available again. + got, err := h.PickHpCpus(0, 0, 4, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus after release: %v", err) + } + if got.Size() != 4 { + t.Errorf("post-release PickHpCpus returned %d CPUs, want 4", got.Size()) + } +} + +// --------------------------------------------------------------------------- +// AccountHpCpus +// --------------------------------------------------------------------------- + +func TestHandlerAccountHpCpus_NilHandler(t *testing.T) { + var h *cpuclass.Handler + if err := h.AccountHpCpus(0, 0, cpuset.New()); err == nil { + t.Fatal("expected error from nil handler, got nil") + } +} + +func TestHandlerAccountHpCpus_NilPct(t *testing.T) { + h := &cpuclass.Handler{} + if err := h.AccountHpCpus(0, 0, cpuset.New()); err == nil { + t.Fatal("expected error from nil pct, got nil") + } +} + +func TestHandlerAccountHpCpus_InactivePct(t *testing.T) { + h := newInactiveHandler(t) + if err := h.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + t.Fatal("expected error from inactive PCT, got nil") + } +} + +func TestHandlerAccountHpCpus_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + // AccountHpCpus simulates restart reconciliation (union semantics, + // no allocation): pick 2, release, then re-account. + cpus, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus setup: %v", err) + } + h.ReleaseHpCpus(0, 0, cpus) + if err := h.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus(%s) = %v, want nil", cpus, err) + } + // Idempotency: calling again with the same CPUs must not error. + if err := h.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus idempotent call = %v, want nil", err) + } +} + +// --------------------------------------------------------------------------- +// IsHPClass +// --------------------------------------------------------------------------- + +func TestHandlerIsHPClass_NilHandler(t *testing.T) { + var h *cpuclass.Handler + if h.IsHPClass("hp") { + t.Error("IsHPClass on nil handler: got true, want false") + } +} + +func TestHandlerIsHPClass_NilPct(t *testing.T) { + h := &cpuclass.Handler{} + if h.IsHPClass("hp") { + t.Error("IsHPClass with nil pct: got true, want false") + } +} + +func TestHandlerIsHPClass_InactivePct(t *testing.T) { + h := newInactiveHandler(t) + // PCT disabled → all classes report non-HP. + if h.IsHPClass("hp") { + t.Error("IsHPClass with inactive PCT: got true, want false") + } +} + +func TestHandlerIsHPClass_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + // "hp" class has pctPriority=high → must report HP. + if !h.IsHPClass("hp") { + t.Error("IsHPClass(\"hp\") = false, want true") + } + // Unknown / non-HP names must return false. + if h.IsHPClass("lp") { + t.Error("IsHPClass(\"lp\") = true, want false") + } + if h.IsHPClass("") { + t.Error("IsHPClass(\"\") = true, want false") + } +} diff --git a/pkg/resmgr/cpuclass/dra.go b/pkg/resmgr/cpuclass/dra.go new file mode 100644 index 000000000..82d9d51fd --- /dev/null +++ b/pkg/resmgr/cpuclass/dra.go @@ -0,0 +1,355 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cpuclass + +import ( + "fmt" + "regexp" + "sort" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + resapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/resource" + kptr "k8s.io/utils/ptr" + + policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" +) + +// DRA device attribute/capacity keys shared with pkg/resmgr/dra (which reads +// them back out of the published device list) — defined once here, +// referenced everywhere else, mirroring the DRADriverName convention. +const ( + // AttrCPUClass names the cpuClass a device belongs to. + AttrCPUClass resapi.QualifiedName = "nri/cpuClass" + // AttrPackageID names the CPU package a device's punit lives on. + AttrPackageID resapi.QualifiedName = "nri/packageID" + // AttrPunitID names the SST-TF punit a device represents. + AttrPunitID resapi.QualifiedName = "nri/punitID" + // AttrAllowedCPUs carries the CPUSet string (allowed ∩ punit.CPUs) for a + // device, identifying exactly which CPUs it represents. + AttrAllowedCPUs resapi.QualifiedName = "nri/allowedCPUs" + // CapacityCPUs is the capacity key for the number of CPUs a device grants. + CapacityCPUs resapi.QualifiedName = "nri/cpus" +) + +// nonAlphaRe matches runs of characters that are not lowercase letters or digits. +// Used by sanitizeBase to replace them with hyphens. +var nonAlphaRe = regexp.MustCompile(`[^a-z0-9]+`) + +// maxDNSLabel is the Kubernetes DNS label name length limit. +const maxDNSLabel = 63 + +// maxDeviceBase returns the maximum length for the sanitized class-name +// portion of a device name before any dedup suffix is appended. +func maxDeviceBase(punits []pct.PunitInfo, classCount int) int { + maxSuffixLen := len("-pkg0-punit0") + for _, pu := range punits { + if suffixLen := len("-pkg" + strconv.Itoa(pu.PkgID) + "-punit" + strconv.Itoa(pu.PunitID)); suffixLen > maxSuffixLen { + maxSuffixLen = suffixLen + } + } + maxDedupSuffixLen := len("-" + strconv.Itoa(classCount+1)) + return maxDNSLabel - maxSuffixLen - maxDedupSuffixLen +} + +// ValidateCPUClassesForDRA checks that DRA-published PCT classes do not +// overcommit any priority tier. Classes are grouped by tier — the +// pctPriority value for managed PCT classes, or the SstClosID for +// assoc-only classes — and more than one DRA-published class in the same +// tier is an error. Non-PCT classes and managed LP classes (pctPriority +// != "high") are exempt, since buildDRADevices with hpOnly=true never +// publishes them. +// +// sharedCounters is rejected: buildDRADevices always publishes +// independent full-capacity devices per class, so accepting it here would +// silently disable this overcommit guard without implementing the +// KEP-5941 shared-counter model. +// +// Called at driver Configure time, not at config load time. The []Punit +// parameter is absent in v1 — per-punit enforcement is deferred to the +// device-build step where runtime punit topology is available. +func ValidateCPUClassesForDRA(classes []*policyapi.CPUClass, sharedCounters bool) error { + if sharedCounters { + return fmt.Errorf( + "DRA: sharedCounters is not yet supported (KEP-5941 is not " + + "implemented); leave spec.dra.sharedCounters unset or false", + ) + } + + // Group DRA-published PCT classes by tier label. + byTier := map[string][]string{} // tier label → sorted class names + for _, cc := range classes { + if !isPCTClass(cc) { + continue + } + if !cc.DRAPublish() { + continue + } + if cc.PctPriority != "" && cc.PctPriority != "high" { + continue // LP classes are exempt, see doc comment + } + tier := tierLabel(cc) + byTier[tier] = append(byTier[tier], cc.Name) + } + + // Check for conflicts: any tier with more than one published class. + tiers := make([]string, 0, len(byTier)) + for t := range byTier { + tiers = append(tiers, t) + } + sort.Strings(tiers) // deterministic outer ordering + + for _, tier := range tiers { + names := byTier[tier] + if len(names) <= 1 { + continue + } + sort.Strings(names) // deterministic name listing in the error + return fmt.Errorf( + "DRA: tier %q has %d published cpuClasses (%v); "+ + "at most one is allowed. "+ + "Resolution: set cpuClass.dra.publish: false on all but one "+ + "(sharedCounters is not yet supported — see KEP-5941)", + tier, len(names), names, + ) + } + + return nil +} + +// isPCTClass reports whether cc is a PCT class (managed or assoc-only). +func isPCTClass(cc *policyapi.CPUClass) bool { + return cc.PctPriority != "" || cc.SstClosID != nil +} + +// tierLabel returns the tier string used for grouping and error messages. +func tierLabel(cc *policyapi.CPUClass) string { + if cc.PctPriority != "" { + return "pctPriority=" + cc.PctPriority + } + return fmt.Sprintf("closID=%d", *cc.SstClosID) +} + +// sanitizeBase lowercases s, replaces runs of non-alphanumeric characters with +// "-", trims leading/trailing hyphens, and truncates to maxLen (trimming any +// trailing hyphen created by truncation). Returns "class" if the result is empty. +func sanitizeBase(s string, maxLen int) string { + b := strings.ToLower(s) + b = nonAlphaRe.ReplaceAllString(b, "-") + b = strings.Trim(b, "-") + if b == "" { + return "class" + } + if len(b) > maxLen { + b = strings.TrimRight(b[:maxLen], "-") + } + if b == "" { + return "class" + } + return b +} + +// deviceName assembles a DRA device name from a pre-sanitized class base and +// punit topology identifiers. Format: -pkg-punit. +func deviceName(classBase string, pkgID, punitID int) string { + return classBase + "-pkg" + strconv.Itoa(pkgID) + "-punit" + strconv.Itoa(punitID) +} + +// intAttr returns a DeviceAttribute with an integer value. +func intAttr(v int64) resapi.DeviceAttribute { + return resapi.DeviceAttribute{IntValue: kptr.To(v)} +} + +// strAttr returns a DeviceAttribute with a string value. +func strAttr(v string) resapi.DeviceAttribute { + return resapi.DeviceAttribute{StringValue: kptr.To(v)} +} + +// buildDRADevices constructs the []resapi.Device slice (one device per +// published cpuClass × SST-TF punit) to be passed to kubeletplugin.PublishResources. +// +// For each published class, for each punit: emits one device if capacity > 0. +// HP classes use HPCapacity; non-HP classes use NonHPCapacity. +// +// When hpOnly is true, only HP classes (isHP returns true) are emitted. +// Non-HP DRA is deferred because PunitInfo carries no per-punit CPU list. +func buildDRADevices( + classes []*policyapi.CPUClass, + punits []pct.PunitInfo, + isHP func(className string) bool, + hpOnly bool, +) []resapi.Device { + if len(classes) == 0 || len(punits) == 0 { + return []resapi.Device{} + } + + // Pre-compute a stable sanitized base for each published class name. + // Dedup: if two different class names produce the same base, the second + // gets a "-N" suffix (N starting at 2). The same class name across multiple + // punits always reuses the same pre-computed base (no counter increment). + takenBases := map[string]struct{}{} // bases already claimed by some class + baseForClass := map[string]string{} // className -> final sanitized base + publishedClassCount := 0 + for _, cc := range classes { + if cc.DRAPublish() && (!hpOnly || isHP(cc.Name)) { + publishedClassCount++ + } + } + baseMaxLen := maxDeviceBase(punits, publishedClassCount) + + for _, cc := range classes { + if !cc.DRAPublish() { + continue + } + // non-HP DRA deferred — PunitInfo has no per-punit CPU list. + if hpOnly && !isHP(cc.Name) { + continue + } + if _, done := baseForClass[cc.Name]; done { + continue // same class name seen twice — skip (defensive) + } + candidate := sanitizeBase(cc.Name, baseMaxLen) + if _, taken := takenBases[candidate]; !taken { + takenBases[candidate] = struct{}{} + baseForClass[cc.Name] = candidate + } else { + // Collision: find the next available suffixed base. + for n := 2; ; n++ { + suffixed := candidate + "-" + strconv.Itoa(n) + if _, inUse := takenBases[suffixed]; !inUse { + takenBases[suffixed] = struct{}{} + baseForClass[cc.Name] = suffixed + break + } + } + } + } + + var devices []resapi.Device + + for _, cc := range classes { + if !cc.DRAPublish() { + continue + } + if hpOnly && !isHP(cc.Name) { + continue + } + base := baseForClass[cc.Name] + // Always set: duplicate class names are skipped above, and + // validation guarantees no duplicates reach this point. + for _, pu := range punits { + // Select capacity based on HP classification. + var capacity int + if isHP(cc.Name) { + capacity = pu.HPCapacity + } else { + capacity = pu.NonHPCapacity + } + if capacity == 0 { + continue // zero-capacity RequestPolicy is invalid; skip + } + + name := deviceName(base, pu.PkgID, pu.PunitID) + + attrs := map[resapi.QualifiedName]resapi.DeviceAttribute{ + AttrPackageID: intAttr(int64(pu.PkgID)), + AttrPunitID: intAttr(int64(pu.PunitID)), + AttrCPUClass: strAttr(cc.Name), + } + // nri/allowedCPUs captures the CPU identity (allowed ∩ punit.CPUs) + // the device represents. + if pu.AllowedCPUs != "" { + attrs[AttrAllowedCPUs] = strAttr(pu.AllowedCPUs) + } + // nri/pctPriority is only emitted for PCT classes (non-empty PctPriority). + // Omitting it for non-PCT classes avoids CEL false-positives on "" values. + if cc.PctPriority != "" { + attrs["nri/pctPriority"] = strAttr(cc.PctPriority) + } + + capStr := strconv.Itoa(capacity) + dev := resapi.Device{ + Name: name, + Attributes: attrs, + Capacity: map[resapi.QualifiedName]resapi.DeviceCapacity{ + CapacityCPUs: { + Value: resource.MustParse(capStr), + RequestPolicy: &resapi.CapacityRequestPolicy{ + Default: kptr.To(resource.MustParse("1")), + ValidRange: &resapi.CapacityRequestPolicyRange{ + Min: kptr.To(resource.MustParse("1")), + Max: kptr.To(resource.MustParse(capStr)), + Step: kptr.To(resource.MustParse("1")), + }, + }, + }, + }, + AllowMultipleAllocations: kptr.To(true), + NodeAllocatableResources: map[corev1.ResourceName]resapi.NodeAllocatableResource{ + corev1.ResourceCPU: { + Mapping: &resapi.NodeAllocatableMapping{ + CapacityKey: kptr.To(CapacityCPUs), + CapacityMultiplier: kptr.To(resource.MustParse("1")), + }, + }, + }, + } + devices = append(devices, dev) + } + } + + if devices == nil { + return []resapi.Device{} + } + return devices +} + +// DRADevices returns the DRA device slice for the current cpuClass configuration. +// Returns an empty (non-nil) slice when the handler is nil, PCT is inactive, +// or no punits are available. +// +// Must be called on the resmgr goroutine or under the resmgr lock — same as all +// other Handler methods. +func (h *Handler) DRADevices(_ string) ([]resapi.Device, error) { + if h == nil || h.pct == nil { + return []resapi.Device{}, nil + } + // Punits() returns nil when inactive, so len()==0 covers both + // the inactive and the "no punits" cases. + punits := h.pct.Punits() + if len(punits) == 0 { + return []resapi.Device{}, nil + } + return buildDRADevices(h.classes, punits, h.pct.IsHPClass, true), nil +} + +// DRADevicesAtMaxCapacity is like DRADevices but sets HPCapacity to +// GuaranteedHpCpus (ignoring hpUsed). Used by Reconfigure to snapshot +// hardware-level capacity for change detection — comparing workload-adjusted +// capacities would produce false negatives when hpUsed coincidentally equals +// the capacity delta introduced by a hardware change. +func (h *Handler) DRADevicesAtMaxCapacity(_ string) ([]resapi.Device, error) { + if h == nil || h.pct == nil { + return []resapi.Device{}, nil + } + punits := h.pct.MaxPunits() + if len(punits) == 0 { + return []resapi.Device{}, nil + } + return buildDRADevices(h.classes, punits, h.pct.IsHPClass, true), nil +} diff --git a/pkg/resmgr/cpuclass/dra_test.go b/pkg/resmgr/cpuclass/dra_test.go new file mode 100644 index 000000000..99e0e9569 --- /dev/null +++ b/pkg/resmgr/cpuclass/dra_test.go @@ -0,0 +1,875 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cpuclass + +import ( + "regexp" + "strings" + "testing" + + corev1 "k8s.io/api/core/v1" + resapi "k8s.io/api/resource/v1" + + policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +func ptr[T any](v T) *T { return &v } + +func TestValidateCPUClassesForDRA(t *testing.T) { + tests := []struct { + name string + classes []*policyapi.CPUClass + sharedCounters bool + wantErr bool + // errContains is a substring the error must contain (if wantErr). + errContains string + }{ + { + name: "empty class list", + classes: nil, + wantErr: false, + }, + { + name: "non-PCT classes only — all exempt", + classes: []*policyapi.CPUClass{ + {Name: "default"}, + {Name: "idle"}, + {Name: "turbo"}, + }, + wantErr: false, + }, + { + name: "single managed HP class published", + classes: []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + }, + wantErr: false, + }, + { + name: "single managed LP class published", + classes: []*policyapi.CPUClass{ + {Name: "lp", PctPriority: "low"}, + }, + wantErr: false, + }, + { + name: "one HP + one LP managed — different tiers, ok", + classes: []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + {Name: "lp", PctPriority: "low"}, + }, + wantErr: false, + }, + { + // Regression: two published LP classes must not trigger a tier-overcommit + // error. DRADevices uses hpOnly=true, so LP classes produce no devices + // and cannot overcommit any tier. + name: "two managed LP classes, both published — ok (hpOnly skips them)", + classes: []*policyapi.CPUClass{ + {Name: "lp1", PctPriority: "low"}, + {Name: "lp2", PctPriority: "low"}, + }, + wantErr: false, + }, + { + name: "two assoc-only classes with different SstClosIDs — ok", + classes: []*policyapi.CPUClass{ + {Name: "hp-assoc", SstClosID: ptr(0)}, + {Name: "lp-assoc", SstClosID: ptr(3)}, + }, + wantErr: false, + }, + { + name: "two assoc-only classes with same SstClosID, sharedCounters=false — error (same CLOS, can both be HP-published)", + classes: []*policyapi.CPUClass{ + {Name: "class-a", SstClosID: ptr(0)}, + {Name: "class-b", SstClosID: ptr(0)}, + }, + sharedCounters: false, + wantErr: true, + errContains: `closID=0`, + }, + { + name: "two assoc-only classes with same SstClosID, sharedCounters=true — rejected", + classes: []*policyapi.CPUClass{ + {Name: "class-a", SstClosID: ptr(0)}, + {Name: "class-b", SstClosID: ptr(0)}, + }, + sharedCounters: true, + wantErr: true, + errContains: "sharedCounters", + }, + { + name: "two managed HP classes, both published, sharedCounters=false — error", + classes: []*policyapi.CPUClass{ + {Name: "hp-perf", PctPriority: "high"}, + {Name: "hp-turbo", PctPriority: "high"}, + }, + sharedCounters: false, + wantErr: true, + errContains: "hp-perf", + }, + { + name: "two managed HP classes, both published, sharedCounters=true — rejected", + classes: []*policyapi.CPUClass{ + {Name: "hp-perf", PctPriority: "high"}, + {Name: "hp-turbo", PctPriority: "high"}, + }, + sharedCounters: true, + wantErr: true, + errContains: "sharedCounters", + }, + { + name: "two managed HP classes, one opted out — ok", + classes: []*policyapi.CPUClass{ + {Name: "hp-perf", PctPriority: "high"}, + {Name: "hp-turbo", PctPriority: "high", DRA: &policyapi.CPUClassDRA{Publish: ptr(false)}}, + }, + sharedCounters: false, + wantErr: false, + }, + { + name: "three managed HP classes, all published — error names all", + classes: []*policyapi.CPUClass{ + {Name: "hp-a", PctPriority: "high"}, + {Name: "hp-b", PctPriority: "high"}, + {Name: "hp-c", PctPriority: "high"}, + }, + sharedCounters: false, + wantErr: true, + errContains: "hp-a", + }, + { + name: "mixed PCT and non-PCT — only PCT classes checked", + classes: []*policyapi.CPUClass{ + {Name: "default"}, + {Name: "hp", PctPriority: "high"}, + {Name: "another-default"}, + }, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := ValidateCPUClassesForDRA(tc.classes, tc.sharedCounters) + if tc.wantErr { + if err == nil { + t.Fatalf("ValidateCPUClassesForDRA() = nil, want error") + } + if tc.errContains != "" && !strings.Contains(err.Error(), tc.errContains) { + t.Errorf("error %q does not contain %q", err.Error(), tc.errContains) + } + } else { + if err != nil { + t.Fatalf("ValidateCPUClassesForDRA() = %v, want nil", err) + } + } + }) + } +} + +// dnsLabelRe matches valid Kubernetes DNS label names (RFC 1123 subset). +var dnsLabelRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) + +// isDNSLabel reports whether s is a valid DNS label (≤63 chars, lowercase alphanumeric + hyphens). +func isDNSLabel(s string) bool { + return len(s) <= 63 && dnsLabelRe.MatchString(s) +} + +// attrInt retrieves the integer value of a device attribute by key. +// Returns (0, false) if absent or not an int. +func attrInt(dev resapi.Device, key resapi.QualifiedName) (int64, bool) { + a, ok := dev.Attributes[key] + if !ok || a.IntValue == nil { + return 0, false + } + return *a.IntValue, true +} + +// attrStr retrieves the string value of a device attribute by key. +// Returns ("", false) if absent or not a string. +func attrStr(dev resapi.Device, key resapi.QualifiedName) (string, bool) { + a, ok := dev.Attributes[key] + if !ok || a.StringValue == nil { + return "", false + } + return *a.StringValue, true +} + +// cpusMax returns the RequestPolicy.ValidRange.Max quantity for the "nri/cpus" +// capacity entry of a device. Returns (0, false) if missing or malformed. +func cpusMax(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.ValidRange == nil || cap.RequestPolicy.ValidRange.Max == nil { + return 0, false + } + return cap.RequestPolicy.ValidRange.Max.Value(), true +} + +// cpusCapacity returns the DeviceCapacity Value for the "nri/cpus" entry. +func cpusCapacity(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok { + return 0, false + } + return cap.Value.Value(), true +} + +// cpusDefault returns the RequestPolicy.Default value for "nri/cpus". +// Returns (0, false) if missing or malformed. +func cpusDefault(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.Default == nil { + return 0, false + } + return cap.RequestPolicy.Default.Value(), true +} + +// cpusMin returns the RequestPolicy.ValidRange.Min value for "nri/cpus". +// Returns (0, false) if missing or malformed. +func cpusMin(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.ValidRange == nil || cap.RequestPolicy.ValidRange.Min == nil { + return 0, false + } + return cap.RequestPolicy.ValidRange.Min.Value(), true +} + +// cpusStep returns the RequestPolicy.ValidRange.Step value for "nri/cpus". +// Returns (0, false) if missing or malformed. +func cpusStep(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.ValidRange == nil || cap.RequestPolicy.ValidRange.Step == nil { + return 0, false + } + return cap.RequestPolicy.ValidRange.Step.Value(), true +} + +// checkDeviceShape asserts the shape invariants that every emitted device must +// satisfy: AllowMultipleAllocations, DeviceCapacity.Value, the full +// RequestPolicy (Default/Min/Max/Step), and NodeAllocatableResources content +// (Mapping.CapacityKey + Mapping.CapacityMultiplier). It also verifies +// topology attributes and the nri/cpuClass attribute. +// +// wantPctPriorityPresent controls whether nri/pctPriority is expected to exist +// (PCT classes) or must be absent (non-PCT classes). +func checkDeviceShape(t *testing.T, dev resapi.Device, wantCapacity int64, + wantClass, wantPctPriority string, wantPctPriorityPresent bool) { + t.Helper() + + // AllowMultipleAllocations must be true. + if dev.AllowMultipleAllocations == nil || !*dev.AllowMultipleAllocations { + t.Errorf("AllowMultipleAllocations: got %v, want true", dev.AllowMultipleAllocations) + } + + // nri/cpuClass. + if v, ok := attrStr(dev, "nri/cpuClass"); !ok { + t.Error("nri/cpuClass attribute missing") + } else if v != wantClass { + t.Errorf("nri/cpuClass = %q, want %q", v, wantClass) + } + + // nri/pctPriority. + _, hasPct := dev.Attributes["nri/pctPriority"] + if wantPctPriorityPresent { + if v, ok := attrStr(dev, "nri/pctPriority"); !ok { + t.Errorf("nri/pctPriority attribute missing (want %q)", wantPctPriority) + } else if v != wantPctPriority { + t.Errorf("nri/pctPriority = %q, want %q", v, wantPctPriority) + } + } else if hasPct { + t.Errorf("nri/pctPriority must be absent, got %v", dev.Attributes["nri/pctPriority"]) + } + + // DeviceCapacity.Value (outer field, independent of ValidRange.Max). + if cv, ok := cpusCapacity(dev); !ok { + t.Error("nri/cpus capacity Value missing") + } else if cv != wantCapacity { + t.Errorf("nri/cpus capacity Value = %d, want %d", cv, wantCapacity) + } + + // RequestPolicy: Default must be 1. + if dv, ok := cpusDefault(dev); !ok { + t.Error("nri/cpus RequestPolicy.Default missing") + } else if dv != 1 { + t.Errorf("nri/cpus RequestPolicy.Default = %d, want 1", dv) + } + + // RequestPolicy.ValidRange: Min must be 1. + if mv, ok := cpusMin(dev); !ok { + t.Error("nri/cpus RequestPolicy.ValidRange.Min missing") + } else if mv != 1 { + t.Errorf("nri/cpus RequestPolicy.ValidRange.Min = %d, want 1", mv) + } + + // RequestPolicy.ValidRange: Max must equal capacity. + if xv, ok := cpusMax(dev); !ok { + t.Error("nri/cpus RequestPolicy.ValidRange.Max missing") + } else if xv != wantCapacity { + t.Errorf("nri/cpus RequestPolicy.ValidRange.Max = %d, want %d", xv, wantCapacity) + } + + // RequestPolicy.ValidRange: Step must be 1. + if sv, ok := cpusStep(dev); !ok { + t.Error("nri/cpus RequestPolicy.ValidRange.Step missing") + } else if sv != 1 { + t.Errorf("nri/cpus RequestPolicy.ValidRange.Step = %d, want 1", sv) + } + + // NodeAllocatableResources content. + if dev.NodeAllocatableResources == nil { + t.Error("NodeAllocatableResources is nil") + } else { + r, ok := dev.NodeAllocatableResources[corev1.ResourceCPU] + if !ok { + t.Errorf("NodeAllocatableResources: missing %q key", corev1.ResourceCPU) + } else if r.Mapping == nil { + t.Errorf("NodeAllocatableResources[cpu].Mapping is nil") + } else { + if r.Mapping.CapacityKey == nil || *r.Mapping.CapacityKey != "nri/cpus" { + t.Errorf("NodeAllocatableResources[cpu].Mapping.CapacityKey = %v, want \"nri/cpus\"", r.Mapping.CapacityKey) + } + if r.Mapping.CapacityMultiplier == nil || r.Mapping.CapacityMultiplier.Value() != 1 { + t.Errorf("NodeAllocatableResources[cpu].Mapping.CapacityMultiplier = %v, want 1", + r.Mapping.CapacityMultiplier) + } + } + } +} + +func TestBuildDRADevices(t *testing.T) { + // Shorthand helpers used only in test table. + hpClass := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{Name: name, PctPriority: "high"} + } + lpClass := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{Name: name, PctPriority: "low"} + } + nonPCTClass := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{Name: name} + } + unpublished := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{ + Name: name, + PctPriority: "high", + DRA: &policyapi.CPUClassDRA{Publish: ptr(false)}, + } + } + punit := func(pkg, id, hpCap, nonHPCap int) pct.PunitInfo { + return pct.PunitInfo{PkgID: pkg, PunitID: id, HPCapacity: hpCap, NonHPCapacity: nonHPCap} + } + + // isHP returns true only for classes whose name starts with "hp". + isHP := func(name string) bool { return strings.HasPrefix(name, "hp") } + + tests := []struct { + name string + classes []*policyapi.CPUClass + punits []pct.PunitInfo + isHP func(string) bool + hpOnly bool + // wantCount is the expected number of returned devices. + wantCount int + // verify is an optional per-result checker. + verify func(t *testing.T, devices []resapi.Device) + }{ + { + name: "one HP class + one punit (pkg=0 punit=0)", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + // Full device-shape invariants: capacity=HPCapacity=4, PCT class "hp"/"high". + checkDeviceShape(t, dev, 4, "hp", "high", true) + // Topology attributes present and correct. + if v, ok := attrInt(dev, "nri/packageID"); !ok { + t.Errorf("nri/packageID attribute missing") + } else if v != 0 { + t.Errorf("nri/packageID = %d, want 0", v) + } + if v, ok := attrInt(dev, "nri/punitID"); !ok { + t.Errorf("nri/punitID attribute missing") + } else if v != 0 { + t.Errorf("nri/punitID = %d, want 0", v) + } + // Device name must be DNS-valid. + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + name: "one non-HP PCT class + one punit → max=NonHPCapacity", + classes: []*policyapi.CPUClass{lpClass("lp")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + // Full device-shape invariants: capacity=NonHPCapacity=8, PCT class "lp"/"low". + checkDeviceShape(t, dev, 8, "lp", "low", true) + // Topology attributes. + if _, ok := attrInt(dev, "nri/packageID"); !ok { + t.Error("nri/packageID attribute missing") + } + if _, ok := attrInt(dev, "nri/punitID"); !ok { + t.Error("nri/punitID attribute missing") + } + }, + }, + { + name: "HP class + HPCapacity==0 → device skipped for that punit", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: []pct.PunitInfo{punit(0, 0, 0, 8)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "NonHPCapacity==0 → device skipped for that punit", + classes: []*policyapi.CPUClass{lpClass("lp")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 0)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "class with dra.publish: false → excluded", + classes: []*policyapi.CPUClass{unpublished("hp-hidden")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "two classes × two punits → four devices with correct names", + classes: []*policyapi.CPUClass{ + hpClass("hp"), + lpClass("lp"), + }, + punits: []pct.PunitInfo{ + punit(0, 0, 4, 8), + punit(0, 1, 4, 8), + }, + isHP: isHP, + wantCount: 4, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + names := make(map[string]bool, 4) + for _, d := range devices { + names[d.Name] = true + if !isDNSLabel(d.Name) { + t.Errorf("device name %q is not a valid DNS label", d.Name) + } + } + // Same class across punits must use the same sanitized base, + // not trigger the dedup counter (e.g. "hp-2-pkg0-punit0"). + wantNames := []string{ + "hp-pkg0-punit0", + "hp-pkg0-punit1", + "lp-pkg0-punit0", + "lp-pkg0-punit1", + } + for _, want := range wantNames { + if !names[want] { + t.Errorf("expected device name %q not found in %v", want, devices) + } + } + // Every device must have AllowMultipleAllocations=true. + for _, d := range devices { + if d.AllowMultipleAllocations == nil || !*d.AllowMultipleAllocations { + t.Errorf("device %q: AllowMultipleAllocations not true", d.Name) + } + } + // Every device must carry nri/cpuClass pointing to the right class. + for _, d := range devices { + v, ok := attrStr(d, "nri/cpuClass") + if !ok { + t.Errorf("device %q: nri/cpuClass missing", d.Name) + continue + } + // Name encodes the class base: "hp-pkg..." vs "lp-pkg...". + wantClass := "hp" + if strings.HasPrefix(d.Name, "lp-") { + wantClass = "lp" + } + if v != wantClass { + t.Errorf("device %q: nri/cpuClass = %q, want %q", d.Name, v, wantClass) + } + } + }, + }, + { + name: "empty classes → empty result", + classes: nil, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "empty punits → empty result", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: nil, + isHP: isHP, + wantCount: 0, + }, + { + // Class name > 60 chars must produce a device name ≤ 63 chars. + name: "long class name → device name ≤ 63 chars", + classes: []*policyapi.CPUClass{hpClass("hp-this-is-a-very-long-cpuclass-name-that-exceeds-sixty-chars-total-yes")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if len(dev.Name) > 63 { + t.Errorf("device name %q has length %d, want ≤ 63", dev.Name, len(dev.Name)) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + // 2-digit pkg+punit IDs ("-pkg10-punit10" = 14 chars) combined with a + // max-length base and a dedup suffix must still fit in 63 chars. + name: "2-digit pkg and punit IDs → device name ≤ 63 chars", + classes: []*policyapi.CPUClass{hpClass("hp-this-is-a-very-long-cpuclass-name-that-exceeds-sixty-chars-total-yes")}, + punits: []pct.PunitInfo{punit(10, 10, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if len(dev.Name) > 63 { + t.Errorf("device name %q has length %d, want ≤ 63", dev.Name, len(dev.Name)) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + // Two classes whose names sanitize to the same base must get distinct device names. + name: "two classes sanitize to same base → distinct device names", + classes: []*policyapi.CPUClass{ + // Both "hp-class" and "hp_class" sanitize to "hp-class". + hpClass("hp-class"), + hpClass("hp_class"), + }, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + // Both classes have HPCapacity>0 so both emit one device. + wantCount: 2, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + seen := map[string]bool{} + for _, d := range devices { + if seen[d.Name] { + t.Errorf("duplicate device name %q", d.Name) + } + seen[d.Name] = true + if !isDNSLabel(d.Name) { + t.Errorf("device name %q is not a valid DNS label", d.Name) + } + } + }, + }, + { + // Non-PCT class must NOT emit nri/pctPriority attribute; must still + // carry topology and cpuClass attributes, and AllowMultipleAllocations. + name: "non-PCT class → nri/pctPriority absent, topology present", + classes: []*policyapi.CPUClass{nonPCTClass("default")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: func(string) bool { return false }, // non-PCT: never HP + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + // pctPriority must be absent. + if _, ok := dev.Attributes["nri/pctPriority"]; ok { + t.Errorf("nri/pctPriority must be absent for non-PCT class, got %v", + dev.Attributes["nri/pctPriority"]) + } + // AllowMultipleAllocations must still be true. + if dev.AllowMultipleAllocations == nil || !*dev.AllowMultipleAllocations { + t.Errorf("AllowMultipleAllocations: got %v, want true", dev.AllowMultipleAllocations) + } + // Topology attributes must be present. + if _, ok := attrInt(dev, "nri/packageID"); !ok { + t.Error("nri/packageID attribute missing") + } + if _, ok := attrInt(dev, "nri/punitID"); !ok { + t.Error("nri/punitID attribute missing") + } + // nri/cpuClass must still be present. + if v, ok := attrStr(dev, "nri/cpuClass"); !ok { + t.Error("nri/cpuClass attribute missing") + } else if v != "default" { + t.Errorf("nri/cpuClass = %q, want \"default\"", v) + } + }, + }, + { + // Non-zero PkgID and PunitID must appear correctly in attributes and name. + name: "non-zero PkgID and PunitID → correct in attrs and name", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: []pct.PunitInfo{punit(2, 3, 5, 6)}, // pkg=2, punit=3 + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if v, ok := attrInt(dev, "nri/packageID"); !ok || v != 2 { + t.Errorf("nri/packageID = %d (ok=%v), want 2", v, ok) + } + if v, ok := attrInt(dev, "nri/punitID"); !ok || v != 3 { + t.Errorf("nri/punitID = %d (ok=%v), want 3", v, ok) + } + if dev.Name != "hp-pkg2-punit3" { + t.Errorf("device name = %q, want hp-pkg2-punit3", dev.Name) + } + // Full shape check with HPCapacity=5. + checkDeviceShape(t, dev, 5, "hp", "high", true) + }, + }, + { + // Class name that is all non-alphanumeric → sanitizeBase returns "class". + name: "all-special-char class name → 'class' fallback base", + classes: []*policyapi.CPUClass{hpClass("---")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if dev.Name != "class-pkg0-punit0" { + t.Errorf("device name = %q, want class-pkg0-punit0", dev.Name) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + // hpOnly=true: mixed HP/non-HP config → only HP devices emitted. + // Non-HP DRA is deferred; non-HP classes must be silently filtered. + name: "mixed HP/non-HP config with hpOnly=true → only HP devices emitted", + classes: []*policyapi.CPUClass{ + hpClass("hp"), + lpClass("lp"), + nonPCTClass("default"), + }, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + hpOnly: true, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if v, ok := attrStr(dev, "nri/cpuClass"); !ok { + t.Error("nri/cpuClass attribute missing") + } else if v != "hp" { + t.Errorf("nri/cpuClass = %q, want \"hp\"", v) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + // hpOnly=false: non-HP classes are included (base behaviour unchanged). + name: "mixed HP/non-HP config with hpOnly=false → all published devices emitted", + classes: []*policyapi.CPUClass{ + hpClass("hp"), + lpClass("lp"), + }, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + hpOnly: false, + wantCount: 2, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + isHPFn := tc.isHP + if isHPFn == nil { + isHPFn = func(string) bool { return false } + } + devices := buildDRADevices(tc.classes, tc.punits, isHPFn, tc.hpOnly) + if len(devices) != tc.wantCount { + t.Fatalf("buildDRADevices() returned %d devices, want %d; devices=%v", + len(devices), tc.wantCount, deviceNames(devices)) + } + // buildDRADevices must always return a non-nil slice (including the + // empty case) so callers can safely range over it. + if devices == nil { + t.Errorf("buildDRADevices() returned nil, want non-nil []Device{}") + } + if tc.verify != nil { + tc.verify(t, devices) + } + }) + } +} + +// deviceNames returns a slice of device names for use in test error messages. +func deviceNames(devices []resapi.Device) []string { + names := make([]string, len(devices)) + for i, d := range devices { + names[i] = d.Name + } + return names +} + +// sstWithPunit is a JSON seed for OVERRIDE_SST that provides one active +// SST-TF package (pkg=1) with one punit (punit=2), 8 CPUs, and +// GuaranteedHpCpus=3 (i.e. HPCapacity=3, NonHPCapacity=5). +// Using non-zero pkg and punit IDs exercises the full attribute path. +const sstWithPunit = `{"supported":true,"clos_count":4,"packages":[{"id":1,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"cp_priority":"ordered","punits":[{"id":2,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":3}]}]}` + +// TestDRADevices covers Handler.DRADevices: the nil-handler guard, the +// nil-pct guard, the inactive-pct early-return, and the delegation path +// to buildDRADevices when pct is active with classes set. +// +// Tests that require an active pct allocator use OVERRIDE_SST so that +// pct.Configure succeeds without real SST hardware. +func TestDRADevices(t *testing.T) { + t.Run("nil handler returns empty non-nil", func(t *testing.T) { + var h *Handler + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices on nil handler: %v", err) + } + if devs == nil { + t.Error("nil handler: want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("nil handler: want 0 devices, got %d", len(devs)) + } + }) + + t.Run("nil pct returns empty non-nil", func(t *testing.T) { + h := &Handler{} // pct is nil + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices with nil pct: %v", err) + } + if devs == nil { + t.Error("nil pct: want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("nil pct: want 0 devices, got %d", len(devs)) + } + }) + + t.Run("inactive pct returns empty non-nil", func(t *testing.T) { + // OVERRIDE_SST with supported=false → pct stays disabled after Configure. + t.Setenv("OVERRIDE_SST", `{"supported":false,"packages":[]}`) + pctA, err := pct.NewAllocator(nil) + if err != nil { + t.Fatalf("pct.NewAllocator: %v", err) + } + if err := pctA.Configure(nil, cpuset.New()); err != nil { + t.Fatalf("pct.Configure: %v", err) + } + h := &Handler{pct: pctA} + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices inactive pct: %v", err) + } + if devs == nil { + t.Error("inactive pct: want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("inactive pct: want 0 devices, got %d", len(devs)) + } + }) + + t.Run("active pct empty classes returns empty non-nil", func(t *testing.T) { + // pct is active but h.classes is nil → buildDRADevices returns empty. + t.Setenv("OVERRIDE_SST", sstWithPunit) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + pctA, err := pct.NewAllocator(nil) + if err != nil { + t.Fatalf("pct.NewAllocator: %v", err) + } + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + if err := pctA.Configure(classes, cpuset.New()); err != nil { + t.Fatalf("pct.Configure: %v", err) + } + h := &Handler{pct: pctA, classes: nil} // classes not set + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices: %v", err) + } + if devs == nil { + t.Error("want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("want 0 devices (no classes), got %d", len(devs)) + } + }) + + t.Run("active pct with classes delegates to buildDRADevices", func(t *testing.T) { + // OVERRIDE_SST: pkg=1, punit=2, HPCapacity=3. + // DRADevices must pass h.classes and h.pct.IsHPClass to buildDRADevices + // and return the resulting device slice. + t.Setenv("OVERRIDE_SST", sstWithPunit) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + pctA, err := pct.NewAllocator(nil) + if err != nil { + t.Fatalf("pct.NewAllocator: %v", err) + } + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + if err := pctA.Configure(classes, cpuset.New()); err != nil { + t.Fatalf("pct.Configure: %v", err) + } + if !pctA.Active() { + t.Fatal("pct not active after Configure with supported SST — check OVERRIDE_SST JSON") + } + // h.classes is set directly, mirroring what Handler.Configure does + // (post fix: h.classes is assigned after all fallible ops). + h := &Handler{pct: pctA, classes: classes} + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices: %v", err) + } + // pkg=1, punit=2, HPCapacity=3 → one device "hp-pkg1-punit2" with + // capacity 3. NonHPCapacity=5 (8 CPUs − 3 HP) but "hp" is HP class. + if len(devs) != 1 { + t.Fatalf("DRADevices: got %d devices, want 1; names=%v", len(devs), deviceNames(devs)) + } + dev := devs[0] + if dev.Name != "hp-pkg1-punit2" { + t.Errorf("device name = %q, want hp-pkg1-punit2", dev.Name) + } + // Verify topology attributes carry the non-zero IDs. + if v, ok := attrInt(dev, "nri/packageID"); !ok || v != 1 { + t.Errorf("nri/packageID = %d (ok=%v), want 1", v, ok) + } + if v, ok := attrInt(dev, "nri/punitID"); !ok || v != 2 { + t.Errorf("nri/punitID = %d (ok=%v), want 2", v, ok) + } + // Full device-shape invariants (capacity=3, HP class). + checkDeviceShape(t, dev, 3, "hp", "high", true) + }) +} diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index e4d7f979f..68283e293 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -96,8 +96,12 @@ type Allocator struct { // allocator treats them as "no HP knowledge". punitByCpu map[int]int // hpUsed[i] is the set of CPUs currently held by HP-class - // workloads on punits[i]. + // workloads on punits[i] via the non-DRA (hint-driven) path. hpUsed map[int]cpuset.CPUSet + // hpDRAUsed[i] is the set of CPUs currently held by DRA claims + // on punits[i]. Separate from hpUsed so that clearHpUsage (called + // from the non-DRA UseClass path) can never evict DRA holds. + hpDRAUsed map[int]cpuset.CPUSet // hpEligiblePunit[i] reports whether punits[i] can actually // host HP-class CPUs at top turbo. Populated at Configure(). // In managed mode every punit becomes eligible (the plugin @@ -138,6 +142,7 @@ func (a *Allocator) Configure(classes []*policyapi.CPUClass, allowed cpuset.CPUS a.fallbackClos = pctDefaultHpClos // CLOS 0 == default-after-reset a.allowed = allowed a.hpUsed = map[int]cpuset.CPUSet{} + a.hpDRAUsed = map[int]cpuset.CPUSet{} a.hpClasses = map[string]bool{} a.hpEligiblePunit = map[int]bool{} a.punits = nil @@ -520,6 +525,229 @@ func (a *Allocator) FreeClassCapacity(className string, held cpuset.CPUSet) int return total } +// PunitInfo is a snapshot of one SST punit's DRA-relevant capacity. +// Returned by Allocator.Punits(); valid within the current +// Configure/Reconfigure cycle (indices may change on next Configure). +type PunitInfo struct { + PkgID int + PunitID int + HPCapacity int // GuaranteedHpCpus minus non-DRA HP CPUs already in hpUsed, or 0 if the punit is HP-ineligible + NonHPCapacity int // allocatable non-HP CPUs (allowed ∩ punit.CPUs − GuaranteedHpCpus) + AllowedCPUs string // CPUSet string for allowed ∩ punit.CPUs +} + +// Punits returns a snapshot of per-punit DRA-relevant capacity for all +// punits known to the allocator. Returns nil when the allocator is not +// active. +func (a *Allocator) Punits() []PunitInfo { + if !a.Active() { + return nil + } + out := make([]PunitInfo, len(a.punits)) + for i, pu := range a.punits { + out[i] = PunitInfo{ + PkgID: pu.PkgID, + PunitID: pu.PunitID, + HPCapacity: a.punitAvailableHPCapacity(i), + NonHPCapacity: a.punitNonHPCapacity(i), + AllowedCPUs: pu.CPUs.String(), + } + } + return out +} + +// MaxPunits is like Punits but sets HPCapacity to the full GuaranteedHpCpus +// (not reduced by hpUsed). Used by Handler.DRADevicesAtMaxCapacity for +// hardware-change detection in Reconfigure, where comparing workload-adjusted +// capacities would produce false negatives when hpUsed coincidentally equals +// the capacity delta. +func (a *Allocator) MaxPunits() []PunitInfo { + if !a.Active() { + return nil + } + out := make([]PunitInfo, len(a.punits)) + for i, pu := range a.punits { + out[i] = PunitInfo{ + PkgID: pu.PkgID, + PunitID: pu.PunitID, + HPCapacity: a.punitHPCapacity(i), + NonHPCapacity: a.punitNonHPCapacity(i), + AllowedCPUs: pu.CPUs.String(), + } + } + return out +} + +// punitIdxByID returns the index in a.punits for the punit identified +// by (pkgID, punitID), or -1 if not found. +func (a *Allocator) punitIdxByID(pkgID, punitID int) int { + for i, pu := range a.punits { + if pu.PkgID == pkgID && pu.PunitID == punitID { + return i + } + } + return -1 +} + +// punitHPCapacity returns the guaranteed HP CPU count for punits[idx], +// capped by its actual (allowed-intersected) CPU count, or 0 if the +// punit is HP-ineligible or the index is out of range. The cap is +// needed because snapshotPunits intersects CPUs with the allowed set +// but leaves GuaranteedHpCpus at its raw hardware value. +func (a *Allocator) punitHPCapacity(idx int) int { + if !a.Active() || idx < 0 || idx >= len(a.punits) { + return 0 + } + if !a.hpEligiblePunit[idx] { + return 0 + } + return min(a.punits[idx].GuaranteedHpCpus, a.punits[idx].CPUs.Size()) +} + +// punitAvailableHPCapacity returns punitHPCapacity(idx) minus the +// non-DRA HP CPUs already held in hpUsed[idx] — the value that must be +// advertised as DRA-consumable capacity, since Kubernetes only subtracts +// DRA allocations from advertised capacity. Staleness from non-DRA HP +// allocations is handled by the DRA plugin's republisherLoop +// (Plugin.TriggerRepublish). +func (a *Allocator) punitAvailableHPCapacity(idx int) int { + capacity := a.punitHPCapacity(idx) - a.hpUsed[idx].Size() + if capacity < 0 { + return 0 + } + return capacity +} + +// punitNonHPCapacity returns the count of allocatable non-HP CPUs in +// punits[idx]. Applies the allowed.Size()>0 guard consistent with other +// allowed-consuming sites in this file. +func (a *Allocator) punitNonHPCapacity(idx int) int { + if !a.Active() || idx < 0 || idx >= len(a.punits) { + return 0 + } + pu := a.punits[idx] + cpus := pu.CPUs + if a.allowed.Size() > 0 { + cpus = cpus.Intersection(a.allowed) + } + n := cpus.Size() - a.punitHPCapacity(idx) + if n < 0 { + return 0 + } + return n +} + +// PickHpCpus selects n HP-eligible CPUs from the punit identified by +// (pkgID, punitID), excluding CPUs in held and those already tracked in +// hpUsed or hpDRAUsed. Records the selection in hpDRAUsed (not hpUsed, +// so clearHpUsage called from the non-DRA path cannot evict DRA holds). +// Returns an error when the allocator is inactive, the punit is not +// found or not HP-eligible, or fewer than n CPUs are available. +func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { + if !a.Active() { + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: allocator not active") + } + idx := a.punitIdxByID(pkgID, punitID) + if idx < 0 { + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) not found", pkgID, punitID) + } + if !a.hpEligiblePunit[idx] { + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) is not HP-eligible", pkgID, punitID) + } + pu := a.punits[idx] + avail := pu.CPUs + if a.allowed.Size() > 0 { + avail = avail.Intersection(a.allowed) + } + avail = avail.Difference(held).Difference(a.hpUsed[idx]).Difference(a.hpDRAUsed[idx]) + // Also enforce the GuaranteedHpCpus cap: can't pick more HP CPUs than + // the punit guarantees, regardless of how many are physically free. + // Use Union.Size() to avoid double-counting any CPU in both sets. + hpAlreadyHeld := a.hpUsed[idx].Union(a.hpDRAUsed[idx]).Size() + hpRoom := pu.GuaranteedHpCpus - hpAlreadyHeld + if hpRoom < 0 { + hpRoom = 0 + } + if avail.Size() < n || hpRoom < n { + available := avail.Size() + if hpRoom < available { + available = hpRoom + } + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) has %d available HP CPUs (room=%d, free=%d), need %d", + pkgID, punitID, available, hpRoom, avail.Size(), n) + } + // Sort for deterministic selection; take first n. + list := avail.List() + picked := cpuset.New(list[:n]...) + // hpDRAUsed is always non-nil here: Configure() initialises it + // unconditionally, and Active() (checked above) is true only after + // a successful Configure(). + a.hpDRAUsed[idx] = a.hpDRAUsed[idx].Union(picked) + return picked, nil +} + +// ReleaseHpCpus removes cpus from hpDRAUsed[punitIdx] for the punit +// identified by (pkgID, punitID). Silently ignores unknown punits and +// CPUs not present in hpDRAUsed (idempotent). +func (a *Allocator) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { + if !a.Active() { + return + } + idx := a.punitIdxByID(pkgID, punitID) + if idx < 0 { + return + } + // hpDRAUsed is always non-nil here: Configure() initialises it + // unconditionally, and Active() (checked above) is true only after + // a successful Configure(). + remaining := a.hpDRAUsed[idx].Difference(cpus) + if remaining.IsEmpty() { + delete(a.hpDRAUsed, idx) + } else { + a.hpDRAUsed[idx] = remaining + } +} + +// AccountHpCpus records cpus as HP DRA-held on the punit identified by +// (pkgID, punitID). Used during restart reconciliation to rebuild +// hpDRAUsed from persisted claim state without reallocating CPUs. +// Returns an error when the allocator is inactive, the punit is not +// found, the punit is not HP-eligible, or cpus is not a subset of the +// punit's current CPU set (e.g. after a reboot or topology/allowed-set +// change made the persisted CPUs stale). Over-commit (hpDRAUsed + +// hpUsed > GuaranteedHpCpus) is permitted — the container may already +// be running; a warning is logged but no error is returned. Union +// semantics make repeated calls with the same CPUs idempotent. +func (a *Allocator) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { + if !a.Active() { + return fmt.Errorf("pct: AccountHpCpus: allocator not active") + } + idx := a.punitIdxByID(pkgID, punitID) + if idx < 0 { + return fmt.Errorf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) not found", pkgID, punitID) + } + if !a.hpEligiblePunit[idx] { + return fmt.Errorf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) is not HP-eligible", pkgID, punitID) + } + if !cpus.IsSubsetOf(a.punits[idx].CPUs) { + return fmt.Errorf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) does not contain all of %s (has %s)", + pkgID, punitID, cpus, a.punits[idx].CPUs) + } + // Union is idempotent: repeated calls with the same CPUs do not + // double-count them. + a.hpDRAUsed[idx] = a.hpDRAUsed[idx].Union(cpus) + // Warn on over-commit but do not reject — the container may already + // be running with these CPUs. + pu := a.punits[idx] + held := a.hpUsed[idx].Union(a.hpDRAUsed[idx]).Size() + if held > pu.GuaranteedHpCpus { + log.Warnf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) over-committed: "+ + "hpDRAUsed+hpUsed=%d > GuaranteedHpCpus=%d", + pkgID, punitID, held, pu.GuaranteedHpCpus) + } + return nil +} + // useClass associates the given CPUs to the CLOS chosen for className. // In managed mode, CPUs whose className is not a PCT class are // associated to the fallback CLOS. In assoc-only mode such CPUs are @@ -569,8 +797,11 @@ func (a *Allocator) trackHpUsage(className string, cpus cpuset.CPUSet) { perPunit[idx] = append(perPunit[idx], cpu) } for idx, list := range perPunit { - set := a.hpUsed[idx] - a.hpUsed[idx] = set.Union(cpuset.New(list...)) + set := a.hpUsed[idx].Union(cpuset.New(list...)) + if dra := a.hpDRAUsed[idx]; !dra.IsEmpty() { + set = set.Difference(dra) + } + a.hpUsed[idx] = set } } @@ -636,6 +867,13 @@ func (a *Allocator) classIsHighPriority(className string) bool { return a.hpClasses[className] } +// IsHPClass reports whether className is currently classified as PCT +// high priority. It is the exported counterpart of classIsHighPriority. +// Returns false when the allocator is inactive or the class is unknown. +func (a *Allocator) IsHPClass(className string) bool { + return a.classIsHighPriority(className) +} + // hpHintsActive reports whether HP-room reasoning (hpReserveCpus, // hpInUseCpus, trackHpUsage) is currently meaningful. It requires // PCT to be active *and* at least one cpuClass to be classified as @@ -675,11 +913,11 @@ func (a *Allocator) hpInUseCpus() cpuset.CPUSet { return cpuset.New() } out := cpuset.New() - for idx, used := range a.hpUsed { - if used.IsEmpty() { - continue - } - if idx < 0 || idx >= len(a.punits) { + // Range over punits rather than hpUsed so that DRA-only punits + // (present in hpDRAUsed but absent from hpUsed) are not skipped. + for idx := range a.punits { + combined := a.hpUsed[idx].Union(a.hpDRAUsed[idx]) + if combined.IsEmpty() { continue } out = out.Union(a.punits[idx].CPUs) @@ -756,7 +994,9 @@ func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, continue } anyKnown = true - used := a.hpUsed[i] + // Union hpUsed and hpDRAUsed so DRA holds reduce reported HP room, + // preventing HP over-subscription via the hint path. + used := a.hpUsed[i].Union(a.hpDRAUsed[i]) if excludeBln.Size() > 0 { used = used.Difference(excludeBln) } diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index 584d32d36..d1ea60b7c 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -211,6 +211,7 @@ func newManagedPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans map classPlan: plans, allowed: allowed, hpUsed: map[int]cpuset.CPUSet{}, + hpDRAUsed: map[int]cpuset.CPUSet{}, hpClasses: map[string]bool{}, } for _, cc := range classes { @@ -1083,3 +1084,428 @@ func TestFreeClassCapacity_UnknownClassReturnsZero(t *testing.T) { t.Errorf("unknown class capacity = %d, want 0", got) } } + +// makePunitsWithGtdHp returns two punits in the same package, each with the +// given MaxHpCpus and GuaranteedHpCpus values. +func makePunitsWithGtdHp(maxHp0, gtdHp0, maxHp1, gtdHp1 int) []pctPunit { + return []pctPunit{ + {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), MaxHpCpus: maxHp0, GuaranteedHpCpus: gtdHp0}, + {PkgID: 0, PunitID: 1, CPUs: cpuset.MustParse("4-7"), MaxHpCpus: maxHp1, GuaranteedHpCpus: gtdHp1}, + } +} + +// newPickAllocator returns an Allocator pre-wired for PickHpCpus / ReleaseHpCpus tests. +func newPickAllocator(t *testing.T, punits []pctPunit) *Allocator { + t.Helper() + sys := newTwoPunitFakeSys() + sst := &fakeSst{supported: true, punits: punits} + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + plans := map[string]*pctClassPlan{"hp": {ClosID: 0}} + a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-7"), sys, sst) + return a +} + +func TestPunitHPCapacity(t *testing.T) { + // Active() == false: Allocator with mode == disabled + inactiveA := &Allocator{} + if got := inactiveA.punitHPCapacity(0); got != 0 { + t.Errorf("punitHPCapacity on inactive allocator = %d, want 0", got) + } + + a := newPickAllocator(t, makePunitsWithGtdHp(4, 3, 4, 1)) + tests := []struct { + name string + idx int + want int + }{ + {"eligible punit 0", 0, 3}, + {"eligible punit 1", 1, 1}, + {"out-of-range", 99, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := a.punitHPCapacity(tc.idx) + if got != tc.want { + t.Errorf("punitHPCapacity(%d) = %d, want %d", tc.idx, got, tc.want) + } + }) + } + // HP-ineligible punit + a.hpEligiblePunit[0] = false + if got := a.punitHPCapacity(0); got != 0 { + t.Errorf("punitHPCapacity for ineligible punit = %d, want 0", got) + } +} + +// TestPunitHPCapacity_CappedByAllowedIntersection verifies that +// punitHPCapacity caps the raw hardware GuaranteedHpCpus by the punit's +// actual CPU count after intersecting with the allowed/online set — +// otherwise it would advertise more DRA capacity than PickHpCpus can +// supply once some of the punit's CPUs are excluded (e.g. offline or +// outside the reserved/shared pool). +func TestPunitHPCapacity_CappedByAllowedIntersection(t *testing.T) { + sys := newTwoPunitFakeSys() + // Raw punit 0 spans CPUs 0-3 (4 CPUs) with GuaranteedHpCpus=3. + // Restricting "allowed" to CPUs 0-1,4-7 leaves punit 0 with only + // CPUs 0-1 (2 CPUs) after intersection -- less than its raw + // GuaranteedHpCpus of 3. + sst := &fakeSst{supported: true, punits: makePunitsWithGtdHp(4, 3, 4, 1)} + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + plans := map[string]*pctClassPlan{"hp": {ClosID: 0}} + a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-1,4-7"), sys, sst) + + if got := a.punitHPCapacity(0); got != 2 { + t.Errorf("punitHPCapacity(0) = %d, want 2 (capped by allowed intersection, not raw GuaranteedHpCpus=3)", got) + } + // Punit 1 is unaffected: its full range (4-7) is within allowed, and + // its GuaranteedHpCpus=1 stays under the 4-CPU cap. + if got := a.punitHPCapacity(1); got != 1 { + t.Errorf("punitHPCapacity(1) = %d, want 1 (unaffected by allowed restriction)", got) + } +} + +func TestPunitNonHPCapacity(t *testing.T) { + inactiveA := &Allocator{} + if got := inactiveA.punitNonHPCapacity(0); got != 0 { + t.Errorf("punitNonHPCapacity on inactive allocator = %d, want 0", got) + } + + // punit 0: CPUs 0-3 (4 total), GuaranteedHpCpus=3 → 1 non-HP + // punit 1: CPUs 4-7 (4 total), GuaranteedHpCpus=0 → 4 non-HP + a := newPickAllocator(t, makePunitsWithGtdHp(4, 3, 4, 0)) + tests := []struct { + name string + idx int + want int + }{ + {"partial HP", 0, 1}, + {"all non-HP", 1, 4}, + {"out-of-range", 99, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := a.punitNonHPCapacity(tc.idx) + if got != tc.want { + t.Errorf("punitNonHPCapacity(%d) = %d, want %d", tc.idx, got, tc.want) + } + }) + } + // all CPUs HP-guaranteed → 0 non-HP + a2 := newPickAllocator(t, makePunitsWithGtdHp(4, 4, 4, 4)) + if got := a2.punitNonHPCapacity(0); got != 0 { + t.Errorf("all-HP punit nonHPCapacity = %d, want 0", got) + } + + // HP-ineligible punit with non-zero GuaranteedHpCpus: the guard in + // punitHPCapacity must zero out the HP deduction so the full CPU count + // is reported as non-HP capacity. + // punit 0: HP-ineligible, GuaranteedHpCpus=2, CPUs 0-3 → 4 non-HP (not 2) + // punit 1: HP-eligible, GuaranteedHpCpus=2, CPUs 4-7 → 2 non-HP + a3 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + a3.hpEligiblePunit[0] = false + if got := a3.punitNonHPCapacity(0); got != 4 { + t.Errorf("HP-ineligible punit nonHPCapacity = %d, want 4", got) + } + if got := a3.punitNonHPCapacity(1); got != 2 { + t.Errorf("HP-eligible punit nonHPCapacity = %d, want 2", got) + } +} + +func TestAllocatorPunits(t *testing.T) { + // Inactive allocator returns nil. + inactiveA := &Allocator{} + if got := inactiveA.Punits(); got != nil { + t.Errorf("Punits() on inactive allocator = %v, want nil", got) + } + + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 1)) + pi := a.Punits() + if len(pi) != 2 { + t.Fatalf("Punits() len = %d, want 2", len(pi)) + } + want := []PunitInfo{ + {PkgID: 0, PunitID: 0, HPCapacity: 2, NonHPCapacity: 2, AllowedCPUs: "0-3"}, + {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3, AllowedCPUs: "4-7"}, + } + for i, w := range want { + if pi[i] != w { + t.Errorf("Punits()[%d] = %+v, want %+v", i, pi[i], w) + } + } +} + +// TestAllocatorPunits_NonDRAHpUsageReducesCapacity verifies that +// Punits() deducts non-DRA HP CPUs already tracked in hpUsed from the +// advertised HPCapacity, since Kubernetes only subtracts DRA +// allocations from the advertised capacity and would otherwise let a +// claim be placed that PickHpCpus must reject at Prepare time. +func TestAllocatorPunits_NonDRAHpUsageReducesCapacity(t *testing.T) { + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 1)) + // Simulate the NRI path having already claimed 1 HP CPU on punit 0. + a.hpUsed[0] = cpuset.MustParse("0") + + pi := a.Punits() + want := []PunitInfo{ + {PkgID: 0, PunitID: 0, HPCapacity: 1, NonHPCapacity: 2, AllowedCPUs: "0-3"}, + {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3, AllowedCPUs: "4-7"}, + } + for i, w := range want { + if pi[i] != w { + t.Errorf("Punits()[%d] = %+v, want %+v", i, pi[i], w) + } + } +} + +func TestPickHpCpus(t *testing.T) { + // Active()==false + inactiveA := &Allocator{} + if _, err := inactiveA.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus on inactive allocator: expected error, got nil") + } + + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + + // Success: pick 2 CPUs from punit 0 (PkgID=0, PunitID=0, CPUs 0-3). + got, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus success case: %v", err) + } + if got.Size() != 2 { + t.Errorf("PickHpCpus returned %d CPUs, want 2", got.Size()) + } + // hpDRAUsed updated, hpUsed unchanged. + if !a.hpDRAUsed[0].Equals(got) { + t.Errorf("hpDRAUsed[0] = %v, want %v", a.hpDRAUsed[0], got) + } + if a.hpUsed[0].Size() != 0 { + t.Errorf("hpUsed[0] should be untouched, got %v", a.hpUsed[0]) + } + + // Exhaustion: already 2 DRA-held + 0 available for another pick. + if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus exhaustion: expected error, got nil") + } + + // HP-ineligible punit. + a2 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + a2.hpEligiblePunit[0] = false + if _, err := a2.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus ineligible punit: expected error, got nil") + } + + // (PkgID, PunitID) not found. + a3 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + if _, err := a3.PickHpCpus(99, 99, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus not-found: expected error, got nil") + } + + // held exclusion: hold CPUs 0,1 → pick of 2 from a 4-CPU punit + // must return 2,3 (the remaining ones). + a4 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + held := cpuset.MustParse("0-1") + got4, err := a4.PickHpCpus(0, 0, 2, held) + if err != nil { + t.Fatalf("PickHpCpus held-exclusion: %v", err) + } + if got4.Intersection(held).Size() != 0 { + t.Errorf("PickHpCpus returned a held CPU: %v", got4) + } +} + +func TestReleaseHpCpus(t *testing.T) { + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + + // Pick 2 CPUs then release them. + picked, _ := a.PickHpCpus(0, 0, 2, cpuset.New()) + a.ReleaseHpCpus(0, 0, picked) + if a.hpDRAUsed[0].Size() != 0 { + t.Errorf("hpDRAUsed[0] after full release = %v, want empty", a.hpDRAUsed[0]) + } + // Map entry deleted. + if _, ok := a.hpDRAUsed[0]; ok { + t.Error("hpDRAUsed[0] entry should be deleted after full release") + } + + // Release CPUs not held — no-op. + a.ReleaseHpCpus(0, 0, cpuset.MustParse("0-1")) + + // Out-of-range (not found) — no-op, no panic. + a.ReleaseHpCpus(99, 99, cpuset.MustParse("0")) + + // Partial release. + picked2, _ := a.PickHpCpus(0, 0, 2, cpuset.New()) + first := cpuset.New(picked2.UnsortedList()[0]) + a.ReleaseHpCpus(0, 0, first) + if a.hpDRAUsed[0].Size() != 1 { + t.Errorf("hpDRAUsed[0] after partial release size = %d, want 1", a.hpDRAUsed[0].Size()) + } +} + +func TestHpDRAUsedIsolation(t *testing.T) { + // Build an allocator with both HP and LP classes to test that UseClass + // on DRA-held CPUs does not corrupt the hpDRAUsed/hpUsed separation. + sys := newTwoPunitFakeSys() + sst := &fakeSst{supported: true, punits: makePunitsWithGtdHp(4, 2, 4, 2)} + classes := []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + {Name: "lp", PctPriority: "low"}, + } + plans := map[string]*pctClassPlan{ + "hp": {ClosID: 0}, + "lp": {ClosID: 3}, + } + a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-7"), sys, sst) + + // DRA holds 2 CPUs on punit 0; hpUsed[0] is empty. + draHeld, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus: %v", err) + } + before := a.hpDRAUsed[0].Clone() + + // HP UseClass on DRA-held CPUs — must NOT add them to hpUsed (they are + // already accounted in hpDRAUsed; double-counting corrupts Punits capacity). + _ = a.UseClass("hp", draHeld) + if !a.hpDRAUsed[0].Equals(before) { + t.Errorf("hpDRAUsed[0] changed after HP UseClass: got %v, want %v", a.hpDRAUsed[0], before) + } + if !a.hpUsed[0].IsEmpty() { + t.Errorf("hpUsed[0] = %v after HP UseClass on DRA-held CPUs, want empty", a.hpUsed[0]) + } + + // Non-HP UseClass on overlapping CPUs — must NOT remove them from hpDRAUsed. + _ = a.UseClass("lp", draHeld) + if !a.hpDRAUsed[0].Equals(before) { + t.Errorf("hpDRAUsed[0] changed after non-HP UseClass: got %v, want %v", a.hpDRAUsed[0], before) + } + // hpInUseCpus must still report the DRA-held CPUs. + inUse := a.hpInUseCpus() + for _, cpu := range draHeld.UnsortedList() { + if !inUse.Contains(cpu) { + t.Errorf("hpInUseCpus missing DRA-held cpu %d", cpu) + } + } +} + +// TestIsHPClass covers the exported IsHPClass wrapper: HP class returns true; +// non-HP class returns false; unknown class returns false; inactive allocator +// returns false. +func TestIsHPClass(t *testing.T) { + sys := newTwoPackageFakeSys() + sst := &fakeSst{supported: true, maxHp: map[int]int{0: 2, 1: 2}} + classes := []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + {Name: "lp", PctPriority: "low"}, + } + a := newManagedPctForTest(t, classes, + map[string]*pctClassPlan{"hp": {ClosID: 0}, "lp": {ClosID: 3}}, + cpuset.MustParse("0-7"), sys, sst) + + // HP class must return true. + if !a.IsHPClass("hp") { + t.Error("IsHPClass(\"hp\") = false, want true") + } + // Non-HP class must return false. + if a.IsHPClass("lp") { + t.Error("IsHPClass(\"lp\") = true, want false") + } + // Unknown class must return false. + if a.IsHPClass("unknown") { + t.Error("IsHPClass(\"unknown\") = true, want false") + } + + // Inactive allocator must return false. + inactiveA := &Allocator{} + if inactiveA.IsHPClass("hp") { + t.Error("IsHPClass on inactive allocator = true, want false") + } +} + +// TestAccountHpCpus covers AccountHpCpus: used during restart reconciliation +// to rebuild hpDRAUsed from persisted claim state. +func TestAccountHpCpus(t *testing.T) { + // Inactive allocator must return an error. + inactiveA := &Allocator{} + if err := inactiveA.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + t.Error("AccountHpCpus on inactive allocator: expected error, got nil") + } + + // HP-ineligible punit must return an error. + aInelig := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + aInelig.hpEligiblePunit[0] = false + if err := aInelig.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + t.Error("AccountHpCpus on HP-ineligible punit: expected error, got nil") + } + + // Unknown punit must return an error. + aUnknown := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + if err := aUnknown.AccountHpCpus(99, 99, cpuset.MustParse("0")); err == nil { + t.Error("AccountHpCpus unknown punit: expected error, got nil") + } + + // Success: account CPUs on an HP-eligible punit. + aOK := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + cpus := cpuset.MustParse("0-1") + if err := aOK.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus success case: %v", err) + } + if !aOK.hpDRAUsed[0].Equals(cpus) { + t.Errorf("hpDRAUsed[0] = %v, want %v", aOK.hpDRAUsed[0], cpus) + } + // hpUsed must remain untouched. + if aOK.hpUsed[0].Size() != 0 { + t.Errorf("hpUsed[0] should be untouched after AccountHpCpus, got %v", aOK.hpUsed[0]) + } + + // Double-account same CPUs is idempotent (union semantics). + if err := aOK.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus idempotent call: %v", err) + } + if !aOK.hpDRAUsed[0].Equals(cpus) { + t.Errorf("hpDRAUsed[0] after double-account = %v, want %v (must be idempotent)", aOK.hpDRAUsed[0], cpus) + } + + // Over-capacity: account more CPUs than GuaranteedHpCpus allows. + // Must NOT return an error (container may already be running), and + // hpDRAUsed must include all accounted CPUs. + aOver := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) // GuaranteedHpCpus=2 + overCommit := cpuset.MustParse("0-3") // 4 CPUs > GuaranteedHpCpus=2 + if err := aOver.AccountHpCpus(0, 0, overCommit); err != nil { + t.Fatalf("AccountHpCpus over-capacity: expected no error, got %v", err) + } + if !aOver.hpDRAUsed[0].Equals(overCommit) { + t.Errorf("hpDRAUsed[0] = %v, want %v (over-capacity still updates)", aOver.hpDRAUsed[0], overCommit) + } +} + +func TestHpReserveRoomWithDRAHolds(t *testing.T) { + // Two punits, each with MaxHpCpus=2, GuaranteedHpCpus=2. + a := newPickAllocator(t, makePunitsWithGtdHp(2, 2, 2, 2)) + + // Before any holds, room on punit 0 should be 2. + // hpReserveCpus returns Tier-A candidate sets; if room>=requested we + // get a candidate set back. Requesting 2 CPUs from punit 0 should succeed. + free := cpuset.MustParse("0-7") + before := a.hpReserveCpus(free, cpuset.New(), 2) + if len(before) == 0 { + t.Fatal("hpReserveCpus before DRA holds: expected at least one candidate, got none") + } + + // DRA picks 1 CPU on punit 0. + _, err := a.PickHpCpus(0, 0, 1, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus: %v", err) + } + + // Now request 2 CPUs from punit 0: room is 1 (2 - 1 DRA hold), so + // hpReserveCpus should not return punit 0 as a single-punit Tier-A + // candidate for a request of 2. It may return punit 1 (unaffected). + after := a.hpReserveCpus(free, cpuset.New(), 2) + for _, candidate := range after { + // No candidate set should include the DRA-held CPUs as "free" HP room + // for a 2-CPU request on punit 0 alone. + if candidate.Intersection(cpuset.MustParse("0-3")).Size() > 1 { + t.Errorf("hpReserveCpus candidate includes punit 0 CPUs despite DRA hold reducing room to 1") + } + } +} diff --git a/pkg/resmgr/dra/cdi.go b/pkg/resmgr/dra/cdi.go new file mode 100644 index 000000000..ba533ef4d --- /dev/null +++ b/pkg/resmgr/dra/cdi.go @@ -0,0 +1,217 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/types" + + cdilib "tags.cncf.io/container-device-interface/pkg/cdi" + "tags.cncf.io/container-device-interface/pkg/parser" + specs "tags.cncf.io/container-device-interface/specs-go" + + logger "github.com/containers/nri-plugins/pkg/log" +) + +var cdiLog = logger.NewLogger("dra") + +const ( + // defaultCDIDir is the default directory for CDI spec files. + defaultCDIDir = "/var/run/cdi" + // cdiClass is the CDI class name used for all DRA claim specs. + cdiClass = "device" +) + +// cdiWriter implements CDIWriter using the upstream CDI library. +type cdiWriter struct { + cache *cdilib.Cache + vendor string + class string + cdiDir string +} + +// NewCDIWriter creates a CDIWriter that writes CDI specs under cdiDir. +// If cdiDir is empty, it defaults to /var/run/cdi. +// driverName is validated as a CDI vendor name. Returns an error if +// driverName is not a valid CDI vendor name or if the CDI cache cannot +// be created. +func NewCDIWriter(driverName, cdiDir string) (CDIWriter, error) { + if err := parser.ValidateVendorName(driverName); err != nil { + return nil, fmt.Errorf("dra cdi: invalid driverName %q: %w", driverName, err) + } + if cdiDir == "" { + cdiDir = defaultCDIDir + } + if err := os.MkdirAll(cdiDir, 0750); err != nil { + return nil, fmt.Errorf("dra cdi: create CDI dir %q: %w", cdiDir, err) + } + cache, err := cdilib.NewCache( + cdilib.WithAutoRefresh(false), + cdilib.WithSpecDirs(cdiDir), + ) + if err != nil { + return nil, fmt.Errorf("dra cdi: create CDI cache: %w", err) + } + return &cdiWriter{ + cache: cache, + vendor: driverName, + class: cdiClass, + cdiDir: cdiDir, + }, nil +} + +// WriteClaim writes a CDI spec file for the claim identified by uid. +// Each element of devices becomes one CDI device entry in the spec. +// Returns an error if devices is empty, if spec assembly fails, or if +// writing the spec file fails. +func (w *cdiWriter) WriteClaim(uid types.UID, devices []CDIDevice) error { + if len(devices) == 0 { + return fmt.Errorf("dra cdi: WriteClaim %s: devices must not be empty", uid) + } + + cdiDevices := make([]specs.Device, 0, len(devices)) + for _, d := range devices { + env := []string{fmt.Sprintf("NRI_CLASS=%s", d.ClassName)} + for _, cpu := range d.CPUs.List() { + env = append(env, fmt.Sprintf("NRI_CPU%d=1", cpu)) + } + cdiDevices = append(cdiDevices, specs.Device{ + Name: d.Name, + ContainerEdits: specs.ContainerEdits{ + Env: env, + }, + }) + } + + spec := specs.Spec{ + Kind: w.vendor + "/" + w.class, + Devices: cdiDevices, + } + // MinimumRequiredVersion must be called after spec is fully assembled, + // including Kind which it reads to determine requirements. + v, err := specs.MinimumRequiredVersion(&spec) + if err != nil { + return fmt.Errorf("dra cdi: WriteClaim %s: determine CDI version: %w", uid, err) + } + spec.Version = v + + name := cdilib.GenerateTransientSpecName(w.vendor, w.class, string(uid)) + if err := w.cache.WriteSpec(&spec, name); err != nil { + return fmt.Errorf("dra cdi: WriteClaim %s: write spec: %w", uid, err) + } + return nil +} + +// RemoveClaim removes the CDI spec file for the claim identified by uid. +// Returns nil if the spec does not exist (idempotent). +func (w *cdiWriter) RemoveClaim(uid types.UID) error { + name := cdilib.GenerateTransientSpecName(w.vendor, w.class, string(uid)) + if err := w.cache.RemoveSpec(name); err != nil { + return fmt.Errorf("dra cdi: RemoveClaim %s: %w", uid, err) + } + return nil +} + +// ClaimSpecExists reports whether the CDI spec file for uid is present on disk. +// The file path is /-_.yaml, consistent with +// how WriteSpec names files for transient specs. +func (w *cdiWriter) ClaimSpecExists(uid types.UID) bool { + name := cdilib.GenerateTransientSpecName(w.vendor, w.class, string(uid)) + path := filepath.Join(w.cdiDir, name+".yaml") + _, err := os.Stat(path) + return err == nil +} + +// ListClaims returns the UIDs of all claims for which a CDI spec exists in +// the managed CDI directory. Refresh errors are logged at Warn level but do +// not abort the listing — foreign or malformed specs in the directory must not +// prevent our claims from being returned. +func (w *cdiWriter) ListClaims() ([]types.UID, error) { + if err := w.cache.Refresh(); err != nil { + cdiLog.Warnf("dra cdi: ListClaims: cache refresh: %v", err) + } + prefix := w.vendor + "-" + w.class + "_" + suffix := ".yaml" + var uids []types.UID + for _, s := range w.cache.GetVendorSpecs(w.vendor) { + base := filepath.Base(s.GetPath()) + if !strings.HasPrefix(base, prefix) || !strings.HasSuffix(base, suffix) { + continue + } + uidStr := strings.TrimPrefix(base, prefix) + uidStr = strings.TrimSuffix(uidStr, suffix) + if uidStr == "" { + continue + } + uids = append(uids, types.UID(uidStr)) + } + return uids, nil +} + +// CDIDeviceName builds a valid CDI device name for the given allocation result. +// The format is: +// +// "claim----" +// +// where sanitize replaces '/' and any character that is not alphanumeric, +// '_', '-', '.', or ':' with '-', then trims any leading or trailing +// non-alphanumeric characters. +// +// The result can be validated with parser.ValidateDeviceName. Using a per- +// result idx ensures that two results that share the same Request+Device +// (e.g. AllowMultipleAllocations with count > 1) produce distinct names. +func CDIDeviceName(uid types.UID, request, device string, idx int) string { + sanitized := sanitizeCDIName(request) + sanitizedDev := sanitizeCDIName(device) + return "claim-" + string(uid) + "-" + sanitized + "-" + sanitizedDev + "-" + strconv.Itoa(idx) +} + +// sanitizeCDIName replaces any character that is invalid in a CDI device name +// middle position with '-', then trims leading and trailing non-alphanumeric +// characters. '/' is always replaced (it is not a valid CDI device name char). +func sanitizeCDIName(s string) string { + var b strings.Builder + for _, c := range s { + switch { + case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'): + b.WriteRune(c) + case c == '_' || c == '-' || c == '.' || c == ':': + b.WriteRune(c) + default: + b.WriteRune('-') + } + } + result := b.String() + // Trim leading/trailing non-alphanumeric characters. + start := strings.IndexFunc(result, func(r rune) bool { + return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + }) + if start < 0 { + // All characters were replaced by dashes or the string is empty. + // Return a safe placeholder so the caller still gets a non-empty segment. + return "x" + } + end := strings.LastIndexFunc(result, func(r rune) bool { + return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + }) + return result[start : end+1] +} diff --git a/pkg/resmgr/dra/cdi_test.go b/pkg/resmgr/dra/cdi_test.go new file mode 100644 index 000000000..5626270a6 --- /dev/null +++ b/pkg/resmgr/dra/cdi_test.go @@ -0,0 +1,327 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/types" + + "tags.cncf.io/container-device-interface/pkg/parser" + + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// TestCDIDeviceName_BasicCase verifies the basic format of a CDI device name. +func TestCDIDeviceName_BasicCase(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000001") + name := CDIDeviceName(uid, "myrequest", "punit-0-0", 0) + if !strings.HasPrefix(name, "claim-") { + t.Errorf("CDIDeviceName() = %q, want prefix \"claim-\"", name) + } + if err := parser.ValidateDeviceName(name); err != nil { + t.Errorf("CDIDeviceName() = %q, invalid CDI device name: %v", name, err) + } +} + +// TestCDIDeviceName_SubrequestSlash verifies that '/' in request is replaced +// so the result is still a valid CDI device name. +func TestCDIDeviceName_SubrequestSlash(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000002") + name := CDIDeviceName(uid, "first-available/req0", "punit-0-1", 0) + if err := parser.ValidateDeviceName(name); err != nil { + t.Errorf("CDIDeviceName() with slash in request = %q, invalid: %v", name, err) + } + if strings.Contains(name, "/") { + t.Errorf("CDIDeviceName() = %q, should not contain '/'", name) + } +} + +// TestCDIDeviceName_TwoResultsSameRequestDevice verifies that two results with +// the same Request+Device but different idx produce distinct valid names. +func TestCDIDeviceName_TwoResultsSameRequestDevice(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000003") + name0 := CDIDeviceName(uid, "myrequest", "punit-0-0", 0) + name1 := CDIDeviceName(uid, "myrequest", "punit-0-0", 1) + if name0 == name1 { + t.Errorf("CDIDeviceName() idx 0 and 1 produced the same name %q", name0) + } + if err := parser.ValidateDeviceName(name0); err != nil { + t.Errorf("name0 %q invalid: %v", name0, err) + } + if err := parser.ValidateDeviceName(name1); err != nil { + t.Errorf("name1 %q invalid: %v", name1, err) + } +} + +// TestCDIDeviceName_AllSlashRequest verifies that a request of all slashes +// produces a valid name (the sanitize+trim fallback to "x"). +func TestCDIDeviceName_AllSlashRequest(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000004") + name := CDIDeviceName(uid, "///", "punit-0-0", 0) + if err := parser.ValidateDeviceName(name); err != nil { + t.Errorf("CDIDeviceName() with all-slash request = %q, invalid: %v", name, err) + } +} + +// TestNewCDIWriter_InvalidVendor verifies that an invalid vendor name causes +// NewCDIWriter to return an error. +func TestNewCDIWriter_InvalidVendor(t *testing.T) { + dir := t.TempDir() + _, err := NewCDIWriter("123invalid", dir) + if err == nil { + t.Error("NewCDIWriter() with invalid vendor name: expected error, got nil") + } +} + +// TestNewCDIWriter_DefaultDir verifies that NewCDIWriter creates /var/run/cdi +// when cdiDir is empty and permissions allow it. This test is skipped unless +// the process can write to /var/run (it needs root or pre-created dir). +func TestNewCDIWriter_DefaultDir(t *testing.T) { + t.Skip("requires write access to /var/run — skipped in unit test environment") +} + +// TestWriteClaim_EnvVarsOnDisk verifies that WriteClaim writes a YAML spec +// file containing the expected env vars and Kind. +func TestWriteClaim_EnvVarsOnDisk(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-claim-uid-0001") + cpus, _ := cpuset.Parse("0,2,4") + devices := []CDIDevice{ + {Name: "claim-test-claim-uid-0001-myreq-punit-0-0-0", ClassName: "gold", CPUs: cpus}, + } + + if err := w.WriteClaim(uid, devices); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + + // Verify spec file exists with expected name pattern. + pattern := filepath.Join(dir, "intel.com-device_test-claim-uid-0001.yaml") + data, err := os.ReadFile(pattern) + if err != nil { + t.Fatalf("spec file not found at %q: %v", pattern, err) + } + + contents := string(data) + if !strings.Contains(contents, "NRI_CLASS=gold") { + t.Errorf("spec missing NRI_CLASS=gold; contents:\n%s", contents) + } + if !strings.Contains(contents, "NRI_CPU0=1") { + t.Errorf("spec missing NRI_CPU0=1; contents:\n%s", contents) + } + if !strings.Contains(contents, "NRI_CPU2=1") { + t.Errorf("spec missing NRI_CPU2=1; contents:\n%s", contents) + } + if !strings.Contains(contents, "NRI_CPU4=1") { + t.Errorf("spec missing NRI_CPU4=1; contents:\n%s", contents) + } + if !strings.Contains(contents, "kind: intel.com/device") { + t.Errorf("spec missing Kind \"intel.com/device\"; contents:\n%s", contents) + } +} + +// TestWriteClaim_EmptyDevices verifies that WriteClaim returns an error when +// devices is empty (CDI rejects specs with no devices). +func TestWriteClaim_EmptyDevices(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + err = w.WriteClaim("some-uid", nil) + if err == nil { + t.Error("WriteClaim() with empty devices: expected error, got nil") + } +} + +// TestRemoveClaim_Removes verifies that RemoveClaim removes the spec file. +func TestRemoveClaim_Removes(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-uid-remove-0001") + cpus, _ := cpuset.Parse("0") + devs := []CDIDevice{{Name: "claim-test-uid-remove-0001-req-dev-0", ClassName: "silver", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + + if !w.ClaimSpecExists(uid) { + t.Fatal("ClaimSpecExists() = false after WriteClaim, want true") + } + if err := w.RemoveClaim(uid); err != nil { + t.Fatalf("RemoveClaim() unexpected error: %v", err) + } + if w.ClaimSpecExists(uid) { + t.Error("ClaimSpecExists() = true after RemoveClaim, want false") + } +} + +// TestRemoveClaim_Idempotent verifies that RemoveClaim on a non-existent spec +// returns nil (idempotent). +func TestRemoveClaim_Idempotent(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + if err := w.RemoveClaim("nonexistent-uid"); err != nil { + t.Errorf("RemoveClaim() on non-existent spec: unexpected error %v", err) + } +} + +// TestClaimSpecExists_TrueAndFalse verifies ClaimSpecExists for written and +// not-written claims. +func TestClaimSpecExists_TrueAndFalse(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-uid-exists-0001") + if w.ClaimSpecExists(uid) { + t.Error("ClaimSpecExists() = true before WriteClaim, want false") + } + + cpus, _ := cpuset.Parse("1") + devs := []CDIDevice{{Name: "claim-test-uid-exists-0001-req-dev-0", ClassName: "gold", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + if !w.ClaimSpecExists(uid) { + t.Error("ClaimSpecExists() = false after WriteClaim, want true") + } +} + +// TestListClaims_TwoClaims verifies that ListClaims returns all written claim +// UIDs. +func TestListClaims_TwoClaims(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid1 := types.UID("test-uid-list-0001") + uid2 := types.UID("test-uid-list-0002") + cpus, _ := cpuset.Parse("0") + + for _, uid := range []types.UID{uid1, uid2} { + devs := []CDIDevice{{Name: "claim-" + string(uid) + "-req-dev-0", ClassName: "gold", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim(%s) unexpected error: %v", uid, err) + } + } + + uids, err := w.ListClaims() + if err != nil { + t.Fatalf("ListClaims() unexpected error: %v", err) + } + + found := make(map[types.UID]bool) + for _, u := range uids { + found[u] = true + } + if !found[uid1] { + t.Errorf("ListClaims() missing uid1 %s", uid1) + } + if !found[uid2] { + t.Errorf("ListClaims() missing uid2 %s", uid2) + } +} + +// TestListClaims_ForeignSpecSurvives verifies that a malformed/foreign spec +// in the CDI dir does not prevent our claims from being listed, and that the +// foreign spec survives (is not removed). +func TestListClaims_ForeignSpecSurvives(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + // Write a "foreign" malformed spec file that should not be parsed as our claim. + foreignPath := filepath.Join(dir, "other-vendor-device_foreign.yaml") + if err := os.WriteFile(foreignPath, []byte("not valid yaml\n"), 0644); err != nil { + t.Fatalf("write foreign spec: %v", err) + } + + // Write a valid claim. + uid := types.UID("test-uid-foreign-0001") + cpus, _ := cpuset.Parse("0") + devs := []CDIDevice{{Name: "claim-test-uid-foreign-0001-req-dev-0", ClassName: "gold", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + + // ListClaims should return our UID (Refresh logs a warning about the foreign + // malformed file but continues). + uids, err := w.ListClaims() + if err != nil { + t.Fatalf("ListClaims() unexpected error: %v", err) + } + found := false + for _, u := range uids { + if u == uid { + found = true + break + } + } + if !found { + t.Errorf("ListClaims() missing expected uid %s (got %v)", uid, uids) + } + + // Foreign spec file must still be on disk. + if _, err := os.Stat(foreignPath); err != nil { + t.Errorf("foreign spec file was removed: %v", err) + } +} + +// TestWriteClaim_SameRequestDeviceTwoIdx verifies that two results with the +// same Request+Device but different idx produce a spec with two distinct +// CDI device names (no duplicate-name error). +func TestWriteClaim_SameRequestDeviceTwoIdx(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-uid-shared-0001") + cpus, _ := cpuset.Parse("0") + name0 := CDIDeviceName(uid, "myrequest", "punit-0-0", 0) + name1 := CDIDeviceName(uid, "myrequest", "punit-0-0", 1) + devs := []CDIDevice{ + {Name: name0, ClassName: "gold", CPUs: cpus}, + {Name: name1, ClassName: "gold", CPUs: cpus}, + } + if err := w.WriteClaim(uid, devs); err != nil { + t.Errorf("WriteClaim() with two same-request/same-device results: unexpected error %v", err) + } +} diff --git a/pkg/resmgr/dra/deps.go b/pkg/resmgr/dra/deps.go new file mode 100644 index 000000000..f887cf3bb --- /dev/null +++ b/pkg/resmgr/dra/deps.go @@ -0,0 +1,134 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes" + + "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// DeviceLister provides the DRA device list for a given driver name. +type DeviceLister interface { + DRADevices(driverName string) ([]resourceapi.Device, error) +} + +// CDIDevice holds the CDI device information for a single allocation result. +// Name is the CDI device name, precomputed by the caller via CDIDeviceName. +// WriteClaim uses Name and ClassName directly to build the CDI spec entry. +type CDIDevice struct { + // Name is the CDI device name (precomputed by the caller via CDIDeviceName). + Name string + // ClassName is the nri/cpuClass attribute value for the allocated device. + ClassName string + // CPUs is the set of CPUs allocated for this device. + CPUs cpuset.CPUSet +} + +// ClaimAllocator provides HP CPU pick/release/account operations needed +// during PrepareResourceClaims, UnprepareResourceClaims, and restart +// reconciliation. +type ClaimAllocator interface { + // PickHpCpus selects n HP-eligible CPUs from the punit identified by + // (pkgID, punitID), excluding CPUs in held and those already tracked + // in internal accounting. + PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) + // ReleaseHpCpus removes cpus from DRA HP accounting on the given punit. + ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) + // AccountHpCpus records cpus as DRA HP-held on the given punit. Used + // during restart reconciliation to rebuild HP accounting without + // re-allocating CPUs. + AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error + // IsHPClass reports whether className is currently classified as PCT + // high priority. + IsHPClass(className string) bool +} + +// CDIWriter manages CDI spec files on behalf of the DRA plugin. +type CDIWriter interface { + // WriteClaim writes a CDI spec file for the claim identified by uid, + // containing one CDI device entry per element of devices. + WriteClaim(uid types.UID, devices []CDIDevice) error + // RemoveClaim removes the CDI spec file for the claim identified by uid. + // Returns nil if the spec does not exist (idempotent). + RemoveClaim(uid types.UID) error + // ClaimSpecExists reports whether the CDI spec file for uid is present + // on disk. + ClaimSpecExists(uid types.UID) bool + // ListClaims returns the UIDs of all claims for which a CDI spec file + // exists under the managed CDI directory. + ListClaims() ([]types.UID, error) +} + +// ClaimStore persists and loads claim state via the resmgr cache. +// Save is a no-op when the cache is in a BlockSave window; data is still +// in-memory policyData and will persist on the next unblocked Save. +// Callers of Save must tolerate this. +type ClaimStore interface { + // Save persists the given claims map to the backing store. + Save(claims map[types.UID]*ClaimState) error + // Load reads the claims map from the backing store. Returns nil, nil + // when no claim state has been saved yet. + Load() (map[types.UID]*ClaimState, error) +} + +// ClaimUnprepare is called after a prepared claim is removed from plugin +// state, while holding the resmgr lock. +type ClaimUnprepare func(uid types.UID, allocs []ResultAlloc) + +// Deps holds the dependencies a policy binary must supply when constructing +// a Plugin. +type Deps struct { + // KubeClient is the Kubernetes client used to publish ResourceSlice objects. + KubeClient kubernetes.Interface + // NodeName is the name of the node this plugin runs on. + NodeName string + // RegistrarDir is the directory where the plugin registrar socket is created. + // Defaults to kubeletplugin.KubeletRegistryDir when empty. + RegistrarDir string + // PluginDataDir is the directory where the plugin data socket is created. + // Defaults to kubeletplugin.KubeletPluginsDir+"/"+driverName when empty. + PluginDataDir string + // ValidateClasses is a closure that validates the current cpuClass + // configuration for DRA compatibility. + ValidateClasses func() error + // ValidateCPUsInPool is a closure that reports whether cpus can fit + // within a single allocation leaf pool of the policy. Returns an error + // if the CPUs span multiple leaf pools (or otherwise don't fit within + // one), so that a claim which would later fail at container-creation + // time is instead rejected at Prepare time. Must not be nil. + ValidateCPUsInPool func(cpus cpuset.CPUSet) error + // DeviceLister returns the list of DRA devices to publish. + DeviceLister DeviceLister + // ClaimAllocator provides HP CPU pick/release/account operations. + ClaimAllocator ClaimAllocator + // CDIWriter manages CDI spec files for prepared claims. + CDIWriter CDIWriter + // ClaimStore persists and loads claim state via the resmgr cache. + ClaimStore ClaimStore + // ClaimUnprepare notifies the policy that a prepared claim was removed. + ClaimUnprepare ClaimUnprepare + // WithLock executes f while holding the resmgr write lock. All accesses + // to Handler state (ValidateClasses, DRADevices, Prepare, Unprepare, and + // RestoreClaims) must run inside WithLock. + WithLock func(func()) + // Logger is the logger used for all plugin log output. + Logger log.Logger +} diff --git a/pkg/resmgr/dra/doc.go b/pkg/resmgr/dra/doc.go new file mode 100644 index 000000000..a97077c90 --- /dev/null +++ b/pkg/resmgr/dra/doc.go @@ -0,0 +1,23 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package dra provides a policy-agnostic DRA (Dynamic Resource Allocation) +// kubelet plugin used by nri-plugins policies. +// +// This package must never import from github.com/containers/nri-plugins/cmd/ +// so that it can be consumed by any policy binary without introducing import +// cycles. +package dra diff --git a/pkg/resmgr/dra/logging.go b/pkg/resmgr/dra/logging.go new file mode 100644 index 000000000..7d7fa55ab --- /dev/null +++ b/pkg/resmgr/dra/logging.go @@ -0,0 +1,30 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "github.com/go-logr/logr" + + "github.com/containers/nri-plugins/pkg/log" +) + +// newLogr returns a logr.Logger backed by the given pkg/log Logger. +// kubeletplugin.Start expects a logr.Logger injected into the context +// via logr.NewContext; this bridge avoids hand-rolling a LogSink. +func newLogr(l log.Logger) logr.Logger { + return logr.FromSlogHandler(l.SlogHandler()) +} diff --git a/pkg/resmgr/dra/plugin.go b/pkg/resmgr/dra/plugin.go new file mode 100644 index 000000000..af6c355a2 --- /dev/null +++ b/pkg/resmgr/dra/plugin.go @@ -0,0 +1,799 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + "time" + + "github.com/go-logr/logr" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" + + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "tags.cncf.io/container-device-interface/pkg/parser" +) + +// Sentinel errors for PrepareResourceClaims. +var ( + errNilAllocation = errors.New("dra plugin: claim has nil Allocation") + errMissingConsumedCapacity = errors.New("dra plugin: ConsumedCapacity[nri/cpus] absent or zero") + errNonHPNotSupported = errors.New("dra plugin: non-HP CPU class not supported (deferred)") + errMultiPunitNotSupported = errors.New("dra plugin: claim results on multiple punits not supported") +) + +// deviceInfo holds the device attributes looked up from the published device list. +type deviceInfo struct { + ClassName string + PkgID int + PunitID int +} + +// Plugin is the DRA kubelet plugin. +type Plugin struct { + mu sync.Mutex // mu guards helper, draCancel, draLoopDone + driverName string + deps Deps + helper *kubeletplugin.Helper + draCancel context.CancelFunc + draLoopDone chan struct{} + claims map[types.UID]*ClaimState + republishCh chan struct{} // signals the republisherLoop to re-run PublishResources + handleErr atomic.Pointer[error] // set on non-recoverable HandleError; checked by PublishResources +} + +// New constructs a Plugin with the given driver name and dependencies. +// Returns an error if any required dependency is missing. +func New(driverName string, deps Deps) (*Plugin, error) { + if driverName == "" { + return nil, fmt.Errorf("dra plugin: driverName must not be empty") + } + if deps.KubeClient == nil { + return nil, fmt.Errorf("dra plugin: KubeClient must not be nil") + } + if deps.NodeName == "" { + return nil, fmt.Errorf("dra plugin: NodeName must not be empty") + } + if deps.ValidateClasses == nil { + return nil, fmt.Errorf("dra plugin: ValidateClasses must not be nil") + } + if deps.ValidateCPUsInPool == nil { + return nil, fmt.Errorf("dra plugin: ValidateCPUsInPool must not be nil") + } + if deps.DeviceLister == nil { + return nil, fmt.Errorf("dra plugin: DeviceLister must not be nil") + } + if deps.Logger == nil { + return nil, fmt.Errorf("dra plugin: Logger must not be nil") + } + if deps.ClaimAllocator == nil { + return nil, fmt.Errorf("dra plugin: ClaimAllocator must not be nil") + } + if deps.CDIWriter == nil { + return nil, fmt.Errorf("dra plugin: CDIWriter must not be nil") + } + if deps.ClaimStore == nil { + return nil, fmt.Errorf("dra plugin: ClaimStore must not be nil") + } + if deps.WithLock == nil { + return nil, fmt.Errorf("dra plugin: WithLock must not be nil") + } + if deps.ClaimUnprepare == nil { + return nil, fmt.Errorf("dra plugin: ClaimUnprepare must not be nil") + } + return &Plugin{ + driverName: driverName, + deps: deps, + claims: make(map[types.UID]*ClaimState), + republishCh: make(chan struct{}, 1), + }, nil +} + +// DriverName returns the plugin's DRA driver name. +func (p *Plugin) DriverName() string { + return p.driverName +} + +// shareIDPtr converts a ShareID string to *types.UID. Returns nil if s is "". +func shareIDPtr(s string) *types.UID { + if s == "" { + return nil + } + uid := types.UID(s) + return &uid +} + +// deviceIndex builds a map from device name to deviceInfo by calling +// deps.DeviceLister.DRADevices. Must be called inside deps.WithLock. +// Called once per PrepareResourceClaims invocation, not per result. +func (p *Plugin) deviceIndex() (map[string]deviceInfo, error) { + devs, err := p.deps.DeviceLister.DRADevices(p.driverName) + if err != nil { + return nil, fmt.Errorf("dra plugin: DRADevices: %w", err) + } + idx := make(map[string]deviceInfo, len(devs)) + for _, d := range devs { + info := deviceInfo{} + if attr, ok := d.Attributes[cpuclass.AttrCPUClass]; ok && attr.StringValue != nil { + info.ClassName = *attr.StringValue + } + if attr, ok := d.Attributes[cpuclass.AttrPackageID]; ok && attr.IntValue != nil { + info.PkgID = int(*attr.IntValue) + } + if attr, ok := d.Attributes[cpuclass.AttrPunitID]; ok && attr.IntValue != nil { + info.PunitID = int(*attr.IntValue) + } + idx[d.Name] = info + } + return idx, nil +} + +// allClaimedCPUs returns the union of all CPUs currently tracked in p.claims. +// Must be called inside deps.WithLock. +func (p *Plugin) allClaimedCPUs() cpuset.CPUSet { + result := cpuset.New() + for _, cs := range p.claims { + for _, alloc := range cs.Allocs { + parsed, err := cpuset.Parse(alloc.CPUs) + if err == nil { + result = result.Union(parsed) + } + } + } + return result +} + +// PrepareResourceClaims prepares all resource claims allocated for this driver. +// For each claim it picks HP CPUs, writes a CDI spec, and persists claim state. +// The entire body runs inside deps.WithLock to serialize with Reconfigure. +func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi.ResourceClaim) (map[types.UID]kubeletplugin.PrepareResult, error) { + result := make(map[types.UID]kubeletplugin.PrepareResult, len(claims)) + p.deps.WithLock(func() { + devIdx, idxErr := p.deviceIndex() + if idxErr != nil { + // Fill all UIDs with the same error and return. + for _, claim := range claims { + result[claim.UID] = kubeletplugin.PrepareResult{Err: idxErr} + } + return + } + + for _, claim := range claims { + uid := claim.UID + result[uid] = func() kubeletplugin.PrepareResult { + if claim.Status.Allocation == nil { + return kubeletplugin.PrepareResult{Err: errNilAllocation} + } + + // Filter results to our driver. + allResults := claim.Status.Allocation.Devices.Results + var filtered []resourceapi.DeviceRequestAllocationResult + for _, r := range allResults { + if r.Driver == p.driverName { + filtered = append(filtered, r) + } + } + if len(filtered) == 0 { + // No results for our driver — valid, no work to do. + return kubeletplugin.PrepareResult{} + } + + // Idempotency: already prepared? + if _, exists := p.claims[uid]; exists { + if p.deps.CDIWriter.ClaimSpecExists(uid) { + // Spec present — re-build PrepareResult from stored state. + return p.buildPrepareResult(uid, filtered) + } + // Spec missing (e.g. node reboot) — re-write CDI spec from + // existing stored state without re-picking CPUs. + cdiDevices := p.cdiDevicesFromClaims(uid, filtered) + if len(cdiDevices) != len(p.claims[uid].Allocs) { + // Some (or all) stored CPU strings failed to parse — stored + // state is corrupt; writing a partial CDI spec and then + // returning CDIDeviceIDs for all allocs would give the runtime + // device IDs that don't exist in the spec, causing container + // start failures. + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: idempotent re-write: only %d of %d CDI devices rebuilt from stored state (stored CPUs unparsable)", len(cdiDevices), len(p.claims[uid].Allocs))} + } + if len(cdiDevices) > 0 { + if writeErr := p.deps.CDIWriter.WriteClaim(uid, cdiDevices); writeErr != nil { + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: re-write CDI spec: %w", writeErr)} + } + } + return p.buildPrepareResult(uid, filtered) + } + + // Allocate CPUs for each filtered result. + heldCPUs := p.allClaimedCPUs() + var pickedAllocs []ResultAlloc + var cdiDevices []CDIDevice + claimCPUs := cpuset.New() + var punit *deviceInfo + + for i, r := range filtered { + attrs, attrOk := devIdx[r.Device] + if !attrOk { + // Unknown device — rollback and report error. + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: unknown device %q", r.Device)} + } + if attrs.ClassName == "" { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: device %q missing nri/cpuClass attribute", r.Device)} + } + if punit != nil && (attrs.PkgID != punit.PkgID || attrs.PunitID != punit.PunitID) { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMultiPunitNotSupported} + } + punit = &attrs + + // Reject claims whose results span more than one punit — + // the topology-aware consumer requires the union of a + // claim's results to fit a single leaf pool. + if len(pickedAllocs) > 0 { + first := pickedAllocs[0] + if attrs.PkgID != first.PkgID || attrs.PunitID != first.PunitID { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMultiPunitNotSupported} + } + } + + q, ok := r.ConsumedCapacity[cpuclass.CapacityCPUs] + if !ok { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMissingConsumedCapacity} + } + n := int(q.Value()) + if n <= 0 { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMissingConsumedCapacity} + } + + if !p.deps.ClaimAllocator.IsHPClass(attrs.ClassName) { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errNonHPNotSupported} + } + + picked, pickErr := p.deps.ClaimAllocator.PickHpCpus(attrs.PkgID, attrs.PunitID, n, heldCPUs) + if pickErr != nil { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: PickHpCpus: %w", pickErr)} + } + heldCPUs = heldCPUs.Union(picked) + claimCPUs = claimCPUs.Union(picked) + + // Determine ShareID. + shareID := "" + if r.ShareID != nil { + shareID = string(*r.ShareID) + } + + pickedAllocs = append(pickedAllocs, ResultAlloc{ + Request: r.Request, + Pool: r.Pool, + Device: r.Device, + ShareID: shareID, + ClassName: attrs.ClassName, + PkgID: attrs.PkgID, + PunitID: attrs.PunitID, + CPUs: picked.String(), + }) + name := CDIDeviceName(uid, r.Request, r.Device, i) + cdiDevices = append(cdiDevices, CDIDevice{ + Name: name, + ClassName: attrs.ClassName, + CPUs: picked, + }) + } + + // A punit can span multiple leaf pools, so verify the union of + // all result CPUs fits within the policy's allocation domain + // before committing. + if validateErr := p.deps.ValidateCPUsInPool(claimCPUs); validateErr != nil { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: claim CPU set %s is outside supported allocation domain: %w", claimCPUs, validateErr)} + } + + if writeErr := p.deps.CDIWriter.WriteClaim(uid, cdiDevices); writeErr != nil { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: WriteClaim: %w", writeErr)} + } + + p.claims[uid] = &ClaimState{UID: string(uid), Allocs: pickedAllocs} + if saveErr := p.deps.ClaimStore.Save(p.claims); saveErr != nil { + // Roll back symmetrically: the CDI spec and CPU picks + // are already live, so leaving the claim "prepared" + // here would lose durable state on a restart. + delete(p.claims, uid) + if removeErr := p.deps.CDIWriter.RemoveClaim(uid); removeErr != nil { + p.deps.Logger.Warnf("dra plugin: PrepareResourceClaims: rollback after ClaimStore.Save failure: RemoveClaim: %v", removeErr) + } + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: ClaimStore.Save: %w", saveErr)} + } + + return p.buildPrepareResult(uid, filtered) + }() + } + }) + return result, nil +} + +// cdiDevicesFromClaims rebuilds the []CDIDevice slice for uid from the stored +// ClaimState. filtered provides the allocation results for our driver, in the +// same order as the original allocs. Used by the idempotency spec-missing path. +func (p *Plugin) cdiDevicesFromClaims(uid types.UID, filtered []resourceapi.DeviceRequestAllocationResult) []CDIDevice { + cs, ok := p.claims[uid] + if !ok { + return nil + } + devices := make([]CDIDevice, 0, len(cs.Allocs)) + for i, alloc := range cs.Allocs { + if i >= len(filtered) { + break + } + r := filtered[i] + name := CDIDeviceName(uid, r.Request, r.Device, i) + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + continue + } + devices = append(devices, CDIDevice{ + Name: name, + ClassName: alloc.ClassName, + CPUs: cpus, + }) + } + return devices +} + +// rollbackPicks releases all CPU picks accumulated so far for a claim that +// encountered an error mid-way through allocation. +func (p *Plugin) rollbackPicks(allocs []ResultAlloc) { + for _, a := range allocs { + cs, err := cpuset.Parse(a.CPUs) + if err != nil { + continue + } + p.deps.ClaimAllocator.ReleaseHpCpus(a.PkgID, a.PunitID, cs) + } +} + +// buildPrepareResult constructs a kubeletplugin.PrepareResult from the stored +// claim state for uid. filtered contains the allocation results for our driver, +// in the same order they were originally processed (positional index matches +// CDIDeviceName index). +func (p *Plugin) buildPrepareResult(uid types.UID, filtered []resourceapi.DeviceRequestAllocationResult) kubeletplugin.PrepareResult { + cs, ok := p.claims[uid] + if !ok { + return kubeletplugin.PrepareResult{} + } + devices := make([]kubeletplugin.Device, 0, len(cs.Allocs)) + for i, alloc := range cs.Allocs { + if i >= len(filtered) { + break + } + r := filtered[i] + name := CDIDeviceName(uid, r.Request, r.Device, i) + devices = append(devices, kubeletplugin.Device{ + Requests: []string{r.Request}, + PoolName: r.Pool, + DeviceName: r.Device, + CDIDeviceIDs: []string{parser.QualifiedName(p.driverName, "device", name)}, + ShareID: shareIDPtr(alloc.ShareID), + }) + } + return kubeletplugin.PrepareResult{Devices: devices} +} + +// UnprepareResourceClaims releases CPUs and CDI specs for the given claims and +// removes them from persisted state. The entire body runs inside deps.WithLock +// to serialize with Reconfigure. +func (p *Plugin) UnprepareResourceClaims(_ context.Context, claims []kubeletplugin.NamespacedObject) (map[types.UID]error, error) { + perUID := make(map[types.UID]error, len(claims)) + p.deps.WithLock(func() { + for _, obj := range claims { + uid := obj.UID + cs, exists := p.claims[uid] + if !exists { + p.deps.Logger.Warnf("dra plugin: UnprepareResourceClaims: claim %s not found in state", uid) + perUID[uid] = nil + continue + } + delete(p.claims, uid) + if saveErr := p.deps.ClaimStore.Save(p.claims); saveErr != nil { + p.claims[uid] = cs + perUID[uid] = fmt.Errorf("dra plugin: UnprepareResourceClaims: ClaimStore.Save: %w", saveErr) + continue + } + p.deps.ClaimUnprepare(uid, cs.Allocs) + // Remove CDI spec unconditionally; log but do not block deletion. + if err := p.deps.CDIWriter.RemoveClaim(uid); err != nil { + p.deps.Logger.Warnf("dra plugin: UnprepareResourceClaims: claim %s: RemoveClaim: %v", uid, err) + } + perUID[uid] = nil + } + }) + return perUID, nil +} + +// LiveClaimsLocked returns a snapshot of the currently live claims as +// map[types.UID][]ResultAlloc. Caller must hold the resmgr lock (do not call +// from inside a WithLock callback — the resmgr lock is not reentrant). Used +// by the pool-accounting re-apply path (reapplyDRAClaims) after +// Start()/Reconfigure() rebuild policy state. +func (p *Plugin) LiveClaimsLocked() map[types.UID][]ResultAlloc { + result := make(map[types.UID][]ResultAlloc, len(p.claims)) + for uid, cs := range p.claims { + allocs := make([]ResultAlloc, len(cs.Allocs)) + copy(allocs, cs.Allocs) + result[uid] = allocs + } + return result +} + +// RestoreClaimsLocked re-runs AccountHpCpus for every entry in p.claims, +// rebuilding HP accounting after a Reconfigure that reset hpDRAUsed. It does +// not reload from cache and does not acquire any lock. +// +// Caller must already hold the resmgr lock; do not call via WithLock. +// RestoreClaims is the complementary WithLock wrapper for callers that are not +// already holding the lock. +func (p *Plugin) RestoreClaimsLocked() error { + var errs []error + for _, cs := range p.claims { + for _, alloc := range cs.Allocs { + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + p.deps.Logger.Warnf("dra plugin: RestoreClaimsLocked: claim %s device %s: parse CPUs %q: %v (skipping)", cs.UID, alloc.Device, alloc.CPUs, err) + continue + } + if err := p.deps.ClaimAllocator.AccountHpCpus(alloc.PkgID, alloc.PunitID, cpus); err != nil { + p.deps.Logger.Warnf("dra plugin: RestoreClaimsLocked: claim %s device %s: AccountHpCpus: %v", cs.UID, alloc.Device, err) + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +// RestoreClaims wraps RestoreClaimsLocked inside deps.WithLock for callers +// that are not already holding the resmgr lock. +// +// RestoreClaims must not be called while holding the resmgr lock +// (see RestoreClaimsLocked for the complementary lock-already-held variant). +func (p *Plugin) RestoreClaims() error { + var err error + p.deps.WithLock(func() { + err = p.RestoreClaimsLocked() + }) + return err +} + +// Start registers this plugin with the kubelet and begins serving DRA +// requests. It validates cpuClass configuration, loads and reconciles +// persisted claim state, creates the plugin data directory, injects a +// logr.Logger into the context, and calls kubeletplugin.Start. Returns an +// error if the plugin is already started, if ValidateClasses fails, or if +// the kubelet plugin cannot be started. +// +// Start must not be called while holding the resmgr lock +// (see RestoreClaimsLocked for the complementary lock-already-held variant). +// +// cpuclass.Handler must have completed Configure() before Start() so that +// AccountHpCpus finds active punits. If the allocator is inactive when Start +// is called, live claims are kept with a warning. If AccountHpCpus fails for +// any other reason (e.g. punit not found due to topology change), Start returns +// an error rather than registering the driver in an unsafe state. +func (p *Plugin) Start(ctx context.Context) error { + p.mu.Lock() + alreadyStarted := p.helper != nil + p.mu.Unlock() + if alreadyStarted { + return fmt.Errorf("dra plugin: already started") + } + if err := p.deps.ValidateClasses(); err != nil { + return fmt.Errorf("dra plugin: ValidateClasses failed: %w", err) + } + + // Reconcile persisted claims inside the resmgr lock before registering + // with the kubelet so that accounting is consistent before any new + // Prepare/Unprepare calls can arrive. + var startErr error + p.deps.WithLock(func() { + loaded, loadErr := p.deps.ClaimStore.Load() + if loadErr != nil { + // A Load failure must not be treated as "no claims": that would + // let the orphan sweep below remove CDI specs that still belong + // to live claim consumers. Fail startup outright instead. + startErr = fmt.Errorf("dra plugin: Start: ClaimStore.Load: %w", loadErr) + return + } + if loaded != nil { + p.claims = loaded + } + + // Re-account every persisted claim from durable state. A missing CDI spec + // on disk does not by itself prove the claim is stale: kubelet commonly + // clears /var/run/cdi on reboot, while the claim store is the durable + // source of truth for active claims. Keep persisted claims accounted and + // let the next Prepare re-create a missing CDI spec if needed. + for uid, cs := range p.claims { + if !p.deps.CDIWriter.ClaimSpecExists(uid) { + p.deps.Logger.Warnf("dra plugin: Start: claim %s has no CDI spec on disk; keeping persisted claim and re-accounting from durable state", uid) + } + for _, alloc := range cs.Allocs { + if alloc.ClassName != "" && !p.deps.ClaimAllocator.IsHPClass(alloc.ClassName) { + startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: cpuClass %q is no longer a valid HP class (removed or renamed); cannot restore claim", uid, alloc.Device, alloc.ClassName) + return + } + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: parse CPUs %q: %w", uid, alloc.Device, alloc.CPUs, err) + return + } + if err := p.deps.ClaimAllocator.AccountHpCpus(alloc.PkgID, alloc.PunitID, cpus); err != nil { + if errors.Is(err, cpuclass.ErrAllocatorInactive) { + p.deps.Logger.Warnf("dra plugin: Start: claim %s device %s: AccountHpCpus: %v (allocator inactive — keeping claim, container may be running)", uid, alloc.Device, err) + } else { + startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: AccountHpCpus: %w", uid, alloc.Device, err) + return + } + } + } + } + + // Orphan sweep: remove CDI specs that have no corresponding live claim. + listed, listErr := p.deps.CDIWriter.ListClaims() + if listErr != nil { + p.deps.Logger.Warnf("dra plugin: Start: CDIWriter.ListClaims: %v (skipping orphan sweep)", listErr) + return + } + for _, uid := range listed { + if _, ok := p.claims[uid]; !ok { + if removeErr := p.deps.CDIWriter.RemoveClaim(uid); removeErr != nil { + p.deps.Logger.Warnf("dra plugin: Start: orphan sweep: RemoveClaim %s: %v", uid, removeErr) + } + } + } + }) + if startErr != nil { + return startErr + } + + // Resolve the plugin data directory default before creating it: an empty + // string passed to os.MkdirAll would fail immediately. + pluginDataDir := p.deps.PluginDataDir + if pluginDataDir == "" { + pluginDataDir = filepath.Join(kubeletplugin.KubeletPluginsDir, p.driverName) + } + if err := os.MkdirAll(pluginDataDir, 0750); err != nil { + return fmt.Errorf("dra plugin: create plugin data dir %q: %w", pluginDataDir, err) + } + ctx = logr.NewContext(ctx, newLogr(p.deps.Logger)) + opts := []kubeletplugin.Option{ + kubeletplugin.DriverName(p.driverName), + kubeletplugin.KubeClient(p.deps.KubeClient), + kubeletplugin.NodeName(p.deps.NodeName), + kubeletplugin.PluginDataDirectoryPath(pluginDataDir), + kubeletplugin.GRPCVerbosity(-1), + } + // Only override the registrar directory when explicitly set; passing an + // empty string would clobber kubeletplugin's built-in KubeletRegistryDir + // default (the option setter stores the value unconditionally). + if p.deps.RegistrarDir != "" { + opts = append(opts, kubeletplugin.RegistrarDirectoryPath(p.deps.RegistrarDir)) + } + helper, err := kubeletplugin.Start(ctx, p, opts...) + if err != nil { + return fmt.Errorf("dra plugin: kubeletplugin.Start: %w", err) + } + // Re-check under lock to close the TOCTOU window between the initial + // guard (above) and this assignment. If a racing Start() won, stop the + // helper we just created and return an error. + p.mu.Lock() + if p.helper != nil { + p.mu.Unlock() + helper.Stop() + return fmt.Errorf("dra plugin: already started") + } + draCtx, draCancel := context.WithCancel(ctx) + loopDone := make(chan struct{}) + p.helper = helper + p.draCancel = draCancel + p.draLoopDone = loopDone + p.mu.Unlock() + go func() { + defer close(loopDone) + p.republisherLoop(draCtx) + }() + return nil +} + +// TriggerRepublish enqueues a request to re-run PublishResources. It is +// non-blocking and safe to call while holding the resmgr lock: if a +// republish is already pending, the extra request is dropped (the pending +// one covers it). The actual publish runs in republisherLoop, outside +// any lock. +func (p *Plugin) TriggerRepublish() { + select { + case p.republishCh <- struct{}{}: + default: + // A republish is already pending; no need to enqueue another. + } +} + +// republisherLoop runs in a goroutine started by Start(). It drains +// republishCh and calls PublishResources for each signal, stopping when +// ctx is cancelled (i.e. when the plugin's draCtx is cancelled by Stop()). +// On a publication failure the loop re-enqueues itself with exponential +// backoff (1s → 2s → … → 60s) so that the ResourceSlice is eventually +// brought back in sync without hammering the API on persistent errors. +func (p *Plugin) republisherLoop(ctx context.Context) { + const ( + initialBackoff = time.Second + maxBackoff = time.Minute + ) + backoff := initialBackoff + for { + select { + case <-ctx.Done(): + return + case <-p.republishCh: + if err := p.PublishResources(ctx); err != nil { + p.deps.Logger.Warnf("dra plugin: background republish: %v (retrying in %s)", err, backoff) + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + backoff *= 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + select { + case p.republishCh <- struct{}{}: + default: + } + } else { + backoff = initialBackoff + } + } + } +} + +// Stop shuts down the kubelet plugin and releases resources. It is +// idempotent: calling Stop on an already-stopped Plugin is safe. +func (p *Plugin) Stop() { + p.mu.Lock() + h := p.helper + cancel := p.draCancel + done := p.draLoopDone + p.helper = nil + p.draCancel = nil + p.draLoopDone = nil + p.mu.Unlock() + if cancel != nil { + cancel() + } + if done != nil { + <-done + } + if h != nil { + h.Stop() + } +} + +// PublishResources validates classes, lists DRA devices, paginates them into +// ResourceSlice objects and hands the resulting DriverResources to the helper +// for publishing. Even zero devices produce one empty slice so the pool +// remains visible. +// +// PublishResources must not be called while holding the resmgr lock +// (see RestoreClaimsLocked for the complementary lock-already-held variant). +func (p *Plugin) PublishResources(ctx context.Context) error { + if pe := p.handleErr.Load(); pe != nil { + return *pe + } + var ( + validateErr error + devices []resourceapi.Device + devicesErr error + ) + p.deps.WithLock(func() { + validateErr = p.deps.ValidateClasses() + if validateErr != nil { + return + } + devices, devicesErr = p.deps.DeviceLister.DRADevices(p.driverName) + }) + if validateErr != nil { + return fmt.Errorf("dra plugin: ValidateClasses failed: %w", validateErr) + } + p.mu.Lock() + h := p.helper + p.mu.Unlock() + if h == nil { + return fmt.Errorf("dra plugin: PublishResources called before Start") + } + if devicesErr != nil { + return fmt.Errorf("dra plugin: DRADevices: %w", devicesErr) + } + resources := buildDriverResources(p.deps.NodeName, devices) + if err := h.PublishResources(ctx, resources); err != nil { + return fmt.Errorf("dra plugin: helper.PublishResources: %w", err) + } + return nil +} + +// buildDriverResources paginates devices into ResourceSlice objects and +// returns a DriverResources ready for Helper.PublishResources. The pool name +// is the node name. At most resourceapi.ResourceSliceMaxDevices devices are +// placed per slice; even zero devices produce one empty slice. +func buildDriverResources(nodeName string, devices []resourceapi.Device) resourceslice.DriverResources { + maxPerSlice := resourceapi.ResourceSliceMaxDevices + var slices []resourceslice.Slice + if len(devices) == 0 { + slices = []resourceslice.Slice{{}} + } else { + for i := 0; i < len(devices); i += maxPerSlice { + end := i + maxPerSlice + if end > len(devices) { + end = len(devices) + } + slices = append(slices, resourceslice.Slice{ + Devices: devices[i:end], + }) + } + } + return resourceslice.DriverResources{ + Pools: map[string]resourceslice.Pool{ + nodeName: {Slices: slices}, + }, + } +} + +// HandleError handles background errors from the kubelet plugin helper. +// Recoverable errors trigger a re-publish so the republisherLoop retries. +// Non-recoverable errors are stored so that the next PublishResources call +// returns them, causing the republisherLoop to enter its backoff path. +func (p *Plugin) HandleError(_ context.Context, err error, msg string) { + if errors.Is(err, kubeletplugin.ErrRecoverable) { + p.deps.Logger.Warnf("%s: %v", msg, err) + p.TriggerRepublish() + } else { + p.deps.Logger.Errorf("%s: %v", msg, err) + e := fmt.Errorf("dra plugin: resource slice controller: %w", err) + p.handleErr.Store(&e) + } +} + +// WatchHealthStatus is not implemented: this driver does not report +// per-device health, so it returns ErrHealthNotSupported as documented by +// kubeletplugin.DRAPlugin. +func (p *Plugin) WatchHealthStatus(_ context.Context, _ chan<- kubeletplugin.DeviceHealthReport) error { + return kubeletplugin.ErrHealthNotSupported +} diff --git a/pkg/resmgr/dra/plugin_test.go b/pkg/resmgr/dra/plugin_test.go new file mode 100644 index 000000000..5ad7092f1 --- /dev/null +++ b/pkg/resmgr/dra/plugin_test.go @@ -0,0 +1,2204 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "context" + "errors" + "fmt" + "runtime" + "strings" + "sync" + "testing" + "time" + + corev1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + + "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "tags.cncf.io/container-device-interface/pkg/parser" +) + +// TestNewLogr verifies that newLogr returns a usable logr.Logger backed by the +// default pkg/log Logger and that calling Info on it does not panic. +func TestNewLogr(t *testing.T) { + l := newLogr(log.Default()) + if l.IsZero() { + t.Error("newLogr returned a zero logr.Logger") + } + // Calling Info must not panic. + l.Info("test message from TestNewLogr") +} + +// validDeps returns a Deps with all required fields populated. +func validDeps() Deps { + return Deps{ + KubeClient: fake.NewClientset(), + NodeName: "test-node", + ValidateClasses: func() error { return nil }, + ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + DeviceLister: &fixedDeviceLister{}, + ClaimAllocator: &noopClaimAllocator{}, + CDIWriter: &noopCDIWriter{}, + ClaimStore: &noopClaimStore{}, + WithLock: func(f func()) { f() }, + ClaimUnprepare: func(_ types.UID, _ []ResultAlloc) {}, + Logger: log.Default(), + } +} + +// TestNew_Succeeds verifies that New returns a non-nil Plugin when all +// required fields are provided. +func TestNew_Succeeds(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + if p == nil { + t.Fatal("New() returned nil Plugin, want non-nil") + } +} + +// TestNew_Validation verifies that New returns an error when any required +// dependency is absent. +func TestNew_Validation(t *testing.T) { + tests := []struct { + name string + driverName string + mutate func(*Deps) + }{ + { + name: "empty driverName", + driverName: "", + mutate: nil, + }, + { + name: "nil KubeClient", + driverName: "test-driver", + mutate: func(d *Deps) { d.KubeClient = nil }, + }, + { + name: "empty NodeName", + driverName: "test-driver", + mutate: func(d *Deps) { d.NodeName = "" }, + }, + { + name: "nil ValidateClasses", + driverName: "test-driver", + mutate: func(d *Deps) { d.ValidateClasses = nil }, + }, + { + name: "nil ValidateCPUsInPool", + driverName: "test-driver", + mutate: func(d *Deps) { d.ValidateCPUsInPool = nil }, + }, + { + name: "nil DeviceLister", + driverName: "test-driver", + mutate: func(d *Deps) { d.DeviceLister = nil }, + }, + { + name: "nil Logger", + driverName: "test-driver", + mutate: func(d *Deps) { d.Logger = nil }, + }, + { + name: "nil ClaimAllocator", + driverName: "test-driver", + mutate: func(d *Deps) { d.ClaimAllocator = nil }, + }, + { + name: "nil CDIWriter", + driverName: "test-driver", + mutate: func(d *Deps) { d.CDIWriter = nil }, + }, + { + name: "nil ClaimStore", + driverName: "test-driver", + mutate: func(d *Deps) { d.ClaimStore = nil }, + }, + { + name: "nil WithLock", + driverName: "test-driver", + mutate: func(d *Deps) { d.WithLock = nil }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + deps := validDeps() + if tc.mutate != nil { + tc.mutate(&deps) + } + p, err := New(tc.driverName, deps) + if err == nil { + t.Errorf("New() expected error, got nil") + } + if p != nil { + t.Errorf("New() expected nil Plugin on error, got %v", p) + } + }) + } +} + +// TestHandleError_RecoverableLogsWarn verifies that a recoverable error is +// handled without panicking. The method logs at Warn level. +func TestHandleError_RecoverableLogsWarn(t *testing.T) { + p := &Plugin{deps: Deps{Logger: log.Default()}} + recoverableErr := fmt.Errorf("transient failure: %w", kubeletplugin.ErrRecoverable) + // Must not panic. + p.HandleError(t.Context(), recoverableErr, "publish failed") +} + +// TestHandleError_FatalLogsError verifies that a non-recoverable (fatal) error +// is handled without panicking. The method logs at Error level. +func TestHandleError_FatalLogsError(t *testing.T) { + p := &Plugin{deps: Deps{Logger: log.Default()}} + fatalErr := errors.New("fatal background error") + // Must not panic. + p.HandleError(t.Context(), fatalErr, "fatal error encountered") +} + +// TestPublishResources_NilHelper verifies that PublishResources returns an +// error (not a panic) when called before Start, and that the error message +// references "Start" so callers can diagnose the ordering mistake. +func TestPublishResources_NilHelper(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // helper is nil at this point (Start has not been called) + err = p.PublishResources(t.Context()) + if err == nil { + t.Fatal("PublishResources() expected error when helper is nil, got nil") + } + if !strings.Contains(err.Error(), "Start") { + t.Errorf("PublishResources() err = %q, want message containing \"Start\"", err.Error()) + } +} + +// TestPublishResources_ValidationError verifies that a ValidateClasses failure +// is propagated by PublishResources. +func TestPublishResources_ValidationError(t *testing.T) { + validateErr := errors.New("invalid class config") + deps := validDeps() + deps.ValidateClasses = func() error { return validateErr } + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // helper is nil — but ValidateClasses is checked first, so that error + // is returned before the nil-helper guard. + err = p.PublishResources(t.Context()) + if err == nil { + t.Fatal("PublishResources() expected error from ValidateClasses, got nil") + } + if !errors.Is(err, validateErr) { + t.Errorf("PublishResources() err = %v, want to wrap %v", err, validateErr) + } +} + +// TestPublishResources_Pagination_Zero verifies that zero devices produce +// exactly one empty slice. +func TestPublishResources_Pagination_Zero(t *testing.T) { + res := buildDriverResources("node1", nil) + pool, ok := res.Pools["node1"] + if !ok { + t.Fatal("expected pool named 'node1'") + } + if len(pool.Slices) != 1 { + t.Errorf("zero devices: got %d slice(s), want 1", len(pool.Slices)) + } + if len(pool.Slices[0].Devices) != 0 { + t.Errorf("zero devices: slice[0] has %d device(s), want 0", len(pool.Slices[0].Devices)) + } +} + +// TestPublishResources_Pagination_ExactMax verifies that exactly +// ResourceSliceMaxDevices devices fit into one slice. +func TestPublishResources_Pagination_ExactMax(t *testing.T) { + max := resourceapi.ResourceSliceMaxDevices + devices := makeTestDevices(max) + res := buildDriverResources("node1", devices) + pool := res.Pools["node1"] + if len(pool.Slices) != 1 { + t.Errorf("exact max devices: got %d slice(s), want 1", len(pool.Slices)) + } + if len(pool.Slices[0].Devices) != max { + t.Errorf("exact max devices: slice[0] has %d device(s), want %d", len(pool.Slices[0].Devices), max) + } +} + +// TestPublishResources_Pagination_OverMax verifies that +// ResourceSliceMaxDevices+1 devices are split into exactly two slices +// belonging to the same pool. +func TestPublishResources_Pagination_OverMax(t *testing.T) { + max := resourceapi.ResourceSliceMaxDevices + devices := makeTestDevices(max + 1) + res := buildDriverResources("node1", devices) + pool := res.Pools["node1"] + if len(pool.Slices) != 2 { + t.Errorf("max+1 devices: got %d slice(s), want 2", len(pool.Slices)) + } + if len(pool.Slices[0].Devices) != max { + t.Errorf("max+1 devices: slice[0] has %d device(s), want %d", len(pool.Slices[0].Devices), max) + } + if len(pool.Slices[1].Devices) != 1 { + t.Errorf("max+1 devices: slice[1] has %d device(s), want 1", len(pool.Slices[1].Devices)) + } +} + +// TestStop_Idempotent verifies that calling Stop twice on a Plugin does not +// panic. Both calls are made on a plugin that was never started (helper == nil). +func TestStop_Idempotent(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Must not panic. + p.Stop() + p.Stop() +} + +// TestStart_ValidateClassesError verifies that Start returns the ValidateClasses +// error before attempting to call kubeletplugin.Start. +func TestStart_ValidateClassesError(t *testing.T) { + validateErr := errors.New("cpu class config invalid") + deps := validDeps() + deps.ValidateClasses = func() error { return validateErr } + deps.PluginDataDir = t.TempDir() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + err = p.Start(t.Context()) + if err == nil { + t.Fatal("Start() expected error from ValidateClasses, got nil") + } + if !errors.Is(err, validateErr) { + t.Errorf("Start() err = %v, want to wrap %v", err, validateErr) + } + if p.helper != nil { + t.Error("Start() set p.helper on ValidateClasses failure, want nil") + } +} + +// makeTestDevices returns a slice of n named resourceapi.Device objects for +// use in pagination tests. +func makeTestDevices(n int) []resourceapi.Device { + devices := make([]resourceapi.Device, n) + for i := range devices { + devices[i] = resourceapi.Device{Name: fmt.Sprintf("dev-%d", i)} + } + return devices +} + +// fixedDeviceLister is a DeviceLister that always returns a preset list of +// devices, used in integration tests. +type fixedDeviceLister struct { + devices []resourceapi.Device +} + +func (f *fixedDeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { + return f.devices, nil +} + +// errorDeviceLister is a DeviceLister that always returns the configured error. +type errorDeviceLister struct { + err error +} + +func (e *errorDeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { + return nil, e.err +} + +// noopClaimAllocator is a ClaimAllocator that succeeds without doing anything. +type noopClaimAllocator struct{} + +func (*noopClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { + return cpuset.New(), nil +} + +func (*noopClaimAllocator) ReleaseHpCpus(_, _ int, _ cpuset.CPUSet) {} + +func (*noopClaimAllocator) AccountHpCpus(_, _ int, _ cpuset.CPUSet) error { return nil } + +func (*noopClaimAllocator) IsHPClass(_ string) bool { return false } + +// noopCDIWriter is a CDIWriter that succeeds without doing anything. +type noopCDIWriter struct{} + +func (*noopCDIWriter) WriteClaim(_ types.UID, _ []CDIDevice) error { return nil } +func (*noopCDIWriter) RemoveClaim(_ types.UID) error { return nil } +func (*noopCDIWriter) ClaimSpecExists(_ types.UID) bool { return false } +func (*noopCDIWriter) ListClaims() ([]types.UID, error) { return nil, nil } + +// noopClaimStore is a ClaimStore that succeeds without persisting anything. +type noopClaimStore struct{} + +func (*noopClaimStore) Save(_ map[types.UID]*ClaimState) error { return nil } +func (*noopClaimStore) Load() (map[types.UID]*ClaimState, error) { return nil, nil } + +// TestPublishResources_DRADevicesError verifies that an error from +// DeviceLister.DRADevices is propagated by PublishResources. +func TestPublishResources_DRADevicesError(t *testing.T) { + sentinel := errors.New("DRADevices failed") + deps := validDeps() + deps.DeviceLister = &errorDeviceLister{err: sentinel} + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Set helper to a non-nil stub so the nil-helper guard is bypassed, + // allowing the test to reach the DRADevices call. + p.helper = new(kubeletplugin.Helper) + err = p.PublishResources(t.Context()) + if !errors.Is(err, sentinel) { + t.Errorf("PublishResources() err = %v, want to wrap sentinel error", err) + } +} + +// TestPublishResources_ConcurrentNoRace verifies that calling PublishResources +// from multiple goroutines while a simulated Reconfigure goroutine acquires the +// same mutex does not produce a data race. When run with -race, the detector +// will flag any unsynchronized access to shared state. +func TestPublishResources_ConcurrentNoRace(t *testing.T) { + var mu sync.Mutex + + deps := validDeps() + // Replace the simple direct-call WithLock from validDeps with a real mutex + // so the race detector can verify PublishResources holds the lock around + // its ValidateClasses + DRADevices calls. + deps.WithLock = func(f func()) { + mu.Lock() + defer mu.Unlock() + f() + } + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Set helper to non-nil so the nil-helper guard is bypassed. + p.helper = new(kubeletplugin.Helper) + + ctx := context.Background() + const goroutines = 10 + var wg sync.WaitGroup + wg.Add(goroutines) + for range goroutines { + go func() { + defer wg.Done() + // Errors are expected (helper is a zero-value stub); what we are + // testing is the absence of data races. + _ = p.PublishResources(ctx) + }() + } + // Simulate concurrent Reconfigure by acquiring the same mutex for a brief + // hold, which is what resmgr.apply() does when it calls policy.Reconfigure() + // under the write lock. + wg.Add(1) + go func() { + defer wg.Done() + for range 50 { + mu.Lock() + runtime.Gosched() // hold briefly to interleave with PublishResources + mu.Unlock() + } + }() + wg.Wait() +} + +// TestTriggerRepublish_EnqueuesSignal verifies that TriggerRepublish sends +// exactly one signal on the buffered republishCh when the channel is empty. +func TestTriggerRepublish_EnqueuesSignal(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + select { + case <-p.republishCh: + t.Fatal("unexpected signal in republishCh before TriggerRepublish") + default: + } + + p.TriggerRepublish() + + select { + case <-p.republishCh: + default: + t.Fatal("no signal in republishCh after TriggerRepublish") + } +} + +// TestTriggerRepublish_Idempotent verifies that multiple TriggerRepublish +// calls while a publish is already pending enqueue exactly one signal (no +// blocking, no panic, no second entry). +func TestTriggerRepublish_Idempotent(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + p.TriggerRepublish() + p.TriggerRepublish() // second call must not block or add a second signal + + count := 0 + for { + select { + case <-p.republishCh: + count++ + default: + if count != 1 { + t.Errorf("republishCh had %d signals after two TriggerRepublish calls, want 1", count) + } + return + } + } +} + +// TestStart_AlreadyStarted verifies that a second call to Start returns an +// error without spawning a second helper. The guard is tested by setting +// p.helper to a non-nil stub before calling Start. +func TestStart_AlreadyStarted(t *testing.T) { + deps := validDeps() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Simulate an already-started plugin. + p.helper = new(kubeletplugin.Helper) + err = p.Start(t.Context()) + if err == nil { + t.Fatal("Start() expected error on double-call, got nil") + } + if !strings.Contains(err.Error(), "already started") { + t.Errorf("Start() err = %q, want message containing \"already started\"", err.Error()) + } +} + +// TestPublishResources_Integration starts a Plugin against a fake Kubernetes +// clientset, calls PublishResources, and polls until the fake cluster receives +// at least one ResourceSlice create, then validates the driver name field. +// +// Both the kubelet-plugin registration socket and plugin data socket are +// created under t.TempDir() — no real kubelet is required. +func TestPublishResources_Integration(t *testing.T) { + registrarDir := t.TempDir() + pluginDataDir := t.TempDir() + + // Fake clientset pre-loaded with the node object so the + // resourceslice.Controller can look up the node UID. + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + + // Reactor captures every ResourceSlice create call and passes it through + // to the default object tracker so the controller behaves normally. + var ( + mu sync.Mutex + capturedSlices []*resourceapi.ResourceSlice + ) + fakeClient.PrependReactor("create", "resourceslices", + func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + createAction := action.(k8stesting.CreateAction) + if slice, ok := createAction.GetObject().(*resourceapi.ResourceSlice); ok { + mu.Lock() + capturedSlices = append(capturedSlices, slice.DeepCopy()) + mu.Unlock() + } + return false, nil, nil // pass through to default tracker + }, + ) + + deps := Deps{ + KubeClient: fakeClient, + NodeName: "test-node", + RegistrarDir: registrarDir, + PluginDataDir: pluginDataDir, + ValidateClasses: func() error { return nil }, + ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + DeviceLister: &fixedDeviceLister{devices: makeTestDevices(5)}, + ClaimAllocator: &noopClaimAllocator{}, + CDIWriter: &noopCDIWriter{}, + ClaimStore: &noopClaimStore{}, + WithLock: func(f func()) { f() }, + ClaimUnprepare: func(_ types.UID, _ []ResultAlloc) {}, + Logger: log.Default(), + } + + const driverName = "test.driver.io" + p, err := New(driverName, deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + defer p.Stop() + + if err := p.PublishResources(ctx); err != nil { + t.Fatalf("PublishResources() unexpected error: %v", err) + } + + // The resourceslice.Controller drives ResourceSlice creation + // asynchronously; poll with a 5-second deadline. + const pollDeadline = 5 * time.Second + deadline := time.Now().Add(pollDeadline) + for time.Now().Before(deadline) { + mu.Lock() + n := len(capturedSlices) + mu.Unlock() + if n > 0 { + break + } + time.Sleep(50 * time.Millisecond) + } + + mu.Lock() + defer mu.Unlock() + if len(capturedSlices) == 0 { + t.Fatalf("no ResourceSlice was created within the %s deadline", pollDeadline) + } + if got := capturedSlices[0].Spec.Driver; got != driverName { + t.Errorf("ResourceSlice[0].Spec.Driver = %q, want %q", got, driverName) + } +} + +// trackingClaimAllocator tracks PickHpCpus and ReleaseHpCpus calls. +type trackingClaimAllocator struct { + pickResult cpuset.CPUSet + pickErr error + isHP bool + picks []cpuset.CPUSet // CPUSets returned per PickHpCpus call + releases []cpuset.CPUSet // CPUSets released per ReleaseHpCpus call + accounts []cpuset.CPUSet // CPUSets accounted per AccountHpCpus call + accountErr error +} + +func (a *trackingClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { + if a.pickErr != nil { + return cpuset.New(), a.pickErr + } + a.picks = append(a.picks, a.pickResult) + return a.pickResult, nil +} + +func (a *trackingClaimAllocator) ReleaseHpCpus(_, _ int, cpus cpuset.CPUSet) { + a.releases = append(a.releases, cpus) +} + +func (a *trackingClaimAllocator) AccountHpCpus(_, _ int, cpus cpuset.CPUSet) error { + if a.accountErr != nil { + return a.accountErr + } + a.accounts = append(a.accounts, cpus) + return nil +} + +func (a *trackingClaimAllocator) IsHPClass(_ string) bool { return a.isHP } + +// trackingCDIWriter tracks WriteClaim and RemoveClaim calls. +type trackingCDIWriter struct { + writeErr error + removeErr error + existsValue bool + written []types.UID + removed []types.UID +} + +func (w *trackingCDIWriter) WriteClaim(uid types.UID, _ []CDIDevice) error { + if w.writeErr != nil { + return w.writeErr + } + w.written = append(w.written, uid) + return nil +} + +func (w *trackingCDIWriter) RemoveClaim(uid types.UID) error { + w.removed = append(w.removed, uid) + return w.removeErr +} + +func (w *trackingCDIWriter) ClaimSpecExists(_ types.UID) bool { return w.existsValue } +func (w *trackingCDIWriter) ListClaims() ([]types.UID, error) { return nil, nil } + +// trackingClaimStore records Save and Load calls. +type trackingClaimStore struct { + saveErr error + saved int +} + +func (s *trackingClaimStore) Save(_ map[types.UID]*ClaimState) error { + if s.saveErr != nil { + return s.saveErr + } + s.saved++ + return nil +} + +func (s *trackingClaimStore) Load() (map[types.UID]*ClaimState, error) { return nil, nil } + +// strPtr returns a pointer to s, used to build DeviceAttribute.StringValue. +func strPtr(s string) *string { return &s } + +// int64Ptr returns a pointer to v, used to build DeviceAttribute.IntValue. +func int64Ptr(v int64) *int64 { return &v } + +// hpDevice builds a resourceapi.Device with nri/cpuClass, nri/packageID, and +// nri/punitID attributes, plus an nri/cpus capacity. +func hpDevice(name, className string, pkgID, punitID int) resourceapi.Device { + return resourceapi.Device{ + Name: name, + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/cpuClass": {StringValue: strPtr(className)}, + "nri/packageID": {IntValue: int64Ptr(int64(pkgID))}, + "nri/punitID": {IntValue: int64Ptr(int64(punitID))}, + }, + } +} + +// hpDeviceLister returns a DeviceLister that always returns the given devices. +func hpDeviceLister(devs ...resourceapi.Device) *fixedDeviceLister { + return &fixedDeviceLister{devices: devs} +} + +// makeClaim builds a ResourceClaim with a single allocation result for +// driverName/poolName/deviceName, with the given ConsumedCapacity cpu count. +func makeClaim(uid types.UID, driverName, poolName, deviceName, request string, cpus int) *resourceapi.ResourceClaim { + qty := resource.MustParse(fmt.Sprintf("%d", cpus)) + return &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + { + Driver: driverName, + Pool: poolName, + Device: deviceName, + Request: request, + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{ + "nri/cpus": qty, + }, + }, + }, + }, + }, + }, + } +} + +// TestPrepare_SingleHPSuccess verifies that a single HP claim results in a +// PrepareResult with one Device and that CPUs are picked and CDI is written. +func TestPrepare_SingleHPSuccess(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-1") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil { + t.Fatalf("PrepareResourceClaims() unexpected error: %v", err) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing uid-1") + } + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if len(r.Devices) != 1 { + t.Fatalf("PrepareResult.Devices len = %d, want 1", len(r.Devices)) + } + if len(alloc.picks) != 1 { + t.Errorf("PickHpCpus called %d times, want 1", len(alloc.picks)) + } + if len(cdiW.written) != 1 || cdiW.written[0] != uid { + t.Errorf("WriteClaim called for %v, want %v", cdiW.written, []types.UID{uid}) + } + if store.saved != 1 { + t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + } + // Verify the claim is stored. + if _, ok := p.claims[uid]; !ok { + t.Error("claim not stored in p.claims") + } +} + +// TestPrepare_Idempotent_SpecPresent verifies that a second Prepare call for the +// same claim with the CDI spec already present returns the same PrepareResult +// without re-picking CPUs or re-writing the spec. +func TestPrepare_Idempotent_SpecPresent(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-idem") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + + // First Prepare — succeeds. + result1, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result1[uid].Err != nil { + t.Fatalf("first Prepare failed: %v / %v", err, result1[uid].Err) + } + firstPicks := len(alloc.picks) + firstWrites := len(cdiW.written) + + // Simulate CDI spec existing. + cdiW.existsValue = true + + // Second Prepare — should be idempotent. + result2, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result2[uid].Err != nil { + t.Fatalf("second Prepare failed: %v / %v", err, result2[uid].Err) + } + if len(alloc.picks) != firstPicks { + t.Errorf("second Prepare picked CPUs again (picks: %d → %d)", firstPicks, len(alloc.picks)) + } + if len(cdiW.written) != firstWrites { + t.Errorf("second Prepare re-wrote CDI spec (writes: %d → %d)", firstWrites, len(cdiW.written)) + } + // Devices content must be identical between calls. + devs1 := result1[uid].Devices + devs2 := result2[uid].Devices + if len(devs1) != len(devs2) { + t.Errorf("Devices len differs between calls: first=%d second=%d", len(devs1), len(devs2)) + } else { + for i := range devs1 { + if len(devs1[i].CDIDeviceIDs) != len(devs2[i].CDIDeviceIDs) { + t.Errorf("Device[%d] CDIDeviceIDs len differs: first=%d second=%d", i, len(devs1[i].CDIDeviceIDs), len(devs2[i].CDIDeviceIDs)) + continue + } + for j := range devs1[i].CDIDeviceIDs { + if devs1[i].CDIDeviceIDs[j] != devs2[i].CDIDeviceIDs[j] { + t.Errorf("Device[%d].CDIDeviceIDs[%d]: first=%q second=%q", i, j, devs1[i].CDIDeviceIDs[j], devs2[i].CDIDeviceIDs[j]) + } + } + } + } +} + +// TestPrepare_Idempotent_SpecMissing verifies that a second Prepare call for the +// same claim where the CDI spec is missing re-writes the spec without re-picking CPUs. +func TestPrepare_Idempotent_SpecMissing(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-rewrite") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + + // First Prepare. + result1, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result1[uid].Err != nil { + t.Fatalf("first Prepare failed: %v / %v", err, result1[uid].Err) + } + firstPicks := len(alloc.picks) + firstWrites := len(cdiW.written) + + // CDI spec remains missing (existsValue == false by default). + + // Second Prepare — should re-write but not re-pick. + result2, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result2[uid].Err != nil { + t.Fatalf("second Prepare failed: %v / %v", err, result2[uid].Err) + } + if len(alloc.picks) != firstPicks { + t.Errorf("second Prepare re-picked CPUs (picks: %d → %d)", firstPicks, len(alloc.picks)) + } + if len(cdiW.written) != firstWrites+1 { + t.Errorf("second Prepare should have re-written CDI spec (writes: %d → %d)", firstWrites, len(cdiW.written)) + } + // Devices content must be identical between calls. + devs1 := result1[uid].Devices + devs2 := result2[uid].Devices + if len(devs1) != len(devs2) { + t.Errorf("Devices len differs between calls: first=%d second=%d", len(devs1), len(devs2)) + } else { + for i := range devs1 { + if len(devs1[i].CDIDeviceIDs) != len(devs2[i].CDIDeviceIDs) { + t.Errorf("Device[%d] CDIDeviceIDs len differs: first=%d second=%d", i, len(devs1[i].CDIDeviceIDs), len(devs2[i].CDIDeviceIDs)) + continue + } + for j := range devs1[i].CDIDeviceIDs { + if devs1[i].CDIDeviceIDs[j] != devs2[i].CDIDeviceIDs[j] { + t.Errorf("Device[%d].CDIDeviceIDs[%d]: first=%q second=%q", i, j, devs1[i].CDIDeviceIDs[j], devs2[i].CDIDeviceIDs[j]) + } + } + } + } +} + +// TestPrepare_Idempotent_SpecMissing_PartialCorruptCPU verifies that the +// spec-missing idempotency path returns an error when only some stored alloc +// CPUs are parseable (partial corruption), preventing a mismatched CDI spec +// from being written. +func TestPrepare_Idempotent_SpecMissing_PartialCorruptCPU(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + uid := types.UID("uid-partial-corrupt") + // Two stored allocs: first is valid, second has an unparsable CPU string. + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{ + {Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-1", ClassName: "gold"}, + {Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "NOT-A-CPUSET", ClassName: "gold"}, + }, + } + + qty := resource.MustParse("2") + // Claim carries two allocation results for our driver — one per device. + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "req0", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + {Driver: "test-driver", Pool: "p", Device: "dev1", Request: "req1", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + + // Wire in the pre-loaded claim state so the idempotency path fires. + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + // CDI spec is missing (cdiW.existsValue == false by default). + + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing UID") + } + if r.Err == nil { + t.Fatal("PrepareResult.Err = nil; want error for partial corrupt stored CPUs") + } + // No CDI spec should have been written. + if len(cdiW.written) != 0 { + t.Errorf("CDIWriter.WriteClaim called %d time(s), want 0", len(cdiW.written)) + } + // No CPU picks should have been made. + if len(alloc.picks) != 0 { + t.Errorf("PickHpCpus called %d time(s), want 0", len(alloc.picks)) + } +} + +// TestPrepare_NilAllocation verifies that a nil claim.Status.Allocation produces +// a per-claim errNilAllocation in the result map, not a global error. +func TestPrepare_NilAllocation(t *testing.T) { + deps := validDeps() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-nil-alloc") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing UID") + } + if !errors.Is(r.Err, errNilAllocation) { + t.Errorf("PrepareResult.Err = %v, want errNilAllocation", r.Err) + } +} + +// TestPrepare_ForeignDriverOnly verifies that a claim with results only for a +// different driver produces an empty PrepareResult with no error. +func TestPrepare_ForeignDriverOnly(t *testing.T) { + deps := validDeps() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-foreign") + qty := resource.MustParse("4") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "other-driver", Pool: "p", Device: "d", Request: "r", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing UID") + } + if r.Err != nil { + t.Errorf("PrepareResult.Err = %v, want nil for foreign-driver-only claim", r.Err) + } + if len(r.Devices) != 0 { + t.Errorf("PrepareResult.Devices = %v, want empty for foreign-driver-only claim", r.Devices) + } +} + +// TestPrepare_UnknownDevice verifies that a result referencing a device not in +// deviceIndex produces a per-claim error. +func TestPrepare_UnknownDevice(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + // DeviceLister returns no devices. + deps.DeviceLister = hpDeviceLister() + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-unknown") + claim := makeClaim(uid, "test-driver", "pool0", "unknown-dev", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error for unknown device, got nil") + } +} + +// TestPrepare_NilAttr verifies that a device with a missing (empty) nri/cpuClass +// attribute produces a per-claim error. +func TestPrepare_NilAttr(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + // Device has no nri/cpuClass attribute. + deps.DeviceLister = hpDeviceLister(resourceapi.Device{ + Name: "dev0", + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/packageID": {IntValue: int64Ptr(0)}, + "nri/punitID": {IntValue: int64Ptr(0)}, + // nri/cpuClass intentionally absent. + }, + }) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-nil-attr") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error for nil nri/cpuClass attr, got nil") + } +} + +// TestPrepare_NilIntAttr verifies that a device attribute with a nil IntValue +// (for packageID or punitID) is handled gracefully: the field defaults to 0 +// and Prepare succeeds when all other required attrs are present. +func TestPrepare_NilIntAttr(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = &trackingCDIWriter{} + deps.ClaimStore = &trackingClaimStore{} + // Device has cpuClass set but packageID and punitID with nil IntValue. + deps.DeviceLister = hpDeviceLister(resourceapi.Device{ + Name: "dev0", + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/cpuClass": {StringValue: strPtr("gold")}, + "nri/packageID": {IntValue: nil}, // nil IntValue → PkgID defaults to 0 + "nri/punitID": {IntValue: nil}, // nil IntValue → PunitID defaults to 0 + }, + }) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-nil-int-attr") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + // Nil IntValue falls back to zero — Prepare should succeed (pick from pkg 0, punit 0). + if r.Err != nil { + t.Errorf("PrepareResult.Err = %v, want nil for nil IntValue attrs (defaults to 0)", r.Err) + } +} + +// TestPrepare_AbsentConsumedCapacity verifies that a result with no nri/cpus +// entry in ConsumedCapacity produces errMissingConsumedCapacity. +func TestPrepare_AbsentConsumedCapacity(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-no-cap") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "r"}, + // No ConsumedCapacity. + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if !errors.Is(r.Err, errMissingConsumedCapacity) { + t.Errorf("PrepareResult.Err = %v, want errMissingConsumedCapacity", r.Err) + } +} + +// TestPrepare_NonHP verifies that a non-HP class produces errNonHPNotSupported. +func TestPrepare_NonHP(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: false} // isHP == false → not HP + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "silver", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-non-hp") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if !errors.Is(r.Err, errNonHPNotSupported) { + t.Errorf("PrepareResult.Err = %v, want errNonHPNotSupported", r.Err) + } +} + +// TestPrepare_PickFailure verifies that a PickHpCpus failure rolls back +// any previously picked CPUs and returns a per-claim error. +func TestPrepare_PickFailure(t *testing.T) { + pickErr := errors.New("pick failed") + alloc := &trackingClaimAllocator{pickErr: pickErr, isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-pick-fail") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error from PickHpCpus failure, got nil") + } + if !errors.Is(r.Err, pickErr) { + t.Errorf("PrepareResult.Err = %v, want to wrap pickErr", r.Err) + } + // Claim must not be stored after a PickHpCpus failure. + if _, ok := p.claims[uid]; ok { + t.Error("claim stored in p.claims after PickHpCpus failure (should not be present)") + } +} + +// TestPrepare_CDIWriteFailure verifies that a CDI WriteClaim failure rolls back +// the picked CPUs and returns a per-claim error. +func TestPrepare_CDIWriteFailure(t *testing.T) { + writeErr := errors.New("CDI write failed") + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{writeErr: writeErr} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-cdi-fail") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error from CDI write failure, got nil") + } + if !errors.Is(r.Err, writeErr) { + t.Errorf("PrepareResult.Err = %v, want to wrap writeErr", r.Err) + } + // Rollback: CPUs must be released. + if len(alloc.releases) == 0 { + t.Error("expected ReleaseHpCpus to be called on CDI write failure") + } + // Claim must not be stored after a CDI write failure. + if _, ok := p.claims[uid]; ok { + t.Error("claim stored in p.claims after CDI write failure (should have been rolled back)") + } +} + +// TestPrepare_ClaimStoreSaveFailure verifies that a ClaimStore.Save failure +// after a successful CDI write and CPU pick rolls back symmetrically: the +// just-written CDI spec is removed, the picked CPUs are released, and the +// claim is not left in p.claims — otherwise a restart would lose durable +// state for a claim whose CDI spec/CPUs are already live. +func TestPrepare_ClaimStoreSaveFailure(t *testing.T) { + saveErr := errors.New("claim store save failed") + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{saveErr: saveErr} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-save-fail") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Fatal("expected per-claim error from ClaimStore.Save failure, got nil (PrepareResult{} instead)") + } + if !errors.Is(r.Err, saveErr) { + t.Errorf("PrepareResult.Err = %v, want to wrap saveErr", r.Err) + } + // The just-written CDI spec must be removed. + if len(cdiW.removed) != 1 || cdiW.removed[0] != uid { + t.Errorf("CDIWriter.RemoveClaim called for %v, want [%v]", cdiW.removed, uid) + } + // The picked CPUs must be released. + if len(alloc.releases) == 0 { + t.Error("expected ReleaseHpCpus to be called on ClaimStore.Save failure") + } + // The claim must not be left set in p.claims. + if _, ok := p.claims[uid]; ok { + t.Error("claim stored in p.claims after ClaimStore.Save failure (should have been rolled back)") + } +} + +// TestPrepare_MultiResultTwoPunits verifies that a claim spanning punit pools +// is rejected before it can be committed. +func TestPrepare_MultiResultTwoPunits(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-1"), isHP: true} + cdiW := &trackingCDIWriter{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.DeviceLister = hpDeviceLister( + hpDevice("dev0", "gold", 0, 0), + hpDevice("dev1", "gold", 0, 1), + ) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + qty := resource.MustParse("2") + uid := types.UID("uid-multi") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "req0", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + {Driver: "test-driver", Pool: "p", Device: "dev1", Request: "req1", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if !errors.Is(r.Err, errMultiPunitNotSupported) { + t.Fatalf("PrepareResult.Err = %v, want errMultiPunitNotSupported", r.Err) + } + if len(r.Devices) != 0 { + t.Errorf("PrepareResult.Devices len = %d, want 0", len(r.Devices)) + } + if len(alloc.releases) != 1 { + t.Errorf("ReleaseHpCpus called %d times, want 1 rollback", len(alloc.releases)) + } + if len(cdiW.written) != 0 { + t.Errorf("WriteClaim called %d times, want 0", len(cdiW.written)) + } + if _, ok := p.claims[uid]; ok { + t.Error("claim stored after multi-punit rejection") + } +} + +// TestPrepare_ClaimCPUsOutsideAllocationDomain verifies that claim CPU picks +// are rejected before CDI/state commit when the policy reports they don't +// fit a single allocation domain (e.g. a punit spanning more than one +// topology leaf pool). +func TestPrepare_ClaimCPUsOutsideAllocationDomain(t *testing.T) { + validateErr := errors.New("spans multiple leaf pools") + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + deps.ValidateCPUsInPool = func(_ cpuset.CPUSet) error { return validateErr } + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-invalid-domain") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if !errors.Is(r.Err, validateErr) { + t.Fatalf("PrepareResult.Err = %v, want to wrap validateErr", r.Err) + } + if len(alloc.releases) != 1 { + t.Errorf("ReleaseHpCpus called %d times, want 1 rollback", len(alloc.releases)) + } + if len(cdiW.written) != 0 { + t.Errorf("WriteClaim called %d times, want 0", len(cdiW.written)) + } + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0", store.saved) + } + if _, ok := p.claims[uid]; ok { + t.Error("claim stored after allocation-domain rejection") + } +} + +// TestPrepare_ShareIDNil verifies that a result with a nil ShareID produces a +// Device with ShareID == nil. +func TestPrepare_ShareIDNil(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-no-share") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) // no ShareID + result, _ := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + r := result[uid] + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if r.Devices[0].ShareID != nil { + t.Errorf("Device.ShareID = %v, want nil", r.Devices[0].ShareID) + } +} + +// TestPrepare_ShareIDSet verifies that a result with a non-nil ShareID produces a +// Device with a matching non-nil ShareID. +func TestPrepare_ShareIDSet(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + shareUID := types.UID("share-abc") + qty := resource.MustParse("4") + uid := types.UID("uid-share") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "req0", + ShareID: &shareUID, + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, _ := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + r := result[uid] + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if r.Devices[0].ShareID == nil { + t.Error("Device.ShareID = nil, want non-nil") + } else if *r.Devices[0].ShareID != shareUID { + t.Errorf("Device.ShareID = %v, want %v", *r.Devices[0].ShareID, shareUID) + } +} + +// TestPrepare_AllUIDsInResultMap verifies that every claim UID appears in the +// result map even when some claims error. +func TestPrepare_AllUIDsInResultMap(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid1 := types.UID("uid-ok") + uid2 := types.UID("uid-nil-alloc") + + good := makeClaim(uid1, "test-driver", "pool0", "dev0", "req0", 4) + bad := &resourceapi.ResourceClaim{ObjectMeta: metav1.ObjectMeta{UID: uid2}} + + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{good, bad}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + if _, ok := result[uid1]; !ok { + t.Errorf("result map missing %v", uid1) + } + if _, ok := result[uid2]; !ok { + t.Errorf("result map missing %v", uid2) + } + if result[uid1].Err != nil { + t.Errorf("good claim: PrepareResult.Err = %v, want nil", result[uid1].Err) + } + if !errors.Is(result[uid2].Err, errNilAllocation) { + t.Errorf("bad claim: PrepareResult.Err = %v, want errNilAllocation", result[uid2].Err) + } +} + +// TestPrepare_SubrequestSlashInName verifies that a request name containing '/' +// (FirstAvailable subrequest format) produces a valid CDI device name and that +// the spec can be written successfully. +func TestPrepare_SubrequestSlashInName(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + qty := resource.MustParse("4") + uid := types.UID("uid-slash") + // Request name uses the FirstAvailable subrequest format: "main/sub". + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", + Request: "main-req/sub-req", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if len(r.Devices) != 1 { + t.Fatalf("PrepareResult.Devices len = %d, want 1", len(r.Devices)) + } + // Verify the device name part of each CDI qualified ID passes parser.ValidateDeviceName. + // CDI qualified names have the format "vendor/class=name"; split on "=" to get the name. + for _, cdiID := range r.Devices[0].CDIDeviceIDs { + parts := strings.SplitN(cdiID, "=", 2) + if len(parts) != 2 || parts[1] == "" { + t.Errorf("CDI device ID %q has unexpected format (want vendor/class=name)", cdiID) + continue + } + if err := parser.ValidateDeviceName(parts[1]); err != nil { + t.Errorf("CDI device name %q from ID %q failed validation: %v", parts[1], cdiID, err) + } + } + if len(cdiW.written) == 0 { + t.Error("WriteClaim was not called for subrequest claim") + } +} + +// ---- Task 8: UnprepareResourceClaims tests ---- + +// unprepareObj builds a kubeletplugin.NamespacedObject for a given UID, +// used to drive UnprepareResourceClaims calls in tests. +func unprepareObj(uid types.UID) kubeletplugin.NamespacedObject { + return kubeletplugin.NamespacedObject{UID: uid} +} + +// preparePlugin is a helper that creates a Plugin, pre-populates p.claims with +// the given ClaimState entries, and wires the provided allocator, CDI writer, +// and store into Deps. +func preparePlugin(t *testing.T, alloc ClaimAllocator, cdiW CDIWriter, store ClaimStore, + claimsIn map[types.UID]*ClaimState) *Plugin { + t.Helper() + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + for uid, cs := range claimsIn { + p.claims[uid] = cs + } + return p +} + +// TestUnprepare_KnownClaim verifies that an existing claim is released, +// CDI is removed, the claim is deleted from p.claims, and the result map +// contains nil for the UID. +func TestUnprepare_KnownClaim(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + uid := types.UID("uid-known") + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{ + {Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}, + }, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if perErr, ok := result[uid]; !ok { + t.Error("result map missing uid") + } else if perErr != nil { + t.Errorf("result[uid] = %v, want nil", perErr) + } + if len(cdiW.removed) != 1 || cdiW.removed[0] != uid { + t.Errorf("RemoveClaim called for %v, want [%v]", cdiW.removed, uid) + } + if _, exists := p.claims[uid]; exists { + t.Error("claim still in p.claims after Unprepare") + } + if store.saved != 1 { + t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + } +} + +// TestUnprepare_UnknownUID verifies that an unknown UID produces a warning and +// a nil entry in the result map (no panic, no error). +func TestUnprepare_UnknownUID(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + p := preparePlugin(t, alloc, cdiW, store, nil) + + uid := types.UID("uid-unknown") + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if perErr, ok := result[uid]; !ok { + t.Error("result map missing uid") + } else if perErr != nil { + t.Errorf("result[uid] = %v, want nil for unknown UID", perErr) + } + // CDI remove must NOT be called for unknown claims. + if len(cdiW.removed) != 0 { + t.Errorf("RemoveClaim called for unknown UID (removed = %v)", cdiW.removed) + } + // ReleaseHpCpus must NOT be called for unknown claims. + if len(alloc.releases) != 0 { + t.Errorf("ReleaseHpCpus called %d times, want 0 for unknown UID", len(alloc.releases)) + } + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0", store.saved) + } +} + +// TestUnprepare_CDIRemoveError verifies that a CDI RemoveClaim error produces +// a warning but does not surface as an error in the result map (nil for that UID). +func TestUnprepare_CDIRemoveError(t *testing.T) { + removeErr := errors.New("CDI remove failed") + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{removeErr: removeErr} + store := &trackingClaimStore{} + + uid := types.UID("uid-remove-err") + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "4-7", ClassName: "gold"}}, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + // CDI error is a warning only — result must be nil for the UID. + if perErr := result[uid]; perErr != nil { + t.Errorf("result[uid] = %v, want nil on CDI remove error", perErr) + } + // Claim must still be deleted from in-memory state. + if _, exists := p.claims[uid]; exists { + t.Error("claim still in p.claims after Unprepare despite CDI error") + } + // ReleaseHpCpus is now called by the ClaimUnprepare callback (policy layer), + // not directly by UnprepareResourceClaims, so alloc.releases stays 0 here. +} + +func TestUnprepare_ClaimStoreSaveFailure(t *testing.T) { + saveErr := errors.New("claim store save failed") + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{saveErr: saveErr} + uid := types.UID("uid-save-err") + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "4-7", ClassName: "gold"}}, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if !errors.Is(result[uid], saveErr) { + t.Errorf("result[uid] = %v, want to wrap saveErr", result[uid]) + } + if _, exists := p.claims[uid]; !exists { + t.Error("claim removed from memory despite persistence failure") + } + if len(alloc.releases) != 0 { + t.Errorf("ReleaseHpCpus called %d times, want 0 before durable removal", len(alloc.releases)) + } + if len(cdiW.removed) != 0 { + t.Errorf("RemoveClaim called %d times, want 0 before durable removal", len(cdiW.removed)) + } +} + +// TestUnprepare_MixedBatch verifies that a batch with one known and one unknown +// UID both appear in the result map (nil for each), and only the known claim's +// CDI is removed. +func TestUnprepare_MixedBatch(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + known := types.UID("uid-known-batch") + unknown := types.UID("uid-unknown-batch") + claimState := &ClaimState{ + UID: string(known), + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-1", ClassName: "gold"}}, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{known: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), + []kubeletplugin.NamespacedObject{unprepareObj(known), unprepareObj(unknown)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if perErr, ok := result[known]; !ok { + t.Error("result map missing known uid") + } else if perErr != nil { + t.Errorf("result[known] = %v, want nil", perErr) + } + if perErr, ok := result[unknown]; !ok { + t.Error("result map missing unknown uid") + } else if perErr != nil { + t.Errorf("result[unknown] = %v, want nil", perErr) + } + // Only the known UID's CDI should be removed. + if len(cdiW.removed) != 1 || cdiW.removed[0] != known { + t.Errorf("RemoveClaim called for %v, want [%v]", cdiW.removed, known) + } + // One batch save. + if store.saved != 1 { + t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + } +} + +// TestShareIDPtr verifies that shareIDPtr returns nil for "" and a non-nil +// pointer for a non-empty string. +func TestShareIDPtr(t *testing.T) { + if shareIDPtr("") != nil { + t.Error("shareIDPtr(\"\") should return nil") + } + ptr := shareIDPtr("abc") + if ptr == nil { + t.Fatal("shareIDPtr(\"abc\") returned nil, want *types.UID") + } + if *ptr != types.UID("abc") { + t.Errorf("*shareIDPtr(\"abc\") = %v, want %v", *ptr, types.UID("abc")) + } +} + +// ---- Task 9: RestoreClaimsLocked, Start reconciliation ---- + +// startTestCDIWriter supports per-UID ClaimSpecExists state and a fixed +// ListClaims result for Start reconciliation tests. WriteClaim is a no-op. +type startTestCDIWriter struct { + existsByUID map[types.UID]bool + listResult []types.UID + listErr error + removed []types.UID + removeErr error +} + +func (w *startTestCDIWriter) WriteClaim(_ types.UID, _ []CDIDevice) error { return nil } +func (w *startTestCDIWriter) RemoveClaim(uid types.UID) error { + w.removed = append(w.removed, uid) + return w.removeErr +} +func (w *startTestCDIWriter) ClaimSpecExists(uid types.UID) bool { return w.existsByUID[uid] } +func (w *startTestCDIWriter) ListClaims() ([]types.UID, error) { + return w.listResult, w.listErr +} + +// preloadedClaimStore loads a pre-configured map on Load and tracks saves. +type preloadedClaimStore struct { + initial map[types.UID]*ClaimState + loadErr error + saved int + savedClaims map[types.UID]*ClaimState // last map passed to Save +} + +func (s *preloadedClaimStore) Load() (map[types.UID]*ClaimState, error) { + return s.initial, s.loadErr +} + +func (s *preloadedClaimStore) Save(claims map[types.UID]*ClaimState) error { + s.saved++ + cp := make(map[types.UID]*ClaimState, len(claims)) + for k, v := range claims { + cp[k] = v + } + s.savedClaims = cp + return nil +} + +// TestLiveClaimsLocked_Empty verifies that LiveClaimsLocked returns an empty +// map when there are no claims. +func TestLiveClaimsLocked_Empty(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + got := p.LiveClaimsLocked() + if len(got) != 0 { + t.Errorf("LiveClaimsLocked() = %v, want empty map", got) + } +} + +// TestLiveClaimsLocked_Snapshot verifies that LiveClaimsLocked returns a +// snapshot matching p.claims, and that mutating the returned map/slices does +// not corrupt the plugin's internal state (caller holds the resmgr lock, but +// the returned value must still be a defensive copy of the per-claim slice). +func TestLiveClaimsLocked_Snapshot(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + p.claims[types.UID("uid-a")] = &ClaimState{ + UID: "uid-a", + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}, + } + p.claims[types.UID("uid-b")] = &ClaimState{ + UID: "uid-b", + Allocs: []ResultAlloc{ + {Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-5", ClassName: "silver"}, + {Device: "dev2", PkgID: 0, PunitID: 1, CPUs: "6-7", ClassName: "silver"}, + }, + } + + got := p.LiveClaimsLocked() + if len(got) != 2 { + t.Fatalf("LiveClaimsLocked() len = %d, want 2", len(got)) + } + if allocs := got[types.UID("uid-a")]; len(allocs) != 1 || allocs[0].ClassName != "gold" { + t.Errorf("LiveClaimsLocked()[uid-a] = %+v, want one gold alloc", allocs) + } + if allocs := got[types.UID("uid-b")]; len(allocs) != 2 { + t.Errorf("LiveClaimsLocked()[uid-b] len = %d, want 2", len(allocs)) + } + + // Mutating the returned slice must not affect p.claims (defensive copy). + got[types.UID("uid-a")][0].ClassName = "mutated" + if p.claims[types.UID("uid-a")].Allocs[0].ClassName != "gold" { + t.Errorf("LiveClaimsLocked() leaked internal state: p.claims mutated via returned snapshot") + } +} + +// TestRestoreClaimsLocked_RebuildsAccounting verifies that RestoreClaimsLocked +// calls AccountHpCpus for each alloc in p.claims, rebuilding accounting after +// a Reconfigure reset (simulated by a fresh trackingClaimAllocator). +func TestRestoreClaimsLocked_RebuildsAccounting(t *testing.T) { + alloc := &trackingClaimAllocator{} + deps := validDeps() + deps.ClaimAllocator = alloc + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + // Pre-populate two claims with one alloc each. + p.claims[types.UID("uid-a")] = &ClaimState{ + UID: "uid-a", + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}, + } + p.claims[types.UID("uid-b")] = &ClaimState{ + UID: "uid-b", + Allocs: []ResultAlloc{{Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-7", ClassName: "gold"}}, + } + + if err := p.RestoreClaimsLocked(); err != nil { + t.Fatalf("RestoreClaimsLocked() unexpected error: %v", err) + } + if len(alloc.accounts) != 2 { + t.Errorf("AccountHpCpus called %d times, want 2", len(alloc.accounts)) + } + // Verify the union of all accounted CPU sets matches the expected total. + // Map iteration order is non-deterministic so we check the union. + totalAccounted := cpuset.New() + for _, cs := range alloc.accounts { + totalAccounted = totalAccounted.Union(cs) + } + wantTotal := cpuset.MustParse("0-7") + if !totalAccounted.Equals(wantTotal) { + t.Errorf("AccountHpCpus total CPUs = %v, want %v", totalAccounted, wantTotal) + } +} + +// TestRestoreClaims_WithLockWrapper verifies that RestoreClaims acquires the +// lock (via WithLock) and calls AccountHpCpus for each alloc. +func TestRestoreClaims_WithLockWrapper(t *testing.T) { + var mu sync.Mutex + alloc := &trackingClaimAllocator{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.WithLock = func(f func()) { + mu.Lock() + defer mu.Unlock() + f() + } + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + p.claims[types.UID("uid-c")] = &ClaimState{ + UID: "uid-c", + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-1", ClassName: "gold"}}, + } + + if err := p.RestoreClaims(); err != nil { + t.Fatalf("RestoreClaims() unexpected error: %v", err) + } + if len(alloc.accounts) != 1 { + t.Errorf("AccountHpCpus called %d times, want 1", len(alloc.accounts)) + } +} + +// TestStart_Reconciliation verifies that persisted claims are re-accounted from +// durable state even when the CDI spec file is missing on disk. Missing CDI is +// not treated as a proof of staleness; only an actual Unprepare removes a claim +// from the store. Orphan CDI specs (not tracked in the claim store) are still +// swept and removed. +func TestStart_Reconciliation(t *testing.T) { + liveUID := types.UID("uid-live") + staleUID := types.UID("uid-stale") + orphanUID := types.UID("uid-orphan") + + alloc := &trackingClaimAllocator{isHP: true} + + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{ + liveUID: true, + staleUID: false, + }, + // ListClaims returns liveUID and orphanUID (orphan has no claim entry). + listResult: []types.UID{liveUID, orphanUID}, + } + + store := &preloadedClaimStore{ + initial: map[types.UID]*ClaimState{ + liveUID: { + UID: string(liveUID), + Allocs: []ResultAlloc{ + {Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}, + }, + }, + staleUID: { + UID: string(staleUID), + Allocs: []ResultAlloc{ + {Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-7", ClassName: "gold"}, + }, + }, + }, + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + defer p.Stop() + + // Both claims must remain in p.claims — staleUID is kept even though its + // CDI spec is absent; PrepareResourceClaims will recreate it. + if _, ok := p.claims[liveUID]; !ok { + t.Error("liveUID missing from p.claims after Start") + } + if _, ok := p.claims[staleUID]; !ok { + t.Error("staleUID missing from p.claims after Start — missing CDI spec must not imply stale claim") + } + + // Both persisted claims are re-accounted from durable state. + if len(alloc.accounts) != 2 { + t.Errorf("AccountHpCpus called %d times, want 2", len(alloc.accounts)) + } + + // No claim-store writes happen when we keep persisted claims. + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0", store.saved) + } + + // Orphan sweep: orphanUID must have been removed via CDI writer. + found := false + for _, uid := range cdiW.removed { + if uid == orphanUID { + found = true + break + } + } + if !found { + t.Errorf("orphanUID was not removed by orphan sweep (removed = %v)", cdiW.removed) + } + // staleUID should NOT be in cdiW.removed (it is a live persisted claim). + for _, uid := range cdiW.removed { + if uid == staleUID { + t.Error("staleUID was unexpectedly passed to CDI RemoveClaim") + } + } +} + +// TestStart_ClaimStoreLoadError verifies that a ClaimStore.Load error during +// Start's reconciliation causes Start to return a non-nil error and does not +// proceed to the orphan sweep with an empty claim map — otherwise a transient +// Load failure would cause the orphan sweep to remove CDI specs for claims +// whose consuming containers are still running. +func TestStart_ClaimStoreLoadError(t *testing.T) { + liveUID := types.UID("uid-live-on-disk") + + alloc := &trackingClaimAllocator{isHP: true} + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{liveUID: true}, + // ListClaims reports liveUID's CDI spec still exists on disk. + listResult: []types.UID{liveUID}, + } + store := &preloadedClaimStore{ + loadErr: errors.New("claim store load failed"), + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err = p.Start(ctx) + if err == nil { + t.Fatal("Start() expected error on ClaimStore.Load failure, got nil") + } + defer p.Stop() + + // The orphan sweep must not have run: liveUID's CDI spec must not have + // been removed even though p.claims would have been empty. + for _, uid := range cdiW.removed { + if uid == liveUID { + t.Errorf("orphan sweep removed CDI spec for %v after a ClaimStore.Load failure — live claim consumer's CDI spec destroyed", liveUID) + } + } + if len(cdiW.removed) != 0 { + t.Errorf("CDIWriter.RemoveClaim called %d time(s), want 0 (orphan sweep must be skipped on Load failure)", len(cdiW.removed)) + } +} + +// TestStart_InactiveAllocator verifies that if AccountHpCpus returns an error +// (simulating an inactive allocator), all claims with CDI specs present are +// still kept with a warning (not dropped). +func TestStart_InactiveAllocator(t *testing.T) { + uid1 := types.UID("uid-1") + uid2 := types.UID("uid-2") + + alloc := &trackingClaimAllocator{accountErr: fmt.Errorf("test: %w", cpuclass.ErrAllocatorInactive), isHP: true} + + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{uid1: true, uid2: true}, + listResult: []types.UID{uid1, uid2}, + } + + store := &preloadedClaimStore{ + initial: map[types.UID]*ClaimState{ + uid1: {UID: string(uid1), Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}}, + uid2: {UID: string(uid2), Allocs: []ResultAlloc{{Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-7", ClassName: "gold"}}}, + }, + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + defer p.Stop() + + // Both claims must still be present despite AccountHpCpus errors. + if _, ok := p.claims[uid1]; !ok { + t.Error("uid1 missing from p.claims — should be kept despite AccountHpCpus error") + } + if _, ok := p.claims[uid2]; !ok { + t.Error("uid2 missing from p.claims — should be kept despite AccountHpCpus error") + } + // No drops occurred, so ClaimStore.Save must not have been called. + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0 (no drops)", store.saved) + } +} + +// TestStart_AccountHpCpus_FatalError verifies that Start fails when +// AccountHpCpus returns an error that does not wrap ErrAllocatorInactive +// (e.g. punit not found due to topology change). Registering in that state +// would leave CPUs unaccounted and allow double-allocation. +func TestStart_AccountHpCpus_FatalError(t *testing.T) { + uid1 := types.UID("uid-1") + + alloc := &trackingClaimAllocator{accountErr: errors.New("pct: AccountHpCpus: punit (pkg=0, punit=0) not found"), isHP: true} + + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{uid1: true}, + listResult: []types.UID{uid1}, + } + + store := &preloadedClaimStore{ + initial: map[types.UID]*ClaimState{ + uid1: {UID: string(uid1), Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}}, + }, + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err == nil { + p.Stop() + t.Fatal("Start() expected error for non-inactive AccountHpCpus failure, got nil") + } +} diff --git a/pkg/resmgr/dra/state.go b/pkg/resmgr/dra/state.go new file mode 100644 index 000000000..e5dea6921 --- /dev/null +++ b/pkg/resmgr/dra/state.go @@ -0,0 +1,128 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "encoding/json" + "fmt" + + "k8s.io/apimachinery/pkg/types" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" +) + +// draClaimsKey is the cache policy-entry key under which claim state is stored. +const draClaimsKey = "dra/claims" + +// ResultAlloc holds the allocated state for a single DeviceRequestAllocationResult. +// Allocs[i] corresponds to filtered allocation result i; CDI device-name index is +// positional — order must be preserved on rebuild. +type ResultAlloc struct { + // Request is r.Request from DeviceRequestAllocationResult. + Request string `json:"Request"` + // Pool is r.Pool from DeviceRequestAllocationResult. + Pool string `json:"Pool"` + // Device is r.Device (the DRA device name) from DeviceRequestAllocationResult. + Device string `json:"Device"` + // ShareID is string(*r.ShareID) or "" when r.ShareID is nil. + ShareID string `json:"ShareID"` + // ClassName is the value of the nri/cpuClass attribute. + ClassName string `json:"ClassName"` + // PkgID is the value of the nri/packageID attribute. + PkgID int `json:"PkgID"` + // PunitID is the value of the nri/punitID attribute. + PunitID int `json:"PunitID"` + // CPUs is the cpuset.CPUSet.String() representation of the allocated CPUs. + CPUs string `json:"CPUs"` +} + +// ClaimState holds the persisted state for a prepared ResourceClaim. +type ClaimState struct { + // UID is types.UID as string. + UID string `json:"UID"` + Allocs []ResultAlloc `json:"Allocs"` +} + +// marshalClaims encodes the claims map to a map[string]string (uid → JSON of ClaimState) +// suitable for storage via cache.SetPolicyEntry. +func marshalClaims(claims map[types.UID]*ClaimState) (map[string]string, error) { + out := make(map[string]string, len(claims)) + for uid, cs := range claims { + data, err := json.Marshal(cs) + if err != nil { + return nil, fmt.Errorf("dra: marshal claim %s: %w", uid, err) + } + out[string(uid)] = string(data) + } + return out, nil +} + +// unmarshalClaims decodes a map[string]string (as stored by marshalClaims) back +// into the claims map. +func unmarshalClaims(raw map[string]string) (map[types.UID]*ClaimState, error) { + out := make(map[types.UID]*ClaimState, len(raw)) + for uid, data := range raw { + var cs ClaimState + if err := json.Unmarshal([]byte(data), &cs); err != nil { + return nil, fmt.Errorf("dra: unmarshal claim %s: %w", uid, err) + } + out[types.UID(uid)] = &cs + } + return out, nil +} + +// cacheClaimStore implements ClaimStore using the resmgr cache as its backing +// store. Save is a no-op when the cache is in a BlockSave window; the data is +// still in the in-memory policyData map and will persist on the next unblocked +// Save. Callers of Save must tolerate this. +type cacheClaimStore struct { + c cache.Cache +} + +// NewCacheClaimStore returns a ClaimStore backed by the given resmgr cache. +func NewCacheClaimStore(c cache.Cache) ClaimStore { + return &cacheClaimStore{c: c} +} + +// Save persists the given claims map to the backing cache store. +func (s *cacheClaimStore) Save(claims map[types.UID]*ClaimState) error { + previous := map[string]string{} + s.c.GetPolicyEntry(draClaimsKey, &previous) + m, err := marshalClaims(claims) + if err != nil { + return err + } + s.c.SetPolicyEntry(draClaimsKey, m) + if err := s.c.Save(); err != nil { + s.c.SetPolicyEntry(draClaimsKey, previous) + return err + } + return nil +} + +// Load reads the claims map from the backing cache store. Returns nil, nil +// when no claim state has been saved yet. +func (s *cacheClaimStore) Load() (map[types.UID]*ClaimState, error) { + // Named variable is required — a composite-literal address passed directly + // to GetPolicyEntry is not addressable and the unmarshal target would be + // unreachable on a first-load path. + m := map[string]string{} + if !s.c.GetPolicyEntry(draClaimsKey, &m) { + return nil, nil + } + return unmarshalClaims(m) +} diff --git a/pkg/resmgr/dra/state_test.go b/pkg/resmgr/dra/state_test.go new file mode 100644 index 000000000..de7fe5e2a --- /dev/null +++ b/pkg/resmgr/dra/state_test.go @@ -0,0 +1,287 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "os" + "path/filepath" + "testing" + + "k8s.io/apimachinery/pkg/types" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// newTestCache creates a real cache in a temporary directory. The cache rejects +// directories with group/other write bits (reject mask 0022); we create the +// sub-directory with explicit 0700 permissions to avoid a t.TempDir() umask +// clash on systems with umask 0002 (which would give 0775). +func newTestCache(t *testing.T) cache.Cache { + t.Helper() + cacheDir := filepath.Join(t.TempDir(), "cache") + if err := os.Mkdir(cacheDir, 0700); err != nil { + t.Fatalf("newTestCache: mkdir %q: %v", cacheDir, err) + } + c, err := cache.NewCache(cache.Options{CacheDir: cacheDir}) + if err != nil { + t.Fatalf("newTestCache: cache.NewCache() error: %v", err) + } + return c +} + +// TestMarshalUnmarshalClaims_RoundTrip verifies that a multi-alloc claim +// survives a marshal → unmarshal round-trip without data loss. +func TestMarshalUnmarshalClaims_RoundTrip(t *testing.T) { + uid := types.UID("test-uid-1234") + originalCPUs := cpuset.New(0, 1, 2, 3) + + claims := map[types.UID]*ClaimState{ + uid: { + UID: string(uid), + Allocs: []ResultAlloc{ + { + Request: "req-hp", + Pool: "node1", + Device: "sst-cp-0-0", + ShareID: "", + ClassName: "hp", + PkgID: 0, + PunitID: 0, + CPUs: originalCPUs.String(), + }, + { + Request: "req-hp-2", + Pool: "node1", + Device: "sst-cp-0-1", + ShareID: "share-abc", + ClassName: "hp", + PkgID: 0, + PunitID: 1, + CPUs: cpuset.New(4, 5).String(), + }, + }, + }, + } + + raw, err := marshalClaims(claims) + if err != nil { + t.Fatalf("marshalClaims() unexpected error: %v", err) + } + + got, err := unmarshalClaims(raw) + if err != nil { + t.Fatalf("unmarshalClaims() unexpected error: %v", err) + } + + cs, ok := got[uid] + if !ok { + t.Fatalf("unmarshalClaims(): uid %q not found in result", uid) + } + if cs.UID != string(uid) { + t.Errorf("ClaimState.UID = %q, want %q", cs.UID, string(uid)) + } + if len(cs.Allocs) != 2 { + t.Fatalf("len(Allocs) = %d, want 2", len(cs.Allocs)) + } + + // Verify CPUs round-trip via cpuset.Parse. + parsedCPUs, err := cpuset.Parse(cs.Allocs[0].CPUs) + if err != nil { + t.Fatalf("cpuset.Parse(%q) error: %v", cs.Allocs[0].CPUs, err) + } + if !parsedCPUs.Equals(originalCPUs) { + t.Errorf("CPUs round-trip: got %v, want %v", parsedCPUs, originalCPUs) + } + + // Verify other fields. + if cs.Allocs[0].Request != "req-hp" { + t.Errorf("Allocs[0].Request = %q, want %q", cs.Allocs[0].Request, "req-hp") + } + if cs.Allocs[1].ShareID != "share-abc" { + t.Errorf("Allocs[1].ShareID = %q, want %q", cs.Allocs[1].ShareID, "share-abc") + } + if cs.Allocs[1].PunitID != 1 { + t.Errorf("Allocs[1].PunitID = %d, want 1", cs.Allocs[1].PunitID) + } +} + +// TestUnmarshalClaims_Empty verifies that an empty map round-trips to an empty +// map (not nil). +func TestUnmarshalClaims_Empty(t *testing.T) { + raw, err := marshalClaims(map[types.UID]*ClaimState{}) + if err != nil { + t.Fatalf("marshalClaims({}) error: %v", err) + } + got, err := unmarshalClaims(raw) + if err != nil { + t.Fatalf("unmarshalClaims({}) error: %v", err) + } + if len(got) != 0 { + t.Errorf("unmarshalClaims({}) = %v, want empty map", got) + } +} + +// TestCacheClaimStore_RoundTrip verifies that Save → Load preserves multi-alloc +// claim data using a real (file-backed) cache. +func TestCacheClaimStore_RoundTrip(t *testing.T) { + c := newTestCache(t) + + uid := types.UID("claim-abc") + originalCPUs := cpuset.New(10, 11, 12, 13) + + claims := map[types.UID]*ClaimState{ + uid: { + UID: string(uid), + Allocs: []ResultAlloc{ + { + Request: "req-0", + Pool: "node1", + Device: "dev-0", + ShareID: "", + ClassName: "hp", + PkgID: 1, + PunitID: 2, + CPUs: originalCPUs.String(), + }, + }, + }, + } + + store := NewCacheClaimStore(c) + if err := store.Save(claims); err != nil { + t.Fatalf("Save() error: %v", err) + } + + got, err := store.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if got == nil { + t.Fatal("Load() returned nil, want non-nil map") + } + cs, ok := got[uid] + if !ok { + t.Fatalf("Load(): uid %q not found", uid) + } + if len(cs.Allocs) != 1 { + t.Fatalf("Load(): len(Allocs) = %d, want 1", len(cs.Allocs)) + } + + // Verify CPUs survive round-trip. + parsedCPUs, err := cpuset.Parse(cs.Allocs[0].CPUs) + if err != nil { + t.Fatalf("cpuset.Parse(%q) error: %v", cs.Allocs[0].CPUs, err) + } + if !parsedCPUs.Equals(originalCPUs) { + t.Errorf("CPU round-trip: got %v, want %v", parsedCPUs, originalCPUs) + } +} + +// TestCacheClaimStore_EmptyCache verifies that Load returns nil, nil when +// no claim state has ever been saved. +func TestCacheClaimStore_EmptyCache(t *testing.T) { + c := newTestCache(t) + + store := NewCacheClaimStore(c) + got, err := store.Load() + if err != nil { + t.Fatalf("Load() on empty cache error: %v", err) + } + if got != nil { + t.Errorf("Load() on empty cache = %v, want nil", got) + } +} + +// TestCacheClaimStore_LoadReturnsSavedData verifies that Load returns the +// data previously saved (guarding against the composite-literal-address bug +// where the in-parameter to GetPolicyEntry would be unreachable). +func TestCacheClaimStore_LoadReturnsSavedData(t *testing.T) { + c := newTestCache(t) + + uid := types.UID("claim-xyz") + claims := map[types.UID]*ClaimState{ + uid: { + UID: string(uid), + Allocs: []ResultAlloc{ + { + Request: "req-x", + Pool: "pool-0", + Device: "dev-x", + ClassName: "hp", + PkgID: 0, + PunitID: 0, + CPUs: cpuset.New(7, 8).String(), + }, + }, + }, + } + + store := NewCacheClaimStore(c) + if err := store.Save(claims); err != nil { + t.Fatalf("Save() error: %v", err) + } + + // Immediately call Load on the same store instance (tests the in-memory path). + got, err := store.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(got) == 0 { + t.Fatalf("Load() returned empty map — composite-literal-address bug?") + } + if _, ok := got[uid]; !ok { + t.Errorf("Load() result missing uid %q", uid) + } +} + +// TestCacheClaimStore_MultiClaim verifies Save/Load with multiple claims in the +// same store entry. +func TestCacheClaimStore_MultiClaim(t *testing.T) { + c := newTestCache(t) + + uid1 := types.UID("claim-001") + uid2 := types.UID("claim-002") + + claims := map[types.UID]*ClaimState{ + uid1: {UID: string(uid1), Allocs: []ResultAlloc{ + {Request: "r1", Pool: "p", Device: "d1", ClassName: "hp", PkgID: 0, PunitID: 0, CPUs: cpuset.New(0).String()}, + }}, + uid2: {UID: string(uid2), Allocs: []ResultAlloc{ + {Request: "r2", Pool: "p", Device: "d2", ClassName: "hp", PkgID: 0, PunitID: 1, CPUs: cpuset.New(1).String()}, + }}, + } + + store := NewCacheClaimStore(c) + if err := store.Save(claims); err != nil { + t.Fatalf("Save() error: %v", err) + } + + got, err := store.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(got) != 2 { + t.Fatalf("Load() returned %d claims, want 2", len(got)) + } + if _, ok := got[uid1]; !ok { + t.Errorf("Load() missing uid %q", uid1) + } + if _, ok := got[uid2]; !ok { + t.Errorf("Load() missing uid %q", uid2) + } +} diff --git a/pkg/resmgr/main/main.go b/pkg/resmgr/main/main.go index b756366a6..127c62dbc 100644 --- a/pkg/resmgr/main/main.go +++ b/pkg/resmgr/main/main.go @@ -88,8 +88,8 @@ func (m *Main) Run() error { }() defer signal.Stop(sigCh) - err := m.mgr.Start() - return err + defer m.mgr.Stop() + return m.mgr.Start() } func (m *Main) ResourceManager() resmgr.ResourceManager { diff --git a/pkg/resmgr/main/main_test.go b/pkg/resmgr/main/main_test.go new file mode 100644 index 000000000..7ae067793 --- /dev/null +++ b/pkg/resmgr/main/main_test.go @@ -0,0 +1,105 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resmgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/events" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" +) + +// fakeResourceManager is a minimal ResourceManager implementation that +// records the order in which Start()/Stop() are invoked, so tests can +// assert that Main.Run() reaches Stop() after Start() returns. +type fakeResourceManager struct { + startErr error + calls []string +} + +func (f *fakeResourceManager) Start() error { + f.calls = append(f.calls, "start") + return f.startErr +} + +func (f *fakeResourceManager) Stop() { + f.calls = append(f.calls, "stop") +} + +func (f *fakeResourceManager) SendEvent(interface{}) error { return nil } + +// fakeBackend is a minimal policy.Backend implementation, just enough to +// satisfy Main.Run()'s use of m.policy.Name() (for tracing identity). +type fakeBackend struct{} + +func (fakeBackend) Name() string { return "fake" } +func (fakeBackend) Description() string { return "fake backend for tests" } +func (fakeBackend) Setup(*policyapi.BackendOptions) error { return nil } +func (fakeBackend) Reconfigure(interface{}) error { return nil } +func (fakeBackend) Start() error { return nil } +func (fakeBackend) Stop() error { return nil } +func (fakeBackend) Sync([]cache.Container, []cache.Container) error { return nil } +func (fakeBackend) AllocateResources(cache.Container) error { return nil } +func (fakeBackend) ReleaseResources(cache.Container) error { return nil } +func (fakeBackend) UpdateResources(cache.Container) error { return nil } +func (fakeBackend) HandleEvent(*events.Policy) (bool, error) { return false, nil } +func (fakeBackend) ExportResourceData(cache.Container) map[string]string { + return nil +} +func (fakeBackend) GetTopologyZones() []*policyapi.TopologyZone { return nil } +func (fakeBackend) GetExtendedResources() map[string]*resource.Quantity { + return nil +} + +var _ policyapi.Backend = fakeBackend{} + +// TestMainRunStopsResourceManagerAfterStartReturns verifies that Main.Run() +// calls mgr.Stop() once mgr.Start() returns, so that shutdown reaches the +// active policy's Backend.Stop() (e.g. the DRA plugin) even though nothing +// in the SIGTERM/SIGINT path calls mgr.Stop() directly. +func TestMainRunStopsResourceManagerAfterStartReturns(t *testing.T) { + mgr := &fakeResourceManager{} + m := &Main{ + policy: fakeBackend{}, + mgr: mgr, + } + + err := m.Run() + + require.NoError(t, err) + assert.Equal(t, []string{"start", "stop"}, mgr.calls) +} + +// TestMainRunStopsResourceManagerEvenOnStartError verifies Stop() is still +// reached (and the original Start() error still surfaces) when Start() +// itself fails. +func TestMainRunStopsResourceManagerEvenOnStartError(t *testing.T) { + startErr := assert.AnError + mgr := &fakeResourceManager{startErr: startErr} + m := &Main{ + policy: fakeBackend{}, + mgr: mgr, + } + + err := m.Run() + + assert.ErrorIs(t, err, startErr) + assert.Equal(t, []string{"start", "stop"}, mgr.calls) +} diff --git a/pkg/resmgr/nri.go b/pkg/resmgr/nri.go index 8e2c90384..8ac3b4fac 100644 --- a/pkg/resmgr/nri.go +++ b/pkg/resmgr/nri.go @@ -110,7 +110,7 @@ func (p *nriPlugin) start() error { } func (p *nriPlugin) stop() { - if p == nil { + if p == nil || p.stub == nil { return } @@ -302,6 +302,9 @@ func (p *nriPlugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, con m := p.resmgr + m.Lock() + defer m.Unlock() + allocated, released, err := p.syncWithNRI(pods, containers) if err != nil { nri.Errorf("failed to synchronize with NRI: %v", err) diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index a5fd9af76..ff6f4d952 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -20,6 +20,7 @@ import ( "sort" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" @@ -56,6 +57,14 @@ type ConstraintSet map[Domain]Constraint type Options struct { // SendEvent is the function for delivering events back to the resource manager. SendEvent SendEventFn + // KubeClientFn returns the shared kubernetes client, or a nil interface + // if none is available (yet). Evaluated by backends at Setup() time. + KubeClientFn func() kubernetes.Interface + // NodeName is the kubernetes node name the resource manager runs on. + NodeName string + // WithLock runs f while holding the resource manager's write lock. + // It is not re-entrant: calling WithLock again from within f deadlocks. + WithLock func(func()) } // BackendOptions describes the options for a policy backend instance @@ -68,6 +77,14 @@ type BackendOptions struct { SendEvent SendEventFn // Config is the policy-specific configuration. Config any + // KubeClientFn returns the shared kubernetes client, or a nil interface + // if none is available (yet). Evaluated by backends at Setup() time. + KubeClientFn func() kubernetes.Interface + // NodeName is the kubernetes node name the resource manager runs on. + NodeName string + // WithLock runs f while holding the resource manager's write lock. + // It is not re-entrant: calling WithLock again from within f deadlocks. + WithLock func(func()) } // CreateFn is the type for functions used to create a policy instance. @@ -102,6 +119,8 @@ type Backend interface { Reconfigure(any) error // Start up and sycnhronizes the policy, using the given cache and resource constraints. Start() error + // Stop shuts down the policy backend, releasing any resources it holds. + Stop() error // Sync synchronizes the policy, allocating/releasing the given containers. Sync([]cache.Container, []cache.Container) error // AllocateResources allocates resources to/for a container. @@ -140,6 +159,8 @@ type Policy interface { ActivePolicy() string // Start starts up policy, prepare for serving resource management requests. Start(any) error + // Stop shuts down the policy, releasing any resources it holds. + Stop() error // Reconfigure the policy. Reconfigure(any) error // Sync synchronizes the state of the active policy. @@ -267,10 +288,13 @@ func (p *policy) Start(cfg any) error { log.Infof("activating '%s' policy...", p.active.Name()) if err := p.active.Setup(&BackendOptions{ - Cache: p.cache, - System: p.system, - SendEvent: p.options.SendEvent, - Config: cfg, + Cache: p.cache, + System: p.system, + SendEvent: p.options.SendEvent, + Config: cfg, + KubeClientFn: p.options.KubeClientFn, + NodeName: p.options.NodeName, + WithLock: p.options.WithLock, }); err != nil { return err } @@ -284,6 +308,14 @@ func (p *policy) Start(cfg any) error { return p.active.Start() } +// Stop shuts down the active policy backend. +func (p *policy) Stop() error { + if p.active == nil { + return nil + } + return p.active.Stop() +} + // Reconfigure the policy. func (p *policy) Reconfigure(cfg any) error { scollect, err := p.newSystemCollector() diff --git a/pkg/resmgr/policy/policy_test.go b/pkg/resmgr/policy/policy_test.go new file mode 100644 index 000000000..7cc3f2c59 --- /dev/null +++ b/pkg/resmgr/policy/policy_test.go @@ -0,0 +1,249 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/events" +) + +// newTestCache creates a real, temp-dir-backed cache for tests that need a +// non-nil cache.Cache (e.g. because Start() drives metrics collection which +// reads from it). +func newTestCache(t *testing.T) cache.Cache { + t.Helper() + // Use a not-yet-existing subdirectory of t.TempDir(): NewCache + // validates the permissions of a pre-existing CacheDir and rejects + // group-writable ones, but t.TempDir() itself is created 0777&^umask, + // which is group-writable under a permissive umask. Handing NewCache a + // fresh path lets it create the directory itself with the permissions + // it expects. + c, err := cache.NewCache(cache.Options{CacheDir: filepath.Join(t.TempDir(), "cache")}) + require.NoError(t, err) + return c +} + +// mockBackend is a minimal Backend implementation that records calls and +// the options it was set up with, for use by policy-level unit tests. +type mockBackend struct { + setupOpts *BackendOptions + setupErr error + startErr error + stopErr error + reconfigErr error + startCalled bool + stopCalled bool + setupCalled bool + callOrder *[]string + onStart func() + onSetup func(*BackendOptions) +} + +func (m *mockBackend) Name() string { return "mock" } +func (m *mockBackend) Description() string { return "mock backend for tests" } + +func (m *mockBackend) Setup(opts *BackendOptions) error { + m.setupCalled = true + m.setupOpts = opts + if m.callOrder != nil { + *m.callOrder = append(*m.callOrder, "setup") + } + if m.onSetup != nil { + m.onSetup(opts) + } + return m.setupErr +} + +func (m *mockBackend) Reconfigure(interface{}) error { return m.reconfigErr } + +func (m *mockBackend) Start() error { + m.startCalled = true + if m.callOrder != nil { + *m.callOrder = append(*m.callOrder, "start") + } + if m.onStart != nil { + m.onStart() + } + return m.startErr +} + +func (m *mockBackend) Stop() error { + m.stopCalled = true + if m.callOrder != nil { + *m.callOrder = append(*m.callOrder, "stop") + } + return m.stopErr +} + +func (m *mockBackend) Sync([]cache.Container, []cache.Container) error { return nil } +func (m *mockBackend) AllocateResources(cache.Container) error { return nil } +func (m *mockBackend) ReleaseResources(cache.Container) error { return nil } +func (m *mockBackend) UpdateResources(cache.Container) error { return nil } +func (m *mockBackend) HandleEvent(*events.Policy) (bool, error) { return false, nil } +func (m *mockBackend) ExportResourceData(cache.Container) map[string]string { + return nil +} +func (m *mockBackend) GetTopologyZones() []*TopologyZone { return nil } +func (m *mockBackend) GetExtendedResources() map[string]*resource.Quantity { + return nil +} + +var _ Backend = &mockBackend{} + +func TestPolicyStopForwardsToBackend(t *testing.T) { + backend := &mockBackend{} + p, err := NewPolicy(backend, newTestCache(t), &Options{}) + require.NoError(t, err) + + require.NoError(t, p.Start(nil)) + assert.True(t, backend.startCalled) + + require.NoError(t, p.Stop()) + assert.True(t, backend.stopCalled) +} + +func TestPolicyStopNilActiveBackendDoesNotPanic(t *testing.T) { + p := &policy{} + assert.NotPanics(t, func() { + err := p.Stop() + assert.NoError(t, err) + }) +} + +func TestPolicyStartForwardsKubeClientFnNodeNameAndWithLock(t *testing.T) { + backend := &mockBackend{} + wantClient := fakeKubeClient{} + var lockCalls int + withLock := func(f func()) { + lockCalls++ + f() + } + + p, err := NewPolicy(backend, newTestCache(t), &Options{ + KubeClientFn: func() kubernetes.Interface { return wantClient }, + NodeName: "node-under-test", + WithLock: withLock, + }) + require.NoError(t, err) + require.NoError(t, p.Start(nil)) + + require.NotNil(t, backend.setupOpts) + require.NotNil(t, backend.setupOpts.KubeClientFn) + assert.Equal(t, wantClient, backend.setupOpts.KubeClientFn()) + assert.Equal(t, "node-under-test", backend.setupOpts.NodeName) + require.NotNil(t, backend.setupOpts.WithLock) + + // Exercise the forwarded WithLock: it should invoke the callback and + // go through the resmgr-supplied withLock, not some other function. + called := false + backend.setupOpts.WithLock(func() { called = true }) + assert.True(t, called) + assert.Equal(t, 1, lockCalls) +} + +func TestPolicyStartKubeClientFnNilWhenNoClient(t *testing.T) { + backend := &mockBackend{} + p, err := NewPolicy(backend, newTestCache(t), &Options{ + KubeClientFn: func() kubernetes.Interface { return nil }, + }) + require.NoError(t, err) + require.NoError(t, p.Start(nil)) + + require.NotNil(t, backend.setupOpts.KubeClientFn) + // Must be observed as a genuinely nil interface by the backend, not a + // non-nil interface wrapping a typed nil. + assert.Nil(t, backend.setupOpts.KubeClientFn()) +} + +// lockContractStub is a WithLock stand-in that panics if invoked while +// already "held", i.e. re-entrantly. It is used to assert that a backend's +// lifecycle calls made through WithLock never nest. +type lockContractStub struct { + mu sync.Mutex + held bool +} + +func (s *lockContractStub) run(f func()) { + s.mu.Lock() + if s.held { + s.mu.Unlock() + panic("WithLock invoked re-entrantly") + } + s.held = true + s.mu.Unlock() + + defer func() { + s.mu.Lock() + s.held = false + s.mu.Unlock() + }() + + f() +} + +// TestLockContractWithLockNotReentrant asserts that a backend driving two +// logically-sequential operations through WithLock (e.g. a future DRA +// plugin's Start() then PublishResources()) never nests those calls: the +// stub above would panic if it did. +func TestLockContractWithLockNotReentrant(t *testing.T) { + stub := &lockContractStub{} + var order []string + + backend := &mockBackend{ + onStart: func() { + // Simulate a backend that runs two operations under the + // resource manager's write lock, sequentially rather than + // nested. + stub.run(func() { order = append(order, "locked-op-1") }) + stub.run(func() { order = append(order, "locked-op-2") }) + }, + } + + p, err := NewPolicy(backend, newTestCache(t), &Options{WithLock: stub.run}) + require.NoError(t, err) + + assert.NotPanics(t, func() { + require.NoError(t, p.Start(nil)) + }) + assert.Equal(t, []string{"locked-op-1", "locked-op-2"}, order) + assert.False(t, stub.held) +} + +// TestLockContractReentrantCallPanics is a sanity check that the stub +// itself actually detects re-entrancy (guards against a vacuously-true +// contract test above). +func TestLockContractReentrantCallPanics(t *testing.T) { + stub := &lockContractStub{} + assert.Panics(t, func() { + stub.run(func() { + stub.run(func() {}) + }) + }) +} + +// fakeKubeClient is a minimal kubernetes.Interface stand-in used only for +// identity comparison in tests; its methods are never called. +type fakeKubeClient struct { + kubernetes.Interface +} diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index 596ca6dea..e556d933d 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -30,6 +30,7 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/topology" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/yaml" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1" @@ -223,10 +224,32 @@ func (m *resmgr) start(cfg cfgapi.ResmgrConfig) error { func (m *resmgr) Stop() { log.Infof("shutting down...") + // Stop the active policy backend (e.g. the DRA plugin) before + // acquiring the write lock: an in-flight Prepare/AllocateResources + // call may be holding the lock via WithLock, and the backend's + // Stop() must be able to run (and complete) without waiting on it. + // Calling it under the lock would deadlock in that case. + if m.policy != nil { + if err := m.policy.Stop(); err != nil { + log.Warnf("failed to stop policy: %v", err) + } + } + m.Lock() defer m.Unlock() - m.nri.stop() + if m.nri != nil { + m.nri.stop() + } +} + +// withWriteLock runs f while holding the resource manager's write lock. +// It is not re-entrant: calling withWriteLock again from within f deadlocks, +// since the underlying mutex is not recursive. +func (m *resmgr) withWriteLock(f func()) { + m.Lock() + defer m.Unlock() + f() } // setupCache creates a cache and reloads its last saved state if found. @@ -254,7 +277,12 @@ func (m *resmgr) setupPolicy(backend policy.Backend) error { log.Warnf("failed to set active policy: %v", err) } - p, err := policy.NewPolicy(backend, m.cache, &policy.Options{SendEvent: m.SendEvent}) + p, err := policy.NewPolicy(backend, m.cache, &policy.Options{ + SendEvent: m.SendEvent, + KubeClientFn: m.kubeClientFn, + NodeName: m.agent.NodeName(), + WithLock: m.withWriteLock, + }) if err != nil { return resmgrError("failed to create policy %s: %v", backend.Name(), err) } @@ -263,6 +291,19 @@ func (m *resmgr) setupPolicy(backend policy.Backend) error { return nil } +// kubeClientFn returns the resource manager's shared kubernetes client, or +// a nil interface if the agent has none (yet). Wrapping agent.KubeClient() +// directly as a kubernetes.Interface would yield a non-nil interface +// wrapping a typed nil *client.Client in local-config mode; the explicit +// nil check below avoids that trap. +func (m *resmgr) kubeClientFn() kubernetes.Interface { + c := m.agent.KubeClient() + if c == nil { + return nil + } + return c +} + // setupHealthCheck prepares the resource manager for serving health-check requests. func (m *resmgr) setupHealthCheck() { mux := instrumentation.HTTPServer().GetMux() diff --git a/pkg/resmgr/resource_manager_test.go b/pkg/resmgr/resource_manager_test.go new file mode 100644 index 000000000..850a77b93 --- /dev/null +++ b/pkg/resmgr/resource_manager_test.go @@ -0,0 +1,152 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resmgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/containers/nri-plugins/pkg/agent" + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/events" + "github.com/containers/nri-plugins/pkg/resmgr/policy" +) + +// fakePolicy is a minimal policy.Policy implementation that records Stop() +// calls (and, optionally, runs a probe from within Stop()) for use by +// resmgr-level shutdown-ordering tests. +type fakePolicy struct { + stopCalled int + stopErr error + onStop func() +} + +func (f *fakePolicy) ActivePolicy() string { return "fake" } +func (f *fakePolicy) Start(interface{}) error { return nil } +func (f *fakePolicy) Stop() error { + f.stopCalled++ + if f.onStop != nil { + f.onStop() + } + return f.stopErr +} +func (f *fakePolicy) Reconfigure(interface{}) error { return nil } +func (f *fakePolicy) Sync([]cache.Container, []cache.Container) error { return nil } +func (f *fakePolicy) AllocateResources(cache.Container) error { return nil } +func (f *fakePolicy) ReleaseResources(cache.Container) error { return nil } +func (f *fakePolicy) UpdateResources(cache.Container) error { return nil } +func (f *fakePolicy) HandleEvent(*events.Policy) (bool, error) { return false, nil } +func (f *fakePolicy) ExportResourceData(cache.Container) {} +func (f *fakePolicy) GetTopologyZones() []*policy.TopologyZone { return nil } +func (f *fakePolicy) GetExtendedResources() map[string]*resource.Quantity { return nil } + +var _ policy.Policy = &fakePolicy{} + +// TestResmgrStopCallsPolicyStop verifies that resmgr.Stop() reaches the +// active policy's Stop() method (which, in turn, is how Backend.Stop() -- +// e.g. the DRA plugin shutdown -- gets invoked). +func TestResmgrStopCallsPolicyStop(t *testing.T) { + fp := &fakePolicy{} + m := &resmgr{policy: fp} + + m.Stop() + + assert.Equal(t, 1, fp.stopCalled) +} + +// TestResmgrStopCallsPolicyStopBeforeLock verifies that resmgr.Stop() calls +// policy.Stop() before acquiring the resource manager's write lock: an +// in-flight Prepare/AllocateResources call may be holding that lock via +// WithLock, and policy.Stop() must not have to wait for it. +func TestResmgrStopCallsPolicyStopBeforeLock(t *testing.T) { + m := &resmgr{} + fp := &fakePolicy{ + onStop: func() { + // If resmgr.Stop() had already acquired the write lock before + // calling policy.Stop(), this TryLock would fail. + locked := m.TryLock() + require.True(t, locked, "resmgr write lock must not be held while policy.Stop() runs") + m.Unlock() + }, + } + m.policy = fp + + m.Stop() + + assert.Equal(t, 1, fp.stopCalled) +} + +// TestResmgrStopNilPolicyDoesNotPanic verifies that Stop() is safe to call +// when Start() was never called (m.policy is nil). +func TestResmgrStopNilPolicyDoesNotPanic(t *testing.T) { + m := &resmgr{} + assert.NotPanics(t, func() { + m.Stop() + }) +} + +// TestResmgrStopNilNriDoesNotPanic verifies that Stop() remains safe when the +// NRI plugin was never started, so startup failures before m.nri.start() do not +// end up dereferencing a nil stub. +func TestResmgrStopNilNriDoesNotPanic(t *testing.T) { + m := &resmgr{nri: &nriPlugin{}} + assert.NotPanics(t, func() { + m.Stop() + }) +} + +// TestNriPluginStopNilStubDoesNotPanic verifies that shutdown is safe when the +// plugin exists but its stub was never created during a failed startup path. +func TestNriPluginStopNilStubDoesNotPanic(t *testing.T) { + p := &nriPlugin{} + assert.NotPanics(t, func() { + p.stop() + }) +} + +// TestWithWriteLockRunsCallbackUnderLock verifies that withWriteLock +// acquires the resource manager's write lock for the duration of the +// callback. +func TestWithWriteLockRunsCallbackUnderLock(t *testing.T) { + m := &resmgr{} + ran := false + + m.withWriteLock(func() { + ran = true + assert.False(t, m.TryLock(), "lock must be held while the withWriteLock callback runs") + }) + + assert.True(t, ran) + // The lock must have been released once withWriteLock returned. + assert.True(t, m.TryLock()) + m.Unlock() +} + +// TestKubeClientFnNilWhenAgentHasNoClient verifies the typed-nil-safe +// wrapper: in local-config mode agent.KubeClient() returns a nil +// *client.Client, and m.kubeClientFn() must surface that as a genuinely +// nil kubernetes.Interface rather than a non-nil interface wrapping a +// typed nil pointer. +func TestKubeClientFnNilWhenAgentHasNoClient(t *testing.T) { + agt, err := agent.New(agent.TopologyAwareConfigInterface(), agent.WithConfigFile("/nonexistent/config.yaml")) + require.NoError(t, err) + + m := &resmgr{agent: agt} + + assert.Nil(t, m.kubeClientFn()) +} diff --git a/test/e2e/files/Vagrantfile.in b/test/e2e/files/Vagrantfile.in index f40a314a0..1f622d5b8 100644 --- a/test/e2e/files/Vagrantfile.in +++ b/test/e2e/files/Vagrantfile.in @@ -19,6 +19,8 @@ KERNEL_GETSOURCE = "#{ENV['kernel_getsource']}" KERNEL_CONFIG = "#{ENV['kernel_config']}" K8S_RELEASE = "#{ENV['k8s_release']}" K8S_VERSION = "#{ENV['k8s_version']}" +K8S_FEATURE_GATES = "#{ENV['k8s_feature_gates']}" +K8S_LOG_VERBOSITY = "#{ENV['k8s_log_verbosity']}" HELM_RELEASE = "#{ENV['helm_release']}" CRI_RUNTIME = "#{ENV['k8scri']}" CRIO_RELEASE = "1.28.1" @@ -125,6 +127,8 @@ Vagrant.configure("2") do |config| kernel_config: KERNEL_CONFIG, k8s_release: K8S_RELEASE, k8s_version: K8S_VERSION, + k8s_feature_gates: K8S_FEATURE_GATES, + k8s_log_verbosity: K8S_LOG_VERBOSITY, helm_release: HELM_RELEASE, cri_runtime: CRI_RUNTIME, containerd_release: CONTAINERD_RELEASE, diff --git a/test/e2e/playbook/provision.yaml b/test/e2e/playbook/provision.yaml index be961c0bf..0a540bb8f 100644 --- a/test/e2e/playbook/provision.yaml +++ b/test/e2e/playbook/provision.yaml @@ -7,6 +7,12 @@ cri_runtime: "{{ cri_runtime }}" k8s_release: "{{ k8s_release }}" k8s_version: "{{ k8s_version }}" + # Comma-separated "Gate1=true,Gate2=false" list, or empty string. + k8s_feature_gates: "{{ k8s_feature_gates }}" + # klog verbosity level (e.g. "4") for apiserver/scheduler/controller-manager/ + # kubelet, or empty to leave every component at its default verbosity. + # Independent of k8s_feature_gates -- either can be set without the other. + k8s_log_verbosity: "{{ k8s_log_verbosity }}" is_containerd: false is_crio: false containerd_release: "{{ containerd_release }}" @@ -501,9 +507,70 @@ daemon_reload: true state: restarted + - name: Write kubeadm --config with feature gates and/or log verbosity for apiserver/scheduler/controller-manager/kubelet + ansible.builtin.copy: + dest: /root/kubeadm-config.yaml + content: | + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration + networking: + podSubnet: "{{ network }}" + apiServer: + extraArgs: + {% if k8s_feature_gates != "" %} + - name: feature-gates + value: "{{ k8s_feature_gates }}" + {% endif %} + {% if k8s_log_verbosity != "" %} + - name: v + value: "{{ k8s_log_verbosity }}" + {% endif %} + scheduler: + extraArgs: + {% if k8s_feature_gates != "" %} + - name: feature-gates + value: "{{ k8s_feature_gates }}" + {% endif %} + {% if k8s_log_verbosity != "" %} + - name: v + value: "{{ k8s_log_verbosity }}" + {% endif %} + {% if k8s_feature_gates != "" or k8s_log_verbosity != "" %} + controllerManager: + extraArgs: + {% if k8s_feature_gates != "" %} + - name: feature-gates + value: "{{ k8s_feature_gates }}" + {% endif %} + {% if k8s_log_verbosity != "" %} + - name: v + value: "{{ k8s_log_verbosity }}" + {% endif %} + {% endif %} + --- + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + {% if k8s_feature_gates != "" %} + featureGates: + {% for gate in k8s_feature_gates.split(',') if gate.strip() and '=' in gate %} + {{ gate.split('=')[0] }}: {{ gate.split('=')[1] }} + {% endfor %} + {% endif %} + {% if k8s_log_verbosity != "" %} + logging: + verbosity: {{ k8s_log_verbosity }} + {% endif %} + when: k8s_feature_gates != "" or k8s_log_verbosity != "" + - name: Initialize the Kubernetes cluster using kubeadm ansible.builtin.command: cmd: kubeadm init --pod-network-cidr="{{ network }}" + when: k8s_feature_gates == "" and k8s_log_verbosity == "" + + - name: Initialize the Kubernetes cluster using kubeadm (with feature gates and/or log verbosity) + ansible.builtin.command: + cmd: kubeadm init --config /root/kubeadm-config.yaml + when: k8s_feature_gates != "" or k8s_log_verbosity != "" - name: Setup kubeconfig for vagrant user ansible.builtin.command: "{{ item }}" diff --git a/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in b/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in index 894e4dca3..0c9020169 100644 --- a/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in +++ b/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in @@ -29,6 +29,10 @@ config: $([ -n "$CPU_CLASSES" ] && echo " cpuClasses: $CPU_CLASSES ") + $([ -n "$DRA_ENABLED" ] && echo " + dra: + enabled: $DRA_ENABLED + ") $([ -n "$DEFAULT_EXCLUSIVE_CPUCLASS" ] && echo " defaultExclusiveCPUClass: $DEFAULT_EXCLUSIVE_CPUCLASS ") diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test20-dra/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test20-dra/code.var.sh new file mode 100644 index 000000000..b76d5252a --- /dev/null +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test20-dra/code.var.sh @@ -0,0 +1,276 @@ +# test20-dra: DRA (KEP-5075/KEP-5517) integration for the topology-aware +# policy. +# +# Full test: launches the plugin with the SST/cpufreq mock and a +# hp-turbo cpuClass, probes the API server for the KEP-5075/KEP-5517 +# feature gates (skipping cleanly if either is missing), applies a +# ResourceClaim selecting the HP class by its published nri/pctPriority +# attribute plus a pod consuming it, and asserts the claimed CPUs' +# CDI env vars, their CLOS association in the policy log, and the +# scheduler's pod.status.nodeAllocatableResourceClaimStatuses[] record. + +cleanup() { + vm-command "kubectl delete pods --all --now" + vm-command "kubectl delete resourceclaims --all --now" + helm-terminate +} + +# compress-cpulist "8 9 10 12" prints "8-10,12" -- range-collapsing, +# same python3 idiom as test19-cpuclass's expand-cpulist (its inverse). +# Needed because pct.go:786's "associated cpus %s to CLOS %d" log line +# formats its cpuset with +# cpuset.CPUSet.String(), which collapses contiguous ids into ranges, +# so individual/comma-joined ids parsed out of NRI_CPU env vars +# would never match assert-cpu-clos's regex without this. +compress-cpulist() { + local cpus="$1" + + python3 -c ' +import sys +ids = sorted(int(x) for x in sys.argv[1].split()) +ranges = [] +start = prev = None +for i in ids: + if start is None: + start = prev = i + elif i == prev + 1: + prev = i + else: + ranges.append((start, prev)) + start = prev = i +if start is not None: + ranges.append((start, prev)) +print(",".join(str(a) if a == b else "%d-%d" % (a, b) for a, b in ranges)) +' "$cpus" +} + +# wait-node-allocatable-claim-status [timeout=30] [interval=2] +# Polls pod.status.nodeAllocatableResourceClaimStatuses[] until it +# contains an entry with resourceClaimName == , +# listed in .containers, and a .mapping[] entry {name: cpu, quantity: +# }, or fails with command-error on timeout. +wait-node-allocatable-claim-status() { + local pod="$1" claim="$2" ctr="$3" cpus="$4" timeout=${5:-30} interval=${6:-2} elapsed=0 + while [ "$elapsed" -lt "$timeout" ]; do + vm-command "kubectl get pod $pod -o json | jq -c \ + '[(.status.nodeAllocatableResourceClaimStatuses // [])[] | \ + select(.resourceClaimName == \"$claim\" and \ + (.containers // [] | index(\"$ctr\")) and \ + ((.mapping // []) | any(.name == \"cpu\" and .quantity == \"$cpus\")))] | length'" + [ "$COMMAND_OUTPUT" -gt 0 ] 2>/dev/null && return 0 + sleep "$interval" + elapsed=$((elapsed + interval)) + done + vm-command "kubectl get pod $pod -o jsonpath='{.status.nodeAllocatableResourceClaimStatuses}'" + command-error "pod $pod's status.nodeAllocatableResourceClaimStatuses missing entry {resourceClaimName: $claim, containers: [$ctr], mapping: [{name: cpu, quantity: $cpus}]} (got: $COMMAND_OUTPUT)" +} + +# wait-resourceslice-devices [timeout] +# Polls until the DRA driver's published ResourceSlice(s) report at +# least one device, storing a compact JSON array of +# {allowMultipleAllocations, nodeAllocatableResources} objects +# (one per device) in COMMAND_OUTPUT. The driver needs a moment after +# helm-launch to publish its ResourceSlice, so this can't be a single +# one-shot check. +wait-resourceslice-devices() { + local timeout=${1:-30} elapsed=0 + while [ "$elapsed" -lt "$timeout" ]; do + vm-command "kubectl get resourceslices -o json | \ + jq -c '[.items[] | select(.spec.driver == \"nri.topology-aware.cpu\") | (.spec.devices // [])[] | {name, capacity: .capacity[\"nri/cpus\"].value, class: .attributes[\"nri/cpuClass\"].string, priority: .attributes[\"nri/pctPriority\"].string, allowMultipleAllocations, nodeAllocatableResources}]'" + if [ -n "$COMMAND_OUTPUT" ] && [ "$COMMAND_OUTPUT" != "[]" ]; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + return 1 +} + +OVERRIDE_SYS_CPUFREQ='[{"cpus": "0-15", "base": 2900000, "min": 800000, "max": 3800000}]' +OVERRIDE_SST='{"supported": true, "clos_count": 4, "packages": [{"id": 0, "cpus": "0-7", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}, {"id": 1, "cpus": "8-15", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}]}' +OVERRIDE_SST_STATE_DIR="/tmp/nri-pct-mock" + +CPU_CLASSES="[ + { name: hp-turbo, pctPriority: high, pctMinFreq: turbo, pctMaxFreq: turbo }, + { name: shared , minFreq: min, maxFreq: base } ]" +SHARED_CPUCLASS="shared" +DEBUG_LOGGERS="agent cpu cpuclass" +DRA_ENABLED=true + +cleanup + +helm_config=$(COLOCATE_PODS=false \ + DEBUG_LOGGERS="$DEBUG_LOGGERS" \ + CPU_CLASSES="$CPU_CLASSES" \ + SHARED_CPUCLASS="$SHARED_CPUCLASS" \ + DRA_ENABLED="$DRA_ENABLED" \ + EXTRA_ENV_OVERRIDE_SYS_CPUFREQ="$OVERRIDE_SYS_CPUFREQ" \ + EXTRA_ENV_OVERRIDE_SST="$OVERRIDE_SST" \ + EXTRA_ENV_OVERRIDE_SST_STATE_DIR="$OVERRIDE_SST_STATE_DIR" \ + instantiate helm-config.yaml) helm-launch topology-aware + +# +# Feature-gate probe. +# +# If the API server's alpha gates DRAConsumableCapacity/DRANodeAllocatableResources +# are off, it silently strips both fields on write, so they're absent on read back here. +# This requires the plugin to already be running (hence it happens after helm-launch, +# not before), and with the hp-turbo class + SST mock active above so there's +# at least one real device to read back (an empty CPU_CLASSES publishes zero devices). +wait-resourceslice-devices 30 || { + helm-terminate + error "no devices found in any ResourceSlice within timeout (not a feature-gate issue -- check DRA_ENABLED/CPU_CLASSES/SST mock config)" +} + +# Require both fields on the *same* device object, not merely present +# somewhere in the (possibly multi-device) list -- every device this +# driver publishes gets both fields set unconditionally, so checking +# them independently across the whole list would loosely pass even if, +# say, one device kept allowMultipleAllocations while a different +# device kept nodeAllocatableResources; that's not what either gate +# being enabled actually implies. +if jq -e 'any(.[]; .allowMultipleAllocations == true and (.nodeAllocatableResources != null))' \ + >/dev/null 2>&1 <<< "$COMMAND_OUTPUT"; then + echo "DRA feature gates (KEP-5075/KEP-5517) detected as enabled on the API server; continuing." +else + helm-terminate + echo "Test verdict: SKIP (KEP-5075/KEP-5517 feature gate missing)" + exit 0 +fi + +# +# Claim + pod. +# +# A ResourceClaim selecting the HP cpuClass by its published +# nri/pctPriority attribute. +# +# Only "objects create cleanly and the pod reaches Running" is checked +# here + +# Bypassing create() also bypasses its image pre-pull step -- pull the +# pod's image explicitly so a freshly created VM doesn't stall/fail on +# kubectl's own on-demand pull. +vm-command "crictl -i unix://${k8scri_sock} pull quay.io/prometheus/busybox" || + command-error "failed to pre-pull quay.io/prometheus/busybox" + +# Manifests are written to local temp files and copied to the VM with +# vm-put-file, then kubectl apply'd -- this sidesteps vm-command's +# double-quoting/escaping entirely (the YAML below needs literal +# double quotes for its CEL string literals and the "2" capacity +# request). +claim_yaml=$(mktemp) +cat <<'EOF' > "$claim_yaml" +apiVersion: resource.k8s.io/v1 +kind: ResourceClaim +metadata: + name: hp-turbo-cpus +spec: + devices: + requests: + - name: cpus + exactly: + deviceClassName: nri.topology-aware.cpu + capacity: + requests: + nri/cpus: "2" + selectors: + - cel: + expression: | + device.attributes["nri"].pctPriority == "high" +EOF +vm-put-file --cleanup "$claim_yaml" hp-turbo-cpus-claim.yaml +vm-command "kubectl apply -f hp-turbo-cpus-claim.yaml" || + command-error "failed to create ResourceClaim hp-turbo-cpus" + +# The consuming pod. Not routed through create(): create() defaults to +# wait=Ready, which a ResourceClaim never satisfies on its own (it has +# no Ready condition), and guaranteed.yaml.in (the shared template +# create() uses, shared by ~23 other tests) has no +# resourceClaims/resources.claims support -- not modified here. +pod_yaml=$(mktemp) +cat <<'EOF' > "$pod_yaml" +apiVersion: v1 +kind: Pod +metadata: + name: dra-pod0 +spec: + resourceClaims: + - name: cpus + resourceClaimName: hp-turbo-cpus + containers: + - name: dra-pod0c0 + image: quay.io/prometheus/busybox + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo dra-pod0c0 $(sleep inf) + resources: + claims: + - name: cpus + requests: + cpu: "1" + memory: "100M" + limits: + cpu: "1" + memory: "100M" + terminationGracePeriodSeconds: 1 +EOF +vm-put-file --cleanup "$pod_yaml" dra-pod0.yaml +vm-command "kubectl apply -f dra-pod0.yaml" || + command-error "failed to create pod dra-pod0" + +vm-command "kubectl wait --for=condition=Ready pod/dra-pod0 --timeout=60s" || + command-error "pod dra-pod0 did not reach Running" + +# +# CLOS association, env vars, allocatable deduction, cleanup. +# + +# Env vars: CDI writes NRI_CLASS/NRI_CPU into the OCI spec at +# container-creation time (pkg/resmgr/dra/cdi.go); they're never +# written back to the pod object, so this must be +# `kubectl exec ... -- env`, not an `-o json` read of the pod/container status. +vm-command "kubectl exec dra-pod0 -c dra-pod0c0 -- env" +env_output="$COMMAND_OUTPUT" + +grep -q '^NRI_CLASS=hp-turbo$' <<< "$env_output" || + command-error "missing/incorrect NRI_CLASS env var in dra-pod0c0 (expected hp-turbo)" + +# Parse the claimed CPU ids out of the NRI_CPU=1 env vars. +claimed_cpus=$(grep -oE '^NRI_CPU[0-9]+=1$' <<< "$env_output" | \ + sed -E 's/^NRI_CPU([0-9]+)=1$/\1/' | sort -n | tr '\n' ' ') +claimed_cpus="${claimed_cpus% }" + +[ -n "$claimed_cpus" ] || + command-error "no NRI_CPU=1 env vars found in dra-pod0c0's environment" + +claimed_cpu_count=$(wc -w <<< "$claimed_cpus") +[ "$claimed_cpu_count" -eq 2 ] || + command-error "expected exactly 2 claimed CPUs via NRI_CPU env vars (claim requested nri/cpus: \"2\"), got $claimed_cpu_count ($claimed_cpus)" + +# The container's actual cpuset must include every claimed CPU id, not +# just the CDI-injected env vars: applyGrant unions the container's +# claimed CPUs into the cpuset it pins, so dra-pod0c0's live cpuset +# (read via Cpus_allowed_list) must be a superset of claimed_cpus. +missing_cpus=$(cpulist-difference "$claimed_cpus" "$(container-cpus dra-pod0 dra-pod0c0)") +[ -z "$missing_cpus" ] || + command-error "claimed CPUs ($claimed_cpus) missing from dra-pod0c0's actual cpuset (missing: $missing_cpus)" + +# CLOS association: the pct.go "associated cpus %s to CLOS %d" line +# -- not the mock's startup-time ConfigureClos line, which would pass +# even if the claim were never allocated. The log line formats the +# cpuset with cpuset.CPUSet.String(), which collapses contiguous ids +# into ranges, so compress the env-derived ids into the same form +# before matching. +assert-cpu-clos "$(compress-cpulist "$claimed_cpus")" "CLOS 0" \ + "Missing CPU association for dra-pod0c0 (expected CLOS 0)" + +# KEP-5517 allocation record: node.status.allocatable.cpu is never +# mutated by DRANodeAllocatableResources. The actual persisted signal +# is pod.status.nodeAllocatableResourceClaimStatuses[], written by the +# scheduler at PreBind. Assert it names the claim, the consuming container, +# and the claimed CPU count. +wait-node-allocatable-claim-status dra-pod0 hp-turbo-cpus dra-pod0c0 "$claimed_cpu_count" 30 2 + +cleanup diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 37b55c798..fb4d7e15f 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -51,6 +51,24 @@ export k8s_release=${k8s_release:-"latest"} k8s_release="${k8s_release#v}" export k8s_version="" +# Comma-separated "Gate1=true,Gate2=false" list passed to +# kubeadm's kube-apiserver/scheduler --feature-gates extraArgs and to +# the kubelet's KubeletConfiguration.featureGates. Unset (default): +# provision.yaml runs its plain "kubeadm init --pod-network-cidr=..." +# exactly as before, no feature-gate plumbing at all. +# +# NOTE for callers enabling DRAConsumableCapacity/DRANodeAllocatableResources +# (KEP-5075/KEP-5517): also pin k8s_release explicitly to >=1.37, e.g. +# k8s_release=1.37 k8s_feature_gates="DRAConsumableCapacity=true,DRANodeAllocatableResources=true" ... +# as DRANodeAllocatableResources requires Kubernetes 1.37+. +export k8s_feature_gates=${k8s_feature_gates:-} + +# klog verbosity level (e.g. "4") passed to kube-apiserver's, kube-scheduler's, +# and kube-controller-manager's --v extraArgs, and to the kubelet's +# KubeletConfiguration.logging.verbosity. Unset (default): +# no verbosity plumbing, components log at their normal default level. +export k8s_log_verbosity=${k8s_log_verbosity:-} + GH_HELM_REPO="helm/helm" export helm_release=${helm_release:-"latest"} @@ -274,9 +292,11 @@ echo " EFI boot = ${efi:-no}" echo " Distro = $distro" echo " Distro image = ${distro_img:-vagrant default}" echo " Kubernetes" -echo " - release = $k8s_release" -echo " - version = $k8s_version" -echo " - Helm = $helm_release" +echo " - release = $k8s_release" +echo " - version = $k8s_version" +echo " - Helm = $helm_release" +echo " - feature gates = ${k8s_feature_gates:-none}" +echo " - log verbosity = ${k8s_log_verbosity:-none}" echo " Runtime = $k8scri" echo " Output dir = $OUTPUT_DIR" echo " Test output dir = $TEST_OUTPUT_DIR" diff --git a/test/e2e/run_tests.sh b/test/e2e/run_tests.sh index acb30f7a5..a7508db3b 100755 --- a/test/e2e/run_tests.sh +++ b/test/e2e/run_tests.sh @@ -233,8 +233,10 @@ for POLICY_DIR in "$TESTS_ROOT_DIR"/*; do policy_name="$(basename $POLICY_DIR)" - # Create name for the vm. - export vm_name=$(vm-create-name "$k8scri" "$(basename "$TOPOLOGY_DIR")" ${distro}) + # Create name for the vm. A caller-supplied vm_name (e.g. to + # keep a gated and an ungated VM from fighting over one + # vagrant dir) is preserved rather than clobbered. + export vm_name=${vm_name:-$(vm-create-name "$k8scri" "$(basename "$TOPOLOGY_DIR")" ${distro})} export-and-source-dir "$TOPOLOGY_DIR" # Create ansible inventory file from a template