From 8e0dd937c52d71390821242e3a78b9a01700458c Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 17:38:15 +0300 Subject: [PATCH 01/39] dra: introduce pkg/resmgr/dra plugin skeleton Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- pkg/resmgr/dra/deps.go | 21 +++++++++++++++++++++ pkg/resmgr/dra/doc.go | 23 +++++++++++++++++++++++ pkg/resmgr/dra/plugin.go | 29 +++++++++++++++++++++++++++++ pkg/resmgr/dra/plugin_test.go | 35 +++++++++++++++++++++++++++++++++++ 4 files changed, 108 insertions(+) create mode 100644 pkg/resmgr/dra/deps.go create mode 100644 pkg/resmgr/dra/doc.go create mode 100644 pkg/resmgr/dra/plugin.go create mode 100644 pkg/resmgr/dra/plugin_test.go diff --git a/pkg/resmgr/dra/deps.go b/pkg/resmgr/dra/deps.go new file mode 100644 index 000000000..23fd8925d --- /dev/null +++ b/pkg/resmgr/dra/deps.go @@ -0,0 +1,21 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +// Deps holds the dependencies a policy binary must supply when +// constructing a Plugin. +type Deps struct{} diff --git a/pkg/resmgr/dra/doc.go b/pkg/resmgr/dra/doc.go new file mode 100644 index 000000000..a97077c90 --- /dev/null +++ b/pkg/resmgr/dra/doc.go @@ -0,0 +1,23 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package dra provides a policy-agnostic DRA (Dynamic Resource Allocation) +// kubelet plugin used by nri-plugins policies. +// +// This package must never import from github.com/containers/nri-plugins/cmd/ +// so that it can be consumed by any policy binary without introducing import +// cycles. +package dra diff --git a/pkg/resmgr/dra/plugin.go b/pkg/resmgr/dra/plugin.go new file mode 100644 index 000000000..256159559 --- /dev/null +++ b/pkg/resmgr/dra/plugin.go @@ -0,0 +1,29 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import "errors" + +var errNotImplemented = errors.New("dra plugin: not yet implemented") + +// Plugin is the DRA kubelet plugin. +type Plugin struct{} + +// New constructs a Plugin with the given driver name and dependencies. +func New(driverName string, deps Deps) (*Plugin, error) { + return nil, errNotImplemented +} diff --git a/pkg/resmgr/dra/plugin_test.go b/pkg/resmgr/dra/plugin_test.go new file mode 100644 index 000000000..dd19b3591 --- /dev/null +++ b/pkg/resmgr/dra/plugin_test.go @@ -0,0 +1,35 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "errors" + "testing" +) + +func TestNew_ReturnsNotImplemented(t *testing.T) { + p, err := New("test-driver", Deps{}) + if p != nil { + t.Errorf("New() returned non-nil Plugin, want nil") + } + if err == nil { + t.Fatal("New() returned nil error, want non-nil") + } + if !errors.Is(err, errNotImplemented) { + t.Errorf("New() error = %v, want errors.Is(err, errNotImplemented) == true", err) + } +} From c0a2b4c5951669276467eff15753f2626ed2abb7 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 21:43:16 +0300 Subject: [PATCH 02/39] cpuclass: add dra.publish config field and validation Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- .../bases/config.nri_balloonspolicies.yaml | 12 ++ .../config.nri_topologyawarepolicies.yaml | 12 ++ .../crds/config.nri_balloonspolicies.yaml | 12 ++ .../config.nri_topologyawarepolicies.yaml | 12 ++ .../config/v1alpha1/resmgr/policy/cpuclass.go | 23 +++ .../v1alpha1/resmgr/policy/cpuclass_test.go | 82 ++++++++ .../resmgr/policy/zz_generated.deepcopy.go | 25 +++ pkg/resmgr/cpuclass/dra.go | 98 ++++++++++ pkg/resmgr/cpuclass/dra_test.go | 178 ++++++++++++++++++ 9 files changed, 454 insertions(+) create mode 100644 pkg/apis/config/v1alpha1/resmgr/policy/cpuclass_test.go create mode 100644 pkg/resmgr/cpuclass/dra.go create mode 100644 pkg/resmgr/cpuclass/dra_test.go diff --git a/config/crd/bases/config.nri_balloonspolicies.yaml b/config/crd/bases/config.nri_balloonspolicies.yaml index b144e3cc1..30c7e7da0 100644 --- a/config/crd/bases/config.nri_balloonspolicies.yaml +++ b/config/crd/bases/config.nri_balloonspolicies.yaml @@ -781,6 +781,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 diff --git a/config/crd/bases/config.nri_topologyawarepolicies.yaml b/config/crd/bases/config.nri_topologyawarepolicies.yaml index 92c0de2b9..b9143461f 100644 --- a/config/crd/bases/config.nri_topologyawarepolicies.yaml +++ b/config/crd/bases/config.nri_topologyawarepolicies.yaml @@ -482,6 +482,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 diff --git a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml index b144e3cc1..30c7e7da0 100644 --- a/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml +++ b/deployment/helm/balloons/crds/config.nri_balloonspolicies.yaml @@ -781,6 +781,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 diff --git a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml index 92c0de2b9..b9143461f 100644 --- a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml +++ b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml @@ -482,6 +482,18 @@ spec: items: type: string type: array + dra: + description: |- + DRA holds DRA publication options for this class. Optional; defaults + to publish=true (the class is visible to the DRA driver). + properties: + publish: + description: |- + Publish controls whether this cpuClass is exposed as a DRA device. + Defaults to true when unset. Set to false to hide this class from + DRA while keeping it usable via the normal nri-plugins config path. + type: boolean + type: object energyPerformancePreference: description: EnergyPerformancePreference for CPUs in this class. minimum: 0 diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go index 3e4258fda..df6ad8e22 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass.go @@ -19,6 +19,16 @@ import ( "fmt" ) +// CPUClassDRA holds per-cpuClass DRA publication options. +// +k8s:deepcopy-gen=true +type CPUClassDRA struct { + // Publish controls whether this cpuClass is exposed as a DRA device. + // Defaults to true when unset. Set to false to hide this class from + // DRA while keeping it usable via the normal nri-plugins config path. + // +optional + Publish *bool `json:"publish,omitempty"` +} + // CPUClass specifies CPU frequency, C-state, and turbo attributes // for a CPU class. // +k8s:deepcopy-gen=true @@ -99,6 +109,19 @@ type CPUClass struct { // on a single node. Has effect only when the class also carries // PctPriority or SstClosID. Experimental. PublishExtendedResource bool `json:"publishExtendedResource,omitempty"` + // DRA holds DRA publication options for this class. Optional; defaults + // to publish=true (the class is visible to the DRA driver). + // +optional + DRA *CPUClassDRA `json:"dra,omitempty"` +} + +// DRAPublish reports whether this cpuClass should be published as a DRA device. +// Returns true if DRA is nil or DRA.Publish is nil. +func (cc *CPUClass) DRAPublish() bool { + if cc.DRA == nil || cc.DRA.Publish == nil { + return true + } + return *cc.DRA.Publish } func (cc *CPUClass) Validate() error { diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass_test.go b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass_test.go new file mode 100644 index 000000000..87e33a12f --- /dev/null +++ b/pkg/apis/config/v1alpha1/resmgr/policy/cpuclass_test.go @@ -0,0 +1,82 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "testing" +) + +// ptr returns a pointer to v; helper for bool/int literals in tests. +func ptr[T any](v T) *T { return &v } + +func TestCPUClassDRAPublish(t *testing.T) { + tests := []struct { + name string + cc *CPUClass + want bool + }{ + { + name: "nil DRA field defaults to true", + cc: &CPUClass{}, + want: true, + }, + { + name: "DRA set but Publish nil defaults to true", + cc: &CPUClass{DRA: &CPUClassDRA{}}, + want: true, + }, + { + name: "explicit false", + cc: &CPUClass{DRA: &CPUClassDRA{Publish: ptr(false)}}, + want: false, + }, + { + name: "explicit true", + cc: &CPUClass{DRA: &CPUClassDRA{Publish: ptr(true)}}, + want: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := tc.cc.DRAPublish() + if got != tc.want { + t.Errorf("DRAPublish() = %v, want %v", got, tc.want) + } + }) + } +} + +func TestCPUClassDRADeepCopy(t *testing.T) { + orig := &CPUClass{ + Name: "hp", + DRA: &CPUClassDRA{Publish: ptr(false)}, + } + copy := orig.DeepCopy() + if copy == orig { + t.Fatal("DeepCopy returned same pointer") + } + if copy.DRA == orig.DRA { + t.Fatal("DRA pointer not deep-copied") + } + if copy.DRA.Publish == orig.DRA.Publish { + t.Fatal("DRA.Publish pointer not deep-copied (shared storage)") + } + + // Mutating the copy must not affect the original. + *copy.DRA.Publish = true + if *orig.DRA.Publish != false { + t.Error("mutating copy's DRA.Publish affected the original") + } +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go index ebb0af11a..21f947c1f 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/zz_generated.deepcopy.go @@ -33,6 +33,11 @@ func (in *CPUClass) DeepCopyInto(out *CPUClass) { *out = new(int) **out = **in } + if in.DRA != nil { + in, out := &in.DRA, &out.DRA + *out = new(CPUClassDRA) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPUClass. @@ -45,6 +50,26 @@ func (in *CPUClass) DeepCopy() *CPUClass { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *CPUClassDRA) DeepCopyInto(out *CPUClassDRA) { + *out = *in + if in.Publish != nil { + in, out := &in.Publish, &out.Publish + *out = new(bool) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CPUClassDRA. +func (in *CPUClassDRA) DeepCopy() *CPUClassDRA { + if in == nil { + return nil + } + out := new(CPUClassDRA) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SchedulingClass) DeepCopyInto(out *SchedulingClass) { *out = *in diff --git a/pkg/resmgr/cpuclass/dra.go b/pkg/resmgr/cpuclass/dra.go new file mode 100644 index 000000000..b503993ca --- /dev/null +++ b/pkg/resmgr/cpuclass/dra.go @@ -0,0 +1,98 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cpuclass + +import ( + "fmt" + "sort" + + policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" +) + +// ValidateCPUClassesForDRA checks that DRA-published PCT classes do not +// overcommit any priority tier. Classes are grouped by tier — the +// pctPriority value for managed PCT classes, or the SstClosID for +// assoc-only classes — and more than one DRA-published class in the same +// tier is an error. Non-PCT classes and managed LP classes (pctPriority +// != "high") are exempt, since buildDRADevices with hpOnly=true never +// publishes them. +// +// sharedCounters is rejected: buildDRADevices always publishes +// independent full-capacity devices per class, so accepting it here would +// silently disable this overcommit guard without implementing the +// KEP-5941 shared-counter model. +// +// Called at driver Configure time, not at config load time. +func ValidateCPUClassesForDRA(classes []*policyapi.CPUClass, sharedCounters bool) error { + if sharedCounters { + return fmt.Errorf( + "DRA: sharedCounters is not yet supported (Model C / KEP-5941 is not " + + "implemented); leave spec.dra.sharedCounters unset or false", + ) + } + + // Group DRA-published PCT classes by tier label. + byTier := map[string][]string{} // tier label → sorted class names + for _, cc := range classes { + if !isPCTClass(cc) { + continue + } + if !cc.DRAPublish() { + continue + } + if cc.PctPriority != "" && cc.PctPriority != "high" { + continue // LP classes are exempt, see doc comment + } + tier := tierLabel(cc) + byTier[tier] = append(byTier[tier], cc.Name) + } + + // Check for conflicts: any tier with more than one published class. + tiers := make([]string, 0, len(byTier)) + for t := range byTier { + tiers = append(tiers, t) + } + sort.Strings(tiers) // deterministic outer ordering + + for _, tier := range tiers { + names := byTier[tier] + if len(names) <= 1 { + continue + } + sort.Strings(names) // deterministic name listing in the error + return fmt.Errorf( + "DRA: tier %q has %d published cpuClasses (%v); "+ + "at most one is allowed. "+ + "Resolution: set cpuClass.dra.publish: false on all but one "+ + "(sharedCounters is not yet supported — see KEP-5941)", + tier, len(names), names, + ) + } + + return nil +} + +// isPCTClass reports whether cc is a PCT class (managed or assoc-only). +func isPCTClass(cc *policyapi.CPUClass) bool { + return cc.PctPriority != "" || cc.SstClosID != nil +} + +// tierLabel returns the tier string used for grouping and error messages. +func tierLabel(cc *policyapi.CPUClass) string { + if cc.PctPriority != "" { + return "pctPriority=" + cc.PctPriority + } + return fmt.Sprintf("closID=%d", *cc.SstClosID) +} diff --git a/pkg/resmgr/cpuclass/dra_test.go b/pkg/resmgr/cpuclass/dra_test.go new file mode 100644 index 000000000..f64563f20 --- /dev/null +++ b/pkg/resmgr/cpuclass/dra_test.go @@ -0,0 +1,178 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cpuclass + +import ( + "strings" + "testing" + + policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" +) + +func ptr[T any](v T) *T { return &v } + +func TestValidateCPUClassesForDRA(t *testing.T) { + tests := []struct { + name string + classes []*policyapi.CPUClass + sharedCounters bool + wantErr bool + // errContains is a substring the error must contain (if wantErr). + errContains string + }{ + { + name: "empty class list", + classes: nil, + wantErr: false, + }, + { + name: "non-PCT classes only — all exempt", + classes: []*policyapi.CPUClass{ + {Name: "default"}, + {Name: "idle"}, + {Name: "turbo"}, + }, + wantErr: false, + }, + { + name: "single managed HP class published", + classes: []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + }, + wantErr: false, + }, + { + name: "single managed LP class published", + classes: []*policyapi.CPUClass{ + {Name: "lp", PctPriority: "low"}, + }, + wantErr: false, + }, + { + name: "one HP + one LP managed — different tiers, ok", + classes: []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + {Name: "lp", PctPriority: "low"}, + }, + wantErr: false, + }, + { + // Regression: two published LP classes must not trigger a tier-overcommit + // error. DRADevices uses hpOnly=true, so LP classes produce no devices + // and cannot overcommit any tier. + name: "two managed LP classes, both published — ok (hpOnly skips them)", + classes: []*policyapi.CPUClass{ + {Name: "lp1", PctPriority: "low"}, + {Name: "lp2", PctPriority: "low"}, + }, + wantErr: false, + }, + { + name: "two assoc-only classes with different SstClosIDs — ok", + classes: []*policyapi.CPUClass{ + {Name: "hp-assoc", SstClosID: ptr(0)}, + {Name: "lp-assoc", SstClosID: ptr(3)}, + }, + wantErr: false, + }, + { + name: "two assoc-only classes with same SstClosID, sharedCounters=false — error (same CLOS, can both be HP-published)", + classes: []*policyapi.CPUClass{ + {Name: "class-a", SstClosID: ptr(0)}, + {Name: "class-b", SstClosID: ptr(0)}, + }, + sharedCounters: false, + wantErr: true, + errContains: `closID=0`, + }, + { + name: "two assoc-only classes with same SstClosID, sharedCounters=true — rejected", + classes: []*policyapi.CPUClass{ + {Name: "class-a", SstClosID: ptr(0)}, + {Name: "class-b", SstClosID: ptr(0)}, + }, + sharedCounters: true, + wantErr: true, + errContains: "sharedCounters", + }, + { + name: "two managed HP classes, both published, sharedCounters=false — error", + classes: []*policyapi.CPUClass{ + {Name: "hp-perf", PctPriority: "high"}, + {Name: "hp-turbo", PctPriority: "high"}, + }, + sharedCounters: false, + wantErr: true, + errContains: "hp-perf", + }, + { + name: "two managed HP classes, both published, sharedCounters=true — rejected", + classes: []*policyapi.CPUClass{ + {Name: "hp-perf", PctPriority: "high"}, + {Name: "hp-turbo", PctPriority: "high"}, + }, + sharedCounters: true, + wantErr: true, + errContains: "sharedCounters", + }, + { + name: "two managed HP classes, one opted out — ok", + classes: []*policyapi.CPUClass{ + {Name: "hp-perf", PctPriority: "high"}, + {Name: "hp-turbo", PctPriority: "high", DRA: &policyapi.CPUClassDRA{Publish: ptr(false)}}, + }, + sharedCounters: false, + wantErr: false, + }, + { + name: "three managed HP classes, all published — error names all", + classes: []*policyapi.CPUClass{ + {Name: "hp-a", PctPriority: "high"}, + {Name: "hp-b", PctPriority: "high"}, + {Name: "hp-c", PctPriority: "high"}, + }, + sharedCounters: false, + wantErr: true, + errContains: "hp-a", + }, + { + name: "mixed PCT and non-PCT — only PCT classes checked", + classes: []*policyapi.CPUClass{ + {Name: "default"}, + {Name: "hp", PctPriority: "high"}, + {Name: "another-default"}, + }, + wantErr: false, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + err := ValidateCPUClassesForDRA(tc.classes, tc.sharedCounters) + if tc.wantErr { + if err == nil { + t.Fatalf("ValidateCPUClassesForDRA() = nil, want error") + } + if tc.errContains != "" && !strings.Contains(err.Error(), tc.errContains) { + t.Errorf("error %q does not contain %q", err.Error(), tc.errContains) + } + } else { + if err != nil { + t.Fatalf("ValidateCPUClassesForDRA() = %v, want nil", err) + } + } + }) + } +} From 21a18e5e68f82636bcea8f62888855c01539de53 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 21:50:02 +0300 Subject: [PATCH 03/39] pct: add PickHpCpus, ReleaseHpCpus, and Punits Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- pkg/resmgr/cpuclass/internal/pct/pct.go | 186 ++++++++++- pkg/resmgr/cpuclass/internal/pct/pct_test.go | 320 +++++++++++++++++++ 2 files changed, 497 insertions(+), 9 deletions(-) diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index e4d7f979f..34ae7a92f 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -96,8 +96,12 @@ type Allocator struct { // allocator treats them as "no HP knowledge". punitByCpu map[int]int // hpUsed[i] is the set of CPUs currently held by HP-class - // workloads on punits[i]. + // workloads on punits[i] via the non-DRA (hint-driven) path. hpUsed map[int]cpuset.CPUSet + // hpDRAUsed[i] is the set of CPUs currently held by DRA claims + // on punits[i]. Separate from hpUsed so that clearHpUsage (called + // from the non-DRA UseClass path) can never evict DRA holds. + hpDRAUsed map[int]cpuset.CPUSet // hpEligiblePunit[i] reports whether punits[i] can actually // host HP-class CPUs at top turbo. Populated at Configure(). // In managed mode every punit becomes eligible (the plugin @@ -138,6 +142,7 @@ func (a *Allocator) Configure(classes []*policyapi.CPUClass, allowed cpuset.CPUS a.fallbackClos = pctDefaultHpClos // CLOS 0 == default-after-reset a.allowed = allowed a.hpUsed = map[int]cpuset.CPUSet{} + a.hpDRAUsed = map[int]cpuset.CPUSet{} a.hpClasses = map[string]bool{} a.hpEligiblePunit = map[int]bool{} a.punits = nil @@ -520,6 +525,164 @@ func (a *Allocator) FreeClassCapacity(className string, held cpuset.CPUSet) int return total } +// PunitInfo is a snapshot of one SST punit's DRA-relevant capacity. +// Returned by Allocator.Punits(); valid within the current +// Configure/Reconfigure cycle (indices may change on next Configure). +type PunitInfo struct { + PkgID int + PunitID int + HPCapacity int // GuaranteedHpCpus minus non-DRA HP CPUs already in hpUsed, or 0 if the punit is HP-ineligible + NonHPCapacity int // allocatable non-HP CPUs (allowed ∩ punit.CPUs − GuaranteedHpCpus) +} + +// Punits returns a snapshot of per-punit DRA-relevant capacity for all +// punits known to the allocator. Returns nil when the allocator is not +// active. +func (a *Allocator) Punits() []PunitInfo { + if !a.Active() { + return nil + } + out := make([]PunitInfo, len(a.punits)) + for i, pu := range a.punits { + out[i] = PunitInfo{ + PkgID: pu.PkgID, + PunitID: pu.PunitID, + HPCapacity: a.punitAvailableHPCapacity(i), + NonHPCapacity: a.punitNonHPCapacity(i), + } + } + return out +} + +// punitIdxByID returns the index in a.punits for the punit identified +// by (pkgID, punitID), or -1 if not found. +func (a *Allocator) punitIdxByID(pkgID, punitID int) int { + for i, pu := range a.punits { + if pu.PkgID == pkgID && pu.PunitID == punitID { + return i + } + } + return -1 +} + +// punitHPCapacity returns the guaranteed HP CPU count for punits[idx], +// capped by its actual (allowed-intersected) CPU count, or 0 if the +// punit is HP-ineligible or the index is out of range. The cap is +// needed because snapshotPunits intersects CPUs with the allowed set +// but leaves GuaranteedHpCpus at its raw hardware value. +func (a *Allocator) punitHPCapacity(idx int) int { + if !a.Active() || idx < 0 || idx >= len(a.punits) { + return 0 + } + if !a.hpEligiblePunit[idx] { + return 0 + } + return min(a.punits[idx].GuaranteedHpCpus, a.punits[idx].CPUs.Size()) +} + +// punitAvailableHPCapacity returns punitHPCapacity(idx) minus the +// non-DRA HP CPUs already held in hpUsed[idx] — the value that must be +// advertised as DRA-consumable capacity, since Kubernetes only subtracts +// DRA allocations from advertised capacity. Staleness from non-DRA HP +// allocations is handled by the DRA plugin's republisherLoop +// (Plugin.TriggerRepublish). +func (a *Allocator) punitAvailableHPCapacity(idx int) int { + capacity := a.punitHPCapacity(idx) - a.hpUsed[idx].Size() + if capacity < 0 { + return 0 + } + return capacity +} + +// punitNonHPCapacity returns the count of allocatable non-HP CPUs in +// punits[idx]. Applies the allowed.Size()>0 guard consistent with other +// allowed-consuming sites in this file. +func (a *Allocator) punitNonHPCapacity(idx int) int { + if !a.Active() || idx < 0 || idx >= len(a.punits) { + return 0 + } + pu := a.punits[idx] + cpus := pu.CPUs + if a.allowed.Size() > 0 { + cpus = cpus.Intersection(a.allowed) + } + n := cpus.Size() - pu.GuaranteedHpCpus + if n < 0 { + return 0 + } + return n +} + +// PickHpCpus selects n HP-eligible CPUs from the punit identified by +// (pkgID, punitID), excluding CPUs in held and those already tracked in +// hpUsed or hpDRAUsed. Records the selection in hpDRAUsed (not hpUsed, +// so clearHpUsage called from the non-DRA path cannot evict DRA holds). +// Returns an error when the allocator is inactive, the punit is not +// found or not HP-eligible, or fewer than n CPUs are available. +func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { + if !a.Active() { + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: allocator not active") + } + idx := a.punitIdxByID(pkgID, punitID) + if idx < 0 { + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) not found", pkgID, punitID) + } + if !a.hpEligiblePunit[idx] { + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) is not HP-eligible", pkgID, punitID) + } + pu := a.punits[idx] + avail := pu.CPUs + if a.allowed.Size() > 0 { + avail = avail.Intersection(a.allowed) + } + avail = avail.Difference(held).Difference(a.hpUsed[idx]).Difference(a.hpDRAUsed[idx]) + // Also enforce the GuaranteedHpCpus cap: can't pick more HP CPUs than + // the punit guarantees, regardless of how many are physically free. + hpAlreadyHeld := a.hpUsed[idx].Size() + a.hpDRAUsed[idx].Size() + hpRoom := pu.GuaranteedHpCpus - hpAlreadyHeld + if hpRoom < 0 { + hpRoom = 0 + } + if avail.Size() < n || hpRoom < n { + available := avail.Size() + if hpRoom < available { + available = hpRoom + } + return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) has %d available HP CPUs (room=%d, free=%d), need %d", + pkgID, punitID, available, hpRoom, avail.Size(), n) + } + // Sort for deterministic selection; take first n. + list := avail.List() + picked := cpuset.New(list[:n]...) + if a.hpDRAUsed == nil { + a.hpDRAUsed = map[int]cpuset.CPUSet{} + } + a.hpDRAUsed[idx] = a.hpDRAUsed[idx].Union(picked) + return picked, nil +} + +// ReleaseHpCpus removes cpus from hpDRAUsed[punitIdx] for the punit +// identified by (pkgID, punitID). Silently ignores unknown punits and +// CPUs not present in hpDRAUsed (idempotent). +func (a *Allocator) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { + if !a.Active() { + return + } + idx := a.punitIdxByID(pkgID, punitID) + if idx < 0 { + return + } + if a.hpDRAUsed == nil { + return + } + remaining := a.hpDRAUsed[idx].Difference(cpus) + if remaining.IsEmpty() { + delete(a.hpDRAUsed, idx) + } else { + a.hpDRAUsed[idx] = remaining + } +} + // useClass associates the given CPUs to the CLOS chosen for className. // In managed mode, CPUs whose className is not a PCT class are // associated to the fallback CLOS. In assoc-only mode such CPUs are @@ -569,8 +732,11 @@ func (a *Allocator) trackHpUsage(className string, cpus cpuset.CPUSet) { perPunit[idx] = append(perPunit[idx], cpu) } for idx, list := range perPunit { - set := a.hpUsed[idx] - a.hpUsed[idx] = set.Union(cpuset.New(list...)) + set := a.hpUsed[idx].Union(cpuset.New(list...)) + if dra := a.hpDRAUsed[idx]; !dra.IsEmpty() { + set = set.Difference(dra) + } + a.hpUsed[idx] = set } } @@ -675,11 +841,11 @@ func (a *Allocator) hpInUseCpus() cpuset.CPUSet { return cpuset.New() } out := cpuset.New() - for idx, used := range a.hpUsed { - if used.IsEmpty() { - continue - } - if idx < 0 || idx >= len(a.punits) { + // Range over punits rather than hpUsed so that DRA-only punits + // (present in hpDRAUsed but absent from hpUsed) are not skipped. + for idx := range a.punits { + combined := a.hpUsed[idx].Union(a.hpDRAUsed[idx]) + if combined.IsEmpty() { continue } out = out.Union(a.punits[idx].CPUs) @@ -756,7 +922,9 @@ func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, continue } anyKnown = true - used := a.hpUsed[i] + // Union hpUsed and hpDRAUsed so DRA holds reduce reported HP room, + // preventing HP over-subscription via the hint path. + used := a.hpUsed[i].Union(a.hpDRAUsed[i]) if excludeBln.Size() > 0 { used = used.Difference(excludeBln) } diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index 584d32d36..e1ec821ae 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -1083,3 +1083,323 @@ func TestFreeClassCapacity_UnknownClassReturnsZero(t *testing.T) { t.Errorf("unknown class capacity = %d, want 0", got) } } + +// makePunitsWithGtdHp returns two punits in the same package, each with the +// given MaxHpCpus and GuaranteedHpCpus values. +func makePunitsWithGtdHp(maxHp0, gtdHp0, maxHp1, gtdHp1 int) []pctPunit { + return []pctPunit{ + {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), MaxHpCpus: maxHp0, GuaranteedHpCpus: gtdHp0}, + {PkgID: 0, PunitID: 1, CPUs: cpuset.MustParse("4-7"), MaxHpCpus: maxHp1, GuaranteedHpCpus: gtdHp1}, + } +} + +// newPickAllocator returns an Allocator pre-wired for PickHpCpus / ReleaseHpCpus tests. +func newPickAllocator(t *testing.T, punits []pctPunit) *Allocator { + t.Helper() + sys := newTwoPunitFakeSys() + sst := &fakeSst{supported: true, punits: punits} + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + plans := map[string]*pctClassPlan{"hp": {ClosID: 0}} + a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-7"), sys, sst) + return a +} + +func TestPunitHPCapacity(t *testing.T) { + // Active() == false: Allocator with mode == disabled + inactiveA := &Allocator{} + if got := inactiveA.punitHPCapacity(0); got != 0 { + t.Errorf("punitHPCapacity on inactive allocator = %d, want 0", got) + } + + a := newPickAllocator(t, makePunitsWithGtdHp(4, 3, 4, 1)) + tests := []struct { + name string + idx int + want int + }{ + {"eligible punit 0", 0, 3}, + {"eligible punit 1", 1, 1}, + {"out-of-range", 99, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := a.punitHPCapacity(tc.idx) + if got != tc.want { + t.Errorf("punitHPCapacity(%d) = %d, want %d", tc.idx, got, tc.want) + } + }) + } + // HP-ineligible punit + a.hpEligiblePunit[0] = false + if got := a.punitHPCapacity(0); got != 0 { + t.Errorf("punitHPCapacity for ineligible punit = %d, want 0", got) + } +} + +// TestPunitHPCapacity_CappedByAllowedIntersection verifies that +// punitHPCapacity caps the raw hardware GuaranteedHpCpus by the punit's +// actual CPU count after intersecting with the allowed/online set — +// otherwise it would advertise more DRA capacity than PickHpCpus can +// supply once some of the punit's CPUs are excluded (e.g. offline or +// outside the reserved/shared pool). +func TestPunitHPCapacity_CappedByAllowedIntersection(t *testing.T) { + sys := newTwoPunitFakeSys() + // Raw punit 0 spans CPUs 0-3 (4 CPUs) with GuaranteedHpCpus=3. + // Restricting "allowed" to CPUs 0-1,4-7 leaves punit 0 with only + // CPUs 0-1 (2 CPUs) after intersection -- less than its raw + // GuaranteedHpCpus of 3. + sst := &fakeSst{supported: true, punits: makePunitsWithGtdHp(4, 3, 4, 1)} + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + plans := map[string]*pctClassPlan{"hp": {ClosID: 0}} + a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-1,4-7"), sys, sst) + + if got := a.punitHPCapacity(0); got != 2 { + t.Errorf("punitHPCapacity(0) = %d, want 2 (capped by allowed intersection, not raw GuaranteedHpCpus=3)", got) + } + // Punit 1 is unaffected: its full range (4-7) is within allowed, and + // its GuaranteedHpCpus=1 stays under the 4-CPU cap. + if got := a.punitHPCapacity(1); got != 1 { + t.Errorf("punitHPCapacity(1) = %d, want 1 (unaffected by allowed restriction)", got) + } +} + +func TestPunitNonHPCapacity(t *testing.T) { + inactiveA := &Allocator{} + if got := inactiveA.punitNonHPCapacity(0); got != 0 { + t.Errorf("punitNonHPCapacity on inactive allocator = %d, want 0", got) + } + + // punit 0: CPUs 0-3 (4 total), GuaranteedHpCpus=3 → 1 non-HP + // punit 1: CPUs 4-7 (4 total), GuaranteedHpCpus=0 → 4 non-HP + a := newPickAllocator(t, makePunitsWithGtdHp(4, 3, 4, 0)) + tests := []struct { + name string + idx int + want int + }{ + {"partial HP", 0, 1}, + {"all non-HP", 1, 4}, + {"out-of-range", 99, 0}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + got := a.punitNonHPCapacity(tc.idx) + if got != tc.want { + t.Errorf("punitNonHPCapacity(%d) = %d, want %d", tc.idx, got, tc.want) + } + }) + } + // all CPUs HP-guaranteed → 0 non-HP + a2 := newPickAllocator(t, makePunitsWithGtdHp(4, 4, 4, 4)) + if got := a2.punitNonHPCapacity(0); got != 0 { + t.Errorf("all-HP punit nonHPCapacity = %d, want 0", got) + } +} + +func TestAllocatorPunits(t *testing.T) { + // Inactive allocator returns nil. + inactiveA := &Allocator{} + if got := inactiveA.Punits(); got != nil { + t.Errorf("Punits() on inactive allocator = %v, want nil", got) + } + + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 1)) + pi := a.Punits() + if len(pi) != 2 { + t.Fatalf("Punits() len = %d, want 2", len(pi)) + } + want := []PunitInfo{ + {PkgID: 0, PunitID: 0, HPCapacity: 2, NonHPCapacity: 2}, + {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3}, + } + for i, w := range want { + if pi[i] != w { + t.Errorf("Punits()[%d] = %+v, want %+v", i, pi[i], w) + } + } +} + +// TestAllocatorPunits_NonDRAHpUsageReducesCapacity verifies that +// Punits() deducts non-DRA HP CPUs already tracked in hpUsed from the +// advertised HPCapacity, since Kubernetes only subtracts DRA +// allocations from the advertised capacity and would otherwise let a +// claim be placed that PickHpCpus must reject at Prepare time. +func TestAllocatorPunits_NonDRAHpUsageReducesCapacity(t *testing.T) { + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 1)) + // Simulate the NRI path having already claimed 1 HP CPU on punit 0. + a.hpUsed[0] = cpuset.MustParse("0") + + pi := a.Punits() + want := []PunitInfo{ + {PkgID: 0, PunitID: 0, HPCapacity: 1, NonHPCapacity: 2}, + {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3}, + } + for i, w := range want { + if pi[i] != w { + t.Errorf("Punits()[%d] = %+v, want %+v", i, pi[i], w) + } + } +} + +func TestPickHpCpus(t *testing.T) { + // Active()==false + inactiveA := &Allocator{} + if _, err := inactiveA.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus on inactive allocator: expected error, got nil") + } + + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + + // Success: pick 2 CPUs from punit 0 (PkgID=0, PunitID=0, CPUs 0-3). + got, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus success case: %v", err) + } + if got.Size() != 2 { + t.Errorf("PickHpCpus returned %d CPUs, want 2", got.Size()) + } + // hpDRAUsed updated, hpUsed unchanged. + if !a.hpDRAUsed[0].Equals(got) { + t.Errorf("hpDRAUsed[0] = %v, want %v", a.hpDRAUsed[0], got) + } + if a.hpUsed[0].Size() != 0 { + t.Errorf("hpUsed[0] should be untouched, got %v", a.hpUsed[0]) + } + + // Exhaustion: already 2 DRA-held + 0 available for another pick. + if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus exhaustion: expected error, got nil") + } + + // HP-ineligible punit. + a2 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + a2.hpEligiblePunit[0] = false + if _, err := a2.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus ineligible punit: expected error, got nil") + } + + // (PkgID, PunitID) not found. + a3 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + if _, err := a3.PickHpCpus(99, 99, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus not-found: expected error, got nil") + } + + // held exclusion: hold CPUs 0,1 → pick of 2 from a 4-CPU punit + // must return 2,3 (the remaining ones). + a4 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + held := cpuset.MustParse("0-1") + got4, err := a4.PickHpCpus(0, 0, 2, held) + if err != nil { + t.Fatalf("PickHpCpus held-exclusion: %v", err) + } + if got4.Intersection(held).Size() != 0 { + t.Errorf("PickHpCpus returned a held CPU: %v", got4) + } +} + +func TestReleaseHpCpus(t *testing.T) { + a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + + // Pick 2 CPUs then release them. + picked, _ := a.PickHpCpus(0, 0, 2, cpuset.New()) + a.ReleaseHpCpus(0, 0, picked) + if a.hpDRAUsed[0].Size() != 0 { + t.Errorf("hpDRAUsed[0] after full release = %v, want empty", a.hpDRAUsed[0]) + } + // Map entry deleted. + if _, ok := a.hpDRAUsed[0]; ok { + t.Error("hpDRAUsed[0] entry should be deleted after full release") + } + + // Release CPUs not held — no-op. + a.ReleaseHpCpus(0, 0, cpuset.MustParse("0-1")) + + // Out-of-range (not found) — no-op, no panic. + a.ReleaseHpCpus(99, 99, cpuset.MustParse("0")) + + // Partial release. + picked2, _ := a.PickHpCpus(0, 0, 2, cpuset.New()) + first := cpuset.New(picked2.UnsortedList()[0]) + a.ReleaseHpCpus(0, 0, first) + if a.hpDRAUsed[0].Size() != 1 { + t.Errorf("hpDRAUsed[0] after partial release size = %d, want 1", a.hpDRAUsed[0].Size()) + } +} + +func TestHpDRAUsedIsolation(t *testing.T) { + // Build an allocator with both HP and LP classes to test that UseClass + // on DRA-held CPUs does not corrupt the hpDRAUsed/hpUsed separation. + sys := newTwoPunitFakeSys() + sst := &fakeSst{supported: true, punits: makePunitsWithGtdHp(4, 2, 4, 2)} + classes := []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + {Name: "lp", PctPriority: "low"}, + } + plans := map[string]*pctClassPlan{ + "hp": {ClosID: 0}, + "lp": {ClosID: 3}, + } + a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-7"), sys, sst) + + // DRA holds 2 CPUs on punit 0; hpUsed[0] is empty. + draHeld, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus: %v", err) + } + before := a.hpDRAUsed[0].Clone() + + // HP UseClass on DRA-held CPUs — must NOT add them to hpUsed (they are + // already accounted in hpDRAUsed; double-counting corrupts Punits capacity). + _ = a.UseClass("hp", draHeld) + if !a.hpDRAUsed[0].Equals(before) { + t.Errorf("hpDRAUsed[0] changed after HP UseClass: got %v, want %v", a.hpDRAUsed[0], before) + } + if !a.hpUsed[0].IsEmpty() { + t.Errorf("hpUsed[0] = %v after HP UseClass on DRA-held CPUs, want empty", a.hpUsed[0]) + } + + // Non-HP UseClass on overlapping CPUs — must NOT remove them from hpDRAUsed. + _ = a.UseClass("lp", draHeld) + if !a.hpDRAUsed[0].Equals(before) { + t.Errorf("hpDRAUsed[0] changed after non-HP UseClass: got %v, want %v", a.hpDRAUsed[0], before) + } + // hpInUseCpus must still report the DRA-held CPUs. + inUse := a.hpInUseCpus() + for _, cpu := range draHeld.UnsortedList() { + if !inUse.Contains(cpu) { + t.Errorf("hpInUseCpus missing DRA-held cpu %d", cpu) + } + } +} + +func TestHpReserveRoomWithDRAHolds(t *testing.T) { + // Two punits, each with MaxHpCpus=2, GuaranteedHpCpus=2. + a := newPickAllocator(t, makePunitsWithGtdHp(2, 2, 2, 2)) + + // Before any holds, room on punit 0 should be 2. + // hpReserveCpus returns Tier-A candidate sets; if room>=requested we + // get a candidate set back. Requesting 2 CPUs from punit 0 should succeed. + free := cpuset.MustParse("0-7") + before := a.hpReserveCpus(free, cpuset.New(), 2) + if len(before) == 0 { + t.Fatal("hpReserveCpus before DRA holds: expected at least one candidate, got none") + } + + // DRA picks 1 CPU on punit 0. + _, err := a.PickHpCpus(0, 0, 1, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus: %v", err) + } + + // Now request 2 CPUs from punit 0: room is 1 (2 - 1 DRA hold), so + // hpReserveCpus should not return punit 0 as a single-punit Tier-A + // candidate for a request of 2. It may return punit 1 (unaffected). + after := a.hpReserveCpus(free, cpuset.New(), 2) + for _, candidate := range after { + // No candidate set should include the DRA-held CPUs as "free" HP room + // for a 2-CPU request on punit 0 alone. + if candidate.Intersection(cpuset.MustParse("0-3")).Size() > 1 { + t.Errorf("hpReserveCpus candidate includes punit 0 CPUs despite DRA hold reducing room to 1") + } + } +} From cab2022982fe93ed776a6ce2b58907726b4dec36 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 18:53:35 +0300 Subject: [PATCH 04/39] cpuclass: implement DRA device builder Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- go.mod | 49 +- go.sum | 114 ++-- pkg/resmgr/cpuclass/cpuclass.go | 9 + pkg/resmgr/cpuclass/dra.go | 211 +++++- pkg/resmgr/cpuclass/dra_test.go | 664 +++++++++++++++++++ pkg/resmgr/cpuclass/internal/pct/pct.go | 24 +- pkg/resmgr/cpuclass/internal/pct/pct_test.go | 49 ++ 7 files changed, 1038 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index eebdbe85b..07937ec76 100644 --- a/go.mod +++ b/go.mod @@ -37,12 +37,12 @@ require ( go.opentelemetry.io/otel/trace v1.45.0 golang.org/x/sys v0.47.0 google.golang.org/grpc v1.83.1 - k8s.io/api v0.34.11 - k8s.io/apimachinery v0.34.11 - k8s.io/client-go v0.34.11 + k8s.io/api v0.37.0 + k8s.io/apimachinery v0.37.0 + k8s.io/client-go v0.37.0 k8s.io/klog/v2 v2.140.0 - k8s.io/kubelet v0.34.11 - k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 + k8s.io/kubelet v0.37.0 + k8s.io/utils v0.0.0-20260626114624-be93311217bd sigs.k8s.io/controller-runtime v0.22.5 sigs.k8s.io/yaml v1.6.0 ) @@ -54,13 +54,24 @@ require ( github.com/cespare/xxhash/v2 v2.3.0 // indirect github.com/containerd/log v0.1.0 // indirect github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect - github.com/emicklei/go-restful/v3 v3.12.2 // indirect - github.com/fxamacker/cbor/v2 v2.9.0 // indirect + github.com/emicklei/go-restful/v3 v3.13.0 // indirect + github.com/fxamacker/cbor/v2 v2.9.1 // indirect github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.21.0 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.23.0 // indirect + github.com/go-openapi/jsonpointer v1.0.0 // indirect + github.com/go-openapi/jsonreference v1.0.0 // indirect + github.com/go-openapi/swag v0.27.1 // indirect + github.com/go-openapi/swag/cmdutils v0.27.1 // indirect + github.com/go-openapi/swag/conv v0.27.1 // indirect + github.com/go-openapi/swag/fileutils v0.27.1 // indirect + github.com/go-openapi/swag/jsonutils v0.27.1 // indirect + github.com/go-openapi/swag/loading v0.27.1 // indirect + github.com/go-openapi/swag/mangling v0.27.1 // indirect + github.com/go-openapi/swag/netutils v0.27.1 // indirect + github.com/go-openapi/swag/pools v0.27.1 // indirect + github.com/go-openapi/swag/stringutils v0.27.1 // indirect + github.com/go-openapi/swag/typeutils v0.27.1 // indirect + github.com/go-openapi/swag/yamlutils v0.27.1 // indirect github.com/go-task/slim-sprig/v3 v3.0.0 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect @@ -68,21 +79,17 @@ require ( github.com/google/go-cmp v0.7.0 // indirect github.com/google/pprof v0.0.0-20260402051712-545e8a4df936 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect - github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knqyf263/go-plugin v0.9.0 // indirect - github.com/mailru/easyjson v0.7.7 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.3.0 // indirect - github.com/pkg/errors v0.9.1 // indirect - github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect github.com/prometheus/procfs v0.21.1 // indirect - github.com/spf13/pflag v1.0.6 // indirect + github.com/spf13/pflag v1.0.10 // indirect github.com/tetratelabs/wazero v1.11.0 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect @@ -96,20 +103,20 @@ require ( golang.org/x/sync v0.22.0 // indirect golang.org/x/term v0.45.0 // indirect golang.org/x/text v0.40.0 // indirect - golang.org/x/time v0.9.0 // indirect + golang.org/x/time v0.15.0 // indirect golang.org/x/tools v0.47.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20260803160001-6ac0973c030d // indirect google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af // indirect - gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect + gopkg.in/evanphx/json-patch.v4 v4.13.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/code-generator v0.34.11 // indirect - k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f // indirect - k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b // indirect - sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 // indirect + k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b // indirect + k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad // indirect + sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect - sigs.k8s.io/structured-merge-diff/v6 v6.3.0 // indirect + sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect ) replace ( diff --git a/go.sum b/go.sum index 19e7836e8..f0ad744e8 100644 --- a/go.sum +++ b/go.sum @@ -20,17 +20,16 @@ github.com/containerd/ttrpc v1.2.9 h1:ha0ak962T0s3CA/RoZ6S6xiWZQF24GrBaEpiGX1uih github.com/containerd/ttrpc v1.2.9/go.mod h1:jjtQRwXm4DL3KsHKW8vDiUOV6wO0hi6IPhmJhxU7aEs= github.com/coreos/go-systemd/v22 v22.7.0 h1:LAEzFkke61DFROc7zNLX/WA2i5J8gYqe0rSj9KI28KA= github.com/coreos/go-systemd/v22 v22.7.0/go.mod h1:xNUYtjHu2EDXbsxz1i41wouACIwT7Ybq9o0BQhMwD0w= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.12.2 h1:DhwDP0vY3k8ZzE0RunuJy8GhNpPL6zqLkDf9B/a0/xU= -github.com/emicklei/go-restful/v3 v3.12.2/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.13.0 h1:C4Bl2xDndpU6nJ4bc1jXd+uTmYPVUwkD6bFY/oTyCes= +github.com/emicklei/go-restful/v3 v3.13.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= github.com/fsnotify/fsnotify v1.10.1 h1:b0/UzAf9yR5rhf3RPm9gf3ehBPpf0oZKIjtpKrx59Ho= github.com/fsnotify/fsnotify v1.10.1/go.mod h1:TLheqan6HD6GBK6PrDWyDPBaEV8LspOxvPSjC+bVfgo= -github.com/fxamacker/cbor/v2 v2.9.0 h1:NpKPmjDBgUfBms6tr6JZkTHtfFGcMKsw3eGcmD/sapM= -github.com/fxamacker/cbor/v2 v2.9.0/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= +github.com/fxamacker/cbor/v2 v2.9.1 h1:2rWm8B193Ll4VdjsJY28jxs70IdDsHRWgQYAI80+rMQ= +github.com/fxamacker/cbor/v2 v2.9.1/go.mod h1:vM4b+DJCtHn+zz7h3FFp/hDAI9WNWCsZj23V5ytsSxQ= github.com/gkampitakis/ciinfo v0.3.2 h1:JcuOPk8ZU7nZQjdUhctuhQofk7BGHuIy0c9Ez8BNhXs= github.com/gkampitakis/ciinfo v0.3.2/go.mod h1:1NIwaOcFChN4fa/B0hEBdAb6npDlFL8Bwx4dfRLRqAo= github.com/gkampitakis/go-diff v1.3.2 h1:Qyn0J9XJSDTgnsgHRdz9Zp24RaJeKMUHg2+PDZZdC4M= @@ -42,14 +41,40 @@ github.com/go-logr/logr v1.4.4 h1:tG4xh9yMsRCAiodLVTxyrkzSZ9+o0L1Kg/+cPVcbP/8= github.com/go-logr/logr v1.4.4/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= -github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= -github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= +github.com/go-openapi/jsonpointer v1.0.0 h1:kR9tHqY0CtZaOPVFm622dPVNhrvYpwr4uCxgL3h1H8s= +github.com/go-openapi/jsonpointer v1.0.0/go.mod h1:Z3rw7dWu1p9IgitXCFamSlA5lmDiklEB6vkaxcNZW5Y= +github.com/go-openapi/jsonreference v1.0.0 h1:jlmTr6torcd1YgDQvSfNmRtKzYDO4FGBkrAdlAVWnpY= +github.com/go-openapi/jsonreference v1.0.0/go.mod h1:jtwdyGbJk0Xhe5Y+rwtglQP6Sb1WZST4rT32LWB+sv0= +github.com/go-openapi/swag v0.27.1 h1:VotvOLWW8q/EAxB0YdsBBGC8XYyeL1YwBj2ungAGPNg= +github.com/go-openapi/swag v0.27.1/go.mod h1:GTkJPwHfhJp6MWr4/rCh64HVI3Ofu+tcsbfjfHmTxpE= +github.com/go-openapi/swag/cmdutils v0.27.1 h1:I7sYqaWVl5mq0NEmNQkAmFDyNin9ufvMX/p2zwtQaOE= +github.com/go-openapi/swag/cmdutils v0.27.1/go.mod h1:Sm1MVFMkF6guJJ+pQqHnQA3N0j9qALV3NxzDSv6bETM= +github.com/go-openapi/swag/conv v0.27.1 h1:8wi9ZG+olmY1wXphl93EWniPtbSPkXM/feH7FgjsvrU= +github.com/go-openapi/swag/conv v0.27.1/go.mod h1:QbqMivkpKhC3g1B1GGGOJ6ANewI3S62dbzYu3Duowqs= +github.com/go-openapi/swag/fileutils v0.27.1 h1:QQqBSoi5mW4XpU85nS0mLcA+zAE6vLzrb0QkmLKf9oM= +github.com/go-openapi/swag/fileutils v0.27.1/go.mod h1:VvJFZLTZS0AI854gEQz5tk7dBESdLjiNUMSZ/th2ry8= +github.com/go-openapi/swag/jsonutils v0.27.1 h1:SVgK3i4USzCU5mibOOS/l4ea2h9UQXy7J7RNLTjuXjU= +github.com/go-openapi/swag/jsonutils v0.27.1/go.mod h1:tdlEpZqdcQ17uj6J4YdK9vd8It5qWMwjWXOs0tjpRlk= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1 h1:mJu3COL9WEaZVp/Kf2PRMi7tPszPEJfSr/OO75ynCs8= +github.com/go-openapi/swag/jsonutils/fixtures_test v0.27.1/go.mod h1:mofwUWx70wvskwESqRJ//k/9kURmCgyJl5m5Ppoh5kY= +github.com/go-openapi/swag/loading v0.27.1 h1:/DxUgDXKbBX4bcn7r9uEXfJyzN5XpiJmZplzQTjrRCY= +github.com/go-openapi/swag/loading v0.27.1/go.mod h1:jvGh3iA2+zyUUycB5fgJWzeHnhrpvGnJJM0RVE9ZShE= +github.com/go-openapi/swag/mangling v0.27.1 h1:yC9D0HyUE8gbP+BfmGx9+AA89ikwZTMjESK3OnnoaqA= +github.com/go-openapi/swag/mangling v0.27.1/go.mod h1:jtBE2+V+3pILxOR7Vgce+Cwp6A2PgZbvVqfNntbVs0w= +github.com/go-openapi/swag/netutils v0.27.1 h1:mICMFoS82F5TZ4Zy3cqmcQk+BFeCp3Uyq3Np7GI0/qU= +github.com/go-openapi/swag/netutils v0.27.1/go.mod h1:J+WYyFMLtvtCGqa6jLv+YNUmIKI3ZRQRrvfNDMoQoEQ= +github.com/go-openapi/swag/pools v0.27.1 h1:9LeadcMyb2GJCbXX5hVQDbZ2Lq9TL4dCs/nx1j5DO0E= +github.com/go-openapi/swag/pools v0.27.1/go.mod h1:kVQefhSK5RWuRe7BXsL8htgBPAMpN7HDGpGEknqugeE= +github.com/go-openapi/swag/stringutils v0.27.1 h1:ZXePZ0r2p1qSjo8tD3Un4vFj8+FqlCkczxDrJIhYUp8= +github.com/go-openapi/swag/stringutils v0.27.1/go.mod h1:lzRN95CxXmA03XcDWHLOb6nOMcxCqR5rGY0lOgsfRoM= +github.com/go-openapi/swag/typeutils v0.27.1 h1:KSTdFlfnse4r6dP9IrEnwMldjE+zs71UeEB3//PtVXc= +github.com/go-openapi/swag/typeutils v0.27.1/go.mod h1:Srm0xFNRZ1Y+vCxJclo5qzx8aj+1pAKda/YfFPrG0dQ= +github.com/go-openapi/swag/yamlutils v0.27.1 h1:ftxv6xvXb1E3zohUc+okZ9nSqNb9StQX/FXnKZ98sQA= +github.com/go-openapi/swag/yamlutils v0.27.1/go.mod h1:bnxFIB1qewGRiZHypXGZ3fNgf13/0HfRgnS/iZBDrOo= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0 h1:gGHwAJ0R/5jU8BEGDbfRNR3hL68dAVi84WuOApp29B0= +github.com/go-openapi/testify/enable/yaml/v2 v2.6.0/go.mod h1:tY+St1SGq4NFl0QIqdTY4aEdbChAHxhyB77XQi9iJCo= +github.com/go-openapi/testify/v2 v2.6.0 h1:5PKH2HE7YJ/LuRPQGvSxBRlFXNQhSetBLlGAgUEu3ug= +github.com/go-openapi/testify/v2 v2.6.0/go.mod h1:SgsVHtfooshd0tublTtJ50FPKhujf47YRqauXXOUxfw= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/goccy/go-yaml v1.18.0 h1:8W7wMFS12Pcas7KU+VVkaiCng+kG8QiFeFwzFb+rwuw= @@ -75,8 +100,6 @@ github.com/intel/goresctrl v0.13.0 h1:5fhKjNq4V5MYDFHa//6M6x0jP6Iq5EXwZc6/eYxdEt github.com/intel/goresctrl v0.13.0/go.mod h1:KFHS91JGOmeeuEog+nTQcsGjLC81nRqdsdhcqf69fjU= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= github.com/intel/memtierd v0.1.1/go.mod h1:NFDBvjoDS42gBK/c9q/CYCJ2pt/+g7UQwOOBvQli4z0= -github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= -github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/joshdk/go-junit v1.0.0 h1:S86cUKIdwBHWwA6xCmFlf3RTLfVXYQfvanM5Uh+K6GE= github.com/joshdk/go-junit v1.0.0/go.mod h1:TiiV0PqkaNfFXjEiyjWM3XXrhVyCa1K4Zfga6W52ung= github.com/json-iterator/go v1.1.12 h1:PV8peI4a0ysnczrg+LtxykD8LfKY9ML6u2jnxaEnrnM= @@ -89,17 +112,12 @@ github.com/klauspost/compress v1.19.1 h1:VsB4HPswih7mmZ8WleSFQ75c/Ui1M4trX5oAsJn github.com/klauspost/compress v1.19.1/go.mod h1:cwPg85FWrGar70rWktvGQj8/hthj3wpl0PGDogxkrSQ= github.com/knqyf263/go-plugin v0.9.0 h1:CQs2+lOPIlkZVtcb835ZYDEoyyWJWLbSTWeCs0EwTwI= github.com/knqyf263/go-plugin v0.9.0/go.mod h1:2z5lCO1/pez6qGo8CvCxSlBFSEat4MEp1DrnA+f7w8Q= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= -github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= -github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= @@ -120,8 +138,6 @@ github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5 github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= -github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= -github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -139,17 +155,12 @@ github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0t github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc= github.com/sirupsen/logrus v1.10.1 h1:xi4336Zh11WpU14fXR6I67V3yaTPQYwRx2WEtHbRg4Q= github.com/sirupsen/logrus v1.10.1/go.mod h1:vsQHnG7xzNsxk3NrwboUiWPnIC3dmbjcGPykD7+tiHk= -github.com/spf13/pflag v1.0.6 h1:jFzHGLGAlb3ruxLB8MhbI6A8+AQX/2eW4qeyNZXNp2o= -github.com/spf13/pflag v1.0.6/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= +github.com/spf13/pflag v1.0.10 h1:4EBh2KAYBwaONj6b2Ye1GiHfwjqyROoF4RwYO+vPwFk= +github.com/spf13/pflag v1.0.10/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/objx v0.5.3 h1:jmXUvGomnU1o3W/V5h2VEradbpJDwGrzugQQvL0POH4= github.com/stretchr/objx v0.5.3/go.mod h1:rDQraq+vQZU7Fde9LOZLr8Tax6zZvy4kuNKF+QYS+U0= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.12.1 h1:EuwCh5fleGS7H32xRwO3wRGT7DxrDhLAT6FF8MpWDWE= github.com/stretchr/testify v1.12.1/go.mod h1:MDEgiDPPsNp5cuIrHPPCyornHKgEVbtFUmoNlxoYthg= github.com/tetratelabs/wazero v1.11.0 h1:+gKemEuKCTevU4d7ZTzlsvgd1uaToIDtlQlmNbwqYhA= @@ -245,8 +256,8 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.40.0 h1:Ub2Z6/xjgF1WrYQz2nuITOEegKFtiIy+rieRJ5lHZKs= golang.org/x/text v0.40.0/go.mod h1:hpnzDAfGV753zIKo+wk3u1bVKCGPbrnF7+7LBF/UHVY= -golang.org/x/time v0.9.0 h1:EsRrnYcQiGH+5FfbgvV4AP7qEZstoyrHB0DzarOQ4ZY= -golang.org/x/time v0.9.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= +golang.org/x/time v0.15.0 h1:bbrp8t3bGUeFOx08pvsMYRTCVSMk89u4tKbNOZbp88U= +golang.org/x/time v0.15.0/go.mod h1:Y4YMaQmXwGQZoFaVFk4YpCt4FLQMYKZe9oeV/f4MSno= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= @@ -274,38 +285,37 @@ google.golang.org/protobuf v1.36.12-0.20260120151049-f2248ac996af/go.mod h1:HTf+ gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= -gopkg.in/evanphx/json-patch.v4 v4.12.0 h1:n6jtcsulIzXPJaxegRbvFNNrZDjbij7ny3gmSPG+6V4= -gopkg.in/evanphx/json-patch.v4 v4.12.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= +gopkg.in/evanphx/json-patch.v4 v4.13.0 h1:czT3CmqEaQ1aanPc5SdlgQrrEIb8w/wwCvWWnfEbYzo= +gopkg.in/evanphx/json-patch.v4 v4.13.0/go.mod h1:p8EYWUEYMpynmqDbY58zCKCFZw8pRWMG4EsWvDvM72M= gopkg.in/inf.v0 v0.9.1 h1:73M5CoZyi3ZLMOyDlQh031Cx6N9NDJ2Vvfl76EDAgDc= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.34.11 h1:LkxGIHlj06urNrS5Gpgj2iFRRgja+0tUV0+ejZ6j/T8= -k8s.io/api v0.34.11/go.mod h1:b1wzM4EH4H4FtXVDK3zAnG1zG/NBQBwtPeahyyYYDIU= -k8s.io/apimachinery v0.34.11 h1:N29JKPe3cXeLmc8QhFILkKU7jYq47EWG1zAZNOIawnM= -k8s.io/apimachinery v0.34.11/go.mod h1:xfCr+Akw9yI3OXIqWDjOaQCklbC498VcPxtFJpRK+FI= -k8s.io/client-go v0.34.11 h1:ibO8IP6RMp6JImv47JIW6ZUcZky98zLnzaGIsD6yQWM= -k8s.io/client-go v0.34.11/go.mod h1:MxE93lNP2kK12JL4S74ddZctG0Pt6JVLiK8nLwlMxyY= +k8s.io/api v0.37.0 h1:Z//Vj9N7RA/yS2sDmxyeo7h+RR4zbUrd2vrd3Z0TbB4= +k8s.io/api v0.37.0/go.mod h1:LKXgcJWMc+f4OLbP5SFR8rulEg07zZhpi/zMULiBImk= +k8s.io/apimachinery v0.37.0 h1:Np2AbDtf8x6RDHiD8T9LbKJ9gaegeVNa8yNm5FuGKm0= +k8s.io/apimachinery v0.37.0/go.mod h1:RN3nhprFSCxOi5Selxd7oMTXOe/c+ZbcE7Im+TS2zkE= +k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ= +k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0= k8s.io/code-generator v0.34.11 h1:o7jWDfqo27pQygEm7a+g8QytR9ZaVEHE3urIfNmFYhU= k8s.io/code-generator v0.34.11/go.mod h1:ACmWPlRob//4Azs/4/wIi+CFPmnGhYuixpJfKUUhyJs= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f h1:SLb+kxmzfA87x4E4brQzB33VBbT2+x7Zq9ROIHmGn9Q= -k8s.io/gengo/v2 v2.0.0-20250604051438-85fd79dbfd9f/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= +k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= k8s.io/klog/v2 v2.140.0/go.mod h1:o+/RWfJ6PwpnFn7OyAG3QnO47BFsymfEfrz6XyYSSp0= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b h1:MloQ9/bdJyIu9lb1PzujOPolHyvO06MXG5TUIj2mNAA= -k8s.io/kube-openapi v0.0.0-20250710124328-f3f2b991d03b/go.mod h1:UZ2yyWbFTpuhSbFhv24aGNOdoRdJZgsIObGBUaYVsts= -k8s.io/kubelet v0.34.11 h1:jp81mYXdPnHRwEqtRWpaVGpdB3+rDSruVL1A0C90ahQ= -k8s.io/kubelet v0.34.11/go.mod h1:wu6sd9D6svx7r1PLzfDVX8hZ9M3C/4BIUhK87fJyHaY= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397 h1:hwvWFiBzdWw1FhfY1FooPn3kzWuJ8tmbZBHi4zVsl1Y= -k8s.io/utils v0.0.0-20250604170112-4c0f3b243397/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad h1:oXImqH8mQNk7PmvzKhmN3ddJoY6OnyM225MXwGHPm0A= +k8s.io/kube-openapi v0.0.0-20260721132016-d427ff9ee9ad/go.mod h1:0/mqHCVhlumdJ3BhCfnjSZQE037nAhNodh1/hK0T8/I= +k8s.io/kubelet v0.37.0 h1:VhaZanjlE5CkoAPAjKw0DH+Q0BYXVfYekZgzZDAJMMg= +k8s.io/kubelet v0.37.0/go.mod h1:PHXfQuVqsTzFVqOeP67UUNv5Ajri+dLGoeaEnW6gGjE= +k8s.io/utils v0.0.0-20260626114624-be93311217bd h1:Ea7fgQ5we8Y9T0OX5o0dAHzQOBRI07D/dEYRaB9ZZEs= +k8s.io/utils v0.0.0-20260626114624-be93311217bd/go.mod h1:xDxuJ0whA3d0I4mf/C4ppKHxXynQ+fxnkmQH0vTHnuk= sigs.k8s.io/controller-runtime v0.22.5 h1:v3nfSUMowX/2WMp27J9slwGFyAt7IV0YwBxAkrUr0GE= sigs.k8s.io/controller-runtime v0.22.5/go.mod h1:pc5SoYWnWI6I+cBHYYdZ7B6YHZVY5xNfll88JB+vniI= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8 h1:gBQPwqORJ8d8/YNZWEjoZs7npUVDpVXUUOFfW6CgAqE= -sigs.k8s.io/json v0.0.0-20241014173422-cfa47c3a1cc8/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 h1:IpInykpT6ceI+QxKBbEflcR5EXP7sU1kvOlxwZh5txg= +sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730/go.mod h1:mdzfpAEoE6DHQEN0uh9ZbOCuHbLK5wOm7dK4ctXE9Tg= sigs.k8s.io/randfill v1.0.0 h1:JfjMILfT8A6RbawdsK2JXGBR5AQVfd+9TbzrlneTyrU= sigs.k8s.io/randfill v1.0.0/go.mod h1:XeLlZ/jmk4i1HRopwe7/aU3H5n1zNUcX6TM94b3QxOY= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0 h1:jTijUJbW353oVOd9oTlifJqOGEkUw2jB/fXCbTiQEco= -sigs.k8s.io/structured-merge-diff/v6 v6.3.0/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3CIki3aW8c4Q= +sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index 34548cea5..b123aeac9 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -78,6 +78,11 @@ type Handler struct { cpufreq *cpufreq.Allocator pct *pct.Allocator + // classes is the last-applied cpuClass list. Updated on every + // Configure() call. Used by DRADevices to enumerate published classes. + // Caller-owned slice; not deep-copied (consistent with pct.Configure behavior). + classes []*policyapi.CPUClass + // defs maps synthetic class name -> resolved class definition. // Populated by SetClassDef calls from the cpufreq allocator. defs map[string]types.ClassDef @@ -158,6 +163,10 @@ func (h *Handler) Configure(spec ConfigSpec) error { return fmt.Errorf("cpuclass: pct configure: %w", err) } + // h.classes is set after all fallible operations so that a partial + // Configure failure leaves h.classes consistent with the previously + // committed state (not a half-applied new config). + h.classes = spec.Classes h.classNames = map[string]struct{}{} for _, cls := range spec.Classes { h.classNames[cls.Name] = struct{}{} diff --git a/pkg/resmgr/cpuclass/dra.go b/pkg/resmgr/cpuclass/dra.go index b503993ca..826c30d41 100644 --- a/pkg/resmgr/cpuclass/dra.go +++ b/pkg/resmgr/cpuclass/dra.go @@ -16,11 +16,40 @@ package cpuclass import ( "fmt" + "regexp" "sort" + "strconv" + "strings" + + corev1 "k8s.io/api/core/v1" + resapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/resource" + kptr "k8s.io/utils/ptr" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" ) +// nonAlphaRe matches runs of characters that are not lowercase letters or digits. +// Used by sanitizeBase to replace them with hyphens. +var nonAlphaRe = regexp.MustCompile(`[^a-z0-9]+`) + +// maxDNSLabel is the Kubernetes DNS label name length limit. +const maxDNSLabel = 63 + +// maxDeviceBase returns the maximum length for the sanitized class-name +// portion of a device name before any dedup suffix is appended. +func maxDeviceBase(punits []pct.PunitInfo, classCount int) int { + maxSuffixLen := len("-pkg0-punit0") + for _, pu := range punits { + if suffixLen := len("-pkg" + strconv.Itoa(pu.PkgID) + "-punit" + strconv.Itoa(pu.PunitID)); suffixLen > maxSuffixLen { + maxSuffixLen = suffixLen + } + } + maxDedupSuffixLen := len("-" + strconv.Itoa(classCount+1)) + return maxDNSLabel - maxSuffixLen - maxDedupSuffixLen +} + // ValidateCPUClassesForDRA checks that DRA-published PCT classes do not // overcommit any priority tier. Classes are grouped by tier — the // pctPriority value for managed PCT classes, or the SstClosID for @@ -38,7 +67,7 @@ import ( func ValidateCPUClassesForDRA(classes []*policyapi.CPUClass, sharedCounters bool) error { if sharedCounters { return fmt.Errorf( - "DRA: sharedCounters is not yet supported (Model C / KEP-5941 is not " + + "DRA: sharedCounters is not yet supported (KEP-5941 is not " + "implemented); leave spec.dra.sharedCounters unset or false", ) } @@ -96,3 +125,183 @@ func tierLabel(cc *policyapi.CPUClass) string { } return fmt.Sprintf("closID=%d", *cc.SstClosID) } + +// sanitizeBase lowercases s, replaces runs of non-alphanumeric characters with +// "-", trims leading/trailing hyphens, and truncates to maxLen (trimming any +// trailing hyphen created by truncation). Returns "class" if the result is empty. +func sanitizeBase(s string, maxLen int) string { + b := strings.ToLower(s) + b = nonAlphaRe.ReplaceAllString(b, "-") + b = strings.Trim(b, "-") + if b == "" { + return "class" + } + if len(b) > maxLen { + b = strings.TrimRight(b[:maxLen], "-") + } + if b == "" { + return "class" + } + return b +} + +// deviceName assembles a DRA device name from a pre-sanitized class base and +// punit topology identifiers. Format: -pkg-punit. +func deviceName(classBase string, pkgID, punitID int) string { + return classBase + "-pkg" + strconv.Itoa(pkgID) + "-punit" + strconv.Itoa(punitID) +} + +// intAttr returns a DeviceAttribute with an integer value. +func intAttr(v int64) resapi.DeviceAttribute { + return resapi.DeviceAttribute{IntValue: kptr.To(v)} +} + +// strAttr returns a DeviceAttribute with a string value. +func strAttr(v string) resapi.DeviceAttribute { + return resapi.DeviceAttribute{StringValue: kptr.To(v)} +} + +// buildDRADevices constructs the []resapi.Device slice (one device per +// published cpuClass × SST-TF punit) to be passed to kubeletplugin.PublishResources. +// +// For each published class, for each punit: emits one device if capacity > 0. +// HP classes use HPCapacity; non-HP classes use NonHPCapacity. hpOnly only +// affects the device-name length budget for now. +func buildDRADevices( + driverName string, + classes []*policyapi.CPUClass, + punits []pct.PunitInfo, + isHP func(className string) bool, + hpOnly bool, +) []resapi.Device { + if len(classes) == 0 || len(punits) == 0 { + return []resapi.Device{} + } + + // Pre-compute a stable sanitized base for each published class name. + // Dedup: if two different class names produce the same base, the second + // gets a "-N" suffix (N starting at 2). The same class name across multiple + // punits always reuses the same pre-computed base (no counter increment). + takenBases := map[string]struct{}{} // bases already claimed by some class + baseForClass := map[string]string{} // className -> final sanitized base + publishedClassCount := 0 + for _, cc := range classes { + if cc.DRAPublish() && (!hpOnly || isHP(cc.Name)) { + publishedClassCount++ + } + } + baseMaxLen := maxDeviceBase(punits, publishedClassCount) + + for _, cc := range classes { + if !cc.DRAPublish() { + continue + } + if _, done := baseForClass[cc.Name]; done { + continue // same class name seen twice — skip (defensive) + } + candidate := sanitizeBase(cc.Name, baseMaxLen) + if _, taken := takenBases[candidate]; !taken { + takenBases[candidate] = struct{}{} + baseForClass[cc.Name] = candidate + } else { + // Collision: find the next available suffixed base. + for n := 2; ; n++ { + suffixed := candidate + "-" + strconv.Itoa(n) + if _, inUse := takenBases[suffixed]; !inUse { + takenBases[suffixed] = struct{}{} + baseForClass[cc.Name] = suffixed + break + } + } + } + } + + var devices []resapi.Device + + for _, cc := range classes { + if !cc.DRAPublish() { + continue + } + base := baseForClass[cc.Name] + // Always set: duplicate class names are skipped above, and + // validation guarantees no duplicates reach this point. + for _, pu := range punits { + // Select capacity based on HP classification. + var capacity int + if isHP(cc.Name) { + capacity = pu.HPCapacity + } else { + capacity = pu.NonHPCapacity + } + if capacity == 0 { + continue // zero-capacity RequestPolicy is invalid; skip + } + + name := deviceName(base, pu.PkgID, pu.PunitID) + + attrs := map[resapi.QualifiedName]resapi.DeviceAttribute{ + "nri/packageID": intAttr(int64(pu.PkgID)), + "nri/punitID": intAttr(int64(pu.PunitID)), + "nri/cpuClass": strAttr(cc.Name), + } + // nri/pctPriority is only emitted for PCT classes (non-empty PctPriority). + // Omitting it for non-PCT classes avoids CEL false-positives on "" values. + if cc.PctPriority != "" { + attrs["nri/pctPriority"] = strAttr(cc.PctPriority) + } + + capStr := strconv.Itoa(capacity) + dev := resapi.Device{ + Name: name, + Attributes: attrs, + Capacity: map[resapi.QualifiedName]resapi.DeviceCapacity{ + "nri/cpus": { + Value: resource.MustParse(capStr), + RequestPolicy: &resapi.CapacityRequestPolicy{ + Default: kptr.To(resource.MustParse("1")), + ValidRange: &resapi.CapacityRequestPolicyRange{ + Min: kptr.To(resource.MustParse("1")), + Max: kptr.To(resource.MustParse(capStr)), + Step: kptr.To(resource.MustParse("1")), + }, + }, + }, + }, + AllowMultipleAllocations: kptr.To(true), + NodeAllocatableResources: map[corev1.ResourceName]resapi.NodeAllocatableResource{ + corev1.ResourceCPU: { + Mapping: &resapi.NodeAllocatableMapping{ + CapacityKey: kptr.To(resapi.QualifiedName("nri/cpus")), + CapacityMultiplier: kptr.To(resource.MustParse("1")), + }, + }, + }, + } + devices = append(devices, dev) + } + } + + if devices == nil { + return []resapi.Device{} + } + return devices +} + +// DRADevices returns the DRA device slice for the current cpuClass configuration. +// Returns an empty (non-nil) slice when the handler is nil, PCT is inactive, +// or no punits are available. +// +// Must be called on the resmgr goroutine or under the resmgr lock — same as all +// other Handler methods. +func (h *Handler) DRADevices(driverName string) ([]resapi.Device, error) { + if h == nil || h.pct == nil { + return []resapi.Device{}, nil + } + // Punits() returns nil when inactive, so len()==0 covers both + // the inactive and the "no punits" cases. + punits := h.pct.Punits() + if len(punits) == 0 { + return []resapi.Device{}, nil + } + return buildDRADevices(driverName, h.classes, punits, h.pct.IsHPClass, true), nil +} diff --git a/pkg/resmgr/cpuclass/dra_test.go b/pkg/resmgr/cpuclass/dra_test.go index f64563f20..660f7a2d3 100644 --- a/pkg/resmgr/cpuclass/dra_test.go +++ b/pkg/resmgr/cpuclass/dra_test.go @@ -15,10 +15,16 @@ package cpuclass import ( + "regexp" "strings" "testing" + corev1 "k8s.io/api/core/v1" + resapi "k8s.io/api/resource/v1" + policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" + "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func ptr[T any](v T) *T { return &v } @@ -176,3 +182,661 @@ func TestValidateCPUClassesForDRA(t *testing.T) { }) } } + +// dnsLabelRe matches valid Kubernetes DNS label names (RFC 1123 subset). +var dnsLabelRe = regexp.MustCompile(`^[a-z0-9]([a-z0-9-]*[a-z0-9])?$`) + +// isDNSLabel reports whether s is a valid DNS label (≤63 chars, lowercase alphanumeric + hyphens). +func isDNSLabel(s string) bool { + return len(s) <= 63 && dnsLabelRe.MatchString(s) +} + +// attrInt retrieves the integer value of a device attribute by key. +// Returns (0, false) if absent or not an int. +func attrInt(dev resapi.Device, key resapi.QualifiedName) (int64, bool) { + a, ok := dev.Attributes[key] + if !ok || a.IntValue == nil { + return 0, false + } + return *a.IntValue, true +} + +// attrStr retrieves the string value of a device attribute by key. +// Returns ("", false) if absent or not a string. +func attrStr(dev resapi.Device, key resapi.QualifiedName) (string, bool) { + a, ok := dev.Attributes[key] + if !ok || a.StringValue == nil { + return "", false + } + return *a.StringValue, true +} + +// cpusMax returns the RequestPolicy.ValidRange.Max quantity for the "nri/cpus" +// capacity entry of a device. Returns (0, false) if missing or malformed. +func cpusMax(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.ValidRange == nil || cap.RequestPolicy.ValidRange.Max == nil { + return 0, false + } + return cap.RequestPolicy.ValidRange.Max.Value(), true +} + +// cpusCapacity returns the DeviceCapacity Value for the "nri/cpus" entry. +func cpusCapacity(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok { + return 0, false + } + return cap.Value.Value(), true +} + +// cpusDefault returns the RequestPolicy.Default value for "nri/cpus". +// Returns (0, false) if missing or malformed. +func cpusDefault(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.Default == nil { + return 0, false + } + return cap.RequestPolicy.Default.Value(), true +} + +// cpusMin returns the RequestPolicy.ValidRange.Min value for "nri/cpus". +// Returns (0, false) if missing or malformed. +func cpusMin(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.ValidRange == nil || cap.RequestPolicy.ValidRange.Min == nil { + return 0, false + } + return cap.RequestPolicy.ValidRange.Min.Value(), true +} + +// cpusStep returns the RequestPolicy.ValidRange.Step value for "nri/cpus". +// Returns (0, false) if missing or malformed. +func cpusStep(dev resapi.Device) (int64, bool) { + cap, ok := dev.Capacity["nri/cpus"] + if !ok || cap.RequestPolicy == nil || cap.RequestPolicy.ValidRange == nil || cap.RequestPolicy.ValidRange.Step == nil { + return 0, false + } + return cap.RequestPolicy.ValidRange.Step.Value(), true +} + +// checkDeviceShape asserts the shape invariants that every emitted device must +// satisfy: AllowMultipleAllocations, DeviceCapacity.Value, the full +// RequestPolicy (Default/Min/Max/Step), and NodeAllocatableResources content +// (Mapping.CapacityKey + Mapping.CapacityMultiplier). It also verifies +// topology attributes and the nri/cpuClass attribute. +// +// wantPctPriorityPresent controls whether nri/pctPriority is expected to exist +// (PCT classes) or must be absent (non-PCT classes). +func checkDeviceShape(t *testing.T, dev resapi.Device, wantCapacity int64, + wantClass, wantPctPriority string, wantPctPriorityPresent bool) { + t.Helper() + + // AllowMultipleAllocations must be true. + if dev.AllowMultipleAllocations == nil || !*dev.AllowMultipleAllocations { + t.Errorf("AllowMultipleAllocations: got %v, want true", dev.AllowMultipleAllocations) + } + + // nri/cpuClass. + if v, ok := attrStr(dev, "nri/cpuClass"); !ok { + t.Error("nri/cpuClass attribute missing") + } else if v != wantClass { + t.Errorf("nri/cpuClass = %q, want %q", v, wantClass) + } + + // nri/pctPriority. + _, hasPct := dev.Attributes["nri/pctPriority"] + if wantPctPriorityPresent { + if v, ok := attrStr(dev, "nri/pctPriority"); !ok { + t.Errorf("nri/pctPriority attribute missing (want %q)", wantPctPriority) + } else if v != wantPctPriority { + t.Errorf("nri/pctPriority = %q, want %q", v, wantPctPriority) + } + } else if hasPct { + t.Errorf("nri/pctPriority must be absent, got %v", dev.Attributes["nri/pctPriority"]) + } + + // DeviceCapacity.Value (outer field, independent of ValidRange.Max). + if cv, ok := cpusCapacity(dev); !ok { + t.Error("nri/cpus capacity Value missing") + } else if cv != wantCapacity { + t.Errorf("nri/cpus capacity Value = %d, want %d", cv, wantCapacity) + } + + // RequestPolicy: Default must be 1. + if dv, ok := cpusDefault(dev); !ok { + t.Error("nri/cpus RequestPolicy.Default missing") + } else if dv != 1 { + t.Errorf("nri/cpus RequestPolicy.Default = %d, want 1", dv) + } + + // RequestPolicy.ValidRange: Min must be 1. + if mv, ok := cpusMin(dev); !ok { + t.Error("nri/cpus RequestPolicy.ValidRange.Min missing") + } else if mv != 1 { + t.Errorf("nri/cpus RequestPolicy.ValidRange.Min = %d, want 1", mv) + } + + // RequestPolicy.ValidRange: Max must equal capacity. + if xv, ok := cpusMax(dev); !ok { + t.Error("nri/cpus RequestPolicy.ValidRange.Max missing") + } else if xv != wantCapacity { + t.Errorf("nri/cpus RequestPolicy.ValidRange.Max = %d, want %d", xv, wantCapacity) + } + + // RequestPolicy.ValidRange: Step must be 1. + if sv, ok := cpusStep(dev); !ok { + t.Error("nri/cpus RequestPolicy.ValidRange.Step missing") + } else if sv != 1 { + t.Errorf("nri/cpus RequestPolicy.ValidRange.Step = %d, want 1", sv) + } + + // NodeAllocatableResources content. + if dev.NodeAllocatableResources == nil { + t.Error("NodeAllocatableResources is nil") + } else { + r, ok := dev.NodeAllocatableResources[corev1.ResourceCPU] + if !ok { + t.Errorf("NodeAllocatableResources: missing %q key", corev1.ResourceCPU) + } else if r.Mapping == nil { + t.Errorf("NodeAllocatableResources[cpu].Mapping is nil") + } else { + if r.Mapping.CapacityKey == nil || *r.Mapping.CapacityKey != "nri/cpus" { + t.Errorf("NodeAllocatableResources[cpu].Mapping.CapacityKey = %v, want \"nri/cpus\"", r.Mapping.CapacityKey) + } + if r.Mapping.CapacityMultiplier == nil || r.Mapping.CapacityMultiplier.Value() != 1 { + t.Errorf("NodeAllocatableResources[cpu].Mapping.CapacityMultiplier = %v, want 1", + r.Mapping.CapacityMultiplier) + } + } + } +} + +func TestBuildDRADevices(t *testing.T) { + // Shorthand helpers used only in test table. + hpClass := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{Name: name, PctPriority: "high"} + } + lpClass := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{Name: name, PctPriority: "low"} + } + nonPCTClass := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{Name: name} + } + unpublished := func(name string) *policyapi.CPUClass { + return &policyapi.CPUClass{ + Name: name, + PctPriority: "high", + DRA: &policyapi.CPUClassDRA{Publish: ptr(false)}, + } + } + punit := func(pkg, id, hpCap, nonHPCap int) pct.PunitInfo { + return pct.PunitInfo{PkgID: pkg, PunitID: id, HPCapacity: hpCap, NonHPCapacity: nonHPCap} + } + + // isHP returns true only for classes whose name starts with "hp". + isHP := func(name string) bool { return strings.HasPrefix(name, "hp") } + + tests := []struct { + name string + driverName string + classes []*policyapi.CPUClass + punits []pct.PunitInfo + isHP func(string) bool + // wantCount is the expected number of returned devices. + wantCount int + // verify is an optional per-result checker. + verify func(t *testing.T, devices []resapi.Device) + }{ + { + name: "one HP class + one punit (pkg=0 punit=0)", + driverName: "test.driver", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + // Full device-shape invariants: capacity=HPCapacity=4, PCT class "hp"/"high". + checkDeviceShape(t, dev, 4, "hp", "high", true) + // Topology attributes present and correct. + if v, ok := attrInt(dev, "nri/packageID"); !ok { + t.Errorf("nri/packageID attribute missing") + } else if v != 0 { + t.Errorf("nri/packageID = %d, want 0", v) + } + if v, ok := attrInt(dev, "nri/punitID"); !ok { + t.Errorf("nri/punitID attribute missing") + } else if v != 0 { + t.Errorf("nri/punitID = %d, want 0", v) + } + // Device name must be DNS-valid. + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + name: "one non-HP PCT class + one punit → max=NonHPCapacity", + classes: []*policyapi.CPUClass{lpClass("lp")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + // Full device-shape invariants: capacity=NonHPCapacity=8, PCT class "lp"/"low". + checkDeviceShape(t, dev, 8, "lp", "low", true) + // Topology attributes. + if _, ok := attrInt(dev, "nri/packageID"); !ok { + t.Error("nri/packageID attribute missing") + } + if _, ok := attrInt(dev, "nri/punitID"); !ok { + t.Error("nri/punitID attribute missing") + } + }, + }, + { + name: "HP class + HPCapacity==0 → device skipped for that punit", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: []pct.PunitInfo{punit(0, 0, 0, 8)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "NonHPCapacity==0 → device skipped for that punit", + classes: []*policyapi.CPUClass{lpClass("lp")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 0)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "class with dra.publish: false → excluded", + classes: []*policyapi.CPUClass{unpublished("hp-hidden")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "two classes × two punits → four devices with correct names", + classes: []*policyapi.CPUClass{ + hpClass("hp"), + lpClass("lp"), + }, + punits: []pct.PunitInfo{ + punit(0, 0, 4, 8), + punit(0, 1, 4, 8), + }, + isHP: isHP, + wantCount: 4, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + names := make(map[string]bool, 4) + for _, d := range devices { + names[d.Name] = true + if !isDNSLabel(d.Name) { + t.Errorf("device name %q is not a valid DNS label", d.Name) + } + } + // Same class across punits must use the same sanitized base, + // not trigger the dedup counter (e.g. "hp-2-pkg0-punit0"). + wantNames := []string{ + "hp-pkg0-punit0", + "hp-pkg0-punit1", + "lp-pkg0-punit0", + "lp-pkg0-punit1", + } + for _, want := range wantNames { + if !names[want] { + t.Errorf("expected device name %q not found in %v", want, devices) + } + } + // Every device must have AllowMultipleAllocations=true. + for _, d := range devices { + if d.AllowMultipleAllocations == nil || !*d.AllowMultipleAllocations { + t.Errorf("device %q: AllowMultipleAllocations not true", d.Name) + } + } + // Every device must carry nri/cpuClass pointing to the right class. + for _, d := range devices { + v, ok := attrStr(d, "nri/cpuClass") + if !ok { + t.Errorf("device %q: nri/cpuClass missing", d.Name) + continue + } + // Name encodes the class base: "hp-pkg..." vs "lp-pkg...". + wantClass := "hp" + if strings.HasPrefix(d.Name, "lp-") { + wantClass = "lp" + } + if v != wantClass { + t.Errorf("device %q: nri/cpuClass = %q, want %q", d.Name, v, wantClass) + } + } + }, + }, + { + name: "empty classes → empty result", + classes: nil, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 0, + }, + { + name: "empty punits → empty result", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: nil, + isHP: isHP, + wantCount: 0, + }, + { + // Class name > 60 chars must produce a device name ≤ 63 chars. + name: "long class name → device name ≤ 63 chars", + classes: []*policyapi.CPUClass{hpClass("hp-this-is-a-very-long-cpuclass-name-that-exceeds-sixty-chars-total-yes")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if len(dev.Name) > 63 { + t.Errorf("device name %q has length %d, want ≤ 63", dev.Name, len(dev.Name)) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + // 2-digit pkg+punit IDs ("-pkg10-punit10" = 14 chars) combined with a + // max-length base and a dedup suffix must still fit in 63 chars. + name: "2-digit pkg and punit IDs → device name ≤ 63 chars", + classes: []*policyapi.CPUClass{hpClass("hp-this-is-a-very-long-cpuclass-name-that-exceeds-sixty-chars-total-yes")}, + punits: []pct.PunitInfo{punit(10, 10, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if len(dev.Name) > 63 { + t.Errorf("device name %q has length %d, want ≤ 63", dev.Name, len(dev.Name)) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + // Two classes whose names sanitize to the same base must get distinct device names. + name: "two classes sanitize to same base → distinct device names", + classes: []*policyapi.CPUClass{ + // Both "hp-class" and "hp_class" sanitize to "hp-class". + hpClass("hp-class"), + hpClass("hp_class"), + }, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + // Both classes have HPCapacity>0 so both emit one device. + wantCount: 2, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + seen := map[string]bool{} + for _, d := range devices { + if seen[d.Name] { + t.Errorf("duplicate device name %q", d.Name) + } + seen[d.Name] = true + if !isDNSLabel(d.Name) { + t.Errorf("device name %q is not a valid DNS label", d.Name) + } + } + }, + }, + { + // Non-PCT class must NOT emit nri/pctPriority attribute; must still + // carry topology and cpuClass attributes, and AllowMultipleAllocations. + name: "non-PCT class → nri/pctPriority absent, topology present", + classes: []*policyapi.CPUClass{nonPCTClass("default")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: func(string) bool { return false }, // non-PCT: never HP + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + // pctPriority must be absent. + if _, ok := dev.Attributes["nri/pctPriority"]; ok { + t.Errorf("nri/pctPriority must be absent for non-PCT class, got %v", + dev.Attributes["nri/pctPriority"]) + } + // AllowMultipleAllocations must still be true. + if dev.AllowMultipleAllocations == nil || !*dev.AllowMultipleAllocations { + t.Errorf("AllowMultipleAllocations: got %v, want true", dev.AllowMultipleAllocations) + } + // Topology attributes must be present. + if _, ok := attrInt(dev, "nri/packageID"); !ok { + t.Error("nri/packageID attribute missing") + } + if _, ok := attrInt(dev, "nri/punitID"); !ok { + t.Error("nri/punitID attribute missing") + } + // nri/cpuClass must still be present. + if v, ok := attrStr(dev, "nri/cpuClass"); !ok { + t.Error("nri/cpuClass attribute missing") + } else if v != "default" { + t.Errorf("nri/cpuClass = %q, want \"default\"", v) + } + }, + }, + { + // Non-zero PkgID and PunitID must appear correctly in attributes and name. + name: "non-zero PkgID and PunitID → correct in attrs and name", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: []pct.PunitInfo{punit(2, 3, 5, 6)}, // pkg=2, punit=3 + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if v, ok := attrInt(dev, "nri/packageID"); !ok || v != 2 { + t.Errorf("nri/packageID = %d (ok=%v), want 2", v, ok) + } + if v, ok := attrInt(dev, "nri/punitID"); !ok || v != 3 { + t.Errorf("nri/punitID = %d (ok=%v), want 3", v, ok) + } + if dev.Name != "hp-pkg2-punit3" { + t.Errorf("device name = %q, want hp-pkg2-punit3", dev.Name) + } + // Full shape check with HPCapacity=5. + checkDeviceShape(t, dev, 5, "hp", "high", true) + }, + }, + { + // Class name that is all non-alphanumeric → sanitizeBase returns "class". + name: "all-special-char class name → 'class' fallback base", + classes: []*policyapi.CPUClass{hpClass("---")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if dev.Name != "class-pkg0-punit0" { + t.Errorf("device name = %q, want class-pkg0-punit0", dev.Name) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + isHPFn := tc.isHP + if isHPFn == nil { + isHPFn = func(string) bool { return false } + } + driverName := tc.driverName + if driverName == "" { + driverName = "test.driver" + } + devices := buildDRADevices(driverName, tc.classes, tc.punits, isHPFn, true) + if len(devices) != tc.wantCount { + t.Fatalf("buildDRADevices() returned %d devices, want %d; devices=%v", + len(devices), tc.wantCount, deviceNames(devices)) + } + // buildDRADevices must always return a non-nil slice (including the + // empty case) so callers can safely range over it. + if devices == nil { + t.Errorf("buildDRADevices() returned nil, want non-nil []Device{}") + } + if tc.verify != nil { + tc.verify(t, devices) + } + }) + } +} + +// deviceNames returns a slice of device names for use in test error messages. +func deviceNames(devices []resapi.Device) []string { + names := make([]string, len(devices)) + for i, d := range devices { + names[i] = d.Name + } + return names +} + +// sstWithPunit is a JSON seed for OVERRIDE_SST that provides one active +// SST-TF package (pkg=1) with one punit (punit=2), 8 CPUs, and +// GuaranteedHpCpus=3 (i.e. HPCapacity=3, NonHPCapacity=5). +// Using non-zero pkg and punit IDs exercises the full attribute path. +const sstWithPunit = `{"supported":true,"clos_count":4,"packages":[{"id":1,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"cp_priority":"ordered","punits":[{"id":2,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":3}]}]}` + +// TestDRADevices covers Handler.DRADevices: the nil-handler guard, the +// nil-pct guard, the inactive-pct early-return, and the delegation path +// to buildDRADevices when pct is active with classes set. +// +// Tests that require an active pct allocator use OVERRIDE_SST so that +// pct.Configure succeeds without real SST hardware. +func TestDRADevices(t *testing.T) { + t.Run("nil handler returns empty non-nil", func(t *testing.T) { + var h *Handler + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices on nil handler: %v", err) + } + if devs == nil { + t.Error("nil handler: want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("nil handler: want 0 devices, got %d", len(devs)) + } + }) + + t.Run("nil pct returns empty non-nil", func(t *testing.T) { + h := &Handler{} // pct is nil + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices with nil pct: %v", err) + } + if devs == nil { + t.Error("nil pct: want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("nil pct: want 0 devices, got %d", len(devs)) + } + }) + + t.Run("inactive pct returns empty non-nil", func(t *testing.T) { + // OVERRIDE_SST with supported=false → pct stays disabled after Configure. + t.Setenv("OVERRIDE_SST", `{"supported":false,"packages":[]}`) + pctA, err := pct.NewAllocator(nil) + if err != nil { + t.Fatalf("pct.NewAllocator: %v", err) + } + if err := pctA.Configure(nil, cpuset.New()); err != nil { + t.Fatalf("pct.Configure: %v", err) + } + h := &Handler{pct: pctA} + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices inactive pct: %v", err) + } + if devs == nil { + t.Error("inactive pct: want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("inactive pct: want 0 devices, got %d", len(devs)) + } + }) + + t.Run("active pct empty classes returns empty non-nil", func(t *testing.T) { + // pct is active but h.classes is nil → buildDRADevices returns empty. + t.Setenv("OVERRIDE_SST", sstWithPunit) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + pctA, err := pct.NewAllocator(nil) + if err != nil { + t.Fatalf("pct.NewAllocator: %v", err) + } + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + if err := pctA.Configure(classes, cpuset.New()); err != nil { + t.Fatalf("pct.Configure: %v", err) + } + h := &Handler{pct: pctA, classes: nil} // classes not set + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices: %v", err) + } + if devs == nil { + t.Error("want non-nil empty slice, got nil") + } + if len(devs) != 0 { + t.Errorf("want 0 devices (no classes), got %d", len(devs)) + } + }) + + t.Run("active pct with classes delegates to buildDRADevices", func(t *testing.T) { + // OVERRIDE_SST: pkg=1, punit=2, HPCapacity=3. + // DRADevices must pass h.classes and h.pct.IsHPClass to buildDRADevices + // and return the resulting device slice. + t.Setenv("OVERRIDE_SST", sstWithPunit) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + pctA, err := pct.NewAllocator(nil) + if err != nil { + t.Fatalf("pct.NewAllocator: %v", err) + } + classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} + if err := pctA.Configure(classes, cpuset.New()); err != nil { + t.Fatalf("pct.Configure: %v", err) + } + if !pctA.Active() { + t.Fatal("pct not active after Configure with supported SST — check OVERRIDE_SST JSON") + } + // h.classes is set directly, mirroring what Handler.Configure does + // (post fix: h.classes is assigned after all fallible ops). + h := &Handler{pct: pctA, classes: classes} + devs, err := h.DRADevices("test.driver") + if err != nil { + t.Fatalf("DRADevices: %v", err) + } + // pkg=1, punit=2, HPCapacity=3 → one device "hp-pkg1-punit2" with + // capacity 3. NonHPCapacity=5 (8 CPUs − 3 HP) but "hp" is HP class. + if len(devs) != 1 { + t.Fatalf("DRADevices: got %d devices, want 1; names=%v", len(devs), deviceNames(devs)) + } + dev := devs[0] + if dev.Name != "hp-pkg1-punit2" { + t.Errorf("device name = %q, want hp-pkg1-punit2", dev.Name) + } + // Verify topology attributes carry the non-zero IDs. + if v, ok := attrInt(dev, "nri/packageID"); !ok || v != 1 { + t.Errorf("nri/packageID = %d (ok=%v), want 1", v, ok) + } + if v, ok := attrInt(dev, "nri/punitID"); !ok || v != 2 { + t.Errorf("nri/punitID = %d (ok=%v), want 2", v, ok) + } + // Full device-shape invariants (capacity=3, HP class). + checkDeviceShape(t, dev, 3, "hp", "high", true) + }) +} diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index 34ae7a92f..62cc319c8 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -606,7 +606,7 @@ func (a *Allocator) punitNonHPCapacity(idx int) int { if a.allowed.Size() > 0 { cpus = cpus.Intersection(a.allowed) } - n := cpus.Size() - pu.GuaranteedHpCpus + n := cpus.Size() - a.punitHPCapacity(idx) if n < 0 { return 0 } @@ -638,7 +638,8 @@ func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuse avail = avail.Difference(held).Difference(a.hpUsed[idx]).Difference(a.hpDRAUsed[idx]) // Also enforce the GuaranteedHpCpus cap: can't pick more HP CPUs than // the punit guarantees, regardless of how many are physically free. - hpAlreadyHeld := a.hpUsed[idx].Size() + a.hpDRAUsed[idx].Size() + // Use Union.Size() to avoid double-counting any CPU in both sets. + hpAlreadyHeld := a.hpUsed[idx].Union(a.hpDRAUsed[idx]).Size() hpRoom := pu.GuaranteedHpCpus - hpAlreadyHeld if hpRoom < 0 { hpRoom = 0 @@ -654,9 +655,9 @@ func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuse // Sort for deterministic selection; take first n. list := avail.List() picked := cpuset.New(list[:n]...) - if a.hpDRAUsed == nil { - a.hpDRAUsed = map[int]cpuset.CPUSet{} - } + // hpDRAUsed is always non-nil here: Configure() initialises it + // unconditionally, and Active() (checked above) is true only after + // a successful Configure(). a.hpDRAUsed[idx] = a.hpDRAUsed[idx].Union(picked) return picked, nil } @@ -672,9 +673,9 @@ func (a *Allocator) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { if idx < 0 { return } - if a.hpDRAUsed == nil { - return - } + // hpDRAUsed is always non-nil here: Configure() initialises it + // unconditionally, and Active() (checked above) is true only after + // a successful Configure(). remaining := a.hpDRAUsed[idx].Difference(cpus) if remaining.IsEmpty() { delete(a.hpDRAUsed, idx) @@ -802,6 +803,13 @@ func (a *Allocator) classIsHighPriority(className string) bool { return a.hpClasses[className] } +// IsHPClass reports whether className is currently classified as PCT +// high priority. It is the exported counterpart of classIsHighPriority. +// Returns false when the allocator is inactive or the class is unknown. +func (a *Allocator) IsHPClass(className string) bool { + return a.classIsHighPriority(className) +} + // hpHintsActive reports whether HP-room reasoning (hpReserveCpus, // hpInUseCpus, trackHpUsage) is currently meaningful. It requires // PCT to be active *and* at least one cpuClass to be classified as diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index e1ec821ae..a5221081b 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -211,6 +211,7 @@ func newManagedPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans map classPlan: plans, allowed: allowed, hpUsed: map[int]cpuset.CPUSet{}, + hpDRAUsed: map[int]cpuset.CPUSet{}, hpClasses: map[string]bool{}, } for _, cc := range classes { @@ -1194,6 +1195,20 @@ func TestPunitNonHPCapacity(t *testing.T) { if got := a2.punitNonHPCapacity(0); got != 0 { t.Errorf("all-HP punit nonHPCapacity = %d, want 0", got) } + + // HP-ineligible punit with non-zero GuaranteedHpCpus: the guard in + // punitHPCapacity must zero out the HP deduction so the full CPU count + // is reported as non-HP capacity. + // punit 0: HP-ineligible, GuaranteedHpCpus=2, CPUs 0-3 → 4 non-HP (not 2) + // punit 1: HP-eligible, GuaranteedHpCpus=2, CPUs 4-7 → 2 non-HP + a3 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + a3.hpEligiblePunit[0] = false + if got := a3.punitNonHPCapacity(0); got != 4 { + t.Errorf("HP-ineligible punit nonHPCapacity = %d, want 4", got) + } + if got := a3.punitNonHPCapacity(1); got != 2 { + t.Errorf("HP-eligible punit nonHPCapacity = %d, want 2", got) + } } func TestAllocatorPunits(t *testing.T) { @@ -1372,6 +1387,40 @@ func TestHpDRAUsedIsolation(t *testing.T) { } } +// TestIsHPClass covers the exported IsHPClass wrapper: HP class returns true; +// non-HP class returns false; unknown class returns false; inactive allocator +// returns false. +func TestIsHPClass(t *testing.T) { + sys := newTwoPackageFakeSys() + sst := &fakeSst{supported: true, maxHp: map[int]int{0: 2, 1: 2}} + classes := []*policyapi.CPUClass{ + {Name: "hp", PctPriority: "high"}, + {Name: "lp", PctPriority: "low"}, + } + a := newManagedPctForTest(t, classes, + map[string]*pctClassPlan{"hp": {ClosID: 0}, "lp": {ClosID: 3}}, + cpuset.MustParse("0-7"), sys, sst) + + // HP class must return true. + if !a.IsHPClass("hp") { + t.Error("IsHPClass(\"hp\") = false, want true") + } + // Non-HP class must return false. + if a.IsHPClass("lp") { + t.Error("IsHPClass(\"lp\") = true, want false") + } + // Unknown class must return false. + if a.IsHPClass("unknown") { + t.Error("IsHPClass(\"unknown\") = true, want false") + } + + // Inactive allocator must return false. + inactiveA := &Allocator{} + if inactiveA.IsHPClass("hp") { + t.Error("IsHPClass on inactive allocator = true, want false") + } +} + func TestHpReserveRoomWithDRAHolds(t *testing.T) { // Two punits, each with MaxHpCpus=2, GuaranteedHpCpus=2. a := newPickAllocator(t, makePunitsWithGtdHp(2, 2, 2, 2)) From a95d0f8c8021d871410edc0070c3be8abdea76f4 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 18:56:41 +0300 Subject: [PATCH 05/39] dra: implement kubelet plugin lifecycle Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- go.mod | 8 +- go.sum | 14 +- pkg/resmgr/dra/deps.go | 36 ++- pkg/resmgr/dra/logging.go | 30 +++ pkg/resmgr/dra/plugin.go | 189 ++++++++++++++- pkg/resmgr/dra/plugin_test.go | 429 +++++++++++++++++++++++++++++++++- 6 files changed, 691 insertions(+), 15 deletions(-) create mode 100644 pkg/resmgr/dra/logging.go diff --git a/go.mod b/go.mod index 07937ec76..3650b729a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/containers/nri-plugins/pkg/topology v0.0.0 github.com/coreos/go-systemd/v22 v22.7.0 github.com/fsnotify/fsnotify v1.10.1 + github.com/go-logr/logr v1.4.4 github.com/google/uuid v1.6.0 github.com/intel/goresctrl v0.13.0 github.com/intel/memtierd v0.1.1 @@ -40,6 +41,7 @@ require ( k8s.io/api v0.37.0 k8s.io/apimachinery v0.37.0 k8s.io/client-go v0.37.0 + k8s.io/dynamic-resource-allocation v0.37.0 k8s.io/klog/v2 v2.140.0 k8s.io/kubelet v0.37.0 k8s.io/utils v0.0.0-20260626114624-be93311217bd @@ -56,7 +58,6 @@ require ( github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect github.com/emicklei/go-restful/v3 v3.13.0 // indirect github.com/fxamacker/cbor/v2 v2.9.1 // indirect - github.com/go-logr/logr v1.4.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-openapi/jsonpointer v1.0.0 // indirect github.com/go-openapi/jsonreference v1.0.0 // indirect @@ -85,6 +86,7 @@ require ( github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.3.0 // indirect + github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect github.com/prometheus/otlptranslator v1.0.0 // indirect @@ -92,9 +94,12 @@ require ( github.com/spf13/pflag v1.0.10 // indirect github.com/tetratelabs/wazero v1.11.0 // indirect github.com/x448/float16 v0.8.4 // indirect + go.etcd.io/etcd/client/pkg/v3 v3.7.0 // indirect go.opentelemetry.io/auto/sdk v1.2.1 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.45.0 // indirect go.opentelemetry.io/proto/otlp v1.11.0 // indirect + go.uber.org/multierr v1.11.0 // indirect + go.uber.org/zap v1.27.1 // indirect go.yaml.in/yaml/v2 v2.4.4 // indirect go.yaml.in/yaml/v3 v3.0.5 // indirect golang.org/x/mod v0.37.0 // indirect @@ -117,6 +122,7 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect + tags.cncf.io/container-device-interface/specs-go v1.1.0 // indirect ) replace ( diff --git a/go.sum b/go.sum index f0ad744e8..e1747e79d 100644 --- a/go.sum +++ b/go.sum @@ -177,6 +177,8 @@ github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.etcd.io/etcd/client/pkg/v3 v3.7.0 h1:sW9njJzS3vXKcAJjjLQ4nk+avNUJ12Bcijcx8ehUskE= +go.etcd.io/etcd/client/pkg/v3 v3.7.0/go.mod h1:cnzZGIUzSfjEwLC6UBVsSXlEK1eepS/JUD7wE6PLRT0= go.opentelemetry.io/auto/sdk v1.2.1 h1:jXsnJ4Lmnqd11kwkBV2LgLoFMZKizbCi5fNZ/ipaZ64= go.opentelemetry.io/auto/sdk v1.2.1/go.mod h1:KRTj+aOaElaLi+wW1kO/DZRXwkF4C5xPbEe3ZiIhN7Y= go.opentelemetry.io/contrib/bridges/otelslog v0.20.0 h1:oEl2Pw/i4OQwhAuda2pAHFAcOMivA+Xa+iTccBfab/g= @@ -219,6 +221,10 @@ go.opentelemetry.io/proto/otlp v1.11.0 h1:5rrYs0Ykyj50sdU/JU0x8etU+LubXWb+gED6Tb go.opentelemetry.io/proto/otlp v1.11.0/go.mod h1:SmVizdCOAm3XBtG1g1NnOdhW6jtddT72hLMhv8VwA8E= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= +go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.27.1 h1:08RqriUEv8+ArZRYSTXy1LeBScaMpVSTBhCeaZYfMYc= +go.uber.org/zap v1.27.1/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= go.yaml.in/yaml/v2 v2.4.4 h1:tuyd0P+2Ont/d6e2rl3be67goVK4R6deVxCUX5vyPaQ= go.yaml.in/yaml/v2 v2.4.4/go.mod h1:gMZqIpDtDqOfM0uNfy0SkpRhvUryYH0Z6wdMYcacYXQ= go.yaml.in/yaml/v3 v3.0.5 h1:N6y/pJk8buWs9NY5ERU2HSMfm+IuD/OtfdAnq6kESPw= @@ -226,8 +232,8 @@ go.yaml.in/yaml/v3 v3.0.5/go.mod h1:HVTZu1O7/Vkt2N+BFy8Zza+lnLsABggaTM2ZpNIGuKg= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 h1:2dVuKD2vS7b0QIHQbpyTISPd0LeHDbnYEryqj5Q1ug8= -golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56/go.mod h1:M4RDyNAINzryxdtnbRXRL/OHtkFuWGRjvuhBJpk2IlY= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f h1:W3F4c+6OLc6H2lb//N1q4WpJkhzJCK5J6kUi1NTVXfM= +golang.org/x/exp v0.0.0-20260410095643-746e56fc9e2f/go.mod h1:J1xhfL/vlindoeF/aINzNzt2Bket5bjo9sdOYzOsU80= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.37.0 h1:vF1DjpVEshcIqoEaauuHebaLk1O1forxjxBaVn884JQ= @@ -299,6 +305,8 @@ k8s.io/client-go v0.37.0 h1:nsN31fy8wBySuZ+QRnKmrjRSQLOG2rvoGN0tKd12zhQ= k8s.io/client-go v0.37.0/go.mod h1:FcGqw+Ll/gNQiq+nPGY1Oyt9y7SgDh1d3MW3RFDEbn0= k8s.io/code-generator v0.34.11 h1:o7jWDfqo27pQygEm7a+g8QytR9ZaVEHE3urIfNmFYhU= k8s.io/code-generator v0.34.11/go.mod h1:ACmWPlRob//4Azs/4/wIi+CFPmnGhYuixpJfKUUhyJs= +k8s.io/dynamic-resource-allocation v0.37.0 h1:pfn3hsUQVvhqdDh0gA1DpV0w5njRzB68rjYT15AteT4= +k8s.io/dynamic-resource-allocation v0.37.0/go.mod h1:Nkono0X3H5tNnEf3IOK0Ml+I7HB4o0/UXaiiNCgB2Vw= k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b h1:gMplByicHV/TJBizHd9aVEsTYoJBnnUAT5MHlTkbjhQ= k8s.io/gengo/v2 v2.0.0-20250922181213-ec3ebc5fd46b/go.mod h1:CgujABENc3KuTrcsdpGmrrASjtQsWCT7R99mEV4U/fM= k8s.io/klog/v2 v2.140.0 h1:Tf+J3AH7xnUzZyVVXhTgGhEKnFqye14aadWv7bzXdzc= @@ -319,3 +327,5 @@ sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3C sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +tags.cncf.io/container-device-interface/specs-go v1.1.0 h1:QRZVeAceQM+zTZe12eyfuJuuzp524EKYwhmvLd+h+yQ= +tags.cncf.io/container-device-interface/specs-go v1.1.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ= diff --git a/pkg/resmgr/dra/deps.go b/pkg/resmgr/dra/deps.go index 23fd8925d..00b8ae678 100644 --- a/pkg/resmgr/dra/deps.go +++ b/pkg/resmgr/dra/deps.go @@ -16,6 +16,36 @@ limitations under the License. package dra -// Deps holds the dependencies a policy binary must supply when -// constructing a Plugin. -type Deps struct{} +import ( + resourceapi "k8s.io/api/resource/v1" + "k8s.io/client-go/kubernetes" + + "github.com/containers/nri-plugins/pkg/log" +) + +// DeviceLister provides the DRA device list for a given driver name. +type DeviceLister interface { + DRADevices(driverName string) ([]resourceapi.Device, error) +} + +// Deps holds the dependencies a policy binary must supply when constructing +// a Plugin. +type Deps struct { + // KubeClient is the Kubernetes client used to publish ResourceSlice objects. + KubeClient kubernetes.Interface + // NodeName is the name of the node this plugin runs on. + NodeName string + // RegistrarDir is the directory where the plugin registrar socket is created. + // Defaults to kubeletplugin.KubeletRegistryDir when empty. + RegistrarDir string + // PluginDataDir is the directory where the plugin data socket is created. + // Defaults to kubeletplugin.KubeletPluginsDir+"/"+driverName when empty. + PluginDataDir string + // ValidateClasses is a closure that validates the current cpuClass + // configuration for DRA compatibility. + ValidateClasses func() error + // DeviceLister returns the list of DRA devices to publish. + DeviceLister DeviceLister + // Logger is the logger used for all plugin log output. + Logger log.Logger +} diff --git a/pkg/resmgr/dra/logging.go b/pkg/resmgr/dra/logging.go new file mode 100644 index 000000000..7d7fa55ab --- /dev/null +++ b/pkg/resmgr/dra/logging.go @@ -0,0 +1,30 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "github.com/go-logr/logr" + + "github.com/containers/nri-plugins/pkg/log" +) + +// newLogr returns a logr.Logger backed by the given pkg/log Logger. +// kubeletplugin.Start expects a logr.Logger injected into the context +// via logr.NewContext; this bridge avoids hand-rolling a LogSink. +func newLogr(l log.Logger) logr.Logger { + return logr.FromSlogHandler(l.SlogHandler()) +} diff --git a/pkg/resmgr/dra/plugin.go b/pkg/resmgr/dra/plugin.go index 256159559..2525b0bd3 100644 --- a/pkg/resmgr/dra/plugin.go +++ b/pkg/resmgr/dra/plugin.go @@ -16,14 +16,199 @@ limitations under the License. package dra -import "errors" +import ( + "context" + "errors" + "fmt" + "os" + "path/filepath" + "sync" + "sync/atomic" + + "github.com/go-logr/logr" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + "k8s.io/dynamic-resource-allocation/resourceslice" +) var errNotImplemented = errors.New("dra plugin: not yet implemented") // Plugin is the DRA kubelet plugin. -type Plugin struct{} +type Plugin struct { + mu sync.Mutex + driverName string + deps Deps + helper *kubeletplugin.Helper + handleErr atomic.Pointer[error] // set on non-recoverable HandleError; checked by PublishResources +} // New constructs a Plugin with the given driver name and dependencies. +// Returns an error if any required dependency is missing. func New(driverName string, deps Deps) (*Plugin, error) { + if driverName == "" { + return nil, fmt.Errorf("dra plugin: driverName must not be empty") + } + if deps.KubeClient == nil { + return nil, fmt.Errorf("dra plugin: KubeClient must not be nil") + } + if deps.NodeName == "" { + return nil, fmt.Errorf("dra plugin: NodeName must not be empty") + } + if deps.ValidateClasses == nil { + return nil, fmt.Errorf("dra plugin: ValidateClasses must not be nil") + } + if deps.DeviceLister == nil { + return nil, fmt.Errorf("dra plugin: DeviceLister must not be nil") + } + if deps.Logger == nil { + return nil, fmt.Errorf("dra plugin: Logger must not be nil") + } + return &Plugin{driverName: driverName, deps: deps}, nil +} + +// PrepareResourceClaims is a stub that satisfies kubeletplugin.DRAPlugin. +func (p *Plugin) PrepareResourceClaims(_ context.Context, _ []*resourceapi.ResourceClaim) (map[types.UID]kubeletplugin.PrepareResult, error) { + return nil, errNotImplemented +} + +// UnprepareResourceClaims is a stub that satisfies kubeletplugin.DRAPlugin. +func (p *Plugin) UnprepareResourceClaims(_ context.Context, _ []kubeletplugin.NamespacedObject) (map[types.UID]error, error) { return nil, errNotImplemented } + +// Start registers this plugin with the kubelet and begins serving DRA +// requests. It validates cpuClass configuration, creates the plugin data +// directory, injects a logr.Logger into the context, and calls +// kubeletplugin.Start. Returns an error if the plugin is already started, +// if ValidateClasses fails, or if the kubelet plugin cannot be started. +func (p *Plugin) Start(ctx context.Context) error { + p.mu.Lock() + alreadyStarted := p.helper != nil + p.mu.Unlock() + if alreadyStarted { + return fmt.Errorf("dra plugin: already started") + } + if err := p.deps.ValidateClasses(); err != nil { + return fmt.Errorf("dra plugin: ValidateClasses failed: %w", err) + } + // Resolve the plugin data directory default before creating it: an empty + // string passed to os.MkdirAll would fail immediately. + pluginDataDir := p.deps.PluginDataDir + if pluginDataDir == "" { + pluginDataDir = filepath.Join(kubeletplugin.KubeletPluginsDir, p.driverName) + } + if err := os.MkdirAll(pluginDataDir, 0750); err != nil { + return fmt.Errorf("dra plugin: create plugin data dir %q: %w", pluginDataDir, err) + } + ctx = logr.NewContext(ctx, newLogr(p.deps.Logger)) + opts := []kubeletplugin.Option{ + kubeletplugin.DriverName(p.driverName), + kubeletplugin.KubeClient(p.deps.KubeClient), + kubeletplugin.NodeName(p.deps.NodeName), + kubeletplugin.PluginDataDirectoryPath(pluginDataDir), + kubeletplugin.GRPCVerbosity(-1), + } + // Only override the registrar directory when explicitly set; passing an + // empty string would clobber kubeletplugin's built-in KubeletRegistryDir + // default (the option setter stores the value unconditionally). + if p.deps.RegistrarDir != "" { + opts = append(opts, kubeletplugin.RegistrarDirectoryPath(p.deps.RegistrarDir)) + } + helper, err := kubeletplugin.Start(ctx, p, opts...) + if err != nil { + return fmt.Errorf("dra plugin: kubeletplugin.Start: %w", err) + } + p.mu.Lock() + p.helper = helper + p.mu.Unlock() + return nil +} + +// Stop shuts down the kubelet plugin and releases resources. It is +// idempotent: calling Stop on an already-stopped Plugin is safe. +func (p *Plugin) Stop() { + p.mu.Lock() + h := p.helper + p.helper = nil + p.mu.Unlock() + if h != nil { + h.Stop() + } +} + +// PublishResources validates classes, lists DRA devices, paginates them into +// ResourceSlice objects and hands the resulting DriverResources to the helper +// for publishing. Even zero devices produce one empty slice so the pool +// remains visible. +func (p *Plugin) PublishResources(ctx context.Context) error { + if pe := p.handleErr.Load(); pe != nil { + return *pe + } + if err := p.deps.ValidateClasses(); err != nil { + return fmt.Errorf("dra plugin: ValidateClasses failed: %w", err) + } + p.mu.Lock() + h := p.helper + p.mu.Unlock() + if h == nil { + return fmt.Errorf("dra plugin: PublishResources called before Start") + } + devices, err := p.deps.DeviceLister.DRADevices(p.driverName) + if err != nil { + return fmt.Errorf("dra plugin: DRADevices: %w", err) + } + resources := buildDriverResources(p.deps.NodeName, devices) + if err := h.PublishResources(ctx, resources); err != nil { + return fmt.Errorf("dra plugin: helper.PublishResources: %w", err) + } + return nil +} + +// buildDriverResources paginates devices into ResourceSlice objects and +// returns a DriverResources ready for Helper.PublishResources. The pool name +// is the node name. At most resourceapi.ResourceSliceMaxDevices devices are +// placed per slice; even zero devices produce one empty slice. +func buildDriverResources(nodeName string, devices []resourceapi.Device) resourceslice.DriverResources { + maxPerSlice := resourceapi.ResourceSliceMaxDevices + var slices []resourceslice.Slice + if len(devices) == 0 { + slices = []resourceslice.Slice{{}} + } else { + for i := 0; i < len(devices); i += maxPerSlice { + end := i + maxPerSlice + if end > len(devices) { + end = len(devices) + } + slices = append(slices, resourceslice.Slice{ + Devices: devices[i:end], + }) + } + } + return resourceslice.DriverResources{ + Pools: map[string]resourceslice.Pool{ + nodeName: {Slices: slices}, + }, + } +} + +// HandleError handles background errors from the kubelet plugin helper. +// Recoverable errors trigger a re-publish so the republisherLoop retries. +// Non-recoverable errors are stored so that the next PublishResources call +// returns them, causing the republisherLoop to enter its backoff path. +func (p *Plugin) HandleError(_ context.Context, err error, msg string) { + if errors.Is(err, kubeletplugin.ErrRecoverable) { + p.deps.Logger.Warnf("%s: %v", msg, err) + } else { + p.deps.Logger.Errorf("%s: %v", msg, err) + e := fmt.Errorf("dra plugin: resource slice controller: %w", err) + p.handleErr.Store(&e) + } +} + +// WatchHealthStatus is not implemented: this driver does not report +// per-device health, so it returns ErrHealthNotSupported as documented by +// kubeletplugin.DRAPlugin. +func (p *Plugin) WatchHealthStatus(_ context.Context, _ chan<- kubeletplugin.DeviceHealthReport) error { + return kubeletplugin.ErrHealthNotSupported +} diff --git a/pkg/resmgr/dra/plugin_test.go b/pkg/resmgr/dra/plugin_test.go index dd19b3591..02fd16abd 100644 --- a/pkg/resmgr/dra/plugin_test.go +++ b/pkg/resmgr/dra/plugin_test.go @@ -17,19 +17,434 @@ limitations under the License. package dra import ( + "context" "errors" + "fmt" + "strings" + "sync" "testing" + "time" + + corev1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/kubernetes/fake" + k8stesting "k8s.io/client-go/testing" + "k8s.io/dynamic-resource-allocation/kubeletplugin" + + "github.com/containers/nri-plugins/pkg/log" ) -func TestNew_ReturnsNotImplemented(t *testing.T) { - p, err := New("test-driver", Deps{}) - if p != nil { - t.Errorf("New() returned non-nil Plugin, want nil") +// TestNewLogr verifies that newLogr returns a usable logr.Logger backed by the +// default pkg/log Logger and that calling Info on it does not panic. +func TestNewLogr(t *testing.T) { + l := newLogr(log.Default()) + if l.IsZero() { + t.Error("newLogr returned a zero logr.Logger") } - if err == nil { - t.Fatal("New() returned nil error, want non-nil") + // Calling Info must not panic. + l.Info("test message from TestNewLogr") +} + +// validDeps returns a Deps with all required fields populated. +func validDeps() Deps { + return Deps{ + KubeClient: fake.NewClientset(), + NodeName: "test-node", + ValidateClasses: func() error { return nil }, + DeviceLister: &fixedDeviceLister{}, + Logger: log.Default(), + } +} + +// TestNew_Succeeds verifies that New returns a non-nil Plugin when all +// required fields are provided. +func TestNew_Succeeds(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + if p == nil { + t.Fatal("New() returned nil Plugin, want non-nil") + } +} + +// TestNew_Validation verifies that New returns an error when any required +// dependency is absent. +func TestNew_Validation(t *testing.T) { + tests := []struct { + name string + driverName string + mutate func(*Deps) + }{ + { + name: "empty driverName", + driverName: "", + mutate: nil, + }, + { + name: "nil KubeClient", + driverName: "test-driver", + mutate: func(d *Deps) { d.KubeClient = nil }, + }, + { + name: "empty NodeName", + driverName: "test-driver", + mutate: func(d *Deps) { d.NodeName = "" }, + }, + { + name: "nil ValidateClasses", + driverName: "test-driver", + mutate: func(d *Deps) { d.ValidateClasses = nil }, + }, + { + name: "nil DeviceLister", + driverName: "test-driver", + mutate: func(d *Deps) { d.DeviceLister = nil }, + }, + { + name: "nil Logger", + driverName: "test-driver", + mutate: func(d *Deps) { d.Logger = nil }, + }, + } + + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + deps := validDeps() + if tc.mutate != nil { + tc.mutate(&deps) + } + p, err := New(tc.driverName, deps) + if err == nil { + t.Errorf("New() expected error, got nil") + } + if p != nil { + t.Errorf("New() expected nil Plugin on error, got %v", p) + } + }) + } +} + +// TestPrepareResourceClaims_Stub verifies that PrepareResourceClaims returns +// errNotImplemented before real allocation logic is wired in Step 7. +func TestPrepareResourceClaims_Stub(t *testing.T) { + p := &Plugin{} + result, err := p.PrepareResourceClaims(t.Context(), []*resourceapi.ResourceClaim{}) + if result != nil { + t.Errorf("PrepareResourceClaims() result = %v, want nil", result) } if !errors.Is(err, errNotImplemented) { - t.Errorf("New() error = %v, want errors.Is(err, errNotImplemented) == true", err) + t.Errorf("PrepareResourceClaims() err = %v, want errNotImplemented", err) + } +} + +// TestUnprepareResourceClaims_Stub verifies that UnprepareResourceClaims returns +// errNotImplemented before real deallocation logic is wired in Step 7. +func TestUnprepareResourceClaims_Stub(t *testing.T) { + p := &Plugin{} + result, err := p.UnprepareResourceClaims(t.Context(), []kubeletplugin.NamespacedObject{}) + if result != nil { + t.Errorf("UnprepareResourceClaims() result = %v, want nil", result) + } + if !errors.Is(err, errNotImplemented) { + t.Errorf("UnprepareResourceClaims() err = %v, want errNotImplemented", err) + } +} + +// TestHandleError_RecoverableLogsWarn verifies that a recoverable error is +// handled without panicking. The method logs at Warn level. +func TestHandleError_RecoverableLogsWarn(t *testing.T) { + p := &Plugin{deps: Deps{Logger: log.Default()}} + recoverableErr := fmt.Errorf("transient failure: %w", kubeletplugin.ErrRecoverable) + // Must not panic. + p.HandleError(t.Context(), recoverableErr, "publish failed") +} + +// TestHandleError_FatalLogsError verifies that a non-recoverable (fatal) error +// is handled without panicking. The method logs at Error level. +func TestHandleError_FatalLogsError(t *testing.T) { + p := &Plugin{deps: Deps{Logger: log.Default()}} + fatalErr := errors.New("fatal background error") + // Must not panic. + p.HandleError(t.Context(), fatalErr, "fatal error encountered") +} + +// TestPublishResources_NilHelper verifies that PublishResources returns an +// error (not a panic) when called before Start, and that the error message +// references "Start" so callers can diagnose the ordering mistake. +func TestPublishResources_NilHelper(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // helper is nil at this point (Start has not been called) + err = p.PublishResources(t.Context()) + if err == nil { + t.Fatal("PublishResources() expected error when helper is nil, got nil") + } + if !strings.Contains(err.Error(), "Start") { + t.Errorf("PublishResources() err = %q, want message containing \"Start\"", err.Error()) + } +} + +// TestPublishResources_ValidationError verifies that a ValidateClasses failure +// is propagated by PublishResources. +func TestPublishResources_ValidationError(t *testing.T) { + validateErr := errors.New("invalid class config") + deps := validDeps() + deps.ValidateClasses = func() error { return validateErr } + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // helper is nil — but ValidateClasses is checked first, so that error + // is returned before the nil-helper guard. + err = p.PublishResources(t.Context()) + if err == nil { + t.Fatal("PublishResources() expected error from ValidateClasses, got nil") + } + if !errors.Is(err, validateErr) { + t.Errorf("PublishResources() err = %v, want to wrap %v", err, validateErr) + } +} + +// TestPublishResources_Pagination_Zero verifies that zero devices produce +// exactly one empty slice. +func TestPublishResources_Pagination_Zero(t *testing.T) { + res := buildDriverResources("node1", nil) + pool, ok := res.Pools["node1"] + if !ok { + t.Fatal("expected pool named 'node1'") + } + if len(pool.Slices) != 1 { + t.Errorf("zero devices: got %d slice(s), want 1", len(pool.Slices)) + } + if len(pool.Slices[0].Devices) != 0 { + t.Errorf("zero devices: slice[0] has %d device(s), want 0", len(pool.Slices[0].Devices)) + } +} + +// TestPublishResources_Pagination_ExactMax verifies that exactly +// ResourceSliceMaxDevices devices fit into one slice. +func TestPublishResources_Pagination_ExactMax(t *testing.T) { + max := resourceapi.ResourceSliceMaxDevices + devices := makeTestDevices(max) + res := buildDriverResources("node1", devices) + pool := res.Pools["node1"] + if len(pool.Slices) != 1 { + t.Errorf("exact max devices: got %d slice(s), want 1", len(pool.Slices)) + } + if len(pool.Slices[0].Devices) != max { + t.Errorf("exact max devices: slice[0] has %d device(s), want %d", len(pool.Slices[0].Devices), max) + } +} + +// TestPublishResources_Pagination_OverMax verifies that +// ResourceSliceMaxDevices+1 devices are split into exactly two slices +// belonging to the same pool. +func TestPublishResources_Pagination_OverMax(t *testing.T) { + max := resourceapi.ResourceSliceMaxDevices + devices := makeTestDevices(max + 1) + res := buildDriverResources("node1", devices) + pool := res.Pools["node1"] + if len(pool.Slices) != 2 { + t.Errorf("max+1 devices: got %d slice(s), want 2", len(pool.Slices)) + } + if len(pool.Slices[0].Devices) != max { + t.Errorf("max+1 devices: slice[0] has %d device(s), want %d", len(pool.Slices[0].Devices), max) + } + if len(pool.Slices[1].Devices) != 1 { + t.Errorf("max+1 devices: slice[1] has %d device(s), want 1", len(pool.Slices[1].Devices)) + } +} + +// TestStop_Idempotent verifies that calling Stop twice on a Plugin does not +// panic. Both calls are made on a plugin that was never started (helper == nil). +func TestStop_Idempotent(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Must not panic. + p.Stop() + p.Stop() +} + +// TestStart_ValidateClassesError verifies that Start returns the ValidateClasses +// error before attempting to call kubeletplugin.Start. +func TestStart_ValidateClassesError(t *testing.T) { + validateErr := errors.New("cpu class config invalid") + deps := validDeps() + deps.ValidateClasses = func() error { return validateErr } + deps.PluginDataDir = t.TempDir() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + err = p.Start(t.Context()) + if err == nil { + t.Fatal("Start() expected error from ValidateClasses, got nil") + } + if !errors.Is(err, validateErr) { + t.Errorf("Start() err = %v, want to wrap %v", err, validateErr) + } + if p.helper != nil { + t.Error("Start() set p.helper on ValidateClasses failure, want nil") + } +} + +// makeTestDevices returns a slice of n named resourceapi.Device objects for +// use in pagination tests. +func makeTestDevices(n int) []resourceapi.Device { + devices := make([]resourceapi.Device, n) + for i := range devices { + devices[i] = resourceapi.Device{Name: fmt.Sprintf("dev-%d", i)} + } + return devices +} + +// fixedDeviceLister is a DeviceLister that always returns a preset list of +// devices, used in integration tests. +type fixedDeviceLister struct { + devices []resourceapi.Device +} + +func (f *fixedDeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { + return f.devices, nil +} + +// errorDeviceLister is a DeviceLister that always returns the configured error. +type errorDeviceLister struct { + err error +} + +func (e *errorDeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { + return nil, e.err +} + +// TestPublishResources_DRADevicesError verifies that an error from +// DeviceLister.DRADevices is propagated by PublishResources. +func TestPublishResources_DRADevicesError(t *testing.T) { + sentinel := errors.New("DRADevices failed") + deps := validDeps() + deps.DeviceLister = &errorDeviceLister{err: sentinel} + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Set helper to a non-nil stub so the nil-helper guard is bypassed, + // allowing the test to reach the DRADevices call. + p.helper = new(kubeletplugin.Helper) + err = p.PublishResources(t.Context()) + if !errors.Is(err, sentinel) { + t.Errorf("PublishResources() err = %v, want to wrap sentinel error", err) + } +} + +// TestStart_AlreadyStarted verifies that a second call to Start returns an +// error without spawning a second helper. The guard is tested by setting +// p.helper to a non-nil stub before calling Start. +func TestStart_AlreadyStarted(t *testing.T) { + deps := validDeps() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Simulate an already-started plugin. + p.helper = new(kubeletplugin.Helper) + err = p.Start(t.Context()) + if err == nil { + t.Fatal("Start() expected error on double-call, got nil") + } + if !strings.Contains(err.Error(), "already started") { + t.Errorf("Start() err = %q, want message containing \"already started\"", err.Error()) + } +} + +// TestPublishResources_Integration starts a Plugin against a fake Kubernetes +// clientset, calls PublishResources, and polls until the fake cluster receives +// at least one ResourceSlice create, then validates the driver name field. +// +// Both the kubelet-plugin registration socket and plugin data socket are +// created under t.TempDir() — no real kubelet is required. +func TestPublishResources_Integration(t *testing.T) { + registrarDir := t.TempDir() + pluginDataDir := t.TempDir() + + // Fake clientset pre-loaded with the node object so the + // resourceslice.Controller can look up the node UID. + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + + // Reactor captures every ResourceSlice create call and passes it through + // to the default object tracker so the controller behaves normally. + var ( + mu sync.Mutex + capturedSlices []*resourceapi.ResourceSlice + ) + fakeClient.PrependReactor("create", "resourceslices", + func(action k8stesting.Action) (bool, k8sruntime.Object, error) { + createAction := action.(k8stesting.CreateAction) + if slice, ok := createAction.GetObject().(*resourceapi.ResourceSlice); ok { + mu.Lock() + capturedSlices = append(capturedSlices, slice.DeepCopy()) + mu.Unlock() + } + return false, nil, nil // pass through to default tracker + }, + ) + + deps := Deps{ + KubeClient: fakeClient, + NodeName: "test-node", + RegistrarDir: registrarDir, + PluginDataDir: pluginDataDir, + ValidateClasses: func() error { return nil }, + DeviceLister: &fixedDeviceLister{devices: makeTestDevices(5)}, + Logger: log.Default(), + } + + const driverName = "test.driver.io" + p, err := New(driverName, deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(t.Context(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + defer p.Stop() + + if err := p.PublishResources(ctx); err != nil { + t.Fatalf("PublishResources() unexpected error: %v", err) + } + + // The resourceslice.Controller drives ResourceSlice creation + // asynchronously; poll with a 5-second deadline. + const pollDeadline = 5 * time.Second + deadline := time.Now().Add(pollDeadline) + for time.Now().Before(deadline) { + mu.Lock() + n := len(capturedSlices) + mu.Unlock() + if n > 0 { + break + } + time.Sleep(50 * time.Millisecond) + } + + mu.Lock() + defer mu.Unlock() + if len(capturedSlices) == 0 { + t.Fatalf("no ResourceSlice was created within the %s deadline", pollDeadline) + } + if got := capturedSlices[0].Spec.Driver; got != driverName { + t.Errorf("ResourceSlice[0].Spec.Driver = %q, want %q", got, driverName) } } From 51e896140d42d26a12cca065d490d1e9a7ce505d Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 18:58:54 +0300 Subject: [PATCH 06/39] dra: implement Prepare/UnprepareResourceClaims and CDI writer Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- go.mod | 7 +- go.sum | 20 + pkg/resmgr/cpuclass/cpuclass.go | 58 + pkg/resmgr/cpuclass/cpuclass_dra_test.go | 244 +++ pkg/resmgr/cpuclass/dra.go | 22 +- pkg/resmgr/cpuclass/dra_test.go | 65 +- pkg/resmgr/cpuclass/internal/pct/pct.go | 40 + pkg/resmgr/cpuclass/internal/pct/pct_test.go | 57 + pkg/resmgr/dra/cdi.go | 217 +++ pkg/resmgr/dra/cdi_test.go | 327 ++++ pkg/resmgr/dra/deps.go | 71 + pkg/resmgr/dra/plugin.go | 615 ++++++- pkg/resmgr/dra/plugin_test.go | 1729 +++++++++++++++++- pkg/resmgr/dra/state.go | 128 ++ pkg/resmgr/dra/state_test.go | 287 +++ 15 files changed, 3815 insertions(+), 72 deletions(-) create mode 100644 pkg/resmgr/cpuclass/cpuclass_dra_test.go create mode 100644 pkg/resmgr/dra/cdi.go create mode 100644 pkg/resmgr/dra/cdi_test.go create mode 100644 pkg/resmgr/dra/state.go create mode 100644 pkg/resmgr/dra/state_test.go diff --git a/go.mod b/go.mod index 3650b729a..28cffae5e 100644 --- a/go.mod +++ b/go.mod @@ -47,6 +47,8 @@ require ( k8s.io/utils v0.0.0-20260626114624-be93311217bd sigs.k8s.io/controller-runtime v0.22.5 sigs.k8s.io/yaml v1.6.0 + tags.cncf.io/container-device-interface v1.1.0 + tags.cncf.io/container-device-interface/specs-go v1.1.0 ) require ( @@ -82,10 +84,12 @@ require ( github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/knqyf263/go-plugin v0.9.0 // indirect + github.com/moby/sys/capability v0.4.0 // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect github.com/modern-go/reflect2 v1.0.3-0.20250322232337-35a7c28c31ee // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/opencontainers/runtime-spec v1.3.0 // indirect + github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.2 // indirect github.com/prometheus/common v0.70.1 // indirect @@ -122,12 +126,11 @@ require ( sigs.k8s.io/json v0.0.0-20250730193827-2d320260d730 // indirect sigs.k8s.io/randfill v1.0.0 // indirect sigs.k8s.io/structured-merge-diff/v6 v6.4.2 // indirect - tags.cncf.io/container-device-interface/specs-go v1.1.0 // indirect ) replace ( github.com/containers/nri-plugins/pkg/topology v0.0.0 => ./pkg/topology - github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.0.0-20221026201742-946c877fa809 + github.com/opencontainers/runtime-tools => github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 ) tool k8s.io/code-generator diff --git a/go.sum b/go.sum index e1747e79d..0e4b23618 100644 --- a/go.sum +++ b/go.sum @@ -4,6 +4,8 @@ github.com/askervin/gofmbt v0.0.0-20260730061456-f663bfa65cdf h1:0AmJ+APMItCB+TS github.com/askervin/gofmbt v0.0.0-20260730061456-f663bfa65cdf/go.mod h1:1rWH2fCHPoGz1ApWyGyEV9YhZ2ZHeeCPaHcicW3b6uk= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/brianvoe/gofakeit/v7 v7.12.1 h1:df1tiI4SL1dR5Ix4D/r6a3a+nXBJ/OBGU5jEKRBmmqg= github.com/brianvoe/gofakeit/v7 v7.12.1/go.mod h1:QXuPeBw164PJCzCUZVmgpgHJ3Llj49jSLVkKPMtxtxA= github.com/cenkalti/backoff/v5 v5.0.3 h1:ZN+IMa753KfX5hd8vVaMixjnqRZ3y8CuJKRKj1xcsSM= @@ -96,6 +98,10 @@ github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0 h1:5VipnvEpbqr2gA2VbM+nYVbkIF28c5ZQfqCBQ5g2xfk= github.com/grpc-ecosystem/grpc-gateway/v2 v2.29.0/go.mod h1:Hyl3n6Twe1hvtd9XUXDec4pTvgMSEixRuQKPTMH2bNs= +github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA= +github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= +github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= +github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/intel/goresctrl v0.13.0 h1:5fhKjNq4V5MYDFHa//6M6x0jP6Iq5EXwZc6/eYxdEtQ= github.com/intel/goresctrl v0.13.0/go.mod h1:KFHS91JGOmeeuEog+nTQcsGjLC81nRqdsdhcqf69fjU= github.com/intel/memtierd v0.1.1 h1:hGSN0+dzjaUkwgkJrk6B9SU4dntggXLpXgs9Dm+jfz4= @@ -122,6 +128,8 @@ github.com/maruel/natural v1.1.1 h1:Hja7XhhmvEFhcByqDoHz9QZbkWey+COd9xWfCfn1ioo= github.com/maruel/natural v1.1.1/go.mod h1:v+Rfd79xlw1AgVBjbO0BEQmptqb5HvL/k9GRHB7ZKEg= github.com/mfridman/tparse v0.18.0 h1:wh6dzOKaIwkUGyKgOntDW4liXSo37qg5AXbIhkMV3vE= github.com/mfridman/tparse v0.18.0/go.mod h1:gEvqZTuCgEhPbYk/2lS3Kcxg1GmTxxU7kTC8DvP0i/A= +github.com/moby/sys/capability v0.4.0 h1:4D4mI6KlNtWMCM1Z/K0i7RV1FkX+DBDHKVJpCndZoHk= +github.com/moby/sys/capability v0.4.0/go.mod h1:4g9IK291rVkms3LKCDOoYlnV8xKwoDTpIrNEE35Wq0I= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -136,6 +144,10 @@ github.com/onsi/gomega v1.42.1 h1:iN1rCUX+44NZ1Dc97MPoeFYbFR0vh8zxoxMFwKdyZ6I= github.com/onsi/gomega v1.42.1/go.mod h1:REff/hsDsodHoKlWsP2mAPhu1+5/6hVYNf9rIEBpeSg= github.com/opencontainers/runtime-spec v1.3.0 h1:YZupQUdctfhpZy3TM39nN9Ika5CBWT5diQ8ibYCRkxg= github.com/opencontainers/runtime-spec v1.3.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116 h1:tAKu3NkKWZYpqBSOJKwTxT1wIGueiF7gcmcNgr5pNTY= +github.com/opencontainers/runtime-tools v0.9.1-0.20251114084447-edf4cb3d2116/go.mod h1:DKDEfzxvRkoQ6n9TGhxQgg2IM1lY4aM0eaQP4e3oElw= +github.com/opencontainers/selinux v1.10.0 h1:rAiKF8hTcgLI3w0DHm6i0ylVVcOrlgR1kK99DRLDhyU= +github.com/opencontainers/selinux v1.10.0/go.mod h1:2i0OySw99QjzBBQByd1Gr9gSjvuho1lHsJxIJ3gGbJI= github.com/pelletier/go-toml/v2 v2.4.3 h1:GTRvJQutkOSftxIFD5xw9aepkYNuPWmVJpffdDPYVpY= github.com/pelletier/go-toml/v2 v2.4.3/go.mod h1:2gIqNv+qfxSVS7cM2xJQKtLSTLUE9V8t9Stt+h56mCY= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -175,6 +187,12 @@ github.com/tidwall/sjson v1.2.5 h1:kLy8mja+1c9jlljvWTlSazM7cKDRfJuR/bOJhcY5NcY= github.com/tidwall/sjson v1.2.5/go.mod h1:Fvgq9kS/6ociJEDnK0Fk1cpYF4FIW6ZF7LAe+6jwd28= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= +github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415 h1:EzJWgHovont7NscjpAxXsDA8S8BMYve8Y5+7cuRE7R0= +github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= +github.com/xeipuuv/gojsonschema v1.2.0 h1:LhYJRs+L4fBtjZUfuSZIKGeVu0QRy8e5Xi7D17UxZ74= +github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= go.etcd.io/etcd/client/pkg/v3 v3.7.0 h1:sW9njJzS3vXKcAJjjLQ4nk+avNUJ12Bcijcx8ehUskE= @@ -327,5 +345,7 @@ sigs.k8s.io/structured-merge-diff/v6 v6.4.2 h1:qdOxHwrl2Kaag1aQEarlYcOA9vSyGCp3C sigs.k8s.io/structured-merge-diff/v6 v6.4.2/go.mod h1:M3W8sfWvn2HhQDIbGWj3S099YozAsymCo/wrT5ohRUE= sigs.k8s.io/yaml v1.6.0 h1:G8fkbMSAFqgEFgh4b1wmtzDnioxFCUgTZhlbj5P9QYs= sigs.k8s.io/yaml v1.6.0/go.mod h1:796bPqUfzR/0jLAl6XjHl3Ck7MiyVv8dbTdyT3/pMf4= +tags.cncf.io/container-device-interface v1.1.0 h1:RnxNhxF1JOu6CJUVpetTYvrXHdxw9j9jFYgZpI+anSY= +tags.cncf.io/container-device-interface v1.1.0/go.mod h1:76Oj0Yqp9FwTx/pySDc8Bxjpg+VqXfDb50cKAXVJ34Q= tags.cncf.io/container-device-interface/specs-go v1.1.0 h1:QRZVeAceQM+zTZe12eyfuJuuzp524EKYwhmvLd+h+yQ= tags.cncf.io/container-device-interface/specs-go v1.1.0/go.mod h1:u86hoFWqnh3hWz3esofRFKbI261bUlvUfLKGrDhJkgQ= diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index b123aeac9..df9bbf107 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -25,6 +25,7 @@ package cpuclass import ( + "errors" "fmt" "sort" @@ -41,6 +42,12 @@ import ( var log = logger.NewLogger("cpuclass") +// ErrAllocatorInactive is returned by AccountHpCpus when the PCT allocator +// has not been configured yet. Callers that tolerate a deferred Configure() +// (e.g. plugin.Start with an inactive allocator) should treat this as a +// warn-and-continue condition rather than a fatal error. +var ErrAllocatorInactive = errors.New("cpuclass: pct allocator not active") + // AllocationIntent describes an upcoming CPU allocation for which // the caller wants placement preferences. type AllocationIntent = types.AllocationIntent @@ -143,6 +150,57 @@ func (h *Handler) PctActive() bool { return h != nil && h.pct != nil && h.pct.Active() } +// PickHpCpus selects n HP-eligible CPUs from the punit identified by +// (pkgID, punitID), excluding CPUs in held and those already tracked +// in hpUsed or hpDRAUsed. Delegates to the PCT allocator. Returns an +// error when the handler or its PCT allocator is nil, or when the +// underlying pick fails (inactive allocator, punit not found, or +// insufficient HP capacity). +func (h *Handler) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { + if h == nil || h.pct == nil { + return cpuset.New(), fmt.Errorf("cpuclass: PickHpCpus: pct allocator not initialized") + } + return h.pct.PickHpCpus(pkgID, punitID, n, held) +} + +// ReleaseHpCpus removes cpus from DRA HP accounting on the punit +// identified by (pkgID, punitID). Delegates to the PCT allocator. +// No-op when the handler or its PCT allocator is nil, or when the +// punit is unknown (idempotent). +func (h *Handler) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { + if h == nil || h.pct == nil { + return + } + h.pct.ReleaseHpCpus(pkgID, punitID, cpus) +} + +// AccountHpCpus records cpus as DRA HP-held on the punit identified +// by (pkgID, punitID). Used during restart reconciliation to rebuild +// HP accounting from persisted claim state without re-allocating CPUs. +// Delegates to the PCT allocator. Returns an error when the handler +// or its PCT allocator is nil, or when accounting fails (inactive +// allocator, punit not found, or HP-ineligible punit). +func (h *Handler) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { + if h == nil || h.pct == nil { + return fmt.Errorf("cpuclass: AccountHpCpus: pct allocator not initialized: %w", ErrAllocatorInactive) + } + if !h.pct.Active() { + return fmt.Errorf("cpuclass: AccountHpCpus: pct allocator not active: %w", ErrAllocatorInactive) + } + return h.pct.AccountHpCpus(pkgID, punitID, cpus) +} + +// IsHPClass reports whether className is currently classified as PCT +// high priority. Delegates to the PCT allocator. Returns false when +// the handler or its PCT allocator is nil, or when the allocator is +// inactive. +func (h *Handler) IsHPClass(className string) bool { + if h == nil || h.pct == nil { + return false + } + return h.pct.IsHPClass(className) +} + // Configure (re)applies a configuration spec. Idempotent: may be // called repeatedly with changed classes, turbo-domain mode, or // allowed set. diff --git a/pkg/resmgr/cpuclass/cpuclass_dra_test.go b/pkg/resmgr/cpuclass/cpuclass_dra_test.go new file mode 100644 index 000000000..aad99d883 --- /dev/null +++ b/pkg/resmgr/cpuclass/cpuclass_dra_test.go @@ -0,0 +1,244 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Tests for the DRA ClaimAllocator pass-through methods on Handler. + +package cpuclass_test + +import ( + "testing" + + idset "github.com/intel/goresctrl/pkg/utils" + + policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// Compile-time assertion: *Handler must satisfy dra.ClaimAllocator. +var _ dra.ClaimAllocator = (*cpuclass.Handler)(nil) + +// draTestSys is a minimal sysfs.System implementation for DRA pass-through +// tests. Only CPUIDs is overridden; all other methods are delegated to the +// embedded nil interface, which panics if called. In practice, only CPUIDs +// is invoked during Handler.New() (by cpufreq platform discovery). +type draTestSys struct { + sysfs.System +} + +func (s *draTestSys) CPUIDs() []idset.ID { return nil } + +// newConfiguredHandler creates a Handler with an active managed PCT +// allocator using the SST in-memory mock (OVERRIDE_SST). The mock is +// seeded with one package (ID 0), one punit (ID 0), CPUs 0-7, and +// GuaranteedHpCpus=4. t.Setenv restores the env after the test. +func newConfiguredHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", `{"supported":true,"clos_count":4,"packages":[{"id":0,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"punits":[{"id":0,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":4}]}]}`) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + h, err := cpuclass.New(&draTestSys{}) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + if err := h.Configure(cpuclass.ConfigSpec{ + Classes: []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }); err != nil { + t.Fatalf("Configure() failed: %v", err) + } + return h +} + +// newInactiveHandler creates a Handler without OVERRIDE_SST so that +// SST is reported as unsupported and PCT stays in disabled mode. +// Configure is called with an HP class to exercise the "SST not +// supported → PCT disabled" path. +func newInactiveHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + // Ensure OVERRIDE_SST is unset (t.Setenv restores original value). + t.Setenv("OVERRIDE_SST", "") + h, err := cpuclass.New(&draTestSys{}) + if err != nil { + t.Fatalf("New() failed: %v", err) + } + // Configure with a PCT class; SST unsupported → pct stays disabled. + _ = h.Configure(cpuclass.ConfigSpec{ + Classes: []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }) + return h +} + +// --------------------------------------------------------------------------- +// PickHpCpus +// --------------------------------------------------------------------------- + +func TestHandlerPickHpCpus_NilHandler(t *testing.T) { + var h *cpuclass.Handler + _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + if err == nil { + t.Fatal("expected error from nil handler, got nil") + } +} + +func TestHandlerPickHpCpus_NilPct(t *testing.T) { + h := &cpuclass.Handler{} // pct is nil + _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + if err == nil { + t.Fatal("expected error from nil pct, got nil") + } +} + +func TestHandlerPickHpCpus_InactivePct(t *testing.T) { + h := newInactiveHandler(t) + _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + if err == nil { + t.Fatal("expected error from inactive PCT, got nil") + } +} + +func TestHandlerPickHpCpus_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + got, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus(0,0,2) = %v, want nil error", err) + } + if got.Size() != 2 { + t.Errorf("PickHpCpus(0,0,2) returned %d CPUs, want 2", got.Size()) + } + // Requesting more than GuaranteedHpCpus (4) must error. + if _, err := h.PickHpCpus(0, 0, 5, cpuset.New()); err == nil { + t.Error("PickHpCpus(0,0,5) with capacity=4: expected error, got nil") + } +} + +// --------------------------------------------------------------------------- +// ReleaseHpCpus +// --------------------------------------------------------------------------- + +func TestHandlerReleaseHpCpus_NilHandler(t *testing.T) { + var h *cpuclass.Handler + // must not panic + h.ReleaseHpCpus(0, 0, cpuset.New()) +} + +func TestHandlerReleaseHpCpus_NilPct(t *testing.T) { + h := &cpuclass.Handler{} + // must not panic + h.ReleaseHpCpus(0, 0, cpuset.New()) +} + +func TestHandlerReleaseHpCpus_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + // Pick 2 CPUs, then release them. + cpus, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus setup: %v", err) + } + h.ReleaseHpCpus(0, 0, cpus) + // After release, the full 4-CPU capacity must be available again. + got, err := h.PickHpCpus(0, 0, 4, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus after release: %v", err) + } + if got.Size() != 4 { + t.Errorf("post-release PickHpCpus returned %d CPUs, want 4", got.Size()) + } +} + +// --------------------------------------------------------------------------- +// AccountHpCpus +// --------------------------------------------------------------------------- + +func TestHandlerAccountHpCpus_NilHandler(t *testing.T) { + var h *cpuclass.Handler + if err := h.AccountHpCpus(0, 0, cpuset.New()); err == nil { + t.Fatal("expected error from nil handler, got nil") + } +} + +func TestHandlerAccountHpCpus_NilPct(t *testing.T) { + h := &cpuclass.Handler{} + if err := h.AccountHpCpus(0, 0, cpuset.New()); err == nil { + t.Fatal("expected error from nil pct, got nil") + } +} + +func TestHandlerAccountHpCpus_InactivePct(t *testing.T) { + h := newInactiveHandler(t) + if err := h.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + t.Fatal("expected error from inactive PCT, got nil") + } +} + +func TestHandlerAccountHpCpus_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + // AccountHpCpus simulates restart reconciliation (union semantics, + // no allocation): pick 2, release, then re-account. + cpus, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus setup: %v", err) + } + h.ReleaseHpCpus(0, 0, cpus) + if err := h.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus(%s) = %v, want nil", cpus, err) + } + // Idempotency: calling again with the same CPUs must not error. + if err := h.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus idempotent call = %v, want nil", err) + } +} + +// --------------------------------------------------------------------------- +// IsHPClass +// --------------------------------------------------------------------------- + +func TestHandlerIsHPClass_NilHandler(t *testing.T) { + var h *cpuclass.Handler + if h.IsHPClass("hp") { + t.Error("IsHPClass on nil handler: got true, want false") + } +} + +func TestHandlerIsHPClass_NilPct(t *testing.T) { + h := &cpuclass.Handler{} + if h.IsHPClass("hp") { + t.Error("IsHPClass with nil pct: got true, want false") + } +} + +func TestHandlerIsHPClass_InactivePct(t *testing.T) { + h := newInactiveHandler(t) + // PCT disabled → all classes report non-HP. + if h.IsHPClass("hp") { + t.Error("IsHPClass with inactive PCT: got true, want false") + } +} + +func TestHandlerIsHPClass_ActiveDelegates(t *testing.T) { + h := newConfiguredHandler(t) + // "hp" class has pctPriority=high → must report HP. + if !h.IsHPClass("hp") { + t.Error("IsHPClass(\"hp\") = false, want true") + } + // Unknown / non-HP names must return false. + if h.IsHPClass("lp") { + t.Error("IsHPClass(\"lp\") = true, want false") + } + if h.IsHPClass("") { + t.Error("IsHPClass(\"\") = true, want false") + } +} diff --git a/pkg/resmgr/cpuclass/dra.go b/pkg/resmgr/cpuclass/dra.go index 826c30d41..90d4dcf72 100644 --- a/pkg/resmgr/cpuclass/dra.go +++ b/pkg/resmgr/cpuclass/dra.go @@ -63,7 +63,9 @@ func maxDeviceBase(punits []pct.PunitInfo, classCount int) int { // silently disable this overcommit guard without implementing the // KEP-5941 shared-counter model. // -// Called at driver Configure time, not at config load time. +// Called at driver Configure time, not at config load time. The []Punit +// parameter is absent in v1 — per-punit enforcement is deferred to the +// device-build step where runtime punit topology is available. func ValidateCPUClassesForDRA(classes []*policyapi.CPUClass, sharedCounters bool) error { if sharedCounters { return fmt.Errorf( @@ -165,10 +167,11 @@ func strAttr(v string) resapi.DeviceAttribute { // published cpuClass × SST-TF punit) to be passed to kubeletplugin.PublishResources. // // For each published class, for each punit: emits one device if capacity > 0. -// HP classes use HPCapacity; non-HP classes use NonHPCapacity. hpOnly only -// affects the device-name length budget for now. +// HP classes use HPCapacity; non-HP classes use NonHPCapacity. +// +// When hpOnly is true, only HP classes (isHP returns true) are emitted. +// Non-HP DRA is deferred because PunitInfo carries no per-punit CPU list. func buildDRADevices( - driverName string, classes []*policyapi.CPUClass, punits []pct.PunitInfo, isHP func(className string) bool, @@ -196,6 +199,10 @@ func buildDRADevices( if !cc.DRAPublish() { continue } + // non-HP DRA deferred — PunitInfo has no per-punit CPU list. + if hpOnly && !isHP(cc.Name) { + continue + } if _, done := baseForClass[cc.Name]; done { continue // same class name seen twice — skip (defensive) } @@ -222,6 +229,9 @@ func buildDRADevices( if !cc.DRAPublish() { continue } + if hpOnly && !isHP(cc.Name) { + continue + } base := baseForClass[cc.Name] // Always set: duplicate class names are skipped above, and // validation guarantees no duplicates reach this point. @@ -293,7 +303,7 @@ func buildDRADevices( // // Must be called on the resmgr goroutine or under the resmgr lock — same as all // other Handler methods. -func (h *Handler) DRADevices(driverName string) ([]resapi.Device, error) { +func (h *Handler) DRADevices(_ string) ([]resapi.Device, error) { if h == nil || h.pct == nil { return []resapi.Device{}, nil } @@ -303,5 +313,5 @@ func (h *Handler) DRADevices(driverName string) ([]resapi.Device, error) { if len(punits) == 0 { return []resapi.Device{}, nil } - return buildDRADevices(driverName, h.classes, punits, h.pct.IsHPClass, true), nil + return buildDRADevices(h.classes, punits, h.pct.IsHPClass, true), nil } diff --git a/pkg/resmgr/cpuclass/dra_test.go b/pkg/resmgr/cpuclass/dra_test.go index 660f7a2d3..99e0e9569 100644 --- a/pkg/resmgr/cpuclass/dra_test.go +++ b/pkg/resmgr/cpuclass/dra_test.go @@ -378,23 +378,22 @@ func TestBuildDRADevices(t *testing.T) { isHP := func(name string) bool { return strings.HasPrefix(name, "hp") } tests := []struct { - name string - driverName string - classes []*policyapi.CPUClass - punits []pct.PunitInfo - isHP func(string) bool + name string + classes []*policyapi.CPUClass + punits []pct.PunitInfo + isHP func(string) bool + hpOnly bool // wantCount is the expected number of returned devices. wantCount int // verify is an optional per-result checker. verify func(t *testing.T, devices []resapi.Device) }{ { - name: "one HP class + one punit (pkg=0 punit=0)", - driverName: "test.driver", - classes: []*policyapi.CPUClass{hpClass("hp")}, - punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, - isHP: isHP, - wantCount: 1, + name: "one HP class + one punit (pkg=0 punit=0)", + classes: []*policyapi.CPUClass{hpClass("hp")}, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + wantCount: 1, verify: func(t *testing.T, devices []resapi.Device) { t.Helper() dev := devices[0] @@ -669,6 +668,44 @@ func TestBuildDRADevices(t *testing.T) { } }, }, + { + // hpOnly=true: mixed HP/non-HP config → only HP devices emitted. + // Non-HP DRA is deferred; non-HP classes must be silently filtered. + name: "mixed HP/non-HP config with hpOnly=true → only HP devices emitted", + classes: []*policyapi.CPUClass{ + hpClass("hp"), + lpClass("lp"), + nonPCTClass("default"), + }, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + hpOnly: true, + wantCount: 1, + verify: func(t *testing.T, devices []resapi.Device) { + t.Helper() + dev := devices[0] + if v, ok := attrStr(dev, "nri/cpuClass"); !ok { + t.Error("nri/cpuClass attribute missing") + } else if v != "hp" { + t.Errorf("nri/cpuClass = %q, want \"hp\"", v) + } + if !isDNSLabel(dev.Name) { + t.Errorf("device name %q is not a valid DNS label", dev.Name) + } + }, + }, + { + // hpOnly=false: non-HP classes are included (base behaviour unchanged). + name: "mixed HP/non-HP config with hpOnly=false → all published devices emitted", + classes: []*policyapi.CPUClass{ + hpClass("hp"), + lpClass("lp"), + }, + punits: []pct.PunitInfo{punit(0, 0, 4, 8)}, + isHP: isHP, + hpOnly: false, + wantCount: 2, + }, } for _, tc := range tests { @@ -677,11 +714,7 @@ func TestBuildDRADevices(t *testing.T) { if isHPFn == nil { isHPFn = func(string) bool { return false } } - driverName := tc.driverName - if driverName == "" { - driverName = "test.driver" - } - devices := buildDRADevices(driverName, tc.classes, tc.punits, isHPFn, true) + devices := buildDRADevices(tc.classes, tc.punits, isHPFn, tc.hpOnly) if len(devices) != tc.wantCount { t.Fatalf("buildDRADevices() returned %d devices, want %d; devices=%v", len(devices), tc.wantCount, deviceNames(devices)) diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index 62cc319c8..c2825d0b7 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -684,6 +684,46 @@ func (a *Allocator) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { } } +// AccountHpCpus records cpus as HP DRA-held on the punit identified by +// (pkgID, punitID). Used during restart reconciliation to rebuild +// hpDRAUsed from persisted claim state without reallocating CPUs. +// Returns an error when the allocator is inactive, the punit is not +// found, the punit is not HP-eligible, or cpus is not a subset of the +// punit's current CPU set (e.g. after a reboot or topology/allowed-set +// change made the persisted CPUs stale). Over-commit (hpDRAUsed + +// hpUsed > GuaranteedHpCpus) is permitted — the container may already +// be running; a warning is logged but no error is returned. Union +// semantics make repeated calls with the same CPUs idempotent. +func (a *Allocator) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { + if !a.Active() { + return fmt.Errorf("pct: AccountHpCpus: allocator not active") + } + idx := a.punitIdxByID(pkgID, punitID) + if idx < 0 { + return fmt.Errorf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) not found", pkgID, punitID) + } + if !a.hpEligiblePunit[idx] { + return fmt.Errorf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) is not HP-eligible", pkgID, punitID) + } + if !cpus.IsSubsetOf(a.punits[idx].CPUs) { + return fmt.Errorf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) does not contain all of %s (has %s)", + pkgID, punitID, cpus, a.punits[idx].CPUs) + } + // Union is idempotent: repeated calls with the same CPUs do not + // double-count them. + a.hpDRAUsed[idx] = a.hpDRAUsed[idx].Union(cpus) + // Warn on over-commit but do not reject — the container may already + // be running with these CPUs. + pu := a.punits[idx] + held := a.hpUsed[idx].Union(a.hpDRAUsed[idx]).Size() + if held > pu.GuaranteedHpCpus { + log.Warnf("pct: AccountHpCpus: punit (pkg=%d, punit=%d) over-committed: "+ + "hpDRAUsed+hpUsed=%d > GuaranteedHpCpus=%d", + pkgID, punitID, held, pu.GuaranteedHpCpus) + } + return nil +} + // useClass associates the given CPUs to the CLOS chosen for className. // In managed mode, CPUs whose className is not a PCT class are // associated to the fallback CLOS. In assoc-only mode such CPUs are diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index a5221081b..efcfca837 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -1421,6 +1421,63 @@ func TestIsHPClass(t *testing.T) { } } +// TestAccountHpCpus covers AccountHpCpus: used during restart reconciliation +// to rebuild hpDRAUsed from persisted claim state. +func TestAccountHpCpus(t *testing.T) { + // Inactive allocator must return an error. + inactiveA := &Allocator{} + if err := inactiveA.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + t.Error("AccountHpCpus on inactive allocator: expected error, got nil") + } + + // HP-ineligible punit must return an error. + aInelig := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + aInelig.hpEligiblePunit[0] = false + if err := aInelig.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + t.Error("AccountHpCpus on HP-ineligible punit: expected error, got nil") + } + + // Unknown punit must return an error. + aUnknown := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + if err := aUnknown.AccountHpCpus(99, 99, cpuset.MustParse("0")); err == nil { + t.Error("AccountHpCpus unknown punit: expected error, got nil") + } + + // Success: account CPUs on an HP-eligible punit. + aOK := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) + cpus := cpuset.MustParse("0-1") + if err := aOK.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus success case: %v", err) + } + if !aOK.hpDRAUsed[0].Equals(cpus) { + t.Errorf("hpDRAUsed[0] = %v, want %v", aOK.hpDRAUsed[0], cpus) + } + // hpUsed must remain untouched. + if aOK.hpUsed[0].Size() != 0 { + t.Errorf("hpUsed[0] should be untouched after AccountHpCpus, got %v", aOK.hpUsed[0]) + } + + // Double-account same CPUs is idempotent (union semantics). + if err := aOK.AccountHpCpus(0, 0, cpus); err != nil { + t.Fatalf("AccountHpCpus idempotent call: %v", err) + } + if !aOK.hpDRAUsed[0].Equals(cpus) { + t.Errorf("hpDRAUsed[0] after double-account = %v, want %v (must be idempotent)", aOK.hpDRAUsed[0], cpus) + } + + // Over-capacity: account more CPUs than GuaranteedHpCpus allows. + // Must NOT return an error (container may already be running), and + // hpDRAUsed must include all accounted CPUs. + aOver := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) // GuaranteedHpCpus=2 + overCommit := cpuset.MustParse("0-3") // 4 CPUs > GuaranteedHpCpus=2 + if err := aOver.AccountHpCpus(0, 0, overCommit); err != nil { + t.Fatalf("AccountHpCpus over-capacity: expected no error, got %v", err) + } + if !aOver.hpDRAUsed[0].Equals(overCommit) { + t.Errorf("hpDRAUsed[0] = %v, want %v (over-capacity still updates)", aOver.hpDRAUsed[0], overCommit) + } +} + func TestHpReserveRoomWithDRAHolds(t *testing.T) { // Two punits, each with MaxHpCpus=2, GuaranteedHpCpus=2. a := newPickAllocator(t, makePunitsWithGtdHp(2, 2, 2, 2)) diff --git a/pkg/resmgr/dra/cdi.go b/pkg/resmgr/dra/cdi.go new file mode 100644 index 000000000..ba533ef4d --- /dev/null +++ b/pkg/resmgr/dra/cdi.go @@ -0,0 +1,217 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "fmt" + "os" + "path/filepath" + "strconv" + "strings" + + "k8s.io/apimachinery/pkg/types" + + cdilib "tags.cncf.io/container-device-interface/pkg/cdi" + "tags.cncf.io/container-device-interface/pkg/parser" + specs "tags.cncf.io/container-device-interface/specs-go" + + logger "github.com/containers/nri-plugins/pkg/log" +) + +var cdiLog = logger.NewLogger("dra") + +const ( + // defaultCDIDir is the default directory for CDI spec files. + defaultCDIDir = "/var/run/cdi" + // cdiClass is the CDI class name used for all DRA claim specs. + cdiClass = "device" +) + +// cdiWriter implements CDIWriter using the upstream CDI library. +type cdiWriter struct { + cache *cdilib.Cache + vendor string + class string + cdiDir string +} + +// NewCDIWriter creates a CDIWriter that writes CDI specs under cdiDir. +// If cdiDir is empty, it defaults to /var/run/cdi. +// driverName is validated as a CDI vendor name. Returns an error if +// driverName is not a valid CDI vendor name or if the CDI cache cannot +// be created. +func NewCDIWriter(driverName, cdiDir string) (CDIWriter, error) { + if err := parser.ValidateVendorName(driverName); err != nil { + return nil, fmt.Errorf("dra cdi: invalid driverName %q: %w", driverName, err) + } + if cdiDir == "" { + cdiDir = defaultCDIDir + } + if err := os.MkdirAll(cdiDir, 0750); err != nil { + return nil, fmt.Errorf("dra cdi: create CDI dir %q: %w", cdiDir, err) + } + cache, err := cdilib.NewCache( + cdilib.WithAutoRefresh(false), + cdilib.WithSpecDirs(cdiDir), + ) + if err != nil { + return nil, fmt.Errorf("dra cdi: create CDI cache: %w", err) + } + return &cdiWriter{ + cache: cache, + vendor: driverName, + class: cdiClass, + cdiDir: cdiDir, + }, nil +} + +// WriteClaim writes a CDI spec file for the claim identified by uid. +// Each element of devices becomes one CDI device entry in the spec. +// Returns an error if devices is empty, if spec assembly fails, or if +// writing the spec file fails. +func (w *cdiWriter) WriteClaim(uid types.UID, devices []CDIDevice) error { + if len(devices) == 0 { + return fmt.Errorf("dra cdi: WriteClaim %s: devices must not be empty", uid) + } + + cdiDevices := make([]specs.Device, 0, len(devices)) + for _, d := range devices { + env := []string{fmt.Sprintf("NRI_CLASS=%s", d.ClassName)} + for _, cpu := range d.CPUs.List() { + env = append(env, fmt.Sprintf("NRI_CPU%d=1", cpu)) + } + cdiDevices = append(cdiDevices, specs.Device{ + Name: d.Name, + ContainerEdits: specs.ContainerEdits{ + Env: env, + }, + }) + } + + spec := specs.Spec{ + Kind: w.vendor + "/" + w.class, + Devices: cdiDevices, + } + // MinimumRequiredVersion must be called after spec is fully assembled, + // including Kind which it reads to determine requirements. + v, err := specs.MinimumRequiredVersion(&spec) + if err != nil { + return fmt.Errorf("dra cdi: WriteClaim %s: determine CDI version: %w", uid, err) + } + spec.Version = v + + name := cdilib.GenerateTransientSpecName(w.vendor, w.class, string(uid)) + if err := w.cache.WriteSpec(&spec, name); err != nil { + return fmt.Errorf("dra cdi: WriteClaim %s: write spec: %w", uid, err) + } + return nil +} + +// RemoveClaim removes the CDI spec file for the claim identified by uid. +// Returns nil if the spec does not exist (idempotent). +func (w *cdiWriter) RemoveClaim(uid types.UID) error { + name := cdilib.GenerateTransientSpecName(w.vendor, w.class, string(uid)) + if err := w.cache.RemoveSpec(name); err != nil { + return fmt.Errorf("dra cdi: RemoveClaim %s: %w", uid, err) + } + return nil +} + +// ClaimSpecExists reports whether the CDI spec file for uid is present on disk. +// The file path is /-_.yaml, consistent with +// how WriteSpec names files for transient specs. +func (w *cdiWriter) ClaimSpecExists(uid types.UID) bool { + name := cdilib.GenerateTransientSpecName(w.vendor, w.class, string(uid)) + path := filepath.Join(w.cdiDir, name+".yaml") + _, err := os.Stat(path) + return err == nil +} + +// ListClaims returns the UIDs of all claims for which a CDI spec exists in +// the managed CDI directory. Refresh errors are logged at Warn level but do +// not abort the listing — foreign or malformed specs in the directory must not +// prevent our claims from being returned. +func (w *cdiWriter) ListClaims() ([]types.UID, error) { + if err := w.cache.Refresh(); err != nil { + cdiLog.Warnf("dra cdi: ListClaims: cache refresh: %v", err) + } + prefix := w.vendor + "-" + w.class + "_" + suffix := ".yaml" + var uids []types.UID + for _, s := range w.cache.GetVendorSpecs(w.vendor) { + base := filepath.Base(s.GetPath()) + if !strings.HasPrefix(base, prefix) || !strings.HasSuffix(base, suffix) { + continue + } + uidStr := strings.TrimPrefix(base, prefix) + uidStr = strings.TrimSuffix(uidStr, suffix) + if uidStr == "" { + continue + } + uids = append(uids, types.UID(uidStr)) + } + return uids, nil +} + +// CDIDeviceName builds a valid CDI device name for the given allocation result. +// The format is: +// +// "claim----" +// +// where sanitize replaces '/' and any character that is not alphanumeric, +// '_', '-', '.', or ':' with '-', then trims any leading or trailing +// non-alphanumeric characters. +// +// The result can be validated with parser.ValidateDeviceName. Using a per- +// result idx ensures that two results that share the same Request+Device +// (e.g. AllowMultipleAllocations with count > 1) produce distinct names. +func CDIDeviceName(uid types.UID, request, device string, idx int) string { + sanitized := sanitizeCDIName(request) + sanitizedDev := sanitizeCDIName(device) + return "claim-" + string(uid) + "-" + sanitized + "-" + sanitizedDev + "-" + strconv.Itoa(idx) +} + +// sanitizeCDIName replaces any character that is invalid in a CDI device name +// middle position with '-', then trims leading and trailing non-alphanumeric +// characters. '/' is always replaced (it is not a valid CDI device name char). +func sanitizeCDIName(s string) string { + var b strings.Builder + for _, c := range s { + switch { + case (c >= 'A' && c <= 'Z') || (c >= 'a' && c <= 'z') || (c >= '0' && c <= '9'): + b.WriteRune(c) + case c == '_' || c == '-' || c == '.' || c == ':': + b.WriteRune(c) + default: + b.WriteRune('-') + } + } + result := b.String() + // Trim leading/trailing non-alphanumeric characters. + start := strings.IndexFunc(result, func(r rune) bool { + return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + }) + if start < 0 { + // All characters were replaced by dashes or the string is empty. + // Return a safe placeholder so the caller still gets a non-empty segment. + return "x" + } + end := strings.LastIndexFunc(result, func(r rune) bool { + return (r >= 'A' && r <= 'Z') || (r >= 'a' && r <= 'z') || (r >= '0' && r <= '9') + }) + return result[start : end+1] +} diff --git a/pkg/resmgr/dra/cdi_test.go b/pkg/resmgr/dra/cdi_test.go new file mode 100644 index 000000000..5626270a6 --- /dev/null +++ b/pkg/resmgr/dra/cdi_test.go @@ -0,0 +1,327 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "os" + "path/filepath" + "strings" + "testing" + + "k8s.io/apimachinery/pkg/types" + + "tags.cncf.io/container-device-interface/pkg/parser" + + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// TestCDIDeviceName_BasicCase verifies the basic format of a CDI device name. +func TestCDIDeviceName_BasicCase(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000001") + name := CDIDeviceName(uid, "myrequest", "punit-0-0", 0) + if !strings.HasPrefix(name, "claim-") { + t.Errorf("CDIDeviceName() = %q, want prefix \"claim-\"", name) + } + if err := parser.ValidateDeviceName(name); err != nil { + t.Errorf("CDIDeviceName() = %q, invalid CDI device name: %v", name, err) + } +} + +// TestCDIDeviceName_SubrequestSlash verifies that '/' in request is replaced +// so the result is still a valid CDI device name. +func TestCDIDeviceName_SubrequestSlash(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000002") + name := CDIDeviceName(uid, "first-available/req0", "punit-0-1", 0) + if err := parser.ValidateDeviceName(name); err != nil { + t.Errorf("CDIDeviceName() with slash in request = %q, invalid: %v", name, err) + } + if strings.Contains(name, "/") { + t.Errorf("CDIDeviceName() = %q, should not contain '/'", name) + } +} + +// TestCDIDeviceName_TwoResultsSameRequestDevice verifies that two results with +// the same Request+Device but different idx produce distinct valid names. +func TestCDIDeviceName_TwoResultsSameRequestDevice(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000003") + name0 := CDIDeviceName(uid, "myrequest", "punit-0-0", 0) + name1 := CDIDeviceName(uid, "myrequest", "punit-0-0", 1) + if name0 == name1 { + t.Errorf("CDIDeviceName() idx 0 and 1 produced the same name %q", name0) + } + if err := parser.ValidateDeviceName(name0); err != nil { + t.Errorf("name0 %q invalid: %v", name0, err) + } + if err := parser.ValidateDeviceName(name1); err != nil { + t.Errorf("name1 %q invalid: %v", name1, err) + } +} + +// TestCDIDeviceName_AllSlashRequest verifies that a request of all slashes +// produces a valid name (the sanitize+trim fallback to "x"). +func TestCDIDeviceName_AllSlashRequest(t *testing.T) { + uid := types.UID("abc123-0000-0000-0000-000000000004") + name := CDIDeviceName(uid, "///", "punit-0-0", 0) + if err := parser.ValidateDeviceName(name); err != nil { + t.Errorf("CDIDeviceName() with all-slash request = %q, invalid: %v", name, err) + } +} + +// TestNewCDIWriter_InvalidVendor verifies that an invalid vendor name causes +// NewCDIWriter to return an error. +func TestNewCDIWriter_InvalidVendor(t *testing.T) { + dir := t.TempDir() + _, err := NewCDIWriter("123invalid", dir) + if err == nil { + t.Error("NewCDIWriter() with invalid vendor name: expected error, got nil") + } +} + +// TestNewCDIWriter_DefaultDir verifies that NewCDIWriter creates /var/run/cdi +// when cdiDir is empty and permissions allow it. This test is skipped unless +// the process can write to /var/run (it needs root or pre-created dir). +func TestNewCDIWriter_DefaultDir(t *testing.T) { + t.Skip("requires write access to /var/run — skipped in unit test environment") +} + +// TestWriteClaim_EnvVarsOnDisk verifies that WriteClaim writes a YAML spec +// file containing the expected env vars and Kind. +func TestWriteClaim_EnvVarsOnDisk(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-claim-uid-0001") + cpus, _ := cpuset.Parse("0,2,4") + devices := []CDIDevice{ + {Name: "claim-test-claim-uid-0001-myreq-punit-0-0-0", ClassName: "gold", CPUs: cpus}, + } + + if err := w.WriteClaim(uid, devices); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + + // Verify spec file exists with expected name pattern. + pattern := filepath.Join(dir, "intel.com-device_test-claim-uid-0001.yaml") + data, err := os.ReadFile(pattern) + if err != nil { + t.Fatalf("spec file not found at %q: %v", pattern, err) + } + + contents := string(data) + if !strings.Contains(contents, "NRI_CLASS=gold") { + t.Errorf("spec missing NRI_CLASS=gold; contents:\n%s", contents) + } + if !strings.Contains(contents, "NRI_CPU0=1") { + t.Errorf("spec missing NRI_CPU0=1; contents:\n%s", contents) + } + if !strings.Contains(contents, "NRI_CPU2=1") { + t.Errorf("spec missing NRI_CPU2=1; contents:\n%s", contents) + } + if !strings.Contains(contents, "NRI_CPU4=1") { + t.Errorf("spec missing NRI_CPU4=1; contents:\n%s", contents) + } + if !strings.Contains(contents, "kind: intel.com/device") { + t.Errorf("spec missing Kind \"intel.com/device\"; contents:\n%s", contents) + } +} + +// TestWriteClaim_EmptyDevices verifies that WriteClaim returns an error when +// devices is empty (CDI rejects specs with no devices). +func TestWriteClaim_EmptyDevices(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + err = w.WriteClaim("some-uid", nil) + if err == nil { + t.Error("WriteClaim() with empty devices: expected error, got nil") + } +} + +// TestRemoveClaim_Removes verifies that RemoveClaim removes the spec file. +func TestRemoveClaim_Removes(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-uid-remove-0001") + cpus, _ := cpuset.Parse("0") + devs := []CDIDevice{{Name: "claim-test-uid-remove-0001-req-dev-0", ClassName: "silver", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + + if !w.ClaimSpecExists(uid) { + t.Fatal("ClaimSpecExists() = false after WriteClaim, want true") + } + if err := w.RemoveClaim(uid); err != nil { + t.Fatalf("RemoveClaim() unexpected error: %v", err) + } + if w.ClaimSpecExists(uid) { + t.Error("ClaimSpecExists() = true after RemoveClaim, want false") + } +} + +// TestRemoveClaim_Idempotent verifies that RemoveClaim on a non-existent spec +// returns nil (idempotent). +func TestRemoveClaim_Idempotent(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + if err := w.RemoveClaim("nonexistent-uid"); err != nil { + t.Errorf("RemoveClaim() on non-existent spec: unexpected error %v", err) + } +} + +// TestClaimSpecExists_TrueAndFalse verifies ClaimSpecExists for written and +// not-written claims. +func TestClaimSpecExists_TrueAndFalse(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-uid-exists-0001") + if w.ClaimSpecExists(uid) { + t.Error("ClaimSpecExists() = true before WriteClaim, want false") + } + + cpus, _ := cpuset.Parse("1") + devs := []CDIDevice{{Name: "claim-test-uid-exists-0001-req-dev-0", ClassName: "gold", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + if !w.ClaimSpecExists(uid) { + t.Error("ClaimSpecExists() = false after WriteClaim, want true") + } +} + +// TestListClaims_TwoClaims verifies that ListClaims returns all written claim +// UIDs. +func TestListClaims_TwoClaims(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid1 := types.UID("test-uid-list-0001") + uid2 := types.UID("test-uid-list-0002") + cpus, _ := cpuset.Parse("0") + + for _, uid := range []types.UID{uid1, uid2} { + devs := []CDIDevice{{Name: "claim-" + string(uid) + "-req-dev-0", ClassName: "gold", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim(%s) unexpected error: %v", uid, err) + } + } + + uids, err := w.ListClaims() + if err != nil { + t.Fatalf("ListClaims() unexpected error: %v", err) + } + + found := make(map[types.UID]bool) + for _, u := range uids { + found[u] = true + } + if !found[uid1] { + t.Errorf("ListClaims() missing uid1 %s", uid1) + } + if !found[uid2] { + t.Errorf("ListClaims() missing uid2 %s", uid2) + } +} + +// TestListClaims_ForeignSpecSurvives verifies that a malformed/foreign spec +// in the CDI dir does not prevent our claims from being listed, and that the +// foreign spec survives (is not removed). +func TestListClaims_ForeignSpecSurvives(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + // Write a "foreign" malformed spec file that should not be parsed as our claim. + foreignPath := filepath.Join(dir, "other-vendor-device_foreign.yaml") + if err := os.WriteFile(foreignPath, []byte("not valid yaml\n"), 0644); err != nil { + t.Fatalf("write foreign spec: %v", err) + } + + // Write a valid claim. + uid := types.UID("test-uid-foreign-0001") + cpus, _ := cpuset.Parse("0") + devs := []CDIDevice{{Name: "claim-test-uid-foreign-0001-req-dev-0", ClassName: "gold", CPUs: cpus}} + if err := w.WriteClaim(uid, devs); err != nil { + t.Fatalf("WriteClaim() unexpected error: %v", err) + } + + // ListClaims should return our UID (Refresh logs a warning about the foreign + // malformed file but continues). + uids, err := w.ListClaims() + if err != nil { + t.Fatalf("ListClaims() unexpected error: %v", err) + } + found := false + for _, u := range uids { + if u == uid { + found = true + break + } + } + if !found { + t.Errorf("ListClaims() missing expected uid %s (got %v)", uid, uids) + } + + // Foreign spec file must still be on disk. + if _, err := os.Stat(foreignPath); err != nil { + t.Errorf("foreign spec file was removed: %v", err) + } +} + +// TestWriteClaim_SameRequestDeviceTwoIdx verifies that two results with the +// same Request+Device but different idx produce a spec with two distinct +// CDI device names (no duplicate-name error). +func TestWriteClaim_SameRequestDeviceTwoIdx(t *testing.T) { + dir := t.TempDir() + w, err := NewCDIWriter("intel.com", dir) + if err != nil { + t.Fatalf("NewCDIWriter() unexpected error: %v", err) + } + + uid := types.UID("test-uid-shared-0001") + cpus, _ := cpuset.Parse("0") + name0 := CDIDeviceName(uid, "myrequest", "punit-0-0", 0) + name1 := CDIDeviceName(uid, "myrequest", "punit-0-0", 1) + devs := []CDIDevice{ + {Name: name0, ClassName: "gold", CPUs: cpus}, + {Name: name1, ClassName: "gold", CPUs: cpus}, + } + if err := w.WriteClaim(uid, devs); err != nil { + t.Errorf("WriteClaim() with two same-request/same-device results: unexpected error %v", err) + } +} diff --git a/pkg/resmgr/dra/deps.go b/pkg/resmgr/dra/deps.go index 00b8ae678..f4b977a48 100644 --- a/pkg/resmgr/dra/deps.go +++ b/pkg/resmgr/dra/deps.go @@ -18,9 +18,11 @@ package dra import ( resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // DeviceLister provides the DRA device list for a given driver name. @@ -28,6 +30,65 @@ type DeviceLister interface { DRADevices(driverName string) ([]resourceapi.Device, error) } +// CDIDevice holds the CDI device information for a single allocation result. +// Name is the CDI device name, precomputed by the caller via CDIDeviceName. +// WriteClaim uses Name and ClassName directly to build the CDI spec entry. +type CDIDevice struct { + // Name is the CDI device name (precomputed by the caller via CDIDeviceName). + Name string + // ClassName is the nri/cpuClass attribute value for the allocated device. + ClassName string + // CPUs is the set of CPUs allocated for this device. + CPUs cpuset.CPUSet +} + +// ClaimAllocator provides HP CPU pick/release/account operations needed +// during PrepareResourceClaims, UnprepareResourceClaims, and restart +// reconciliation. +type ClaimAllocator interface { + // PickHpCpus selects n HP-eligible CPUs from the punit identified by + // (pkgID, punitID), excluding CPUs in held and those already tracked + // in internal accounting. + PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) + // ReleaseHpCpus removes cpus from DRA HP accounting on the given punit. + ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) + // AccountHpCpus records cpus as DRA HP-held on the given punit. Used + // during restart reconciliation to rebuild HP accounting without + // re-allocating CPUs. + AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error + // IsHPClass reports whether className is currently classified as PCT + // high priority. + IsHPClass(className string) bool +} + +// CDIWriter manages CDI spec files on behalf of the DRA plugin. +type CDIWriter interface { + // WriteClaim writes a CDI spec file for the claim identified by uid, + // containing one CDI device entry per element of devices. + WriteClaim(uid types.UID, devices []CDIDevice) error + // RemoveClaim removes the CDI spec file for the claim identified by uid. + // Returns nil if the spec does not exist (idempotent). + RemoveClaim(uid types.UID) error + // ClaimSpecExists reports whether the CDI spec file for uid is present + // on disk. + ClaimSpecExists(uid types.UID) bool + // ListClaims returns the UIDs of all claims for which a CDI spec file + // exists under the managed CDI directory. + ListClaims() ([]types.UID, error) +} + +// ClaimStore persists and loads claim state via the resmgr cache. +// Save is a no-op when the cache is in a BlockSave window; data is still +// in-memory policyData and will persist on the next unblocked Save. +// Callers of Save must tolerate this. +type ClaimStore interface { + // Save persists the given claims map to the backing store. + Save(claims map[types.UID]*ClaimState) error + // Load reads the claims map from the backing store. Returns nil, nil + // when no claim state has been saved yet. + Load() (map[types.UID]*ClaimState, error) +} + // Deps holds the dependencies a policy binary must supply when constructing // a Plugin. type Deps struct { @@ -46,6 +107,16 @@ type Deps struct { ValidateClasses func() error // DeviceLister returns the list of DRA devices to publish. DeviceLister DeviceLister + // ClaimAllocator provides HP CPU pick/release/account operations. + ClaimAllocator ClaimAllocator + // CDIWriter manages CDI spec files for prepared claims. + CDIWriter CDIWriter + // ClaimStore persists and loads claim state via the resmgr cache. + ClaimStore ClaimStore + // WithLock executes f while holding the resmgr write lock. All accesses + // to Handler state (ValidateClasses, DRADevices, Prepare, Unprepare, and + // RestoreClaims) must run inside WithLock. + WithLock func(func()) // Logger is the logger used for all plugin log output. Logger log.Logger } diff --git a/pkg/resmgr/dra/plugin.go b/pkg/resmgr/dra/plugin.go index 2525b0bd3..b5948255c 100644 --- a/pkg/resmgr/dra/plugin.go +++ b/pkg/resmgr/dra/plugin.go @@ -24,23 +24,45 @@ import ( "path/filepath" "sync" "sync/atomic" + "time" "github.com/go-logr/logr" resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/dynamic-resource-allocation/kubeletplugin" "k8s.io/dynamic-resource-allocation/resourceslice" + + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "tags.cncf.io/container-device-interface/pkg/parser" +) + +// Sentinel errors for PrepareResourceClaims. +var ( + errNilAllocation = errors.New("dra plugin: claim has nil Allocation") + errMissingConsumedCapacity = errors.New("dra plugin: ConsumedCapacity[nri/cpus] absent or zero") + errNonHPNotSupported = errors.New("dra plugin: non-HP CPU class not supported (deferred)") + errMultiPunitNotSupported = errors.New("dra plugin: claim spans multiple punits (leaf pools): unsupported") ) -var errNotImplemented = errors.New("dra plugin: not yet implemented") +// deviceInfo holds the device attributes looked up from the published device list. +type deviceInfo struct { + ClassName string + PkgID int + PunitID int +} // Plugin is the DRA kubelet plugin. type Plugin struct { - mu sync.Mutex - driverName string - deps Deps - helper *kubeletplugin.Helper - handleErr atomic.Pointer[error] // set on non-recoverable HandleError; checked by PublishResources + mu sync.Mutex // mu guards helper, draCancel, draLoopDone + driverName string + deps Deps + helper *kubeletplugin.Helper + draCancel context.CancelFunc + draLoopDone chan struct{} + claims map[types.UID]*ClaimState + republishCh chan struct{} // signals the republisherLoop to re-run PublishResources + handleErr atomic.Pointer[error] // set on non-recoverable HandleError; checked by PublishResources } // New constructs a Plugin with the given driver name and dependencies. @@ -64,24 +86,414 @@ func New(driverName string, deps Deps) (*Plugin, error) { if deps.Logger == nil { return nil, fmt.Errorf("dra plugin: Logger must not be nil") } - return &Plugin{driverName: driverName, deps: deps}, nil + if deps.ClaimAllocator == nil { + return nil, fmt.Errorf("dra plugin: ClaimAllocator must not be nil") + } + if deps.CDIWriter == nil { + return nil, fmt.Errorf("dra plugin: CDIWriter must not be nil") + } + if deps.ClaimStore == nil { + return nil, fmt.Errorf("dra plugin: ClaimStore must not be nil") + } + if deps.WithLock == nil { + return nil, fmt.Errorf("dra plugin: WithLock must not be nil") + } + return &Plugin{ + driverName: driverName, + deps: deps, + claims: make(map[types.UID]*ClaimState), + republishCh: make(chan struct{}, 1), + }, nil +} + +// shareIDPtr converts a ShareID string to *types.UID. Returns nil if s is "". +func shareIDPtr(s string) *types.UID { + if s == "" { + return nil + } + uid := types.UID(s) + return &uid +} + +// deviceIndex builds a map from device name to deviceInfo by calling +// deps.DeviceLister.DRADevices. Must be called inside deps.WithLock. +// Called once per PrepareResourceClaims invocation, not per result. +func (p *Plugin) deviceIndex() (map[string]deviceInfo, error) { + devs, err := p.deps.DeviceLister.DRADevices(p.driverName) + if err != nil { + return nil, fmt.Errorf("dra plugin: DRADevices: %w", err) + } + idx := make(map[string]deviceInfo, len(devs)) + for _, d := range devs { + info := deviceInfo{} + if attr, ok := d.Attributes[resourceapi.QualifiedName("nri/cpuClass")]; ok && attr.StringValue != nil { + info.ClassName = *attr.StringValue + } + if attr, ok := d.Attributes[resourceapi.QualifiedName("nri/packageID")]; ok && attr.IntValue != nil { + info.PkgID = int(*attr.IntValue) + } + if attr, ok := d.Attributes[resourceapi.QualifiedName("nri/punitID")]; ok && attr.IntValue != nil { + info.PunitID = int(*attr.IntValue) + } + idx[d.Name] = info + } + return idx, nil +} + +// allClaimedCPUs returns the union of all CPUs currently tracked in p.claims. +// Must be called inside deps.WithLock. +func (p *Plugin) allClaimedCPUs() cpuset.CPUSet { + result := cpuset.New() + for _, cs := range p.claims { + for _, alloc := range cs.Allocs { + parsed, err := cpuset.Parse(alloc.CPUs) + if err == nil { + result = result.Union(parsed) + } + } + } + return result +} + +// PrepareResourceClaims prepares all resource claims allocated for this driver. +// For each claim it picks HP CPUs, writes a CDI spec, and persists claim state. +// The entire body runs inside deps.WithLock to serialize with Reconfigure. +func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi.ResourceClaim) (map[types.UID]kubeletplugin.PrepareResult, error) { + result := make(map[types.UID]kubeletplugin.PrepareResult, len(claims)) + p.deps.WithLock(func() { + devIdx, idxErr := p.deviceIndex() + if idxErr != nil { + // Fill all UIDs with the same error and return. + for _, claim := range claims { + result[claim.UID] = kubeletplugin.PrepareResult{Err: idxErr} + } + return + } + + for _, claim := range claims { + uid := claim.UID + result[uid] = func() kubeletplugin.PrepareResult { + if claim.Status.Allocation == nil { + return kubeletplugin.PrepareResult{Err: errNilAllocation} + } + + // Filter results to our driver. + allResults := claim.Status.Allocation.Devices.Results + var filtered []resourceapi.DeviceRequestAllocationResult + for _, r := range allResults { + if r.Driver == p.driverName { + filtered = append(filtered, r) + } + } + if len(filtered) == 0 { + // No results for our driver — valid, no work to do. + return kubeletplugin.PrepareResult{} + } + + // Idempotency: already prepared? + if _, exists := p.claims[uid]; exists { + if p.deps.CDIWriter.ClaimSpecExists(uid) { + // Spec present — re-build PrepareResult from stored state. + return p.buildPrepareResult(uid, filtered) + } + // Spec missing (e.g. node reboot) — re-write CDI spec from + // existing stored state without re-picking CPUs. + cdiDevices := p.cdiDevicesFromClaims(uid, filtered) + if len(cdiDevices) != len(p.claims[uid].Allocs) { + // Some (or all) stored CPU strings failed to parse — stored + // state is corrupt; writing a partial CDI spec and then + // returning CDIDeviceIDs for all allocs would give the runtime + // device IDs that don't exist in the spec, causing container + // start failures. + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: idempotent re-write: only %d of %d CDI devices rebuilt from stored state (stored CPUs unparsable)", len(cdiDevices), len(p.claims[uid].Allocs))} + } + if len(cdiDevices) > 0 { + if writeErr := p.deps.CDIWriter.WriteClaim(uid, cdiDevices); writeErr != nil { + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: re-write CDI spec: %w", writeErr)} + } + } + return p.buildPrepareResult(uid, filtered) + } + + // Allocate CPUs for each filtered result. + heldCPUs := p.allClaimedCPUs() + var pickedAllocs []ResultAlloc + var cdiDevices []CDIDevice + + for i, r := range filtered { + attrs, attrOk := devIdx[r.Device] + if !attrOk { + // Unknown device — rollback and report error. + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: unknown device %q", r.Device)} + } + if attrs.ClassName == "" { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: device %q missing nri/cpuClass attribute", r.Device)} + } + + // Reject claims whose results span more than one punit — + // the topology-aware consumer requires the union of a + // claim's results to fit a single leaf pool. + if len(pickedAllocs) > 0 { + first := pickedAllocs[0] + if attrs.PkgID != first.PkgID || attrs.PunitID != first.PunitID { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMultiPunitNotSupported} + } + } + + q, ok := r.ConsumedCapacity[resourceapi.QualifiedName("nri/cpus")] + if !ok { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMissingConsumedCapacity} + } + n := int(q.Value()) + if n <= 0 { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMissingConsumedCapacity} + } + + if !p.deps.ClaimAllocator.IsHPClass(attrs.ClassName) { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errNonHPNotSupported} + } + + picked, pickErr := p.deps.ClaimAllocator.PickHpCpus(attrs.PkgID, attrs.PunitID, n, heldCPUs) + if pickErr != nil { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: PickHpCpus: %w", pickErr)} + } + heldCPUs = heldCPUs.Union(picked) + + // Determine ShareID. + shareID := "" + if r.ShareID != nil { + shareID = string(*r.ShareID) + } + + pickedAllocs = append(pickedAllocs, ResultAlloc{ + Request: r.Request, + Pool: r.Pool, + Device: r.Device, + ShareID: shareID, + ClassName: attrs.ClassName, + PkgID: attrs.PkgID, + PunitID: attrs.PunitID, + CPUs: picked.String(), + }) + name := CDIDeviceName(uid, r.Request, r.Device, i) + cdiDevices = append(cdiDevices, CDIDevice{ + Name: name, + ClassName: attrs.ClassName, + CPUs: picked, + }) + } + + if writeErr := p.deps.CDIWriter.WriteClaim(uid, cdiDevices); writeErr != nil { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: WriteClaim: %w", writeErr)} + } + + p.claims[uid] = &ClaimState{UID: string(uid), Allocs: pickedAllocs} + if saveErr := p.deps.ClaimStore.Save(p.claims); saveErr != nil { + // Roll back symmetrically: the CDI spec and CPU picks + // are already live, so leaving the claim "prepared" + // here would lose durable state on a restart. + delete(p.claims, uid) + if removeErr := p.deps.CDIWriter.RemoveClaim(uid); removeErr != nil { + p.deps.Logger.Warnf("dra plugin: PrepareResourceClaims: rollback after ClaimStore.Save failure: RemoveClaim: %v", removeErr) + } + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: ClaimStore.Save: %w", saveErr)} + } + + return p.buildPrepareResult(uid, filtered) + }() + } + }) + return result, nil } -// PrepareResourceClaims is a stub that satisfies kubeletplugin.DRAPlugin. -func (p *Plugin) PrepareResourceClaims(_ context.Context, _ []*resourceapi.ResourceClaim) (map[types.UID]kubeletplugin.PrepareResult, error) { - return nil, errNotImplemented +// cdiDevicesFromClaims rebuilds the []CDIDevice slice for uid from the stored +// ClaimState. filtered provides the allocation results for our driver, in the +// same order as the original allocs. Used by the idempotency spec-missing path. +func (p *Plugin) cdiDevicesFromClaims(uid types.UID, filtered []resourceapi.DeviceRequestAllocationResult) []CDIDevice { + cs, ok := p.claims[uid] + if !ok { + return nil + } + devices := make([]CDIDevice, 0, len(cs.Allocs)) + for i, alloc := range cs.Allocs { + if i >= len(filtered) { + break + } + r := filtered[i] + name := CDIDeviceName(uid, r.Request, r.Device, i) + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + continue + } + devices = append(devices, CDIDevice{ + Name: name, + ClassName: alloc.ClassName, + CPUs: cpus, + }) + } + return devices } -// UnprepareResourceClaims is a stub that satisfies kubeletplugin.DRAPlugin. -func (p *Plugin) UnprepareResourceClaims(_ context.Context, _ []kubeletplugin.NamespacedObject) (map[types.UID]error, error) { - return nil, errNotImplemented +// rollbackPicks releases all CPU picks accumulated so far for a claim that +// encountered an error mid-way through allocation. +func (p *Plugin) rollbackPicks(allocs []ResultAlloc) { + for _, a := range allocs { + cs, err := cpuset.Parse(a.CPUs) + if err != nil { + continue + } + p.deps.ClaimAllocator.ReleaseHpCpus(a.PkgID, a.PunitID, cs) + } +} + +// buildPrepareResult constructs a kubeletplugin.PrepareResult from the stored +// claim state for uid. filtered contains the allocation results for our driver, +// in the same order they were originally processed (positional index matches +// CDIDeviceName index). +func (p *Plugin) buildPrepareResult(uid types.UID, filtered []resourceapi.DeviceRequestAllocationResult) kubeletplugin.PrepareResult { + cs, ok := p.claims[uid] + if !ok { + return kubeletplugin.PrepareResult{} + } + devices := make([]kubeletplugin.Device, 0, len(cs.Allocs)) + for i, alloc := range cs.Allocs { + if i >= len(filtered) { + break + } + r := filtered[i] + name := CDIDeviceName(uid, r.Request, r.Device, i) + devices = append(devices, kubeletplugin.Device{ + Requests: []string{r.Request}, + PoolName: r.Pool, + DeviceName: r.Device, + CDIDeviceIDs: []string{parser.QualifiedName(p.driverName, "device", name)}, + ShareID: shareIDPtr(alloc.ShareID), + }) + } + return kubeletplugin.PrepareResult{Devices: devices} +} + +// UnprepareResourceClaims releases CPUs and CDI specs for the given claims and +// removes them from persisted state. The entire body runs inside deps.WithLock +// to serialize with Reconfigure. +func (p *Plugin) UnprepareResourceClaims(_ context.Context, claims []kubeletplugin.NamespacedObject) (map[types.UID]error, error) { + perUID := make(map[types.UID]error, len(claims)) + p.deps.WithLock(func() { + for _, obj := range claims { + uid := obj.UID + cs, exists := p.claims[uid] + if !exists { + p.deps.Logger.Warnf("dra plugin: UnprepareResourceClaims: claim %s not found in state", uid) + perUID[uid] = nil + continue + } + // Release CPUs for each allocation result; parse errors are logged + // but do not block CDI removal or claim deletion. + for _, alloc := range cs.Allocs { + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + p.deps.Logger.Warnf("dra plugin: UnprepareResourceClaims: claim %s device %s: parse CPUs %q: %v (skipping release)", uid, alloc.Device, alloc.CPUs, err) + continue + } + p.deps.ClaimAllocator.ReleaseHpCpus(alloc.PkgID, alloc.PunitID, cpus) + } + // Remove CDI spec unconditionally; log but do not block deletion. + if err := p.deps.CDIWriter.RemoveClaim(uid); err != nil { + p.deps.Logger.Warnf("dra plugin: UnprepareResourceClaims: claim %s: RemoveClaim: %v", uid, err) + } + delete(p.claims, uid) + perUID[uid] = nil + } + // Persist the updated claims map in a single batch write. + if saveErr := p.deps.ClaimStore.Save(p.claims); saveErr != nil { + p.deps.Logger.Errorf("dra plugin: UnprepareResourceClaims: ClaimStore.Save: %v", saveErr) + } + }) + return perUID, nil +} + +// LiveClaimClasses returns a map from className to the number of live claims +// using that class. Each claim is counted once per distinct class it uses. +// Caller must hold the resmgr lock (do not call from inside a WithLock +// callback — the resmgr lock is not reentrant). Used to refuse a Reconfigure +// that would change class-derived attributes while claims are live. +func (p *Plugin) LiveClaimClasses() map[string]int { + result := make(map[string]int) + for _, cs := range p.claims { + // Count each claim once per distinct class it uses. + seen := make(map[string]bool) + for _, alloc := range cs.Allocs { + if alloc.ClassName != "" && !seen[alloc.ClassName] { + result[alloc.ClassName]++ + seen[alloc.ClassName] = true + } + } + } + return result +} + +// RestoreClaimsLocked re-runs AccountHpCpus for every entry in p.claims, +// rebuilding HP accounting after a Reconfigure that reset hpDRAUsed. It does +// not reload from cache and does not acquire any lock. +// +// Caller must already hold the resmgr lock; do not call via WithLock. +// RestoreClaims is the complementary WithLock wrapper for callers that are not +// already holding the lock. +func (p *Plugin) RestoreClaimsLocked() error { + var errs []error + for _, cs := range p.claims { + for _, alloc := range cs.Allocs { + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + p.deps.Logger.Warnf("dra plugin: RestoreClaimsLocked: claim %s device %s: parse CPUs %q: %v (skipping)", cs.UID, alloc.Device, alloc.CPUs, err) + continue + } + if err := p.deps.ClaimAllocator.AccountHpCpus(alloc.PkgID, alloc.PunitID, cpus); err != nil { + p.deps.Logger.Warnf("dra plugin: RestoreClaimsLocked: claim %s device %s: AccountHpCpus: %v", cs.UID, alloc.Device, err) + errs = append(errs, err) + } + } + } + return errors.Join(errs...) +} + +// RestoreClaims wraps RestoreClaimsLocked inside deps.WithLock for callers +// that are not already holding the resmgr lock. +// +// RestoreClaims must not be called while holding the resmgr lock +// (see RestoreClaimsLocked for the complementary lock-already-held variant). +func (p *Plugin) RestoreClaims() error { + var err error + p.deps.WithLock(func() { + err = p.RestoreClaimsLocked() + }) + return err } // Start registers this plugin with the kubelet and begins serving DRA -// requests. It validates cpuClass configuration, creates the plugin data -// directory, injects a logr.Logger into the context, and calls -// kubeletplugin.Start. Returns an error if the plugin is already started, -// if ValidateClasses fails, or if the kubelet plugin cannot be started. +// requests. It validates cpuClass configuration, loads and reconciles +// persisted claim state, creates the plugin data directory, injects a +// logr.Logger into the context, and calls kubeletplugin.Start. Returns an +// error if the plugin is already started, if ValidateClasses fails, or if +// the kubelet plugin cannot be started. +// +// Start must not be called while holding the resmgr lock +// (see RestoreClaimsLocked for the complementary lock-already-held variant). +// +// cpuclass.Handler must have completed Configure() before Start() so that +// AccountHpCpus finds active punits. If the allocator is inactive when Start +// is called, live claims are kept with a warning. If AccountHpCpus fails for +// any other reason (e.g. punit not found due to topology change), Start returns +// an error rather than registering the driver in an unsafe state. func (p *Plugin) Start(ctx context.Context) error { p.mu.Lock() alreadyStarted := p.helper != nil @@ -92,6 +504,72 @@ func (p *Plugin) Start(ctx context.Context) error { if err := p.deps.ValidateClasses(); err != nil { return fmt.Errorf("dra plugin: ValidateClasses failed: %w", err) } + + // Reconcile persisted claims inside the resmgr lock before registering + // with the kubelet so that accounting is consistent before any new + // Prepare/Unprepare calls can arrive. + var startErr error + p.deps.WithLock(func() { + loaded, loadErr := p.deps.ClaimStore.Load() + if loadErr != nil { + // A Load failure must not be treated as "no claims": that would + // let the orphan sweep below remove CDI specs that still belong + // to live claim consumers. Fail startup outright instead. + startErr = fmt.Errorf("dra plugin: Start: ClaimStore.Load: %w", loadErr) + return + } + if loaded != nil { + p.claims = loaded + } + + // Re-account every persisted claim from durable state. A missing CDI spec + // on disk does not by itself prove the claim is stale: kubelet commonly + // clears /var/run/cdi on reboot, while the claim store is the durable + // source of truth for active claims. Keep persisted claims accounted and + // let the next Prepare re-create a missing CDI spec if needed. + for uid, cs := range p.claims { + if !p.deps.CDIWriter.ClaimSpecExists(uid) { + p.deps.Logger.Warnf("dra plugin: Start: claim %s has no CDI spec on disk; keeping persisted claim and re-accounting from durable state", uid) + } + for _, alloc := range cs.Allocs { + if alloc.ClassName != "" && !p.deps.ClaimAllocator.IsHPClass(alloc.ClassName) { + startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: cpuClass %q is no longer a valid HP class (removed or renamed); cannot restore claim", uid, alloc.Device, alloc.ClassName) + return + } + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: parse CPUs %q: %w", uid, alloc.Device, alloc.CPUs, err) + return + } + if err := p.deps.ClaimAllocator.AccountHpCpus(alloc.PkgID, alloc.PunitID, cpus); err != nil { + if errors.Is(err, cpuclass.ErrAllocatorInactive) { + p.deps.Logger.Warnf("dra plugin: Start: claim %s device %s: AccountHpCpus: %v (allocator inactive — keeping claim, container may be running)", uid, alloc.Device, err) + } else { + startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: AccountHpCpus: %w", uid, alloc.Device, err) + return + } + } + } + } + + // Orphan sweep: remove CDI specs that have no corresponding live claim. + listed, listErr := p.deps.CDIWriter.ListClaims() + if listErr != nil { + p.deps.Logger.Warnf("dra plugin: Start: CDIWriter.ListClaims: %v (skipping orphan sweep)", listErr) + return + } + for _, uid := range listed { + if _, ok := p.claims[uid]; !ok { + if removeErr := p.deps.CDIWriter.RemoveClaim(uid); removeErr != nil { + p.deps.Logger.Warnf("dra plugin: Start: orphan sweep: RemoveClaim %s: %v", uid, removeErr) + } + } + } + }) + if startErr != nil { + return startErr + } + // Resolve the plugin data directory default before creating it: an empty // string passed to os.MkdirAll would fail immediately. pluginDataDir := p.deps.PluginDataDir @@ -119,19 +597,97 @@ func (p *Plugin) Start(ctx context.Context) error { if err != nil { return fmt.Errorf("dra plugin: kubeletplugin.Start: %w", err) } + // Re-check under lock to close the TOCTOU window between the initial + // guard (above) and this assignment. If a racing Start() won, stop the + // helper we just created and return an error. p.mu.Lock() + if p.helper != nil { + p.mu.Unlock() + helper.Stop() + return fmt.Errorf("dra plugin: already started") + } + draCtx, draCancel := context.WithCancel(ctx) + loopDone := make(chan struct{}) p.helper = helper + p.draCancel = draCancel + p.draLoopDone = loopDone p.mu.Unlock() + go func() { + defer close(loopDone) + p.republisherLoop(draCtx) + }() return nil } +// TriggerRepublish enqueues a request to re-run PublishResources. It is +// non-blocking and safe to call while holding the resmgr lock: if a +// republish is already pending, the extra request is dropped (the pending +// one covers it). The actual publish runs in republisherLoop, outside +// any lock. +func (p *Plugin) TriggerRepublish() { + select { + case p.republishCh <- struct{}{}: + default: + // A republish is already pending; no need to enqueue another. + } +} + +// republisherLoop runs in a goroutine started by Start(). It drains +// republishCh and calls PublishResources for each signal, stopping when +// ctx is cancelled (i.e. when the plugin's draCtx is cancelled by Stop()). +// On a publication failure the loop re-enqueues itself with exponential +// backoff (1s → 2s → … → 60s) so that the ResourceSlice is eventually +// brought back in sync without hammering the API on persistent errors. +func (p *Plugin) republisherLoop(ctx context.Context) { + const ( + initialBackoff = time.Second + maxBackoff = time.Minute + ) + backoff := initialBackoff + for { + select { + case <-ctx.Done(): + return + case <-p.republishCh: + if err := p.PublishResources(ctx); err != nil { + p.deps.Logger.Warnf("dra plugin: background republish: %v (retrying in %s)", err, backoff) + select { + case <-ctx.Done(): + return + case <-time.After(backoff): + } + backoff *= 2 + if backoff > maxBackoff { + backoff = maxBackoff + } + select { + case p.republishCh <- struct{}{}: + default: + } + } else { + backoff = initialBackoff + } + } + } +} + // Stop shuts down the kubelet plugin and releases resources. It is // idempotent: calling Stop on an already-stopped Plugin is safe. func (p *Plugin) Stop() { p.mu.Lock() h := p.helper + cancel := p.draCancel + done := p.draLoopDone p.helper = nil + p.draCancel = nil + p.draLoopDone = nil p.mu.Unlock() + if cancel != nil { + cancel() + } + if done != nil { + <-done + } if h != nil { h.Stop() } @@ -141,12 +697,27 @@ func (p *Plugin) Stop() { // ResourceSlice objects and hands the resulting DriverResources to the helper // for publishing. Even zero devices produce one empty slice so the pool // remains visible. +// +// PublishResources must not be called while holding the resmgr lock +// (see RestoreClaimsLocked for the complementary lock-already-held variant). func (p *Plugin) PublishResources(ctx context.Context) error { if pe := p.handleErr.Load(); pe != nil { return *pe } - if err := p.deps.ValidateClasses(); err != nil { - return fmt.Errorf("dra plugin: ValidateClasses failed: %w", err) + var ( + validateErr error + devices []resourceapi.Device + devicesErr error + ) + p.deps.WithLock(func() { + validateErr = p.deps.ValidateClasses() + if validateErr != nil { + return + } + devices, devicesErr = p.deps.DeviceLister.DRADevices(p.driverName) + }) + if validateErr != nil { + return fmt.Errorf("dra plugin: ValidateClasses failed: %w", validateErr) } p.mu.Lock() h := p.helper @@ -154,9 +725,8 @@ func (p *Plugin) PublishResources(ctx context.Context) error { if h == nil { return fmt.Errorf("dra plugin: PublishResources called before Start") } - devices, err := p.deps.DeviceLister.DRADevices(p.driverName) - if err != nil { - return fmt.Errorf("dra plugin: DRADevices: %w", err) + if devicesErr != nil { + return fmt.Errorf("dra plugin: DRADevices: %w", devicesErr) } resources := buildDriverResources(p.deps.NodeName, devices) if err := h.PublishResources(ctx, resources); err != nil { @@ -199,6 +769,7 @@ func buildDriverResources(nodeName string, devices []resourceapi.Device) resourc func (p *Plugin) HandleError(_ context.Context, err error, msg string) { if errors.Is(err, kubeletplugin.ErrRecoverable) { p.deps.Logger.Warnf("%s: %v", msg, err) + p.TriggerRepublish() } else { p.deps.Logger.Errorf("%s: %v", msg, err) e := fmt.Errorf("dra plugin: resource slice controller: %w", err) diff --git a/pkg/resmgr/dra/plugin_test.go b/pkg/resmgr/dra/plugin_test.go index 02fd16abd..4b5047d72 100644 --- a/pkg/resmgr/dra/plugin_test.go +++ b/pkg/resmgr/dra/plugin_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + "runtime" "strings" "sync" "testing" @@ -27,13 +28,18 @@ import ( corev1 "k8s.io/api/core/v1" resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes/fake" k8stesting "k8s.io/client-go/testing" "k8s.io/dynamic-resource-allocation/kubeletplugin" "github.com/containers/nri-plugins/pkg/log" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "tags.cncf.io/container-device-interface/pkg/parser" ) // TestNewLogr verifies that newLogr returns a usable logr.Logger backed by the @@ -54,6 +60,10 @@ func validDeps() Deps { NodeName: "test-node", ValidateClasses: func() error { return nil }, DeviceLister: &fixedDeviceLister{}, + ClaimAllocator: &noopClaimAllocator{}, + CDIWriter: &noopCDIWriter{}, + ClaimStore: &noopClaimStore{}, + WithLock: func(f func()) { f() }, Logger: log.Default(), } } @@ -108,6 +118,26 @@ func TestNew_Validation(t *testing.T) { driverName: "test-driver", mutate: func(d *Deps) { d.Logger = nil }, }, + { + name: "nil ClaimAllocator", + driverName: "test-driver", + mutate: func(d *Deps) { d.ClaimAllocator = nil }, + }, + { + name: "nil CDIWriter", + driverName: "test-driver", + mutate: func(d *Deps) { d.CDIWriter = nil }, + }, + { + name: "nil ClaimStore", + driverName: "test-driver", + mutate: func(d *Deps) { d.ClaimStore = nil }, + }, + { + name: "nil WithLock", + driverName: "test-driver", + mutate: func(d *Deps) { d.WithLock = nil }, + }, } for _, tc := range tests { @@ -127,32 +157,6 @@ func TestNew_Validation(t *testing.T) { } } -// TestPrepareResourceClaims_Stub verifies that PrepareResourceClaims returns -// errNotImplemented before real allocation logic is wired in Step 7. -func TestPrepareResourceClaims_Stub(t *testing.T) { - p := &Plugin{} - result, err := p.PrepareResourceClaims(t.Context(), []*resourceapi.ResourceClaim{}) - if result != nil { - t.Errorf("PrepareResourceClaims() result = %v, want nil", result) - } - if !errors.Is(err, errNotImplemented) { - t.Errorf("PrepareResourceClaims() err = %v, want errNotImplemented", err) - } -} - -// TestUnprepareResourceClaims_Stub verifies that UnprepareResourceClaims returns -// errNotImplemented before real deallocation logic is wired in Step 7. -func TestUnprepareResourceClaims_Stub(t *testing.T) { - p := &Plugin{} - result, err := p.UnprepareResourceClaims(t.Context(), []kubeletplugin.NamespacedObject{}) - if result != nil { - t.Errorf("UnprepareResourceClaims() result = %v, want nil", result) - } - if !errors.Is(err, errNotImplemented) { - t.Errorf("UnprepareResourceClaims() err = %v, want errNotImplemented", err) - } -} - // TestHandleError_RecoverableLogsWarn verifies that a recoverable error is // handled without panicking. The method logs at Warn level. func TestHandleError_RecoverableLogsWarn(t *testing.T) { @@ -324,6 +328,33 @@ func (e *errorDeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { return nil, e.err } +// noopClaimAllocator is a ClaimAllocator that succeeds without doing anything. +type noopClaimAllocator struct{} + +func (*noopClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { + return cpuset.New(), nil +} + +func (*noopClaimAllocator) ReleaseHpCpus(_, _ int, _ cpuset.CPUSet) {} + +func (*noopClaimAllocator) AccountHpCpus(_, _ int, _ cpuset.CPUSet) error { return nil } + +func (*noopClaimAllocator) IsHPClass(_ string) bool { return false } + +// noopCDIWriter is a CDIWriter that succeeds without doing anything. +type noopCDIWriter struct{} + +func (*noopCDIWriter) WriteClaim(_ types.UID, _ []CDIDevice) error { return nil } +func (*noopCDIWriter) RemoveClaim(_ types.UID) error { return nil } +func (*noopCDIWriter) ClaimSpecExists(_ types.UID) bool { return false } +func (*noopCDIWriter) ListClaims() ([]types.UID, error) { return nil, nil } + +// noopClaimStore is a ClaimStore that succeeds without persisting anything. +type noopClaimStore struct{} + +func (*noopClaimStore) Save(_ map[types.UID]*ClaimState) error { return nil } +func (*noopClaimStore) Load() (map[types.UID]*ClaimState, error) { return nil, nil } + // TestPublishResources_DRADevicesError verifies that an error from // DeviceLister.DRADevices is propagated by PublishResources. func TestPublishResources_DRADevicesError(t *testing.T) { @@ -343,6 +374,105 @@ func TestPublishResources_DRADevicesError(t *testing.T) { } } +// TestPublishResources_ConcurrentNoRace verifies that calling PublishResources +// from multiple goroutines while a simulated Reconfigure goroutine acquires the +// same mutex does not produce a data race. When run with -race, the detector +// will flag any unsynchronized access to shared state. +func TestPublishResources_ConcurrentNoRace(t *testing.T) { + var mu sync.Mutex + + deps := validDeps() + // Replace the simple direct-call WithLock from validDeps with a real mutex + // so the race detector can verify PublishResources holds the lock around + // its ValidateClasses + DRADevices calls. + deps.WithLock = func(f func()) { + mu.Lock() + defer mu.Unlock() + f() + } + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + // Set helper to non-nil so the nil-helper guard is bypassed. + p.helper = new(kubeletplugin.Helper) + + ctx := context.Background() + const goroutines = 10 + var wg sync.WaitGroup + wg.Add(goroutines) + for range goroutines { + go func() { + defer wg.Done() + // Errors are expected (helper is a zero-value stub); what we are + // testing is the absence of data races. + _ = p.PublishResources(ctx) + }() + } + // Simulate concurrent Reconfigure by acquiring the same mutex for a brief + // hold, which is what resmgr.apply() does when it calls policy.Reconfigure() + // under the write lock. + wg.Add(1) + go func() { + defer wg.Done() + for range 50 { + mu.Lock() + runtime.Gosched() // hold briefly to interleave with PublishResources + mu.Unlock() + } + }() + wg.Wait() +} + +// TestTriggerRepublish_EnqueuesSignal verifies that TriggerRepublish sends +// exactly one signal on the buffered republishCh when the channel is empty. +func TestTriggerRepublish_EnqueuesSignal(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + select { + case <-p.republishCh: + t.Fatal("unexpected signal in republishCh before TriggerRepublish") + default: + } + + p.TriggerRepublish() + + select { + case <-p.republishCh: + default: + t.Fatal("no signal in republishCh after TriggerRepublish") + } +} + +// TestTriggerRepublish_Idempotent verifies that multiple TriggerRepublish +// calls while a publish is already pending enqueue exactly one signal (no +// blocking, no panic, no second entry). +func TestTriggerRepublish_Idempotent(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + p.TriggerRepublish() + p.TriggerRepublish() // second call must not block or add a second signal + + count := 0 + for { + select { + case <-p.republishCh: + count++ + default: + if count != 1 { + t.Errorf("republishCh had %d signals after two TriggerRepublish calls, want 1", count) + } + return + } + } +} + // TestStart_AlreadyStarted verifies that a second call to Start returns an // error without spawning a second helper. The guard is tested by setting // p.helper to a non-nil stub before calling Start. @@ -404,6 +534,10 @@ func TestPublishResources_Integration(t *testing.T) { PluginDataDir: pluginDataDir, ValidateClasses: func() error { return nil }, DeviceLister: &fixedDeviceLister{devices: makeTestDevices(5)}, + ClaimAllocator: &noopClaimAllocator{}, + CDIWriter: &noopCDIWriter{}, + ClaimStore: &noopClaimStore{}, + WithLock: func(f func()) { f() }, Logger: log.Default(), } @@ -448,3 +582,1546 @@ func TestPublishResources_Integration(t *testing.T) { t.Errorf("ResourceSlice[0].Spec.Driver = %q, want %q", got, driverName) } } + +// trackingClaimAllocator tracks PickHpCpus and ReleaseHpCpus calls. +type trackingClaimAllocator struct { + pickResult cpuset.CPUSet + pickErr error + isHP bool + picks []cpuset.CPUSet // CPUSets returned per PickHpCpus call + releases []cpuset.CPUSet // CPUSets released per ReleaseHpCpus call + accounts []cpuset.CPUSet // CPUSets accounted per AccountHpCpus call + accountErr error +} + +func (a *trackingClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { + if a.pickErr != nil { + return cpuset.New(), a.pickErr + } + a.picks = append(a.picks, a.pickResult) + return a.pickResult, nil +} + +func (a *trackingClaimAllocator) ReleaseHpCpus(_, _ int, cpus cpuset.CPUSet) { + a.releases = append(a.releases, cpus) +} + +func (a *trackingClaimAllocator) AccountHpCpus(_, _ int, cpus cpuset.CPUSet) error { + if a.accountErr != nil { + return a.accountErr + } + a.accounts = append(a.accounts, cpus) + return nil +} + +func (a *trackingClaimAllocator) IsHPClass(_ string) bool { return a.isHP } + +// trackingCDIWriter tracks WriteClaim and RemoveClaim calls. +type trackingCDIWriter struct { + writeErr error + removeErr error + existsValue bool + written []types.UID + removed []types.UID +} + +func (w *trackingCDIWriter) WriteClaim(uid types.UID, _ []CDIDevice) error { + if w.writeErr != nil { + return w.writeErr + } + w.written = append(w.written, uid) + return nil +} + +func (w *trackingCDIWriter) RemoveClaim(uid types.UID) error { + w.removed = append(w.removed, uid) + return w.removeErr +} + +func (w *trackingCDIWriter) ClaimSpecExists(_ types.UID) bool { return w.existsValue } +func (w *trackingCDIWriter) ListClaims() ([]types.UID, error) { return nil, nil } + +// trackingClaimStore records Save and Load calls. +type trackingClaimStore struct { + saveErr error + saved int +} + +func (s *trackingClaimStore) Save(_ map[types.UID]*ClaimState) error { + if s.saveErr != nil { + return s.saveErr + } + s.saved++ + return nil +} + +func (s *trackingClaimStore) Load() (map[types.UID]*ClaimState, error) { return nil, nil } + +// strPtr returns a pointer to s, used to build DeviceAttribute.StringValue. +func strPtr(s string) *string { return &s } + +// int64Ptr returns a pointer to v, used to build DeviceAttribute.IntValue. +func int64Ptr(v int64) *int64 { return &v } + +// hpDevice builds a resourceapi.Device with nri/cpuClass, nri/packageID, and +// nri/punitID attributes, plus an nri/cpus capacity. +func hpDevice(name, className string, pkgID, punitID int) resourceapi.Device { + return resourceapi.Device{ + Name: name, + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/cpuClass": {StringValue: strPtr(className)}, + "nri/packageID": {IntValue: int64Ptr(int64(pkgID))}, + "nri/punitID": {IntValue: int64Ptr(int64(punitID))}, + }, + } +} + +// hpDeviceLister returns a DeviceLister that always returns the given devices. +func hpDeviceLister(devs ...resourceapi.Device) *fixedDeviceLister { + return &fixedDeviceLister{devices: devs} +} + +// makeClaim builds a ResourceClaim with a single allocation result for +// driverName/poolName/deviceName, with the given ConsumedCapacity cpu count. +func makeClaim(uid types.UID, driverName, poolName, deviceName, request string, cpus int) *resourceapi.ResourceClaim { + qty := resource.MustParse(fmt.Sprintf("%d", cpus)) + return &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + { + Driver: driverName, + Pool: poolName, + Device: deviceName, + Request: request, + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{ + "nri/cpus": qty, + }, + }, + }, + }, + }, + }, + } +} + +// TestPrepare_SingleHPSuccess verifies that a single HP claim results in a +// PrepareResult with one Device and that CPUs are picked and CDI is written. +func TestPrepare_SingleHPSuccess(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-1") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil { + t.Fatalf("PrepareResourceClaims() unexpected error: %v", err) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing uid-1") + } + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if len(r.Devices) != 1 { + t.Fatalf("PrepareResult.Devices len = %d, want 1", len(r.Devices)) + } + if len(alloc.picks) != 1 { + t.Errorf("PickHpCpus called %d times, want 1", len(alloc.picks)) + } + if len(cdiW.written) != 1 || cdiW.written[0] != uid { + t.Errorf("WriteClaim called for %v, want %v", cdiW.written, []types.UID{uid}) + } + if store.saved != 1 { + t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + } + // Verify the claim is stored. + if _, ok := p.claims[uid]; !ok { + t.Error("claim not stored in p.claims") + } +} + +// TestPrepare_Idempotent_SpecPresent verifies that a second Prepare call for the +// same claim with the CDI spec already present returns the same PrepareResult +// without re-picking CPUs or re-writing the spec. +func TestPrepare_Idempotent_SpecPresent(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-idem") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + + // First Prepare — succeeds. + result1, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result1[uid].Err != nil { + t.Fatalf("first Prepare failed: %v / %v", err, result1[uid].Err) + } + firstPicks := len(alloc.picks) + firstWrites := len(cdiW.written) + + // Simulate CDI spec existing. + cdiW.existsValue = true + + // Second Prepare — should be idempotent. + result2, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result2[uid].Err != nil { + t.Fatalf("second Prepare failed: %v / %v", err, result2[uid].Err) + } + if len(alloc.picks) != firstPicks { + t.Errorf("second Prepare picked CPUs again (picks: %d → %d)", firstPicks, len(alloc.picks)) + } + if len(cdiW.written) != firstWrites { + t.Errorf("second Prepare re-wrote CDI spec (writes: %d → %d)", firstWrites, len(cdiW.written)) + } + // Devices content must be identical between calls. + devs1 := result1[uid].Devices + devs2 := result2[uid].Devices + if len(devs1) != len(devs2) { + t.Errorf("Devices len differs between calls: first=%d second=%d", len(devs1), len(devs2)) + } else { + for i := range devs1 { + if len(devs1[i].CDIDeviceIDs) != len(devs2[i].CDIDeviceIDs) { + t.Errorf("Device[%d] CDIDeviceIDs len differs: first=%d second=%d", i, len(devs1[i].CDIDeviceIDs), len(devs2[i].CDIDeviceIDs)) + continue + } + for j := range devs1[i].CDIDeviceIDs { + if devs1[i].CDIDeviceIDs[j] != devs2[i].CDIDeviceIDs[j] { + t.Errorf("Device[%d].CDIDeviceIDs[%d]: first=%q second=%q", i, j, devs1[i].CDIDeviceIDs[j], devs2[i].CDIDeviceIDs[j]) + } + } + } + } +} + +// TestPrepare_Idempotent_SpecMissing verifies that a second Prepare call for the +// same claim where the CDI spec is missing re-writes the spec without re-picking CPUs. +func TestPrepare_Idempotent_SpecMissing(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-rewrite") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + + // First Prepare. + result1, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result1[uid].Err != nil { + t.Fatalf("first Prepare failed: %v / %v", err, result1[uid].Err) + } + firstPicks := len(alloc.picks) + firstWrites := len(cdiW.written) + + // CDI spec remains missing (existsValue == false by default). + + // Second Prepare — should re-write but not re-pick. + result2, err := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil || result2[uid].Err != nil { + t.Fatalf("second Prepare failed: %v / %v", err, result2[uid].Err) + } + if len(alloc.picks) != firstPicks { + t.Errorf("second Prepare re-picked CPUs (picks: %d → %d)", firstPicks, len(alloc.picks)) + } + if len(cdiW.written) != firstWrites+1 { + t.Errorf("second Prepare should have re-written CDI spec (writes: %d → %d)", firstWrites, len(cdiW.written)) + } + // Devices content must be identical between calls. + devs1 := result1[uid].Devices + devs2 := result2[uid].Devices + if len(devs1) != len(devs2) { + t.Errorf("Devices len differs between calls: first=%d second=%d", len(devs1), len(devs2)) + } else { + for i := range devs1 { + if len(devs1[i].CDIDeviceIDs) != len(devs2[i].CDIDeviceIDs) { + t.Errorf("Device[%d] CDIDeviceIDs len differs: first=%d second=%d", i, len(devs1[i].CDIDeviceIDs), len(devs2[i].CDIDeviceIDs)) + continue + } + for j := range devs1[i].CDIDeviceIDs { + if devs1[i].CDIDeviceIDs[j] != devs2[i].CDIDeviceIDs[j] { + t.Errorf("Device[%d].CDIDeviceIDs[%d]: first=%q second=%q", i, j, devs1[i].CDIDeviceIDs[j], devs2[i].CDIDeviceIDs[j]) + } + } + } + } +} + +// TestPrepare_Idempotent_SpecMissing_PartialCorruptCPU verifies that the +// spec-missing idempotency path returns an error when only some stored alloc +// CPUs are parseable (partial corruption), preventing a mismatched CDI spec +// from being written. +func TestPrepare_Idempotent_SpecMissing_PartialCorruptCPU(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + uid := types.UID("uid-partial-corrupt") + // Two stored allocs: first is valid, second has an unparsable CPU string. + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{ + {Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-1", ClassName: "gold"}, + {Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "NOT-A-CPUSET", ClassName: "gold"}, + }, + } + + qty := resource.MustParse("2") + // Claim carries two allocation results for our driver — one per device. + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "req0", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + {Driver: "test-driver", Pool: "p", Device: "dev1", Request: "req1", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + + // Wire in the pre-loaded claim state so the idempotency path fires. + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + // CDI spec is missing (cdiW.existsValue == false by default). + + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing UID") + } + if r.Err == nil { + t.Fatal("PrepareResult.Err = nil; want error for partial corrupt stored CPUs") + } + // No CDI spec should have been written. + if len(cdiW.written) != 0 { + t.Errorf("CDIWriter.WriteClaim called %d time(s), want 0", len(cdiW.written)) + } + // No CPU picks should have been made. + if len(alloc.picks) != 0 { + t.Errorf("PickHpCpus called %d time(s), want 0", len(alloc.picks)) + } +} + +// TestPrepare_NilAllocation verifies that a nil claim.Status.Allocation produces +// a per-claim errNilAllocation in the result map, not a global error. +func TestPrepare_NilAllocation(t *testing.T) { + deps := validDeps() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-nil-alloc") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing UID") + } + if !errors.Is(r.Err, errNilAllocation) { + t.Errorf("PrepareResult.Err = %v, want errNilAllocation", r.Err) + } +} + +// TestPrepare_ForeignDriverOnly verifies that a claim with results only for a +// different driver produces an empty PrepareResult with no error. +func TestPrepare_ForeignDriverOnly(t *testing.T) { + deps := validDeps() + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-foreign") + qty := resource.MustParse("4") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "other-driver", Pool: "p", Device: "d", Request: "r", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r, ok := result[uid] + if !ok { + t.Fatal("result map missing UID") + } + if r.Err != nil { + t.Errorf("PrepareResult.Err = %v, want nil for foreign-driver-only claim", r.Err) + } + if len(r.Devices) != 0 { + t.Errorf("PrepareResult.Devices = %v, want empty for foreign-driver-only claim", r.Devices) + } +} + +// TestPrepare_UnknownDevice verifies that a result referencing a device not in +// deviceIndex produces a per-claim error. +func TestPrepare_UnknownDevice(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + // DeviceLister returns no devices. + deps.DeviceLister = hpDeviceLister() + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-unknown") + claim := makeClaim(uid, "test-driver", "pool0", "unknown-dev", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error for unknown device, got nil") + } +} + +// TestPrepare_NilAttr verifies that a device with a missing (empty) nri/cpuClass +// attribute produces a per-claim error. +func TestPrepare_NilAttr(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + // Device has no nri/cpuClass attribute. + deps.DeviceLister = hpDeviceLister(resourceapi.Device{ + Name: "dev0", + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/packageID": {IntValue: int64Ptr(0)}, + "nri/punitID": {IntValue: int64Ptr(0)}, + // nri/cpuClass intentionally absent. + }, + }) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-nil-attr") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error for nil nri/cpuClass attr, got nil") + } +} + +// TestPrepare_NilIntAttr verifies that a device attribute with a nil IntValue +// (for packageID or punitID) is handled gracefully: the field defaults to 0 +// and Prepare succeeds when all other required attrs are present. +func TestPrepare_NilIntAttr(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = &trackingCDIWriter{} + deps.ClaimStore = &trackingClaimStore{} + // Device has cpuClass set but packageID and punitID with nil IntValue. + deps.DeviceLister = hpDeviceLister(resourceapi.Device{ + Name: "dev0", + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/cpuClass": {StringValue: strPtr("gold")}, + "nri/packageID": {IntValue: nil}, // nil IntValue → PkgID defaults to 0 + "nri/punitID": {IntValue: nil}, // nil IntValue → PunitID defaults to 0 + }, + }) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-nil-int-attr") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + // Nil IntValue falls back to zero — Prepare should succeed (pick from pkg 0, punit 0). + if r.Err != nil { + t.Errorf("PrepareResult.Err = %v, want nil for nil IntValue attrs (defaults to 0)", r.Err) + } +} + +// TestPrepare_AbsentConsumedCapacity verifies that a result with no nri/cpus +// entry in ConsumedCapacity produces errMissingConsumedCapacity. +func TestPrepare_AbsentConsumedCapacity(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-no-cap") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "r"}, + // No ConsumedCapacity. + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if !errors.Is(r.Err, errMissingConsumedCapacity) { + t.Errorf("PrepareResult.Err = %v, want errMissingConsumedCapacity", r.Err) + } +} + +// TestPrepare_NonHP verifies that a non-HP class produces errNonHPNotSupported. +func TestPrepare_NonHP(t *testing.T) { + alloc := &trackingClaimAllocator{isHP: false} // isHP == false → not HP + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "silver", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-non-hp") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if !errors.Is(r.Err, errNonHPNotSupported) { + t.Errorf("PrepareResult.Err = %v, want errNonHPNotSupported", r.Err) + } +} + +// TestPrepare_PickFailure verifies that a PickHpCpus failure rolls back +// any previously picked CPUs and returns a per-claim error. +func TestPrepare_PickFailure(t *testing.T) { + pickErr := errors.New("pick failed") + alloc := &trackingClaimAllocator{pickErr: pickErr, isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-pick-fail") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error from PickHpCpus failure, got nil") + } + if !errors.Is(r.Err, pickErr) { + t.Errorf("PrepareResult.Err = %v, want to wrap pickErr", r.Err) + } + // Claim must not be stored after a PickHpCpus failure. + if _, ok := p.claims[uid]; ok { + t.Error("claim stored in p.claims after PickHpCpus failure (should not be present)") + } +} + +// TestPrepare_CDIWriteFailure verifies that a CDI WriteClaim failure rolls back +// the picked CPUs and returns a per-claim error. +func TestPrepare_CDIWriteFailure(t *testing.T) { + writeErr := errors.New("CDI write failed") + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{writeErr: writeErr} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-cdi-fail") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Error("expected per-claim error from CDI write failure, got nil") + } + if !errors.Is(r.Err, writeErr) { + t.Errorf("PrepareResult.Err = %v, want to wrap writeErr", r.Err) + } + // Rollback: CPUs must be released. + if len(alloc.releases) == 0 { + t.Error("expected ReleaseHpCpus to be called on CDI write failure") + } + // Claim must not be stored after a CDI write failure. + if _, ok := p.claims[uid]; ok { + t.Error("claim stored in p.claims after CDI write failure (should have been rolled back)") + } +} + +// TestPrepare_ClaimStoreSaveFailure verifies that a ClaimStore.Save failure +// after a successful CDI write and CPU pick rolls back symmetrically: the +// just-written CDI spec is removed, the picked CPUs are released, and the +// claim is not left in p.claims — otherwise a restart would lose durable +// state for a claim whose CDI spec/CPUs are already live. +func TestPrepare_ClaimStoreSaveFailure(t *testing.T) { + saveErr := errors.New("claim store save failed") + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{saveErr: saveErr} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-save-fail") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Fatal("expected per-claim error from ClaimStore.Save failure, got nil (PrepareResult{} instead)") + } + if !errors.Is(r.Err, saveErr) { + t.Errorf("PrepareResult.Err = %v, want to wrap saveErr", r.Err) + } + // The just-written CDI spec must be removed. + if len(cdiW.removed) != 1 || cdiW.removed[0] != uid { + t.Errorf("CDIWriter.RemoveClaim called for %v, want [%v]", cdiW.removed, uid) + } + // The picked CPUs must be released. + if len(alloc.releases) == 0 { + t.Error("expected ReleaseHpCpus to be called on ClaimStore.Save failure") + } + // The claim must not be left set in p.claims. + if _, ok := p.claims[uid]; ok { + t.Error("claim stored in p.claims after ClaimStore.Save failure (should have been rolled back)") + } +} + +// TestPrepare_MultiResultTwoPunits verifies that a claim with two results +// spanning different punits is rejected: the topology-aware consumer +// requires the union of a claim's results to fit a single leaf pool, so the +// CPU pick for the first result is rolled back and nothing is written. +func TestPrepare_MultiResultTwoPunits(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-1"), isHP: true} + cdiW := &trackingCDIWriter{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.DeviceLister = hpDeviceLister( + hpDevice("dev0", "gold", 0, 0), + hpDevice("dev1", "gold", 0, 1), + ) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + qty := resource.MustParse("2") + uid := types.UID("uid-multi") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "req0", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + {Driver: "test-driver", Pool: "p", Device: "dev1", Request: "req1", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err == nil { + t.Fatal("PrepareResult.Err = nil, want multi-punit error") + } + if len(r.Devices) != 0 { + t.Errorf("PrepareResult.Devices len = %d, want 0", len(r.Devices)) + } + if len(cdiW.written) != 0 { + t.Errorf("WriteClaim called %d times, want 0", len(cdiW.written)) + } + if len(alloc.releases) != 1 { + t.Errorf("ReleaseHpCpus called %d times, want 1 (rollback of first pick)", len(alloc.releases)) + } +} + +// TestPrepare_ShareIDNil verifies that a result with a nil ShareID produces a +// Device with ShareID == nil. +func TestPrepare_ShareIDNil(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-no-share") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) // no ShareID + result, _ := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + r := result[uid] + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if r.Devices[0].ShareID != nil { + t.Errorf("Device.ShareID = %v, want nil", r.Devices[0].ShareID) + } +} + +// TestPrepare_ShareIDSet verifies that a result with a non-nil ShareID produces a +// Device with a matching non-nil ShareID. +func TestPrepare_ShareIDSet(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + shareUID := types.UID("share-abc") + qty := resource.MustParse("4") + uid := types.UID("uid-share") + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", Request: "req0", + ShareID: &shareUID, + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, _ := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + r := result[uid] + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if r.Devices[0].ShareID == nil { + t.Error("Device.ShareID = nil, want non-nil") + } else if *r.Devices[0].ShareID != shareUID { + t.Errorf("Device.ShareID = %v, want %v", *r.Devices[0].ShareID, shareUID) + } +} + +// TestPrepare_AllUIDsInResultMap verifies that every claim UID appears in the +// result map even when some claims error. +func TestPrepare_AllUIDsInResultMap(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid1 := types.UID("uid-ok") + uid2 := types.UID("uid-nil-alloc") + + good := makeClaim(uid1, "test-driver", "pool0", "dev0", "req0", 4) + bad := &resourceapi.ResourceClaim{ObjectMeta: metav1.ObjectMeta{UID: uid2}} + + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{good, bad}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + if _, ok := result[uid1]; !ok { + t.Errorf("result map missing %v", uid1) + } + if _, ok := result[uid2]; !ok { + t.Errorf("result map missing %v", uid2) + } + if result[uid1].Err != nil { + t.Errorf("good claim: PrepareResult.Err = %v, want nil", result[uid1].Err) + } + if !errors.Is(result[uid2].Err, errNilAllocation) { + t.Errorf("bad claim: PrepareResult.Err = %v, want errNilAllocation", result[uid2].Err) + } +} + +// TestPrepare_SubrequestSlashInName verifies that a request name containing '/' +// (FirstAvailable subrequest format) produces a valid CDI device name and that +// the spec can be written successfully. +func TestPrepare_SubrequestSlashInName(t *testing.T) { + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + qty := resource.MustParse("4") + uid := types.UID("uid-slash") + // Request name uses the FirstAvailable subrequest format: "main/sub". + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + {Driver: "test-driver", Pool: "p", Device: "dev0", + Request: "main-req/sub-req", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{"nri/cpus": qty}}, + }, + }, + }, + }, + } + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if r.Err != nil { + t.Fatalf("PrepareResult.Err = %v, want nil", r.Err) + } + if len(r.Devices) != 1 { + t.Fatalf("PrepareResult.Devices len = %d, want 1", len(r.Devices)) + } + // Verify the device name part of each CDI qualified ID passes parser.ValidateDeviceName. + // CDI qualified names have the format "vendor/class=name"; split on "=" to get the name. + for _, cdiID := range r.Devices[0].CDIDeviceIDs { + parts := strings.SplitN(cdiID, "=", 2) + if len(parts) != 2 || parts[1] == "" { + t.Errorf("CDI device ID %q has unexpected format (want vendor/class=name)", cdiID) + continue + } + if err := parser.ValidateDeviceName(parts[1]); err != nil { + t.Errorf("CDI device name %q from ID %q failed validation: %v", parts[1], cdiID, err) + } + } + if len(cdiW.written) == 0 { + t.Error("WriteClaim was not called for subrequest claim") + } +} + +// ---- Task 8: UnprepareResourceClaims tests ---- + +// unprepareObj builds a kubeletplugin.NamespacedObject for a given UID, +// used to drive UnprepareResourceClaims calls in tests. +func unprepareObj(uid types.UID) kubeletplugin.NamespacedObject { + return kubeletplugin.NamespacedObject{UID: uid} +} + +// preparePlugin is a helper that creates a Plugin, pre-populates p.claims with +// the given ClaimState entries, and wires the provided allocator, CDI writer, +// and store into Deps. +func preparePlugin(t *testing.T, alloc ClaimAllocator, cdiW CDIWriter, store ClaimStore, + claimsIn map[types.UID]*ClaimState) *Plugin { + t.Helper() + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + for uid, cs := range claimsIn { + p.claims[uid] = cs + } + return p +} + +// TestUnprepare_KnownClaim verifies that an existing claim is released, +// CDI is removed, the claim is deleted from p.claims, and the result map +// contains nil for the UID. +func TestUnprepare_KnownClaim(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + uid := types.UID("uid-known") + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{ + {Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}, + }, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if perErr, ok := result[uid]; !ok { + t.Error("result map missing uid") + } else if perErr != nil { + t.Errorf("result[uid] = %v, want nil", perErr) + } + if len(alloc.releases) != 1 { + t.Errorf("ReleaseHpCpus called %d times, want 1", len(alloc.releases)) + } + if len(cdiW.removed) != 1 || cdiW.removed[0] != uid { + t.Errorf("RemoveClaim called for %v, want [%v]", cdiW.removed, uid) + } + if _, exists := p.claims[uid]; exists { + t.Error("claim still in p.claims after Unprepare") + } + if store.saved != 1 { + t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + } +} + +// TestUnprepare_UnknownUID verifies that an unknown UID produces a warning and +// a nil entry in the result map (no panic, no error). +func TestUnprepare_UnknownUID(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + p := preparePlugin(t, alloc, cdiW, store, nil) + + uid := types.UID("uid-unknown") + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if perErr, ok := result[uid]; !ok { + t.Error("result map missing uid") + } else if perErr != nil { + t.Errorf("result[uid] = %v, want nil for unknown UID", perErr) + } + // CDI remove must NOT be called for unknown claims. + if len(cdiW.removed) != 0 { + t.Errorf("RemoveClaim called for unknown UID (removed = %v)", cdiW.removed) + } + // ReleaseHpCpus must NOT be called for unknown claims. + if len(alloc.releases) != 0 { + t.Errorf("ReleaseHpCpus called %d times, want 0 for unknown UID", len(alloc.releases)) + } + // Save must still be called once (batch write even with no-ops). + if store.saved != 1 { + t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + } +} + +// TestUnprepare_CDIRemoveError verifies that a CDI RemoveClaim error produces +// a warning but does not surface as an error in the result map (nil for that UID). +func TestUnprepare_CDIRemoveError(t *testing.T) { + removeErr := errors.New("CDI remove failed") + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{removeErr: removeErr} + store := &trackingClaimStore{} + + uid := types.UID("uid-remove-err") + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "4-7", ClassName: "gold"}}, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + // CDI error is a warning only — result must be nil for the UID. + if perErr := result[uid]; perErr != nil { + t.Errorf("result[uid] = %v, want nil on CDI remove error", perErr) + } + // Claim must still be deleted from in-memory state. + if _, exists := p.claims[uid]; exists { + t.Error("claim still in p.claims after Unprepare despite CDI error") + } + // ReleaseHpCpus must have been called once — CPU leak on CDI error goes undetected otherwise. + if len(alloc.releases) != 1 { + t.Errorf("ReleaseHpCpus called %d times, want 1 (must release CPUs even on CDI error)", len(alloc.releases)) + } +} + +// TestUnprepare_MixedBatch verifies that a batch with one known and one unknown +// UID both appear in the result map (nil for each), and only the known claim's +// CDI is removed. +func TestUnprepare_MixedBatch(t *testing.T) { + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + + known := types.UID("uid-known-batch") + unknown := types.UID("uid-unknown-batch") + claimState := &ClaimState{ + UID: string(known), + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-1", ClassName: "gold"}}, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{known: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), + []kubeletplugin.NamespacedObject{unprepareObj(known), unprepareObj(unknown)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if perErr, ok := result[known]; !ok { + t.Error("result map missing known uid") + } else if perErr != nil { + t.Errorf("result[known] = %v, want nil", perErr) + } + if perErr, ok := result[unknown]; !ok { + t.Error("result map missing unknown uid") + } else if perErr != nil { + t.Errorf("result[unknown] = %v, want nil", perErr) + } + // Only the known UID's CDI should be removed. + if len(cdiW.removed) != 1 || cdiW.removed[0] != known { + t.Errorf("RemoveClaim called for %v, want [%v]", cdiW.removed, known) + } + // One batch save. + if store.saved != 1 { + t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + } +} + +// TestShareIDPtr verifies that shareIDPtr returns nil for "" and a non-nil +// pointer for a non-empty string. +func TestShareIDPtr(t *testing.T) { + if shareIDPtr("") != nil { + t.Error("shareIDPtr(\"\") should return nil") + } + ptr := shareIDPtr("abc") + if ptr == nil { + t.Fatal("shareIDPtr(\"abc\") returned nil, want *types.UID") + } + if *ptr != types.UID("abc") { + t.Errorf("*shareIDPtr(\"abc\") = %v, want %v", *ptr, types.UID("abc")) + } +} + +// ---- Task 9: LiveClaimClasses, RestoreClaimsLocked, Start reconciliation ---- + +// startTestCDIWriter supports per-UID ClaimSpecExists state and a fixed +// ListClaims result for Start reconciliation tests. WriteClaim is a no-op. +type startTestCDIWriter struct { + existsByUID map[types.UID]bool + listResult []types.UID + listErr error + removed []types.UID + removeErr error +} + +func (w *startTestCDIWriter) WriteClaim(_ types.UID, _ []CDIDevice) error { return nil } +func (w *startTestCDIWriter) RemoveClaim(uid types.UID) error { + w.removed = append(w.removed, uid) + return w.removeErr +} +func (w *startTestCDIWriter) ClaimSpecExists(uid types.UID) bool { return w.existsByUID[uid] } +func (w *startTestCDIWriter) ListClaims() ([]types.UID, error) { + return w.listResult, w.listErr +} + +// preloadedClaimStore loads a pre-configured map on Load and tracks saves. +type preloadedClaimStore struct { + initial map[types.UID]*ClaimState + loadErr error + saved int + savedClaims map[types.UID]*ClaimState // last map passed to Save +} + +func (s *preloadedClaimStore) Load() (map[types.UID]*ClaimState, error) { + return s.initial, s.loadErr +} + +func (s *preloadedClaimStore) Save(claims map[types.UID]*ClaimState) error { + s.saved++ + cp := make(map[types.UID]*ClaimState, len(claims)) + for k, v := range claims { + cp[k] = v + } + s.savedClaims = cp + return nil +} + +// TestLiveClaimClasses_Empty verifies that LiveClaimClasses returns an empty +// map when there are no claims. +func TestLiveClaimClasses_Empty(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + got := p.LiveClaimClasses() + if len(got) != 0 { + t.Errorf("LiveClaimClasses() = %v, want empty map", got) + } +} + +// TestLiveClaimClasses_SameClass verifies that two claims using the same class +// produce a count of 2. +func TestLiveClaimClasses_SameClass(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + p.claims[types.UID("a")] = &ClaimState{UID: "a", Allocs: []ResultAlloc{{ClassName: "gold"}}} + p.claims[types.UID("b")] = &ClaimState{UID: "b", Allocs: []ResultAlloc{{ClassName: "gold"}}} + + got := p.LiveClaimClasses() + if got["gold"] != 2 { + t.Errorf("LiveClaimClasses()[gold] = %d, want 2", got["gold"]) + } + if len(got) != 1 { + t.Errorf("LiveClaimClasses() len = %d, want 1", len(got)) + } +} + +// TestLiveClaimClasses_DifferentClasses verifies that two claims using +// different classes produce two entries in the result map. +func TestLiveClaimClasses_DifferentClasses(t *testing.T) { + p, err := New("test-driver", validDeps()) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + p.claims[types.UID("a")] = &ClaimState{UID: "a", Allocs: []ResultAlloc{{ClassName: "gold"}}} + p.claims[types.UID("b")] = &ClaimState{UID: "b", Allocs: []ResultAlloc{{ClassName: "silver"}}} + + got := p.LiveClaimClasses() + if len(got) != 2 { + t.Errorf("LiveClaimClasses() len = %d, want 2", len(got)) + } + if got["gold"] != 1 { + t.Errorf("LiveClaimClasses()[gold] = %d, want 1", got["gold"]) + } + if got["silver"] != 1 { + t.Errorf("LiveClaimClasses()[silver] = %d, want 1", got["silver"]) + } +} + +// TestRestoreClaimsLocked_RebuildsAccounting verifies that RestoreClaimsLocked +// calls AccountHpCpus for each alloc in p.claims, rebuilding accounting after +// a Reconfigure reset (simulated by a fresh trackingClaimAllocator). +func TestRestoreClaimsLocked_RebuildsAccounting(t *testing.T) { + alloc := &trackingClaimAllocator{} + deps := validDeps() + deps.ClaimAllocator = alloc + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + // Pre-populate two claims with one alloc each. + p.claims[types.UID("uid-a")] = &ClaimState{ + UID: "uid-a", + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}, + } + p.claims[types.UID("uid-b")] = &ClaimState{ + UID: "uid-b", + Allocs: []ResultAlloc{{Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-7", ClassName: "gold"}}, + } + + if err := p.RestoreClaimsLocked(); err != nil { + t.Fatalf("RestoreClaimsLocked() unexpected error: %v", err) + } + if len(alloc.accounts) != 2 { + t.Errorf("AccountHpCpus called %d times, want 2", len(alloc.accounts)) + } + // Verify the union of all accounted CPU sets matches the expected total. + // Map iteration order is non-deterministic so we check the union. + totalAccounted := cpuset.New() + for _, cs := range alloc.accounts { + totalAccounted = totalAccounted.Union(cs) + } + wantTotal := cpuset.MustParse("0-7") + if !totalAccounted.Equals(wantTotal) { + t.Errorf("AccountHpCpus total CPUs = %v, want %v", totalAccounted, wantTotal) + } +} + +// TestRestoreClaims_WithLockWrapper verifies that RestoreClaims acquires the +// lock (via WithLock) and calls AccountHpCpus for each alloc. +func TestRestoreClaims_WithLockWrapper(t *testing.T) { + var mu sync.Mutex + alloc := &trackingClaimAllocator{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.WithLock = func(f func()) { + mu.Lock() + defer mu.Unlock() + f() + } + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + p.claims[types.UID("uid-c")] = &ClaimState{ + UID: "uid-c", + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-1", ClassName: "gold"}}, + } + + if err := p.RestoreClaims(); err != nil { + t.Fatalf("RestoreClaims() unexpected error: %v", err) + } + if len(alloc.accounts) != 1 { + t.Errorf("AccountHpCpus called %d times, want 1", len(alloc.accounts)) + } +} + +// TestStart_Reconciliation verifies that persisted claims are re-accounted from +// durable state even when the CDI spec file is missing on disk. Missing CDI is +// not treated as a proof of staleness; only an actual Unprepare removes a claim +// from the store. Orphan CDI specs (not tracked in the claim store) are still +// swept and removed. +func TestStart_Reconciliation(t *testing.T) { + liveUID := types.UID("uid-live") + staleUID := types.UID("uid-stale") + orphanUID := types.UID("uid-orphan") + + alloc := &trackingClaimAllocator{isHP: true} + + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{ + liveUID: true, + staleUID: false, + }, + // ListClaims returns liveUID and orphanUID (orphan has no claim entry). + listResult: []types.UID{liveUID, orphanUID}, + } + + store := &preloadedClaimStore{ + initial: map[types.UID]*ClaimState{ + liveUID: { + UID: string(liveUID), + Allocs: []ResultAlloc{ + {Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}, + }, + }, + staleUID: { + UID: string(staleUID), + Allocs: []ResultAlloc{ + {Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-7", ClassName: "gold"}, + }, + }, + }, + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + defer p.Stop() + + // Both claims must remain in p.claims — staleUID is kept even though its + // CDI spec is absent; PrepareResourceClaims will recreate it. + if _, ok := p.claims[liveUID]; !ok { + t.Error("liveUID missing from p.claims after Start") + } + if _, ok := p.claims[staleUID]; !ok { + t.Error("staleUID missing from p.claims after Start — missing CDI spec must not imply stale claim") + } + + // Both persisted claims are re-accounted from durable state. + if len(alloc.accounts) != 2 { + t.Errorf("AccountHpCpus called %d times, want 2", len(alloc.accounts)) + } + + // No claim-store writes happen when we keep persisted claims. + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0", store.saved) + } + + // Orphan sweep: orphanUID must have been removed via CDI writer. + found := false + for _, uid := range cdiW.removed { + if uid == orphanUID { + found = true + break + } + } + if !found { + t.Errorf("orphanUID was not removed by orphan sweep (removed = %v)", cdiW.removed) + } + // staleUID should NOT be in cdiW.removed (it is a live persisted claim). + for _, uid := range cdiW.removed { + if uid == staleUID { + t.Error("staleUID was unexpectedly passed to CDI RemoveClaim") + } + } +} + +// TestStart_ClaimStoreLoadError verifies that a ClaimStore.Load error during +// Start's reconciliation causes Start to return a non-nil error and does not +// proceed to the orphan sweep with an empty claim map — otherwise a transient +// Load failure would cause the orphan sweep to remove CDI specs for claims +// whose consuming containers are still running. +func TestStart_ClaimStoreLoadError(t *testing.T) { + liveUID := types.UID("uid-live-on-disk") + + alloc := &trackingClaimAllocator{isHP: true} + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{liveUID: true}, + // ListClaims reports liveUID's CDI spec still exists on disk. + listResult: []types.UID{liveUID}, + } + store := &preloadedClaimStore{ + loadErr: errors.New("claim store load failed"), + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + err = p.Start(ctx) + if err == nil { + t.Fatal("Start() expected error on ClaimStore.Load failure, got nil") + } + defer p.Stop() + + // The orphan sweep must not have run: liveUID's CDI spec must not have + // been removed even though p.claims would have been empty. + for _, uid := range cdiW.removed { + if uid == liveUID { + t.Errorf("orphan sweep removed CDI spec for %v after a ClaimStore.Load failure — live claim consumer's CDI spec destroyed", liveUID) + } + } + if len(cdiW.removed) != 0 { + t.Errorf("CDIWriter.RemoveClaim called %d time(s), want 0 (orphan sweep must be skipped on Load failure)", len(cdiW.removed)) + } +} + +// TestStart_InactiveAllocator verifies that if AccountHpCpus returns an error +// (simulating an inactive allocator), all claims with CDI specs present are +// still kept with a warning (not dropped). +func TestStart_InactiveAllocator(t *testing.T) { + uid1 := types.UID("uid-1") + uid2 := types.UID("uid-2") + + alloc := &trackingClaimAllocator{accountErr: fmt.Errorf("test: %w", cpuclass.ErrAllocatorInactive), isHP: true} + + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{uid1: true, uid2: true}, + listResult: []types.UID{uid1, uid2}, + } + + store := &preloadedClaimStore{ + initial: map[types.UID]*ClaimState{ + uid1: {UID: string(uid1), Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}}, + uid2: {UID: string(uid2), Allocs: []ResultAlloc{{Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-7", ClassName: "gold"}}}, + }, + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + defer p.Stop() + + // Both claims must still be present despite AccountHpCpus errors. + if _, ok := p.claims[uid1]; !ok { + t.Error("uid1 missing from p.claims — should be kept despite AccountHpCpus error") + } + if _, ok := p.claims[uid2]; !ok { + t.Error("uid2 missing from p.claims — should be kept despite AccountHpCpus error") + } + // No drops occurred, so ClaimStore.Save must not have been called. + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0 (no drops)", store.saved) + } +} + +// TestStart_AccountHpCpus_FatalError verifies that Start fails when +// AccountHpCpus returns an error that does not wrap ErrAllocatorInactive +// (e.g. punit not found due to topology change). Registering in that state +// would leave CPUs unaccounted and allow double-allocation. +func TestStart_AccountHpCpus_FatalError(t *testing.T) { + uid1 := types.UID("uid-1") + + alloc := &trackingClaimAllocator{accountErr: errors.New("pct: AccountHpCpus: punit (pkg=0, punit=0) not found"), isHP: true} + + cdiW := &startTestCDIWriter{ + existsByUID: map[types.UID]bool{uid1: true}, + listResult: []types.UID{uid1}, + } + + store := &preloadedClaimStore{ + initial: map[types.UID]*ClaimState{ + uid1: {UID: string(uid1), Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}}, + }, + } + + fakeClient := fake.NewClientset(&corev1.Node{ + ObjectMeta: metav1.ObjectMeta{Name: "test-node"}, + }) + deps := validDeps() + deps.KubeClient = fakeClient + deps.NodeName = "test-node" + deps.RegistrarDir = t.TempDir() + deps.PluginDataDir = t.TempDir() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) + defer cancel() + + if err := p.Start(ctx); err == nil { + p.Stop() + t.Fatal("Start() expected error for non-inactive AccountHpCpus failure, got nil") + } +} diff --git a/pkg/resmgr/dra/state.go b/pkg/resmgr/dra/state.go new file mode 100644 index 000000000..e5dea6921 --- /dev/null +++ b/pkg/resmgr/dra/state.go @@ -0,0 +1,128 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "encoding/json" + "fmt" + + "k8s.io/apimachinery/pkg/types" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" +) + +// draClaimsKey is the cache policy-entry key under which claim state is stored. +const draClaimsKey = "dra/claims" + +// ResultAlloc holds the allocated state for a single DeviceRequestAllocationResult. +// Allocs[i] corresponds to filtered allocation result i; CDI device-name index is +// positional — order must be preserved on rebuild. +type ResultAlloc struct { + // Request is r.Request from DeviceRequestAllocationResult. + Request string `json:"Request"` + // Pool is r.Pool from DeviceRequestAllocationResult. + Pool string `json:"Pool"` + // Device is r.Device (the DRA device name) from DeviceRequestAllocationResult. + Device string `json:"Device"` + // ShareID is string(*r.ShareID) or "" when r.ShareID is nil. + ShareID string `json:"ShareID"` + // ClassName is the value of the nri/cpuClass attribute. + ClassName string `json:"ClassName"` + // PkgID is the value of the nri/packageID attribute. + PkgID int `json:"PkgID"` + // PunitID is the value of the nri/punitID attribute. + PunitID int `json:"PunitID"` + // CPUs is the cpuset.CPUSet.String() representation of the allocated CPUs. + CPUs string `json:"CPUs"` +} + +// ClaimState holds the persisted state for a prepared ResourceClaim. +type ClaimState struct { + // UID is types.UID as string. + UID string `json:"UID"` + Allocs []ResultAlloc `json:"Allocs"` +} + +// marshalClaims encodes the claims map to a map[string]string (uid → JSON of ClaimState) +// suitable for storage via cache.SetPolicyEntry. +func marshalClaims(claims map[types.UID]*ClaimState) (map[string]string, error) { + out := make(map[string]string, len(claims)) + for uid, cs := range claims { + data, err := json.Marshal(cs) + if err != nil { + return nil, fmt.Errorf("dra: marshal claim %s: %w", uid, err) + } + out[string(uid)] = string(data) + } + return out, nil +} + +// unmarshalClaims decodes a map[string]string (as stored by marshalClaims) back +// into the claims map. +func unmarshalClaims(raw map[string]string) (map[types.UID]*ClaimState, error) { + out := make(map[types.UID]*ClaimState, len(raw)) + for uid, data := range raw { + var cs ClaimState + if err := json.Unmarshal([]byte(data), &cs); err != nil { + return nil, fmt.Errorf("dra: unmarshal claim %s: %w", uid, err) + } + out[types.UID(uid)] = &cs + } + return out, nil +} + +// cacheClaimStore implements ClaimStore using the resmgr cache as its backing +// store. Save is a no-op when the cache is in a BlockSave window; the data is +// still in the in-memory policyData map and will persist on the next unblocked +// Save. Callers of Save must tolerate this. +type cacheClaimStore struct { + c cache.Cache +} + +// NewCacheClaimStore returns a ClaimStore backed by the given resmgr cache. +func NewCacheClaimStore(c cache.Cache) ClaimStore { + return &cacheClaimStore{c: c} +} + +// Save persists the given claims map to the backing cache store. +func (s *cacheClaimStore) Save(claims map[types.UID]*ClaimState) error { + previous := map[string]string{} + s.c.GetPolicyEntry(draClaimsKey, &previous) + m, err := marshalClaims(claims) + if err != nil { + return err + } + s.c.SetPolicyEntry(draClaimsKey, m) + if err := s.c.Save(); err != nil { + s.c.SetPolicyEntry(draClaimsKey, previous) + return err + } + return nil +} + +// Load reads the claims map from the backing cache store. Returns nil, nil +// when no claim state has been saved yet. +func (s *cacheClaimStore) Load() (map[types.UID]*ClaimState, error) { + // Named variable is required — a composite-literal address passed directly + // to GetPolicyEntry is not addressable and the unmarshal target would be + // unreachable on a first-load path. + m := map[string]string{} + if !s.c.GetPolicyEntry(draClaimsKey, &m) { + return nil, nil + } + return unmarshalClaims(m) +} diff --git a/pkg/resmgr/dra/state_test.go b/pkg/resmgr/dra/state_test.go new file mode 100644 index 000000000..de7fe5e2a --- /dev/null +++ b/pkg/resmgr/dra/state_test.go @@ -0,0 +1,287 @@ +/* +Copyright The NRI Plugins Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package dra + +import ( + "os" + "path/filepath" + "testing" + + "k8s.io/apimachinery/pkg/types" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// newTestCache creates a real cache in a temporary directory. The cache rejects +// directories with group/other write bits (reject mask 0022); we create the +// sub-directory with explicit 0700 permissions to avoid a t.TempDir() umask +// clash on systems with umask 0002 (which would give 0775). +func newTestCache(t *testing.T) cache.Cache { + t.Helper() + cacheDir := filepath.Join(t.TempDir(), "cache") + if err := os.Mkdir(cacheDir, 0700); err != nil { + t.Fatalf("newTestCache: mkdir %q: %v", cacheDir, err) + } + c, err := cache.NewCache(cache.Options{CacheDir: cacheDir}) + if err != nil { + t.Fatalf("newTestCache: cache.NewCache() error: %v", err) + } + return c +} + +// TestMarshalUnmarshalClaims_RoundTrip verifies that a multi-alloc claim +// survives a marshal → unmarshal round-trip without data loss. +func TestMarshalUnmarshalClaims_RoundTrip(t *testing.T) { + uid := types.UID("test-uid-1234") + originalCPUs := cpuset.New(0, 1, 2, 3) + + claims := map[types.UID]*ClaimState{ + uid: { + UID: string(uid), + Allocs: []ResultAlloc{ + { + Request: "req-hp", + Pool: "node1", + Device: "sst-cp-0-0", + ShareID: "", + ClassName: "hp", + PkgID: 0, + PunitID: 0, + CPUs: originalCPUs.String(), + }, + { + Request: "req-hp-2", + Pool: "node1", + Device: "sst-cp-0-1", + ShareID: "share-abc", + ClassName: "hp", + PkgID: 0, + PunitID: 1, + CPUs: cpuset.New(4, 5).String(), + }, + }, + }, + } + + raw, err := marshalClaims(claims) + if err != nil { + t.Fatalf("marshalClaims() unexpected error: %v", err) + } + + got, err := unmarshalClaims(raw) + if err != nil { + t.Fatalf("unmarshalClaims() unexpected error: %v", err) + } + + cs, ok := got[uid] + if !ok { + t.Fatalf("unmarshalClaims(): uid %q not found in result", uid) + } + if cs.UID != string(uid) { + t.Errorf("ClaimState.UID = %q, want %q", cs.UID, string(uid)) + } + if len(cs.Allocs) != 2 { + t.Fatalf("len(Allocs) = %d, want 2", len(cs.Allocs)) + } + + // Verify CPUs round-trip via cpuset.Parse. + parsedCPUs, err := cpuset.Parse(cs.Allocs[0].CPUs) + if err != nil { + t.Fatalf("cpuset.Parse(%q) error: %v", cs.Allocs[0].CPUs, err) + } + if !parsedCPUs.Equals(originalCPUs) { + t.Errorf("CPUs round-trip: got %v, want %v", parsedCPUs, originalCPUs) + } + + // Verify other fields. + if cs.Allocs[0].Request != "req-hp" { + t.Errorf("Allocs[0].Request = %q, want %q", cs.Allocs[0].Request, "req-hp") + } + if cs.Allocs[1].ShareID != "share-abc" { + t.Errorf("Allocs[1].ShareID = %q, want %q", cs.Allocs[1].ShareID, "share-abc") + } + if cs.Allocs[1].PunitID != 1 { + t.Errorf("Allocs[1].PunitID = %d, want 1", cs.Allocs[1].PunitID) + } +} + +// TestUnmarshalClaims_Empty verifies that an empty map round-trips to an empty +// map (not nil). +func TestUnmarshalClaims_Empty(t *testing.T) { + raw, err := marshalClaims(map[types.UID]*ClaimState{}) + if err != nil { + t.Fatalf("marshalClaims({}) error: %v", err) + } + got, err := unmarshalClaims(raw) + if err != nil { + t.Fatalf("unmarshalClaims({}) error: %v", err) + } + if len(got) != 0 { + t.Errorf("unmarshalClaims({}) = %v, want empty map", got) + } +} + +// TestCacheClaimStore_RoundTrip verifies that Save → Load preserves multi-alloc +// claim data using a real (file-backed) cache. +func TestCacheClaimStore_RoundTrip(t *testing.T) { + c := newTestCache(t) + + uid := types.UID("claim-abc") + originalCPUs := cpuset.New(10, 11, 12, 13) + + claims := map[types.UID]*ClaimState{ + uid: { + UID: string(uid), + Allocs: []ResultAlloc{ + { + Request: "req-0", + Pool: "node1", + Device: "dev-0", + ShareID: "", + ClassName: "hp", + PkgID: 1, + PunitID: 2, + CPUs: originalCPUs.String(), + }, + }, + }, + } + + store := NewCacheClaimStore(c) + if err := store.Save(claims); err != nil { + t.Fatalf("Save() error: %v", err) + } + + got, err := store.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if got == nil { + t.Fatal("Load() returned nil, want non-nil map") + } + cs, ok := got[uid] + if !ok { + t.Fatalf("Load(): uid %q not found", uid) + } + if len(cs.Allocs) != 1 { + t.Fatalf("Load(): len(Allocs) = %d, want 1", len(cs.Allocs)) + } + + // Verify CPUs survive round-trip. + parsedCPUs, err := cpuset.Parse(cs.Allocs[0].CPUs) + if err != nil { + t.Fatalf("cpuset.Parse(%q) error: %v", cs.Allocs[0].CPUs, err) + } + if !parsedCPUs.Equals(originalCPUs) { + t.Errorf("CPU round-trip: got %v, want %v", parsedCPUs, originalCPUs) + } +} + +// TestCacheClaimStore_EmptyCache verifies that Load returns nil, nil when +// no claim state has ever been saved. +func TestCacheClaimStore_EmptyCache(t *testing.T) { + c := newTestCache(t) + + store := NewCacheClaimStore(c) + got, err := store.Load() + if err != nil { + t.Fatalf("Load() on empty cache error: %v", err) + } + if got != nil { + t.Errorf("Load() on empty cache = %v, want nil", got) + } +} + +// TestCacheClaimStore_LoadReturnsSavedData verifies that Load returns the +// data previously saved (guarding against the composite-literal-address bug +// where the in-parameter to GetPolicyEntry would be unreachable). +func TestCacheClaimStore_LoadReturnsSavedData(t *testing.T) { + c := newTestCache(t) + + uid := types.UID("claim-xyz") + claims := map[types.UID]*ClaimState{ + uid: { + UID: string(uid), + Allocs: []ResultAlloc{ + { + Request: "req-x", + Pool: "pool-0", + Device: "dev-x", + ClassName: "hp", + PkgID: 0, + PunitID: 0, + CPUs: cpuset.New(7, 8).String(), + }, + }, + }, + } + + store := NewCacheClaimStore(c) + if err := store.Save(claims); err != nil { + t.Fatalf("Save() error: %v", err) + } + + // Immediately call Load on the same store instance (tests the in-memory path). + got, err := store.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(got) == 0 { + t.Fatalf("Load() returned empty map — composite-literal-address bug?") + } + if _, ok := got[uid]; !ok { + t.Errorf("Load() result missing uid %q", uid) + } +} + +// TestCacheClaimStore_MultiClaim verifies Save/Load with multiple claims in the +// same store entry. +func TestCacheClaimStore_MultiClaim(t *testing.T) { + c := newTestCache(t) + + uid1 := types.UID("claim-001") + uid2 := types.UID("claim-002") + + claims := map[types.UID]*ClaimState{ + uid1: {UID: string(uid1), Allocs: []ResultAlloc{ + {Request: "r1", Pool: "p", Device: "d1", ClassName: "hp", PkgID: 0, PunitID: 0, CPUs: cpuset.New(0).String()}, + }}, + uid2: {UID: string(uid2), Allocs: []ResultAlloc{ + {Request: "r2", Pool: "p", Device: "d2", ClassName: "hp", PkgID: 0, PunitID: 1, CPUs: cpuset.New(1).String()}, + }}, + } + + store := NewCacheClaimStore(c) + if err := store.Save(claims); err != nil { + t.Fatalf("Save() error: %v", err) + } + + got, err := store.Load() + if err != nil { + t.Fatalf("Load() error: %v", err) + } + if len(got) != 2 { + t.Fatalf("Load() returned %d claims, want 2", len(got)) + } + if _, ok := got[uid1]; !ok { + t.Errorf("Load() missing uid %q", uid1) + } + if _, ok := got[uid2]; !ok { + t.Errorf("Load() missing uid %q", uid2) + } +} From 3eda3594fcacc82f7d71ea9671b415e45e676568 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 22:24:23 +0300 Subject: [PATCH 07/39] topology-aware: wire DRA plugin into the policy Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- .../balloons/policy/balloons-policy.go | 6 + .../template/policy/template-policy.go | 6 + cmd/plugins/topology-aware/policy/dra.go | 123 +++ .../topology-aware/policy/dra_adapter.go | 75 ++ .../topology-aware/policy/dra_adapter_test.go | 162 ++++ cmd/plugins/topology-aware/policy/dra_test.go | 511 +++++++++++ .../topology-aware/policy/mocks_test.go | 15 +- cmd/plugins/topology-aware/policy/pools.go | 532 ++++++++++- .../topology-aware/policy/pools_test.go | 616 +++++++++++++ .../topology-aware/policy/resources.go | 93 +- .../topology-aware/policy/resources_test.go | 462 ++++++++++ .../policy/topology-aware-policy.go | 373 ++++++++ .../policy/topology-aware-policy_test.go | 863 ++++++++++++++++++ .../config.nri_topologyawarepolicies.yaml | 19 + .../config.nri_topologyawarepolicies.yaml | 19 + .../resmgr/policy/topologyaware/config.go | 40 + .../policy/topologyaware/config_test.go | 45 + .../topologyaware/zz_generated.deepcopy.go | 20 + pkg/resmgr/cache/cache.go | 3 + pkg/resmgr/cache/container.go | 14 + pkg/resmgr/cache/container_test.go | 55 ++ pkg/resmgr/cpuclass/cpuclass.go | 23 + pkg/resmgr/cpuclass/dra.go | 48 +- pkg/resmgr/cpuclass/internal/pct/pct.go | 28 +- pkg/resmgr/cpuclass/internal/pct/pct_test.go | 8 +- pkg/resmgr/dra/deps.go | 12 + pkg/resmgr/dra/plugin.go | 84 +- pkg/resmgr/dra/plugin_test.go | 215 +++-- pkg/resmgr/main/main.go | 4 +- pkg/resmgr/main/main_test.go | 105 +++ pkg/resmgr/nri.go | 5 +- pkg/resmgr/policy/policy.go | 40 +- pkg/resmgr/policy/policy_test.go | 249 +++++ pkg/resmgr/resource-manager.go | 45 +- pkg/resmgr/resource_manager_test.go | 152 +++ 35 files changed, 4934 insertions(+), 136 deletions(-) create mode 100644 cmd/plugins/topology-aware/policy/dra.go create mode 100644 cmd/plugins/topology-aware/policy/dra_adapter.go create mode 100644 cmd/plugins/topology-aware/policy/dra_adapter_test.go create mode 100644 cmd/plugins/topology-aware/policy/dra_test.go create mode 100644 cmd/plugins/topology-aware/policy/resources_test.go create mode 100644 cmd/plugins/topology-aware/policy/topology-aware-policy_test.go create mode 100644 pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config_test.go create mode 100644 pkg/resmgr/main/main_test.go create mode 100644 pkg/resmgr/policy/policy_test.go create mode 100644 pkg/resmgr/resource_manager_test.go diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 735c3dd1f..37f0a01e0 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -250,6 +250,12 @@ func (p *balloons) Start() error { return nil } +// Stop shuts down this policy. The balloons policy holds no resources +// to release. +func (p *balloons) Stop() error { + return nil +} + // Sync synchronizes the active policy state. func (p *balloons) Sync(add []cache.Container, del []cache.Container) error { irq.BlockWrites() diff --git a/cmd/plugins/template/policy/template-policy.go b/cmd/plugins/template/policy/template-policy.go index 443be4d5a..6c04c295d 100644 --- a/cmd/plugins/template/policy/template-policy.go +++ b/cmd/plugins/template/policy/template-policy.go @@ -75,6 +75,12 @@ func (p *policy) Start() error { return nil } +// Stop shuts down this policy. The template policy holds no resources +// to release. +func (p *policy) Stop() error { + return nil +} + // Reconfigure this policy. func (p *policy) Reconfigure(newCfg any) error { cfg, ok := newCfg.(*cfgapi.Config) diff --git a/cmd/plugins/topology-aware/policy/dra.go b/cmd/plugins/topology-aware/policy/dra.go new file mode 100644 index 000000000..23d960106 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra.go @@ -0,0 +1,123 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" + + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// buildDRAPlugin constructs p.draPlugin from the current policy +// configuration (p.cfg, p.cpuClasses, p.cache) and the given backend +// options. Called once from Setup() when cfg.DRAEnabled() is true, and +// never from Reconfigure() (which refuses any change to DRAEnabled outright +// rather than tearing down/rebuilding p.draPlugin). +// +// A missing kube client or node name is treated as "DRA not ready yet" +// rather than a hard Setup() failure: this logs a warning and leaves +// p.draPlugin nil. Every other Backend lifecycle method already nil-checks +// p.draPlugin, so this degrades to "DRA disabled" rather than crashing. +// +// An empty (nil) p.cpuClasses is different: the plugin is still built, just +// with an empty device set (every *cpuclass.Handler method the adapter +// calls is nil-receiver-safe, see policyDRAAdapter's doc comment). Since +// buildDRAPlugin only ever runs once, leaving p.draPlugin nil here would +// prevent a later Reconfigure() that adds cpuClasses from ever enabling DRA. +// +// Genuine construction failures (CDI writer setup, dra.New's own +// dependency validation) are returned as errors, since those indicate a +// real misconfiguration rather than a timing issue. +func (p *policy) buildDRAPlugin(opts *policyapi.BackendOptions) error { + if opts.KubeClientFn == nil { + log.Warnf("dra: no KubeClientFn provided, DRA plugin not started") + return nil + } + kubeClient := opts.KubeClientFn() + if kubeClient == nil { + log.Warnf("dra: no kube client available yet, DRA plugin not started") + return nil + } + if opts.NodeName == "" { + log.Warnf("dra: node name not known yet, DRA plugin not started") + return nil + } + + adapter := &policyDRAAdapter{p: p} + + cdiWriter, err := dra.NewCDIWriter(DRADriverName, p.cdiDir) + if err != nil { + return policyError("failed to create DRA CDI writer: %w", err) + } + + deps := dra.Deps{ + KubeClient: kubeClient, + NodeName: opts.NodeName, + // ValidateClasses captures p, not a config snapshot, so it observes + // whatever p.cfg is live when called, including after Reconfigure() + // swaps it. DRASharedCounters() is the nil-safe getter — without it, + // a Reconfigure that removes the dra: section would panic on + // p.cfg.DRA.SharedCounters here. + ValidateClasses: func() error { + return cpuclass.ValidateCPUClassesForDRA(p.cfg.CPUClasses, p.cfg.DRASharedCounters()) + }, + // ValidateCPUsInPool mirrors the check allocateClaim performs via + // poolForCPUs at container-creation time, so a claim whose picked + // CPUs straddle more than one leaf pool (e.g. a punit spanning an + // entire package while leaf pools are NUMA or L3 nodes) is rejected + // at Prepare time instead of being persisted and always failing + // later when its container is created. + ValidateCPUsInPool: func(cpus cpuset.CPUSet) error { + _, err := p.poolForCPUs(cpus) + return err + }, + DeviceLister: adapter, + ClaimAllocator: adapter, + CDIWriter: cdiWriter, + ClaimStore: dra.NewCacheClaimStore(p.cache), + ClaimUnprepare: p.unprepareDRAClaim, + WithLock: opts.WithLock, + Logger: log, + } + + plugin, err := dra.New(DRADriverName, deps) + if err != nil { + return policyError("failed to create DRA plugin: %w", err) + } + + p.draPlugin = plugin + + return nil +} + +// triggerDRARepublish enqueues a DRA ResourceSlice republication when +// PCT-based HP capacity is active. Called (deferred) from AllocateResources, +// ReleaseResources, and UpdateResources so that any change to non-DRA HP CPU +// usage (hpUsed) caused by those NRI-path allocations is reflected in the +// next ResourceSlice. +// The enqueue is non-blocking and safe to call while holding the resmgr lock; +// the actual publish runs in the DRA plugin's republisherLoop goroutine, which +// acquires the lock itself after the NRI handler releases it. +func (p *policy) triggerDRARepublish() { + if p.draPlugin == nil { + return + } + if p.cpuClasses == nil || !p.cpuClasses.PctActive() { + return + } + p.draPlugin.TriggerRepublish() +} diff --git a/cmd/plugins/topology-aware/policy/dra_adapter.go b/cmd/plugins/topology-aware/policy/dra_adapter.go new file mode 100644 index 000000000..d4ed7e639 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra_adapter.go @@ -0,0 +1,75 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + resourceapi "k8s.io/api/resource/v1" + + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// DRADriverName is the DRA driver name used to publish and identify CPU +// devices for the topology-aware policy. Defined once here; referenced +// everywhere else DRA devices for this policy are published or identified. +const DRADriverName = "nri.topology-aware.cpu" + +// policyDRAAdapter implements dra.ClaimAllocator and dra.DeviceLister by +// forwarding every call to the *current* p.cpuClasses at call time. +// +// A field holding *cpuclass.Handler directly (captured once, at +// construction) would go stale: initialize() sets p.cpuClasses = nil and +// installs a brand new *cpuclass.Handler on every Reconfigure. Because this +// adapter only ever holds the *policy back-pointer and dereferences +// p.cpuClasses fresh on each call, it always observes the handler that is +// live at call time. +// +// All *cpuclass.Handler methods used here are nil-receiver-safe (they check +// h == nil internally and return zero-values/errors), so no additional nil +// guard is required in the adapter for a nil p.cpuClasses. +type policyDRAAdapter struct { + p *policy +} + +// Make sure policyDRAAdapter implements the interfaces the DRA plugin needs. +var ( + _ dra.ClaimAllocator = &policyDRAAdapter{} + _ dra.DeviceLister = &policyDRAAdapter{} +) + +// PickHpCpus routes to the current p.cpuClasses handler's PickHpCpus. +func (a *policyDRAAdapter) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { + return a.p.cpuClasses.PickHpCpus(pkgID, punitID, n, held) +} + +// ReleaseHpCpus routes to the current p.cpuClasses handler's ReleaseHpCpus. +func (a *policyDRAAdapter) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { + a.p.cpuClasses.ReleaseHpCpus(pkgID, punitID, cpus) +} + +// AccountHpCpus routes to the current p.cpuClasses handler's AccountHpCpus. +func (a *policyDRAAdapter) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { + return a.p.cpuClasses.AccountHpCpus(pkgID, punitID, cpus) +} + +// IsHPClass routes to the current p.cpuClasses handler's IsHPClass. +func (a *policyDRAAdapter) IsHPClass(className string) bool { + return a.p.cpuClasses.IsHPClass(className) +} + +// DRADevices routes to the current p.cpuClasses handler's DRADevices. +func (a *policyDRAAdapter) DRADevices(driverName string) ([]resourceapi.Device, error) { + return a.p.cpuClasses.DRADevices(driverName) +} diff --git a/cmd/plugins/topology-aware/policy/dra_adapter_test.go b/cmd/plugins/topology-aware/policy/dra_adapter_test.go new file mode 100644 index 000000000..8d1803971 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra_adapter_test.go @@ -0,0 +1,162 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "testing" + + idset "github.com/intel/goresctrl/pkg/utils" + + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// adapterTestSys is a minimal sysfs.System implementation sufficient for +// cpuclass.New()/Configure(). Only CPUIDs is overridden; every other method +// is delegated to the embedded nil interface, which panics if called (and is +// never called by cpuclass.New/Configure in practice). +type adapterTestSys struct { + sysfs.System +} + +func (s *adapterTestSys) CPUIDs() []idset.ID { return nil } + +// newActiveClassHandler builds a *cpuclass.Handler with an active managed +// PCT allocator (via the goresctrl SST in-memory mock), one HP class named +// "hp". Mirrors cpuclass.newConfiguredHandler (internal, unexported), kept +// in sync intentionally — this package cannot import that internal test +// helper. +func newActiveClassHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", `{"supported":true,"clos_count":4,"packages":[{"id":0,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"punits":[{"id":0,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":4}]}]}`) + t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) + h, err := cpuclass.New(&adapterTestSys{}) + if err != nil { + t.Fatalf("cpuclass.New() failed: %v", err) + } + if err := h.Configure(cpuclass.ConfigSpec{ + Classes: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }); err != nil { + t.Fatalf("Configure() failed: %v", err) + } + return h +} + +// newInactiveClassHandler builds a *cpuclass.Handler without SST support, so +// PCT stays disabled ("inactive"): all DRA-facing methods report their +// nil/zero-value defaults. +func newInactiveClassHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", "") + h, err := cpuclass.New(&adapterTestSys{}) + if err != nil { + t.Fatalf("cpuclass.New() failed: %v", err) + } + _ = h.Configure(cpuclass.ConfigSpec{ + Classes: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + Allowed: cpuset.MustParse("0-7"), + }) + return h +} + +// TestPolicyDRAAdapterRoutesToCurrentHandler verifies the adapter forwards +// to whatever *cpuclass.Handler is installed in p.cpuClasses *at call time*, +// not to a handler captured once at construction. This matters because +// initialize() sets p.cpuClasses = nil and installs a brand new Handler on +// every Reconfigure — a cached pointer would go stale. +func TestPolicyDRAAdapterRoutesToCurrentHandler(t *testing.T) { + p := &policy{} + a := &policyDRAAdapter{p: p} + + // 1. cpuClasses nil ("no cpuClass config yet"): all methods must return + // safe zero-values, no panic. + p.cpuClasses = nil + if a.IsHPClass("hp") { + t.Error("IsHPClass with nil cpuClasses: got true, want false") + } + if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus with nil cpuClasses: got nil error, want error") + } + if devs, err := a.DRADevices(DRADriverName); err != nil || len(devs) != 0 { + t.Errorf("DRADevices with nil cpuClasses: got (%v, %v), want (empty, nil)", devs, err) + } + a.ReleaseHpCpus(0, 0, cpuset.New()) // must not panic + if err := a.AccountHpCpus(0, 0, cpuset.New()); err == nil { + t.Error("AccountHpCpus with nil cpuClasses: got nil error, want error") + } + + // 2. Swap in an inactive handler (PCT unsupported): still all + // zero-value defaults, but now backed by a real (non-nil) Handler. + p.cpuClasses = newInactiveClassHandler(t) + if a.IsHPClass("hp") { + t.Error("IsHPClass with inactive handler: got true, want false") + } + if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + t.Error("PickHpCpus with inactive handler: got nil error, want error") + } + + // 3. Swap in an active handler with class "hp": the adapter must + // immediately observe the new handler's behavior — proving it reads + // p.cpuClasses fresh on every call rather than a cached pointer. + p.cpuClasses = newActiveClassHandler(t) + if !a.IsHPClass("hp") { + t.Error("IsHPClass with active handler: got false, want true") + } + cpus, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + if err != nil { + t.Fatalf("PickHpCpus with active handler: %v", err) + } + if cpus.Size() != 2 { + t.Errorf("PickHpCpus with active handler: got %d CPUs, want 2", cpus.Size()) + } + devs, err := a.DRADevices(DRADriverName) + if err != nil { + t.Fatalf("DRADevices with active handler: %v", err) + } + if len(devs) == 0 { + t.Error("DRADevices with active handler: got 0 devices, want > 0") + } + a.ReleaseHpCpus(0, 0, cpus) + if err := a.AccountHpCpus(0, 0, cpus); err != nil { + t.Errorf("AccountHpCpus with active handler: %v", err) + } + + // 4. Swap back to nil: must return to safe zero-values, proving the + // forwarding is not sticky/cached from step 3. + p.cpuClasses = nil + if a.IsHPClass("hp") { + t.Error("IsHPClass after swap back to nil: got true, want false") + } +} + +func TestPolicyDRAAdapterNilCpuClassesNoPanic(t *testing.T) { + p := &policy{cpuClasses: nil} + a := &policyDRAAdapter{p: p} + + defer func() { + if r := recover(); r != nil { + t.Fatalf("adapter method panicked with nil cpuClasses: %v", r) + } + }() + + _, _ = a.PickHpCpus(0, 0, 1, cpuset.New()) + a.ReleaseHpCpus(0, 0, cpuset.New()) + _ = a.AccountHpCpus(0, 0, cpuset.New()) + _ = a.IsHPClass("hp") + _, _ = a.DRADevices(DRADriverName) +} diff --git a/cmd/plugins/topology-aware/policy/dra_test.go b/cmd/plugins/topology-aware/policy/dra_test.go new file mode 100644 index 000000000..7b97e3084 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/dra_test.go @@ -0,0 +1,511 @@ +// Copyright 2019 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "context" + "os" + "path" + "strings" + "testing" + + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/kubernetes/fake" + + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" + system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/testutils" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// setupDRATestPolicy builds a real *policy from the "server" sysfs test +// data (same fixture as newDRATestPolicy in resources_test.go) and runs it +// through Setup(). mutateCfg and mutateOpts (both optional) let each test +// case tweak the Config / BackendOptions before Setup() runs; preSetup +// (optional) runs on the freshly constructed, not-yet-Setup *policy — used +// to inject p.cdiDir before buildDRAPlugin would otherwise fall back to the +// real /var/run/cdi default. +func setupDRATestPolicy( + t *testing.T, + mutateCfg func(*cfgapi.Config), + mutateOpts func(*policyapi.BackendOptions), + preSetup func(*policy), +) (*policy, error) { + t.Helper() + + dir, err := os.MkdirTemp("", "nri-resource-policy-test-sysfs-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + t.Cleanup(func() { removeAll(t, dir) }) + + if err := testutils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { + t.Fatalf("failed to uncompress test sysfs data: %v", err) + } + + sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + if err != nil { + t.Fatalf("failed to discover test system: %v", err) + } + + cfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{ + cfgapi.CPU: "750m", + }, + } + if mutateCfg != nil { + mutateCfg(cfg) + } + + opts := &policyapi.BackendOptions{ + Cache: &mockCache{}, + System: sys, + Config: cfg, + } + if mutateOpts != nil { + mutateOpts(opts) + } + + p := New().(*policy) + if preSetup != nil { + preSetup(p) + } + + return p, p.Setup(opts) +} + +// withOneHPClass sets a single valid HP cpuClass, sufficient to make +// initialize() install a non-nil p.cpuClasses handler. +func withOneHPClass(cfg *cfgapi.Config) { + cfg.CPUClasses = []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}} +} + +// withDRAEnabled sets cfg.DRA.Enabled = true (SharedCounters left false). +func withDRAEnabled(cfg *cfgapi.Config) { + cfg.DRA = &cfgapi.TopologyAwareDRA{Enabled: true} +} + +// TestSetupDRADisabledLeavesPluginNil verifies that when DRA is disabled +// (the default zero-value Config.DRA == nil), Setup() never calls +// buildDRAPlugin, leaving p.draPlugin nil, and that the DRA-adjacent +// lifecycle methods (Start, Stop) remain no-ops/no-panics against that nil +// state — i.e. behavior is unchanged from before Step 8. +func TestSetupDRADisabledLeavesPluginNil(t *testing.T) { + p, err := setupDRATestPolicy(t, withOneHPClass, nil, nil) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin != nil { + t.Error("draPlugin: got non-nil, want nil (DRA disabled)") + } + + if err := p.Start(); err != nil { + t.Fatalf("Start() with DRA disabled: got %v, want nil", err) + } + if err := p.Stop(); err != nil { + t.Fatalf("Stop() with DRA disabled: got %v, want nil", err) + } +} + +// TestSetupDRAEnabledNilKubeClientLeavesPluginNil verifies that when DRA is +// enabled but opts.KubeClientFn() returns a nil kubernetes.Interface (e.g. +// local-config mode, or too early during agent startup), Setup() logs a +// warning and leaves p.draPlugin nil rather than failing. +func TestSetupDRAEnabledNilKubeClientLeavesPluginNil(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return nil } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + nil, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin != nil { + t.Error("draPlugin: got non-nil, want nil (no kube client)") + } +} + +// TestSetupDRAEnabledEmptyNodeNameLeavesPluginNil verifies that when DRA is +// enabled, a kube client is available, but opts.NodeName is empty (also +// possible in local-config mode), Setup() logs a warning and leaves +// p.draPlugin nil rather than failing or calling dra.New with an empty +// NodeName (which would itself error). +func TestSetupDRAEnabledEmptyNodeNameLeavesPluginNil(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "" + opts.WithLock = func(f func()) { f() } + }, + nil, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin != nil { + t.Error("draPlugin: got non-nil, want nil (empty node name)") + } +} + +// TestSetupDRAEnabledNoCPUClassesBuildsPluginWithNoDevices verifies that +// when DRA is enabled but no cpuClasses are configured (p.cpuClasses stays +// nil after initialize()), Setup() still builds a non-nil p.draPlugin, +// publishing zero devices — rather than leaving it permanently nil, which +// would prevent a later Reconfigure() that adds cpuClasses from ever +// recovering (buildDRAPlugin only ever runs once, from Setup()). +func TestSetupDRAEnabledNoCPUClassesBuildsPluginWithNoDevices(t *testing.T) { + p, err := setupDRATestPolicy(t, + withDRAEnabled, // no withOneHPClass: CPUClasses stays empty + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.cpuClasses != nil { + t.Fatal("test setup invariant violated: p.cpuClasses is non-nil, want nil") + } + if p.draPlugin == nil { + t.Fatal("draPlugin: got nil, want non-nil (empty cpuClasses must not prevent construction)") + } + + // *dra.Plugin doesn't expose DeviceLister directly; go through the same + // adapter path buildDRAPlugin wired in, to confirm the nil-cpuClasses + // case really does yield an empty (not erroring) device list rather + // than merely a non-nil plugin. + adapter := &policyDRAAdapter{p: p} + devs, err := adapter.DRADevices(DRADriverName) + if err != nil { + t.Fatalf("DRADevices() unexpected error: %v", err) + } + if len(devs) != 0 { + t.Errorf("DRADevices() = %v, want empty for nil p.cpuClasses", devs) + } +} + +// TestSetupDRAEnabledValidDepsBuildsPlugin verifies that when DRA is +// enabled and every dependency (kube client, node name, cpuClass +// configuration) is available, Setup() builds a non-nil p.draPlugin. +func TestSetupDRAEnabledValidDepsBuildsPlugin(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin == nil { + t.Fatal("draPlugin: got nil, want non-nil") + } +} + +// TestSetupDRAEnabledCDIWriterFailureReturnsError verifies buildDRAPlugin's +// genuine-hard-failure path: unlike the four "not ready yet" guards above +// (nil kube client, empty node name, no cpuClasses — all warn-and-nil), a +// real construction failure in one of its own dependencies (here, +// dra.NewCDIWriter failing because p.cdiDir cannot be created) must be +// returned as an error from Setup(), not swallowed. +func TestSetupDRAEnabledCDIWriterFailureReturnsError(t *testing.T) { + // A regular file can't be MkdirAll'd into: NewCDIWriter's os.MkdirAll on + // p.cdiDir (or a path beneath it) will fail with ENOTDIR. + tmp := t.TempDir() + blocker := path.Join(tmp, "not-a-directory") + if err := os.WriteFile(blocker, []byte("x"), 0644); err != nil { + t.Fatalf("failed to create blocking file: %v", err) + } + cdiDir := path.Join(blocker, "cdi") + + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = cdiDir }, + ) + if err == nil { + t.Fatalf("Setup() with an unusable cdiDir: got nil error, want a descriptive error") + } + if p.draPlugin != nil { + t.Errorf("draPlugin: got non-nil after a failed Setup(), want nil") + } +} + +// TestStopCancelsContextAndStopsDRAPlugin verifies that Stop() cancels the +// context draCtxCancel was set with and calls draPlugin.Stop(), and that +// calling Stop() a second time is safe (both context.CancelFunc and +// dra.Plugin.Stop are documented as idempotent). +func TestStopCancelsContextAndStopsDRAPlugin(t *testing.T) { + p := &policy{} + p.draPlugin = newTestDRAPlugin(t, cpuset.New(0), "dev0") + + ctx, cancel := context.WithCancel(context.Background()) + p.draCtxCancel = cancel + + if err := p.Stop(); err != nil { + t.Fatalf("Stop() = %v, want nil", err) + } + if ctx.Err() == nil { + t.Error("Stop() did not cancel the context") + } + + if err := p.Stop(); err != nil { + t.Fatalf("second Stop() = %v, want nil", err) + } +} + +// newConflictingTierClassHandler builds a *cpuclass.Handler configured with +// two managed PCT classes at the same tier (both PctPriority: "high"), the +// exact shape ValidateCPUClassesForDRA rejects when sharedCounters is +// false. SST support is left disabled (PCT inactive) since +// ValidateCPUClassesForDRA inspects the class list itself, not device +// activity — mirrors newInactiveClassHandler in dra_adapter_test.go. +func newConflictingTierClassHandler(t *testing.T) *cpuclass.Handler { + t.Helper() + t.Setenv("OVERRIDE_SST", "") + h, err := cpuclass.New(&adapterTestSys{}) + if err != nil { + t.Fatalf("cpuclass.New() failed: %v", err) + } + classes := []*cfgapi.CPUClass{ + {Name: "hp1", PctPriority: "high"}, + {Name: "hp2", PctPriority: "high"}, + } + if err := h.Configure(cpuclass.ConfigSpec{ + Classes: classes, + Allowed: cpuset.MustParse("0-7"), + }); err != nil { + t.Fatalf("Configure() failed: %v", err) + } + return h +} + +// TestBuildDRAPluginValidateClassesUsesLiveConfig verifies that the +// ValidateClasses closure buildDRAPlugin hands to the DRA plugin reads +// p.cfg live (via the nil-safe DRASharedCounters() getter) rather than a +// config snapshot taken at buildDRAPlugin call time — required so that a +// later Reconfigure() (which swaps p.cfg for a new *Config) is observed +// without rebuilding the plugin. The second phase resolves the tier +// conflict by dropping to a single published class, not by setting +// SharedCounters: true — that option is rejected outright regardless of +// conflicts, since Model C (KEP-5941) isn't implemented. +// +// PublishResources is used as the probe: it runs ValidateClasses before +// checking whether Start() has been called, so the distinction between "a +// tier-conflict error" (ValidateClasses failed) and "called before Start" +// (ValidateClasses passed) is directly observable without ever calling +// Start() (which would require real kubelet registration directories). +func TestBuildDRAPluginValidateClassesUsesLiveConfig(t *testing.T) { + classes := []*cfgapi.CPUClass{ + {Name: "hp1", PctPriority: "high"}, + {Name: "hp2", PctPriority: "high"}, + } + p := &policy{ + cache: &mockCache{}, + cpuClasses: newConflictingTierClassHandler(t), + cfg: &cfgapi.Config{ + CPUClasses: classes, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true, SharedCounters: false}, + }, + cdiDir: t.TempDir(), + } + + opts := &policyapi.BackendOptions{ + KubeClientFn: func() kubernetes.Interface { return fake.NewClientset() }, + NodeName: "test-node", + WithLock: func(f func()) { f() }, + } + + if err := p.buildDRAPlugin(opts); err != nil { + t.Fatalf("buildDRAPlugin() = %v, want nil", err) + } + if p.draPlugin == nil { + t.Fatal("draPlugin: got nil, want non-nil") + } + + // SharedCounters is false and both classes are at the same PCT tier: + // ValidateClasses must fail with a tier-conflict error. + if err := p.draPlugin.PublishResources(context.Background()); err == nil || !strings.Contains(err.Error(), "tier") { + t.Fatalf("PublishResources() before simulated Reconfigure = %v, want tier-conflict error", err) + } + + // Simulate a Reconfigure that swaps p.cfg for a new *Config whose + // CPUClasses no longer conflict (down to a single HP class). + // buildDRAPlugin's ValidateClasses closure captured p, not cfg, so it + // must observe this change immediately — with no need to rebuild + // p.draPlugin. (SharedCounters stays false: it is rejected outright by + // ValidateCPUClassesForDRA regardless of tier conflicts, since Model C + // isn't implemented — it can no longer be used to "fix" a conflict.) + p.cfg = &cfgapi.Config{ + CPUClasses: []*cfgapi.CPUClass{classes[0]}, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true, SharedCounters: false}, + } + + err := p.draPlugin.PublishResources(context.Background()) + if err == nil || !strings.Contains(err.Error(), "called before Start") { + t.Fatalf("PublishResources() after simulated Reconfigure = %v, want \"called before Start\" error (proves ValidateClasses passed)", err) + } +} + +// ---- Reconfigure refusal: DRA config changes require a restart. ---- + +// TestReconfigureRefusesAnyChangeWhileDRAEnabled verifies that Reconfigure() +// refuses a config change unrelated to DRA itself (here: ReservedResources) +// once DRA is enabled -- any config change while DRA is (or was) enabled now +// requires a restart, not just a dra.enabled flip or a live-claim conflict. +func TestReconfigureRefusesAnyChangeWhileDRAEnabled(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + if p.draPlugin == nil { + t.Fatal("test setup error: draPlugin unexpectedly nil") + } + + oldCfg := p.cfg + oldDRAPlugin := p.draPlugin + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "1000m"}, + CPUClasses: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true}, + } + + err = p.Reconfigure(newCfg) + if err == nil || !strings.Contains(err.Error(), "restart") { + t.Fatalf("Reconfigure() = %v, want an error naming the restart requirement", err) + } + if p.cfg != oldCfg { + t.Error("p.cfg was replaced despite the refused Reconfigure") + } + if opt != oldCfg { + t.Error("opt was replaced despite the refused Reconfigure") + } + if p.draPlugin != oldDRAPlugin { + t.Error("draPlugin was replaced despite the refused Reconfigure") + } +} + +// TestReconfigureRefusesDRAEnabledFlipToTrue verifies that Reconfigure() +// refuses a config change that turns DRA on when it was off at Setup() time. +func TestReconfigureRefusesDRAEnabledFlipToTrue(t *testing.T) { + p := newDRATestPolicy(t) // DRA disabled by default (cfg.DRA == nil) + if p.draPlugin != nil { + t.Fatal("test setup error: draPlugin unexpectedly non-nil") + } + + oldCfg := p.cfg + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "750m"}, + DRA: &cfgapi.TopologyAwareDRA{Enabled: true}, + } + + err := p.Reconfigure(newCfg) + if err == nil { + t.Fatal("Reconfigure() = nil, want an error (DRAEnabled() flip false -> true)") + } + if opt != oldCfg { + t.Error("opt not restored to the pre-Reconfigure config after the refused DRAEnabled flip") + } + if p.draPlugin != nil { + t.Error("draPlugin unexpectedly built by a refused Reconfigure") + } +} + +// TestReconfigureRefusesDRAEnabledFlipToFalse mirrors +// TestReconfigureRefusesDRAEnabledFlipToTrue for the opposite direction: DRA +// was enabled (and successfully built) at Setup() time, and a later +// Reconfigure() tries to turn it off. +func TestReconfigureRefusesDRAEnabledFlipToFalse(t *testing.T) { + p, err := setupDRATestPolicy(t, + func(cfg *cfgapi.Config) { withOneHPClass(cfg); withDRAEnabled(cfg) }, + func(opts *policyapi.BackendOptions) { + opts.KubeClientFn = func() kubernetes.Interface { return fake.NewClientset() } + opts.NodeName = "test-node" + opts.WithLock = func(f func()) { f() } + }, + func(p *policy) { p.cdiDir = t.TempDir() }, + ) + if err != nil { + t.Fatalf("Setup() failed: %v", err) + } + oldCfg := p.cfg + oldDRAPlugin := p.draPlugin + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "750m"}, + CPUClasses: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, + // DRA left nil: DRAEnabled() == false. + } + + err = p.Reconfigure(newCfg) + if err == nil { + t.Fatal("Reconfigure() = nil, want an error (DRAEnabled() flip true -> false)") + } + if opt != oldCfg { + t.Error("opt not restored to the pre-Reconfigure config after the refused DRAEnabled flip") + } + if p.draPlugin != oldDRAPlugin { + t.Error("draPlugin was replaced/cleared despite the refused Reconfigure") + } +} + +// TestReconfigureUnaffectedWhenDRANeverEnabled verifies that Reconfigure() +// still succeeds for an ordinary config change when DRA was never enabled +// (nil in both the old and new config) -- guards against the DRA restart +// guard accidentally widening to non-DRA reconfigures. +func TestReconfigureUnaffectedWhenDRANeverEnabled(t *testing.T) { + p := newDRATestPolicy(t) + if p.cfg.DRAEnabled() { + t.Fatal("test setup error: DRA unexpectedly enabled") + } + + newCfg := &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{cfgapi.CPU: "1000m"}, + } + + if err := p.Reconfigure(newCfg); err != nil { + t.Fatalf("Reconfigure() = %v, want nil (DRA never enabled)", err) + } + if p.cfg != newCfg { + t.Error("p.cfg was not updated by a successful Reconfigure") + } +} diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 28b074dbf..91f26197d 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -367,6 +367,9 @@ type mockContainer struct { returnValueForGetID string returnValueForQOSClass v1.PodQOSClass pod cache.Pod + cdiDeviceNames []string + cpusetCpus string + setCpusetCpusCalls []string } func (m *mockContainer) GetPod() (cache.Pod, bool) { @@ -428,6 +431,9 @@ func (m *mockContainer) GetMounts() []*cache.Mount { func (m *mockContainer) GetDevices() []*cache.Device { panic("unimplemented") } +func (m *mockContainer) GetCDIDeviceNames() []string { + return m.cdiDeviceNames +} func (m *mockContainer) PrettyName() string { return m.name } @@ -490,7 +496,9 @@ func (m *mockContainer) SetCPUPeriod(int64) { func (m *mockContainer) SetCPUQuota(int64) { panic("unimplemented") } -func (m *mockContainer) SetCpusetCpus(string) { +func (m *mockContainer) SetCpusetCpus(cpus string) { + m.setCpusetCpusCalls = append(m.setCpusetCpusCalls, cpus) + m.cpusetCpus = cpus } func (m *mockContainer) SetCpusetMems(string) { } @@ -585,7 +593,7 @@ func (m *mockContainer) GetCPUPeriod() int64 { panic("unimplemented") } func (m *mockContainer) GetCpusetCpus() string { - panic("unimplemented") + return m.cpusetCpus } func (m *mockContainer) GetCpusetMems() string { panic("unimplemented") @@ -719,6 +727,7 @@ type mockCache struct { returnValueForGetPolicyEntry bool returnValue1ForLookupContainer cache.Container returnValue2ForLookupContainer bool + containers []cache.Container } func (m *mockCache) InsertPod(*nri.PodSandbox, <-chan *podresapi.PodResources) cache.Pod { @@ -749,7 +758,7 @@ func (m *mockCache) GetPods() []cache.Pod { panic("unimplemented") } func (m *mockCache) GetContainers() []cache.Container { - panic("unimplemented") + return m.containers } func (m *mockCache) GetContainerIds() []string { panic("unimplemented") diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index 2eb1a7c4c..00567da83 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -18,15 +18,19 @@ import ( "fmt" "math" "sort" + "strings" "github.com/containers/nri-plugins/pkg/utils/cpuset" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/types" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/dra" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" system "github.com/containers/nri-plugins/pkg/sysfs" idset "github.com/intel/goresctrl/pkg/utils" + "tags.cncf.io/container-device-interface/pkg/parser" ) // buildPoolsByTopology builds a hierarchical tree of pools based on HW topology. @@ -257,7 +261,7 @@ func (p *policy) getCpuSupply(node Node, cpus cpuset.CPUSet) (Supply, Supply) { log.Infof(" %s CPU: %s", node.Name(), s.DumpCapacity()) - return s, s.Clone() + return s, newSupply(node, isolated, reserved, sharable, 0, 0) } func (p *policy) getMemSupply(node Node, cpus cpuset.CPUSet) (dram, pmem, hbm idset.IDSet) { @@ -513,8 +517,9 @@ func (p *policy) allocatePool(container cache.Container, poolHint string) (Grant // setPreferredCpusetCpus pins container's CPUs according to what has been // allocated for it, taking into account if the container should run -// with hyperthreads hidden. -func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated cpuset.CPUSet, info string) { +// with hyperthreads hidden. CPUs in preserve are always included in the +// final cpuset regardless of hide-hyperthreads filtering (e.g. DRA claimed CPUs). +func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated, preserve cpuset.CPUSet, info string) { allow := allocated hidingInfo := "" pod, ok := container.GetPod() @@ -526,6 +531,7 @@ func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated cpu hidingInfo = " (no hyperthreads to hide)" } } + allow = allow.Union(preserve) log.Infof("%s%s", info, hidingInfo) container.SetCpusetCpus(allow.String()) } @@ -568,6 +574,13 @@ func (p *policy) applyGrant(grant Grant) { return } + // Collect CPUs this container holds via a live DRA claim. These are + // removed from pool free supply by allocateClaim, but that never + // touches the container's cgroup cpuset — this is the only place that + // pins them. They are passed as preserve to setPreferredCpusetCpus so + // that hide-hyperthreads filtering on the normal grant cannot drop them. + claimed := p.claimedCPUsByContainer[container.GetID()] + mems := libmem.NodeMask(0) if opt.PinMemory { mems = grant.GetMemoryZone() @@ -575,10 +588,20 @@ func (p *policy) applyGrant(grant Grant) { if opt.PinCPU { if cpuType == cpuPreserve { + if !claimed.IsEmpty() { + preserved, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + log.Errorf(" => failed to parse %s cpuset %q while adding DRA claim %s: %v", + container.PrettyName(), container.GetCpusetCpus(), claimed, err) + } else { + preserved = preserved.Union(claimed) + container.SetCpusetCpus(preserved.String()) + } + } log.Infof(" => preserving %s cpuset %s", container.PrettyName(), container.GetCpusetCpus()) } else { - if cpus.Size() > 0 { - p.setPreferredCpusetCpus(container, cpus, + if cpus.Size() > 0 || !claimed.IsEmpty() { + p.setPreferredCpusetCpus(container, cpus, claimed, fmt.Sprintf(" => pinning %s to (%s) cpuset %s", container.PrettyName(), kind, cpus)) } else { @@ -701,12 +724,17 @@ func (p *policy) updateSharedAllocations(grant *Grant) { if opt.PinCPU { shared := other.GetCPUNode().FreeSupply().SharableCPUs() exclusive := other.ExclusiveCPUs() + // Pass claimed CPUs as preserve so hide-hyperthreads filtering + // on the normal grant cpuset cannot drop them. + claimed := p.claimedCPUsByContainer[other.GetContainer().GetID()] if exclusive.IsEmpty() { - p.setPreferredCpusetCpus(other.GetContainer(), shared, + cpus := shared + p.setPreferredCpusetCpus(other.GetContainer(), cpus, claimed, fmt.Sprintf(" => updating %s with shared CPUs of %s: %s...", - other, other.GetCPUNode().Name(), shared.String())) + other, other.GetCPUNode().Name(), cpus.String())) } else { - p.setPreferredCpusetCpus(other.GetContainer(), exclusive.Union(shared), + cpus := exclusive.Union(shared) + p.setPreferredCpusetCpus(other.GetContainer(), cpus, claimed, fmt.Sprintf(" => updating %s with exclusive+shared CPUs of %s: %s+%s...", other, other.GetCPUNode().Name(), exclusive.String(), shared.String())) } @@ -1339,3 +1367,491 @@ func combineHintScores(scores map[string]float64) (float64, float64) { } return combined, filtered } + +// +// DRA claim identification and pool accounting. +// + +// cdiClaimDeviceNamePrefix is the fixed prefix of every qualified CDI device +// name the DRA plugin (pkg/resmgr/dra) generates for this driver: the CDI +// "device" class is hardcoded there (cdi.go: cdiClass = "device"), and every +// device name it builds starts with "claim-" (cdi.go: cdiDeviceName). This is +// duplicated here (rather than imported) because cdiClass and cdiDeviceName +// are unexported in package dra. +const cdiClaimDeviceNamePrefix = DRADriverName + "/device=claim-" + +// parseCDIClaimUID extracts the DRA ResourceClaim UID from a qualified CDI +// device name of the form "nri.topology-aware.cpu/device=claim----" +// (see pkg/resmgr/dra/cdi.go's cdiDeviceName), by trimming the trailing +// "---" three '-'-separated tokens and returning what's +// left. +// +// This is a best-effort fast path, not a lossless inverse of cdiDeviceName: +// and are themselves sanitized names that may contain +// '-' (see sanitizeCDIName), so when they do, the split boundary can land in +// the wrong place and the returned string will include extra trailing +// tokens that actually belong to /. It is exact whenever +// and are single tokens (the common case), which is all +// this function alone can guarantee. claimCPUsFromContainer compensates for +// the ambiguous case by falling back to matching against the caller's known +// set of live claim UIDs, which this function has no access to. +func parseCDIClaimUID(deviceName string) (string, bool) { + rest, ok := strings.CutPrefix(deviceName, cdiClaimDeviceNamePrefix) + if !ok { + return "", false + } + + tokens := strings.Split(rest, "-") + if len(tokens) < 4 { + return "", false + } + + uid := strings.Join(tokens[:len(tokens)-3], "-") + if uid == "" { + return "", false + } + + return uid, true +} + +// claimLister is the minimal slice of *dra.Plugin's API that +// claimCPUsFromContainer needs. Declaring it locally (instead of taking a +// *dra.Plugin directly) lets tests exercise the CDI-name-to-claim lookup +// logic with a lightweight fake instead of standing up a full dra.Plugin +// (kubelet registration, CDI writer, claim store, etc.); *dra.Plugin +// satisfies this interface, so production call sites are unaffected. +// +// Callers must nil-check the concrete *dra.Plugin *before* passing it in +// here: a nil *dra.Plugin wrapped in a non-nil claimLister interface value +// would panic inside LiveClaimsLocked (typed-nil trap). +type claimLister interface { + LiveClaimsLocked() map[types.UID][]dra.ResultAlloc + DriverName() string +} + +// classifyClaimCPUs parses every alloc's CPUs field and returns the union of +// all of them plus a grouping of the same CPUs by cpuClass name. +// +// A prepared claim's DeviceRequestAllocationResults are constrained by +// pkg/resmgr/dra to a single punit (PrepareResourceClaims rejects any claim +// whose results span more than one), but that punit can still publish more +// than one HP class: cpuclass.ValidateCPUClassesForDRA allows at most one +// published class per tier, not one class overall, so a punit with several +// tiers publishes a device per (class, tier) pair. Grouping by class here +// lets callers apply each class only to the CPUs that actually belong to it +// (see allocateClaim/remarkClaimInSupply), so a multi-class claim never has +// the wrong physical class silently applied to part of its CPUs. +// +// Allocs whose CPUs field fails to parse are logged and skipped; they +// contribute to neither the returned union nor the per-class grouping. +func classifyClaimCPUs(uid types.UID, allocs []dra.ResultAlloc) (cpuset.CPUSet, map[string]cpuset.CPUSet) { + cpus := cpuset.New() + classCPUs := map[string]cpuset.CPUSet{} + + for _, a := range allocs { + parsed, err := cpuset.Parse(a.CPUs) + if err != nil { + log.Warnf("dra: claim %s: failed to parse allocated CPUs %q: %v", uid, a.CPUs, err) + continue + } + cpus = cpus.Union(parsed) + if existing, ok := classCPUs[a.ClassName]; ok { + classCPUs[a.ClassName] = existing.Union(parsed) + } else { + classCPUs[a.ClassName] = parsed + } + } + + return cpus, classCPUs +} + +// containerClaim describes one live DRA claim's contribution to a +// container: its UID, the union of its allocated CPUs, and those CPUs +// grouped by cpuClass name (see classifyClaimCPUs — a single claim can span +// more than one class). +type containerClaim struct { + UID types.UID + CPUs cpuset.CPUSet + ClassCPUs map[string]cpuset.CPUSet +} + +// claimCPUsFromContainer looks for CDI device names on c that identify live +// DRA claims and returns one containerClaim per distinct live claim UID +// found. Returns an empty slice if c carries no recognizable claim device +// name, or if every embedded UID has no corresponding entry in +// plugin.LiveClaimsLocked() (e.g. a foreign/stale CDI device, or a claim +// that has already been unprepared). +// +// A container is most commonly backed by a single live TA CPU ResourceClaim, +// but Kubernetes' general pod.spec.resourceClaims plumbing does not forbid a +// container from referencing more than one distinct ResourceClaim for this +// driver (this is orthogonal to a single ResourceClaim with +// AllowMultipleAllocations fanning out to *many containers*, which is +// handled by the claimContainerRefs refcount in allocateClaim/releaseClaim). +// Callers must loop over every entry in the returned slice, not just the +// first, to avoid leaving a distinct claim's CPUs unaccounted for (and thus +// double-bookable by a subsequent, unrelated allocation). +func claimCPUsFromContainer(c cache.Container, plugin claimLister) []containerClaim { + if plugin == nil { + return nil + } + + live := plugin.LiveClaimsLocked() + + var result []containerClaim + for _, name := range c.GetCDIDeviceNames() { + for uid, allocs := range live { + for i, alloc := range allocs { + expected := parser.QualifiedName(plugin.DriverName(), "device", + dra.CDIDeviceName(uid, alloc.Request, alloc.Device, i)) + if name != expected { + continue + } + claimCPUs, claimClassCPUs := classifyClaimCPUs(uid, []dra.ResultAlloc{alloc}) + if claimCPUs.IsEmpty() { + continue + } + for j := range result { + if result[j].UID != uid { + continue + } + result[j].CPUs = result[j].CPUs.Union(claimCPUs) + for class, cpus := range claimClassCPUs { + result[j].ClassCPUs[class] = result[j].ClassCPUs[class].Union(cpus) + } + goto nextDevice + } + result = append(result, containerClaim{UID: uid, CPUs: claimCPUs, ClassCPUs: claimClassCPUs}) + goto nextDevice + } + } + nextDevice: + } + + return result +} + +// poolForCPUs returns the tightest (deepest) *leaf* pool whose statically +// assigned CPU range (GetSupply(), which — unlike FreeSupply() — never +// changes with allocation or claim accounting) fully contains cpus. Returns +// an error if no single pool's range is a superset of cpus: either cpus lies +// (at least partly) outside p.allowed altogether, or it straddles more than +// one leaf pool (which a legitimate single-punit DRA CPU pick never does). +// +// Candidates are restricted to leaf pools (Node.IsLeafNode()) deliberately: +// every non-leaf ancestor's static range is, by construction, the union of +// its descendants' ranges, so it is always a superset of any cpus subset of +// p.allowed — including a cpus set that straddles two *different* leaf +// pools. Without this restriction, such a straddling cpus set would +// incorrectly resolve to the lowest common ancestor (worst case, root) +// instead of being rejected: Supply.ClaimCPUs only walks *up* the +// node.Parent() chain from whatever pool it's called on, never down into +// children, so marking the CPUs claimed at the ancestor would fail to +// exclude them from either leaf's own FreeSupply() — a double-booking gap +// this restriction closes. +func (p *policy) poolForCPUs(cpus cpuset.CPUSet) (Node, error) { + var ( + best Node + bestDepth = -1 + ) + + for _, n := range p.pools { + if !n.IsLeafNode() { + continue + } + full := n.GetSupply() + total := full.IsolatedCPUs().Union(full.ReservedCPUs()).Union(full.SharableCPUs()) + if !cpus.IsSubsetOf(total) { + continue + } + if n.RootDistance() > bestDepth { + best = n + bestDepth = n.RootDistance() + } + } + + if best == nil { + return nil, policyError("no single pool contains CPUs %s (outside allowed CPUs, or spanning more than one pool)", cpus) + } + + return best, nil +} + +// applyClassCPUs applies each (className, subset) pair in classCPUs via +// cpuClasses.UseClass, one call per class. This is a no-op if p.cpuClasses +// is nil. Empty class names (an alloc whose device carried no nri/cpuClass +// attribute — should not happen given upstream validation, but defensively +// skipped rather than trusted) and empty subsets are skipped. verb is used +// only for logging ("apply" vs "re-apply"). +// +// classCPUs groups a single DRA claim's CPUs by cpuClass (see +// classifyClaimCPUs): a claim's DeviceRequestAllocationResults can resolve +// to devices of different classes published for the same punit, so the +// physical class must be applied per subset — applying one class to the +// claim's entire unioned CPU set would silently mis-apply it to part of the +// claim. +func (p *policy) applyClassCPUs(verb string, uid types.UID, classCPUs map[string]cpuset.CPUSet) error { + if p.cpuClasses == nil { + return nil + } + for className, subset := range classCPUs { + if className == "" || subset.IsEmpty() { + continue + } + if err := p.cpuClasses.UseClass(className, subset); err != nil { + return fmt.Errorf("dra: failed to %s CPU class %q to claim %s CPUs %s: %w", + verb, className, uid, subset, err) + } + } + return nil +} + +// allocateClaim marks cpus as claimed by DRA ResourceClaim uid in the +// tightest pool that fully contains them, evicting and requeueing for +// reallocation any exclusive grant that overlaps those CPUs. +// +// Safe to call more than once for the same uid: a ResourceClaim with +// AllowMultipleAllocations can back more than one container, and +// AllocateResources calls this once per container. The pool marking itself +// (and any eviction it triggers) is only performed for the first container; +// subsequent calls just bump the per-claim container refcount so that +// releaseClaim knows to keep the CPUs marked until the last referencing +// container is released. +func (p *policy) allocateClaim(uid types.UID, cpus cpuset.CPUSet, classCPUs map[string]cpuset.CPUSet) error { + if cpus.IsEmpty() { + return policyError("cannot allocate DRA claim %s: empty CPU set", uid) + } + + if p.claimContainerRefs == nil { + p.claimContainerRefs = make(map[types.UID]int) + } + + if p.claimContainerRefs[uid] == 0 { + pool, err := p.poolForCPUs(cpus) + if err != nil { + return policyError("cannot allocate DRA claim %s (CPUs %s): %v", uid, cpus, err) + } + + evicted, evictedCpusets := p.evictOverlappingGrants(cpus, fmt.Sprintf("claim %s", uid)) + + pool.FreeSupply().ClaimCPUs(uid, cpus) + + // Apply the physical cpuClass (SST-CP CLOS association, EPP, + // governor, ...) to the claimed CPUs. Without this, pool accounting + // excludes the CPUs from regular grants but the hardware is left in + // whatever class it was in before — the whole point of associating a + // DRA claim with a cpuClass would otherwise have no physical effect. + // classCPUs groups the claimed CPUs by class (see classifyClaimCPUs): + // a single claim can span more than one class (e.g. requests + // resolving to devices of different classes published for the same + // punit), so UseClass is applied once per (class, subset) pair + // rather than once for the whole unioned cpus with a single, + // possibly-wrong class. + if err := p.applyClassCPUs("apply", uid, classCPUs); err != nil { + // Roll back the supply mark so pool accounting stays consistent. + pool.FreeSupply().UnclaimCPUs(uid) + p.resetCpuClass(fmt.Sprintf("dra: rollback claim %s", uid), cpus) + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + log.Errorf("dra: claim %s: failed to restore evicted grants during CPU class rollback: %v", uid, reallocErr) + } + return policyError("dra: claim %s: failed to apply CPU class: %v", uid, err) + } + + // The claimed CPUs may have been subtracted from the sharable pool + // (not just isolated/exclusive), in which case containers already + // pinned (via applyGrant) to the pool's previous, wider sharable + // cpuset need to be re-pinned to the now-reduced set — otherwise they + // keep running on CPUs the pool considers exclusively owned by this + // DRA claim. Run this unconditionally: it is a no-op for containers + // whose cpuset does not include any of the reserved-CPU-type grants + // affected here (see updateSharedAllocations). + p.updateSharedAllocations(nil) + + if err := p.reallocateEvicted(evicted, evictedCpusets, cpus, uid); err != nil { + pool.FreeSupply().UnclaimCPUs(uid) + p.resetCpuClass(fmt.Sprintf("dra: rollback claim %s", uid), cpus) + p.updateSharedAllocations(nil) + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + log.Errorf("dra: claim %s: failed to restore evicted grants during rollback: %v", uid, reallocErr) + } + return policyError("dra: claim %s: evicted %d container(s) to free CPUs %s but failed "+ + "to fully reallocate them: %v", uid, len(evicted), cpus, err) + } + } + + p.claimContainerRefs[uid]++ + + return nil +} + +// evictOverlappingGrants releases the exclusive grant of every container +// whose ExclusiveCPUs() overlaps cpus, returning the evicted containers and +// a snapshot of their cgroup cpuset.cpus (as it was right before eviction) — +// the latter is needed by reallocateEvicted's safety net if reallocation +// later fails for one of them. reason is used only for logging. +func (p *policy) evictOverlappingGrants(cpus cpuset.CPUSet, reason string) ([]cache.Container, map[string]string) { + var evicted []cache.Container + evictedCpusets := map[string]string{} + for _, g := range p.allocations.grants { + if g.ExclusiveCPUs().Intersection(cpus).IsEmpty() { + continue + } + c := g.GetContainer() + evicted = append(evicted, c) + evictedCpusets[c.GetID()] = c.GetCpusetCpus() + } + + for _, c := range evicted { + log.Infof("dra: evicting %s to free CPUs %s for %s", c.PrettyName(), cpus, reason) + p.releasePool(c) + } + + return evicted, evictedCpusets +} + +// reallocateEvicted attempts to reallocate the containers evicted (by +// evictOverlappingGrants) to free cpus for DRA claim uid. If reallocation +// fails for one or more of them, it forcibly strips cpus out of their +// last-known cgroup cpuset (evictedCpusets, as captured before eviction) as +// a safety net: an evicted container that ends up with no grant must not be +// left pinned to a cpuset that overlaps the CPUs a DRA claim now exclusively +// owns — that would let two workloads run on the same physical CPUs +// simultaneously. Returns the (possibly partial-reallocation) error from +// reallocateResources, or nil if evicted is empty or reallocation succeeded. +func (p *policy) reallocateEvicted(evicted []cache.Container, evictedCpusets map[string]string, cpus cpuset.CPUSet, uid types.UID) error { + if len(evicted) == 0 { + return nil + } + + if err := p.reallocateResources(evicted, nil); err != nil { + log.Errorf("dra: failed to fully reallocate %d container(s) evicted for claim %s: %v", + len(evicted), uid, err) + + for _, c := range evicted { + if _, ok := p.allocations.getGrant(c.GetID()); ok { + continue + } + prev, perr := cpuset.Parse(evictedCpusets[c.GetID()]) + if perr != nil { + log.Errorf("dra: claim %s: cannot safely re-pin %s off claimed CPUs %s: %v", + uid, c.PrettyName(), cpus, perr) + continue + } + safe := prev.Difference(cpus) + if safe.IsEmpty() { + // The victim's entire previous cpuset is within the claimed CPUs. + // An empty cpuset string is treated by NRI as "no restriction", so + // we must not call SetCpusetCpus with it — doing so would leave the + // container running on the DRA-claimed CPUs. Try to find at least + // one non-claimed CPU from the system-wide sharable pool as a + // fallback to pin the container to. + fallback := p.root.FreeSupply().SharableCPUs() + if fallback.IsEmpty() { + log.Errorf("dra: claim %s: cannot safely re-pin %s off claimed CPUs %s: "+ + "previous cpuset %s is entirely claimed and no sharable fallback CPU exists", + uid, c.PrettyName(), cpus, prev) + continue + } + safe = cpuset.New(fallback.List()[0]) + } + log.Warnf("dra: claim %s: %s could not be reallocated after eviction; "+ + "forcing cpuset from %s to %s to avoid overlap with claimed CPUs %s", + uid, c.PrettyName(), prev, safe, cpus) + c.SetCpusetCpus(safe.String()) + } + + return err + } + + return nil +} + +// releaseClaim decrements the container refcount for DRA claim uid and, +// once the last referencing container has been released, restores cpus to +// the pool that had them marked as claimed. A no-op (not an error) for a uid +// that allocateClaim was never called for, or that has already been fully +// released — ReleaseResources may run for containers the policy never saw +// AllocateResources for (e.g. across a restart). +func (p *policy) releaseClaim(uid types.UID, cpus cpuset.CPUSet) error { + if p.claimContainerRefs == nil || p.claimContainerRefs[uid] == 0 { + return nil + } + + p.claimContainerRefs[uid]-- + if p.claimContainerRefs[uid] > 0 { + return nil + } + + delete(p.claimContainerRefs, uid) + + // If unprepareDRAClaim ran first (while we still had refs), it tombstoned + // the allocs rather than releasing the HP hold. Do that now. + if allocs, ok := p.tombstonedDRAClaims[uid]; ok { + delete(p.tombstonedDRAClaims, uid) + p.releaseHpCPUsForAllocs(uid, allocs) + } + + pool, err := p.poolForCPUs(cpus) + if err != nil { + return policyError("cannot release DRA claim %s (CPUs %s): %v", uid, cpus, err) + } + + pool.FreeSupply().UnclaimCPUs(uid) + + // Mirror releasePool's resetCpuClass: allocateClaim applied a physical + // cpuClass (SST-CP CLOS, EPP, governor, ...) to these CPUs via + // cpuClasses.UseClass. Without resetting it here, the CPUs go back into + // the pool's regular exclusive/shared rotation still carrying whatever + // class the claim used — silently affecting the next, unrelated + // container the pool hands them to, until the next full + // Reconfigure/restart resets every allowed CPU's class from scratch. + p.resetCpuClass(fmt.Sprintf("dra: release claim %s", uid), cpus) + + // The released CPUs may have been restored to the sharable pool (not + // just isolated/exclusive), widening the cpuset available to containers + // already running in that pool's shared allocation. Let them reclaim it + // — mirrors the same call in allocateClaim/reapplyDRAClaims. + p.updateSharedAllocations(nil) + + return nil +} + +func (p *policy) unprepareDRAClaim(uid types.UID, allocs []dra.ResultAlloc) { + if p.claimContainerRefs != nil && p.claimContainerRefs[uid] > 0 { + // Containers still using this claim's CPUs. Defer the HP-CPU and pool + // release until the last container drops (releaseClaim will drain it). + if p.tombstonedDRAClaims == nil { + p.tombstonedDRAClaims = make(map[types.UID][]dra.ResultAlloc) + } + p.tombstonedDRAClaims[uid] = allocs + return + } + + cpus, _ := classifyClaimCPUs(uid, allocs) + if cpus.IsEmpty() { + return + } + pool, err := p.poolForCPUs(cpus) + if err != nil { + log.Errorf("dra: cannot unclaim prepared claim %s (CPUs %s): %v", uid, cpus, err) + return + } + p.releaseHpCPUsForAllocs(uid, allocs) + pool.FreeSupply().UnclaimCPUs(uid) + p.resetCpuClass(fmt.Sprintf("dra: unprepare claim %s", uid), cpus) + p.updateSharedAllocations(nil) +} + +// releaseHpCPUsForAllocs calls cpuClasses.ReleaseHpCpus for every allocation +// in allocs, parsing CPUs from the stored string. Parse errors are logged and +// skipped (matching UnprepareResourceClaims' own parse-error policy). +func (p *policy) releaseHpCPUsForAllocs(uid types.UID, allocs []dra.ResultAlloc) { + for _, alloc := range allocs { + cpus, err := cpuset.Parse(alloc.CPUs) + if err != nil { + log.Warnf("dra: release claim %s device %s: parse CPUs %q: %v (skipping HP release)", uid, alloc.Device, alloc.CPUs, err) + continue + } + p.cpuClasses.ReleaseHpCpus(alloc.PkgID, alloc.PunitID, cpus) + } +} diff --git a/cmd/plugins/topology-aware/policy/pools_test.go b/cmd/plugins/topology-aware/policy/pools_test.go index e11b978f5..46d191540 100644 --- a/cmd/plugins/topology-aware/policy/pools_test.go +++ b/cmd/plugins/topology-aware/policy/pools_test.go @@ -18,13 +18,19 @@ import ( "fmt" "os" "path" + "strings" "testing" + "k8s.io/apimachinery/pkg/types" + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/dra" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/testutils" + "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func findNodeWithName(name string, nodes []Node) Node { @@ -556,3 +562,613 @@ func TestAffinities(t *testing.T) { }) } } + +// +// DRA claim identification and pool accounting (Step 8, Task 6). +// + +func TestParseCDIClaimUID(t *testing.T) { + tcases := []struct { + name string + deviceName string + wantUID string + wantOK bool + }{ + { + name: "simple uid", + deviceName: "nri.topology-aware.cpu/device=claim-abc123-req-dev-0", + wantUID: "abc123", + wantOK: true, + }, + { + // A uid containing '-' (the real Kubernetes UID shape) is still + // recovered exactly, as long as and are each + // single tokens: the split only has to strip a fixed 3 trailing + // tokens, so it doesn't matter how many tokens the leading uid + // part itself has. + name: "uid with embedded dashes, single-token request/device", + deviceName: "nri.topology-aware.cpu/device=claim-13f7db45-eb2a-4dd1-b0cb-1234567890ab-myreq-dev0-0", + wantUID: "13f7db45-eb2a-4dd1-b0cb-1234567890ab", + wantOK: true, + }, + { + // Documents the known best-effort limitation: when is + // itself multi-token after sanitization (e.g. "punit-0-0"), the + // fixed trailing-3-token strip lands in the wrong place and + // "swallows" part of into the returned uid. + // claimCPUsFromContainer compensates for this by iterating over + // its known-live UIDs and doing exact device-name construction + // instead of relying on parseCDIClaimUID alone. + name: "documents limitation: multi-token device swallows the request", + deviceName: "nri.topology-aware.cpu/device=claim-13f7db45-myreq-punit-0-0-0", + wantUID: "13f7db45-myreq-punit", + wantOK: true, + }, + { + name: "wrong driver prefix", + deviceName: "other.driver/device=claim-abc123-req-dev-0", + wantOK: false, + }, + { + name: "not a CDI qualified name at all", + deviceName: "not-a-cdi-device-name", + wantOK: false, + }, + { + name: "right prefix, too few remaining tokens", + deviceName: "nri.topology-aware.cpu/device=claim-dev-0", + wantOK: false, + }, + } + + for _, tc := range tcases { + t.Run(tc.name, func(t *testing.T) { + uid, ok := parseCDIClaimUID(tc.deviceName) + if ok != tc.wantOK { + t.Fatalf("parseCDIClaimUID(%q) ok = %v, want %v", tc.deviceName, ok, tc.wantOK) + } + if ok && uid != tc.wantUID { + t.Errorf("parseCDIClaimUID(%q) = %q, want %q", tc.deviceName, uid, tc.wantUID) + } + }) + } +} + +// fakeClaimLister is a minimal claimLister for tests, avoiding the need to +// stand up a full *dra.Plugin (kubelet registration, CDI writer, claim +// store, etc.) just to exercise claimCPUsFromContainer's lookup logic. +type fakeClaimLister struct { + claims map[types.UID][]dra.ResultAlloc +} + +func (f *fakeClaimLister) LiveClaimsLocked() map[types.UID][]dra.ResultAlloc { + return f.claims +} + +func (*fakeClaimLister) DriverName() string { + return DRADriverName +} + +func cdiClaimDeviceName(uid types.UID, request, device string, idx int) string { + return DRADriverName + "/device=" + dra.CDIDeviceName(uid, request, device, idx) +} + +func TestClaimCPUsFromContainer(t *testing.T) { + uid := types.UID("claim-uid-1") + name := cdiClaimDeviceName(uid, "myreq", "dev0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: {{Request: "myreq", Device: "dev0", ClassName: "gold", CPUs: "0-1"}}, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if got.UID != uid { + t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) + } + if want := cpuset.MustParse("0-1"); !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) + } + if len(got.ClassCPUs) != 1 { + t.Errorf("claimCPUsFromContainer() classCPUs = %v, want exactly one class", got.ClassCPUs) + } + if want := cpuset.MustParse("0-1"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s", got.CPUs, want) + } +} + +func TestClaimCPUsFromContainerDashedUIDAndDashedDeviceName(t *testing.T) { + // A real k8s UID (embedded dashes) combined with a device name that + // is itself multi-token after sanitization (e.g. "punit-0-0") is exactly + // the case parseCDIClaimUID's fast path alone mis-splits (see + // TestParseCDIClaimUID's "documents limitation" case). claimCPUsFromContainer + // must still resolve it correctly via exact device-name construction. + uid := types.UID("13f7db45-eb2a-4dd1-b0cb-1234567890ab") + name := cdiClaimDeviceName(uid, "myreq", "punit-0-0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: {{Request: "myreq", Device: "punit-0-0", ClassName: "gold", CPUs: "4-5"}}, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if got.UID != uid { + t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) + } + if want := cpuset.MustParse("4-5"); !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) + } + if want := cpuset.MustParse("4-5"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s", got.CPUs, want) + } +} + +func TestClaimCPUsFromContainerUsesOnlyConsumedAllocs(t *testing.T) { + // A single claim can back more than one DeviceRequestAllocationResult + // (e.g. multiple requests in one claim); the union of their CPUs must be + // returned, and (since every alloc here shares the same class) a single + // classCPUs entry covering that whole union. + uid := types.UID("claim-uid-multi") + name0 := cdiClaimDeviceName(uid, "req0", "dev0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name0}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: { + {Request: "req0", Device: "dev0", ClassName: "gold", CPUs: "0-1"}, + {Request: "req1", Device: "dev1", ClassName: "gold", CPUs: "2-3"}, + }, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if want := cpuset.MustParse("0-1"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s (only the consumed allocation)", got.CPUs, want) + } + if want := cpuset.MustParse("0-1"); len(got.ClassCPUs) != 1 || !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs = %v, want {gold: %s}", got.ClassCPUs, want) + } +} + +// TestClaimCPUsFromContainerMultipleClasses covers the MAJOR finding this +// fix addresses: a single ResourceClaim can legitimately contain +// DeviceRequestAllocationResults resolving to devices of *different* +// cpuClasses (the per-punit multi-class-overcommit validation only guards a +// single punit's classes against each other, it does not forbid a claim's +// requests from spanning more than one punit/class). claimCPUsFromContainer +// must group the claimed CPUs by class rather than collapsing to a single +// className, so callers can apply each class only to the CPUs that actually +// belong to it. +func TestClaimCPUsFromContainerMultipleClasses(t *testing.T) { + uid := types.UID("claim-uid-multiclass") + name0 := cdiClaimDeviceName(uid, "req0", "dev0", 0) + name1 := cdiClaimDeviceName(uid, "req1", "dev1", 1) + + c := &mockContainer{cdiDeviceNames: []string{name0, name1}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid: { + {Request: "req0", Device: "dev0", ClassName: "gold", CPUs: "0-1"}, + {Request: "req1", Device: "dev1", ClassName: "silver", CPUs: "2-3"}, + }, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 1 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) + } + got := claims[0] + if got.UID != uid { + t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) + } + if want := cpuset.MustParse("0-3"); !got.CPUs.Equals(want) { + t.Errorf("claimCPUsFromContainer() cpus = %s, want %s (union across classes)", got.CPUs, want) + } + if len(got.ClassCPUs) != 2 { + t.Fatalf("claimCPUsFromContainer() classCPUs = %v, want two distinct classes", got.ClassCPUs) + } + if want := cpuset.MustParse("0-1"); !got.ClassCPUs["gold"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) + } + if want := cpuset.MustParse("2-3"); !got.ClassCPUs["silver"].Equals(want) { + t.Errorf("claimCPUsFromContainer() classCPUs[silver] = %s, want %s", got.ClassCPUs["silver"], want) + } +} + +func TestClaimCPUsFromContainerNoCDIDevices(t *testing.T) { + c := &mockContainer{} + lister := &fakeClaimLister{claims: map[types.UID][]dra.ResultAlloc{}} + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 0 { + t.Errorf("claimCPUsFromContainer() = %v, want empty for a container with no CDI devices", claims) + } +} + +func TestClaimCPUsFromContainerUnknownClaim(t *testing.T) { + // The container carries a well-formed claim device name, but the parsed + // UID has no entry in LiveClaimsLocked() (e.g. the claim was already + // unprepared, or the device belongs to some other driver's namespace). + uid := types.UID("claim-uid-gone") + name := cdiClaimDeviceName(uid, "myreq", "dev0", 0) + + c := &mockContainer{cdiDeviceNames: []string{name}} + lister := &fakeClaimLister{claims: map[types.UID][]dra.ResultAlloc{}} + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 0 { + t.Errorf("claimCPUsFromContainer() = %v, want empty for an unknown/stale claim UID", claims) + } +} + +// TestClaimCPUsFromContainerMultipleDistinctClaims verifies that a +// container whose CDI devices resolve to two distinct live claim UIDs must +// have both claims' CPUs accounted for, not just the first one encountered. +func TestClaimCPUsFromContainerMultipleDistinctClaims(t *testing.T) { + uid1 := types.UID("claim-uid-first") + uid2 := types.UID("claim-uid-second") + name1 := cdiClaimDeviceName(uid1, "req0", "dev0", 0) + name2 := cdiClaimDeviceName(uid2, "req0", "dev1", 0) + + c := &mockContainer{cdiDeviceNames: []string{name1, name2}} + lister := &fakeClaimLister{ + claims: map[types.UID][]dra.ResultAlloc{ + uid1: {{Request: "req0", Device: "dev0", ClassName: "gold", CPUs: "0-1"}}, + uid2: {{Request: "req0", Device: "dev1", ClassName: "silver", CPUs: "2-3"}}, + }, + } + + claims := claimCPUsFromContainer(c, lister) + if len(claims) != 2 { + t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 2 (both distinct live claims must be accounted for)", len(claims)) + } + + byUID := map[types.UID]containerClaim{} + for _, cl := range claims { + byUID[cl.UID] = cl + } + if cl, ok := byUID[uid1]; !ok || !cl.CPUs.Equals(cpuset.MustParse("0-1")) { + t.Errorf("claim %s CPUs = %v, want 0-1", uid1, cl.CPUs) + } + if cl, ok := byUID[uid2]; !ok || !cl.CPUs.Equals(cpuset.MustParse("2-3")) { + t.Errorf("claim %s CPUs = %v, want 2-3", uid2, cl.CPUs) + } +} + +// goldClassCPUs wraps cpus as the single-class classCPUs allocateClaim/ +// remarkClaimInSupply expect, for tests that don't care about the +// multi-class case (see TestAllocateClaimAppliesPerAllocClass for that). +func goldClassCPUs(cpus cpuset.CPUSet) map[string]cpuset.CPUSet { + return map[string]cpuset.CPUSet{"gold": cpus} +} + +// addTestGrant hands out an exclusive grant for container from pool's +// FreeSupply, mirroring (at the level of supply-state mutation) what +// supply.AllocateCPU does for the granting node itself, then records the +// grant in p.allocations so p.releasePool/p.reallocateResources can find it. +// This lets eviction tests exercise the real p.releasePool/grant.Release() +// code path without going through the full container-annotation-driven +// request/offer pipeline (which coldstart_test.go notes is impractical to +// mock with a bare container). +func addTestGrant(t *testing.T, p *policy, pool Node, container cache.Container, exclusive cpuset.CPUSet) Grant { + t.Helper() + + g := newGrant(pool, container, cpuNormal, "", exclusive, 0, memoryDRAM, nil, 0) + + s, ok := pool.FreeSupply().(*supply) + if !ok { + t.Fatalf("pool %q FreeSupply() is not a *supply", pool.Name()) + } + s.isolated = s.isolated.Difference(exclusive) + s.sharable = s.sharable.Difference(exclusive) + g.AccountAllocateCPU() + + p.allocations.addGrant(g) + + return g +} + +func TestAllocateClaimMarksTightestPool(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + cpus := cpuset.New(sharable[0], sharable[1]) + + uid := types.UID("claim-mark") + if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("claimed CPUs %s still present in %q sharable set: %s", cpus, leaf.Name(), got) + } + + // A regular exclusive-CPU request for another container must not be + // able to pick the claimed CPUs. + free := leaf.FreeSupply().AllocatableSharedCPU() + full := free / 1000 + if full < 1 { + t.Fatalf("expected at least 1 full CPU still allocatable on %q, got %dm", leaf.Name(), free) + } + req := &request{full: full, container: &mockContainer{}} + offer, err := leaf.FreeSupply().GetCPUOffer(req) + if err != nil { + t.Fatalf("GetCPUOffer for %d full CPUs failed unexpectedly: %v", full, err) + } + if offer.Intersection(cpus).Size() != 0 { + t.Errorf("CPU offer %s for another container includes claimed CPUs %s", offer, cpus) + } +} + +func TestAllocateClaimOutsideAllowedReturnsError(t *testing.T) { + p := newDRATestPolicy(t) + + // CPU 99999 does not exist on the test system at all, so it can't be a + // subset of any pool's (including root's) statically assigned range. + cpus := cpuset.New(99999) + + if err := p.allocateClaim(types.UID("claim-outside"), cpus, goldClassCPUs(cpus)); err == nil { + t.Fatalf("allocateClaim() with CPUs outside the allowed set: got nil error, want a descriptive error") + } +} + +// TestAllocateClaimSpanningNoPoolReturnsError covers the other poolForCPUs +// failure mode from TestAllocateClaimOutsideAllowedReturnsError: CPUs that +// are individually within p.allowed (each belongs to some pool), but +// straddle two sibling pools so that no single pool's static range is a +// superset of the whole set. A legitimate single-punit DRA CPU pick never +// does this; allocateClaim must still reject it with a descriptive error +// rather than, say, silently marking one of the two pools. +func TestAllocateClaimSpanningNoPoolReturnsError(t *testing.T) { + p := newDRATestPolicy(t) + + // "NUMA node #0" and "NUMA node #2" are siblings under "socket #0" (see + // TestSupplyClaimCPUsAncestorNotDoubleSubtracted): no pool below "root" + // (or "socket #0") is a strict subset spanning both, so a cpuset with + // one CPU from each cannot be contained by any single pool. + leafA := findPoolNode(t, p, "NUMA node #0") + leafB := findPoolNode(t, p, "NUMA node #2") + + cpuA := leafA.GetSupply().SharableCPUs().List() + cpuB := leafB.GetSupply().SharableCPUs().List() + if len(cpuA) < 1 || len(cpuB) < 1 { + t.Fatalf("expected at least 1 CPU on both %q and %q", leafA.Name(), leafB.Name()) + } + spanning := cpuset.New(cpuA[0], cpuB[0]) + + err := p.allocateClaim(types.UID("claim-spanning"), spanning, goldClassCPUs(spanning)) + if err == nil { + t.Fatalf("allocateClaim() with CPUs spanning two pools: got nil error, want a descriptive error") + } + if _, exists := p.claimContainerRefs[types.UID("claim-spanning")]; exists { + t.Errorf("claimContainerRefs unexpectedly populated for a claim that failed to allocate") + } +} + +func TestAllocateClaimRefcountsMultipleContainers(t *testing.T) { + // A ResourceClaim with AllowMultipleAllocations backs more than one + // container; allocateClaim is called once per container sharing it. + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + cpus := cpuset.New(sharable[0]) + uid := types.UID("claim-shared") + + if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { + t.Fatalf("first allocateClaim() failed: %v", err) + } + afterFirst := leaf.FreeSupply().SharableCPUs() + + if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { + t.Fatalf("second allocateClaim() (second container, same claim) failed: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(afterFirst) { + t.Errorf("second allocateClaim() for the same uid changed pool supply: got %s, want unchanged %s", got, afterFirst) + } + if got := p.claimContainerRefs[uid]; got != 2 { + t.Errorf("claimContainerRefs[%s] = %d, want 2 after two containers", uid, got) + } + + // Releasing once (one of the two containers) must not restore the CPUs yet. + if err := p.releaseClaim(uid, cpus); err != nil { + t.Fatalf("first releaseClaim() failed: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("CPUs %s restored after releasing only one of two referencing containers: sharable=%s", cpus, got) + } + + // Releasing the second (last) container must restore the CPUs. + if err := p.releaseClaim(uid, cpus); err != nil { + t.Fatalf("second releaseClaim() failed: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != cpus.Size() { + t.Errorf("CPUs %s not restored after releasing the last referencing container: sharable=%s", cpus, got) + } + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs[%s] still present after refcount reached zero", uid) + } +} + +func TestReleaseClaimUnknownUIDNoop(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + before := leaf.FreeSupply().SharableCPUs() + + if err := p.releaseClaim(types.UID("never-claimed"), cpuset.New(before.List()[0])); err != nil { + t.Errorf("releaseClaim() for an unknown uid: got error %v, want nil (idempotent)", err) + } + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(before) { + t.Errorf("releaseClaim() for an unknown uid changed pool supply: got %s, want unchanged %s", got, before) + } +} + +// TestReleaseClaimResetsCpuClass verifies that releaseClaim is symmetric +// with allocateClaim's cpuClasses.UseClass call: releasing a claim must +// reset the physical cpuClass on the unclaimed CPUs back to the shared-pool +// baseline (mirroring releasePool's resetCpuClass call), not leave them +// stuck in the claim's class for whatever unrelated container the pool +// hands them to next. +func TestReleaseClaimResetsCpuClass(t *testing.T) { + p := newDRATestPolicyWithCPUClasses(t, "shared", "gold") + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + cpu := sharable[0] + claimed := cpuset.New(cpu) + + // A sibling CPU never touched by the claim: its class reflects whatever + // initialize() applied via resetCpuClass("initialize", p.allowed) — the + // shared-pool baseline every allowed CPU starts in. + baselineCPU := sharable[1] + baseline := p.cpuClasses.ClassForCPU(baselineCPU) + + uid := types.UID("claim-cpuclass-reset") + if err := p.allocateClaim(uid, claimed, goldClassCPUs(claimed)); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + if got := p.cpuClasses.ClassForCPU(cpu); got == baseline { + t.Fatalf("test setup error: claimed CPU %d class unchanged (%q) after allocateClaim with class %q", + cpu, got, "gold") + } + + if err := p.releaseClaim(uid, claimed); err != nil { + t.Fatalf("releaseClaim() failed: %v", err) + } + if got := p.cpuClasses.ClassForCPU(cpu); got != baseline { + t.Errorf("claimed CPU %d class = %q after releaseClaim(), want reset back to shared-pool baseline %q", + cpu, got, baseline) + } +} + +// TestAllocateClaimAppliesPerAllocClass covers the MAJOR finding this fix +// addresses: a single DRA claim can resolve to more than one cpuClass +// across its ResultAllocs (see classifyClaimCPUs), and allocateClaim must +// apply each class only to the CPU subset that actually belongs to it, +// rather than applying whichever class happened to come first to the +// claim's entire (unioned) CPU set. +func TestAllocateClaimAppliesPerAllocClass(t *testing.T) { + p := newDRATestPolicyWithCPUClasses(t, "shared", "gold", "silver") + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + goldCPU := cpuset.New(sharable[0]) + silverCPU := cpuset.New(sharable[1]) + claimed := goldCPU.Union(silverCPU) + + uid := types.UID("claim-multiclass") + classCPUs := map[string]cpuset.CPUSet{ + "gold": goldCPU, + "silver": silverCPU, + } + if err := p.allocateClaim(uid, claimed, classCPUs); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + // ClassForCPU may return a synthetic, decorated class name (e.g. with a + // per-die suffix) rather than the literal config name — see + // TestReleaseClaimResetsCpuClass, which compares against a baseline + // instead of a literal string for the same reason. What matters here is + // that the gold- and silver-alloc CPUs end up in *different*, class + // specific buckets: proof that allocateClaim applied each alloc's own + // class to its own CPU subset, instead of the pre-fix behavior of + // picking one class (from the first alloc) and applying it to the whole + // unioned CPU set. + goldGot := p.cpuClasses.ClassForCPU(sharable[0]) + silverGot := p.cpuClasses.ClassForCPU(sharable[1]) + if !strings.HasPrefix(goldGot, "gold") { + t.Errorf("gold-alloc CPU %d class = %q, want a class derived from %q", sharable[0], goldGot, "gold") + } + if !strings.HasPrefix(silverGot, "silver") { + t.Errorf("silver-alloc CPU %d class = %q, want a class derived from %q", sharable[1], silverGot, "silver") + } + if goldGot == silverGot { + t.Errorf("gold-alloc and silver-alloc CPUs ended up in the same class %q; "+ + "per-alloc class application did not take effect", goldGot) + } +} + +func TestAllocateClaimEvictsOverlappingExclusiveGrant(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + victim := &mockContainer{returnValueForGetID: "victim"} + addTestGrant(t, p, leaf, victim, claimed) + + if _, ok := p.allocations.getGrant("victim"); !ok { + t.Fatalf("test setup error: victim grant not present before allocateClaim") + } + + if err := p.allocateClaim(types.UID("evict-claim"), claimed, goldClassCPUs(claimed)); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + // No grant anywhere may still exclusively hold the claimed CPU: either + // the victim's original grant was released outright, or it was + // reallocated to CPUs that no longer overlap the claim. + for _, g := range p.allocations.grants { + if g.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still exclusively granted to %s after eviction", + claimed, g.GetContainer().PrettyName()) + } + } + + // And the claimed CPU must not be free for a new regular allocation. + free := leaf.FreeSupply() + if free.SharableCPUs().Union(free.IsolatedCPUs()).Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still free in %q supply after allocateClaim", claimed, leaf.Name()) + } + + // allocateClaim returned nil, so the evicted victim must actually have + // been reallocated a new grant (not just released and forgotten) — there + // is ample capacity left on this test system for reallocatePool to + // succeed. The new grant's exact shape (exclusive vs. shared/fractional) + // depends on the request reallocatePool derives from the container's own + // declared resource requirements — zero for the bare mockContainer used + // here as "victim" — so only its existence and non-overlap with the + // claimed CPU are asserted, not its exact size/type. + newGrant, ok := p.allocations.getGrant("victim") + if !ok { + t.Fatalf("victim has no grant after allocateClaim() succeeded; eviction must reallocate, not just release") + } + if newGrant.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("victim's new grant %s still overlaps claimed CPU %s", newGrant.ExclusiveCPUs(), claimed) + } +} diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 6307d743e..4be1b32cc 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -20,6 +20,8 @@ import ( "strconv" "time" + "k8s.io/apimachinery/pkg/types" + "github.com/containers/nri-plugins/pkg/agent/podresapi" "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/topology" @@ -77,6 +79,14 @@ type Supply interface { AccountAllocateCPU(Grant) // AccountReleaseCPU accounts for (reinserts) released exclusive capacity into the supply. AccountReleaseCPU(Grant) + // ClaimCPUs marks cpus as claimed by a DRA ResourceClaim (uid), subtracting + // them from isolated/sharable capacity in this supply and, tree-wide, in + // every ancestor supply. Idempotent per uid: a second call for the same + // uid replaces (does not stack on top of) the previous marking. + ClaimCPUs(uid types.UID, cpus cpuset.CPUSet) + // UnclaimCPUs reverses a previous ClaimCPUs marking for uid, restoring the + // claimed CPUs tree-wide. A no-op if uid is unknown. + UnclaimCPUs(uid types.UID) // GetScore calculates how well this supply fits/fulfills the given request. GetScore(Request) Score // AllocatableSharedCPU calculates the allocatable amount of shared CPU of this supply. @@ -236,6 +246,16 @@ type supply struct { sharable cpuset.CPUSet // sharable CPUs at this node grantedReserved int // amount of reserved CPUs allocated grantedShared int // amount of shareable CPUs allocated + + // claimRefs tracks, per DRA claim UID, the CPUs subtracted from isolated/ + // sharable capacity of this supply on behalf of that claim. Marking is + // tree-wide: the same UID is (re)marked in every ancestor supply too (see + // ClaimCPUs/UnclaimCPUs). + claimRefs map[types.UID]cpuset.CPUSet + + // cloned indicates that this supply is a Clone() copy and should not + // propagate ClaimCPUs/UnclaimCPUs to real ancestor nodes. + cloned bool } var _ Supply = &supply{} @@ -322,7 +342,15 @@ func (cs *supply) GetNode() Node { // Clone clones the given CPU supply. func (cs *supply) Clone() Supply { - return newSupply(cs.node, cs.isolated, cs.reserved, cs.sharable, cs.grantedReserved, cs.grantedShared) + clone := newSupply(cs.node, cs.isolated, cs.reserved, cs.sharable, cs.grantedReserved, cs.grantedShared).(*supply) + if len(cs.claimRefs) > 0 { + clone.claimRefs = make(map[types.UID]cpuset.CPUSet, len(cs.claimRefs)) + for uid, cpus := range cs.claimRefs { + clone.claimRefs[uid] = cpus.Clone() + } + } + clone.cloned = true + return clone } // IsolatedCpus returns the isolated CPUSet of this supply. @@ -387,6 +415,69 @@ func (cs *supply) AccountReleaseCPU(g Grant) { cs.sharable = cs.sharable.Union(sharable) } +// ClaimCPUs marks cpus as claimed by a DRA ResourceClaim (uid), subtracting +// them from isolated/reserved/sharable capacity of this supply, then +// propagating the same marking up the node.Parent() chain so every ancestor +// supply also excludes these CPUs from its own isolated/reserved/sharable +// capacity (tree-wide accounting, mirroring AccountAllocateCPU/ +// AccountReleaseCPU). +// +// Reserved CPUs are included because poolForCPUs (pools.go) matches a +// pool's static range as isolated+reserved+sharable: p.allowed (which is +// what the DRA CPU-pick allocator's own "allowed" domain is configured +// from) is not required to exclude p.reserved, so a legitimate DRA pick can +// land on a CPU that is also part of a pool's reserved partition. Without +// subtracting it here too, AllocatableReservedCPU would keep advertising +// that CPU's fractional capacity to ordinary reserved-type grants even +// though a DRA claim already exclusively owns it — a double-booking gap on +// the reserved partition mirroring the one this method already closes for +// isolated/sharable. (Re-pinning any reserved-type container *already* +// running on that CPU is a separate, pre-existing gap: updateSharedAllocations +// explicitly skips cpuType == cpuReserved grants; not addressed here.) +// +// A second ClaimCPUs call for the same uid replaces (rather than stacks on +// top of) the previous marking at each level: the old cpuset for uid is first +// restored, then the new one is subtracted. This makes re-applying claim +// marks after a policy rebuild (Reconfigure/restart) idempotent. +func (cs *supply) ClaimCPUs(uid types.UID, cpus cpuset.CPUSet) { + if old, ok := cs.claimRefs[uid]; ok { + full := cs.node.GetSupply() + cs.isolated = cs.isolated.Union(old.Intersection(full.IsolatedCPUs())) + cs.reserved = cs.reserved.Union(old.Intersection(full.ReservedCPUs())) + cs.sharable = cs.sharable.Union(old.Intersection(full.SharableCPUs())) + } + + if cs.claimRefs == nil { + cs.claimRefs = make(map[types.UID]cpuset.CPUSet) + } + cs.claimRefs[uid] = cpus.Clone() + cs.isolated = cs.isolated.Difference(cpus) + cs.reserved = cs.reserved.Difference(cpus) + cs.sharable = cs.sharable.Difference(cpus) + + if parent := cs.node.Parent(); !cs.cloned && !parent.IsNil() { + parent.FreeSupply().ClaimCPUs(uid, cpus) + } +} + +// UnclaimCPUs reverses a previous ClaimCPUs marking for uid in this supply, +// restoring the claimed CPUs to isolated/reserved/sharable capacity, then +// propagates the same reversal up the node.Parent() chain. A no-op (at every +// level it reaches) for a uid that was never claimed. +func (cs *supply) UnclaimCPUs(uid types.UID) { + if cpus, ok := cs.claimRefs[uid]; ok { + delete(cs.claimRefs, uid) + full := cs.node.GetSupply() + cs.isolated = cs.isolated.Union(cpus.Intersection(full.IsolatedCPUs())) + cs.reserved = cs.reserved.Union(cpus.Intersection(full.ReservedCPUs())) + cs.sharable = cs.sharable.Union(cpus.Intersection(full.SharableCPUs())) + } + + if parent := cs.node.Parent(); !cs.cloned && !parent.IsNil() { + parent.FreeSupply().UnclaimCPUs(uid) + } +} + // Allocate allocates a grant from the supply. func (cs *supply) Allocate(r Request, o *libmem.Offer) (Grant, map[string]libmem.NodeMask, error) { if o == nil { diff --git a/cmd/plugins/topology-aware/policy/resources_test.go b/cmd/plugins/topology-aware/policy/resources_test.go new file mode 100644 index 000000000..ff1035fc7 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/resources_test.go @@ -0,0 +1,462 @@ +// Copyright 2026 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "os" + "path" + "testing" + + "k8s.io/apimachinery/pkg/types" + + cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" + system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/testutils" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// newDRATestPolicy builds a real policy (with a real, multi-level node tree: +// root -> socket -> NUMA node) from the "server" sysfs test data. This gives +// us actual Parent()/Children() wiring, which is what ClaimCPUs/UnclaimCPUs +// tree-wide propagation depends on. +func newDRATestPolicy(t *testing.T) *policy { + t.Helper() + // WithLock must be non-nil: Start() runs reapplyDRAClaims() under + // p.options.WithLock whenever p.draPlugin != nil (tests below set + // p.draPlugin directly), mirroring the real resmgr write lock. + return newDRATestPolicyWithLock(t, func(f func()) { f() }) +} + +// newDRATestPolicyWithLock is newDRATestPolicy with an injectable WithLock, +// letting lock-contract tests (see topology-aware-policy_test.go) observe +// exactly when Start()/Reconfigure() hold the resmgr write lock. +func newDRATestPolicyWithLock(t *testing.T, withLock func(func())) *policy { + t.Helper() + + dir, err := os.MkdirTemp("", "nri-resource-policy-test-sysfs-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + t.Cleanup(func() { removeAll(t, dir) }) + + if err := testutils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { + t.Fatalf("failed to uncompress test sysfs data: %v", err) + } + + sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + if err != nil { + t.Fatalf("failed to discover test system: %v", err) + } + + policyOptions := &policyapi.BackendOptions{ + Cache: &mockCache{}, + System: sys, + Config: &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{ + cfgapi.CPU: "750m", + }, + }, + WithLock: withLock, + } + + p := New().(*policy) + if err := p.Setup(policyOptions); err != nil { + t.Fatalf("failed to set up test policy: %v", err) + } + + return p +} + +// newDRATestPolicyWithCPUClasses is newDRATestPolicy plus a real +// *cpuclass.Handler (p.cpuClasses), configured with sharedClass (used as +// SharedPoolCpuClass, i.e. the class initialize()'s resetCpuClass and +// releaseClaim's resetCpuClass reapply to CPUs no longer exclusively held) +// plus one or more claimClasses (e.g. distinct classes different +// ResultAllocs within one DRA claim can resolve to). Lets tests observe +// cpuClass.UseClass side effects via Handler.ClassForCPU without needing a +// live SST/PCT backend. +func newDRATestPolicyWithCPUClasses(t *testing.T, sharedClass string, claimClasses ...string) *policy { + t.Helper() + + dir, err := os.MkdirTemp("", "nri-resource-policy-test-sysfs-") + if err != nil { + t.Fatalf("failed to create temp dir: %v", err) + } + t.Cleanup(func() { removeAll(t, dir) }) + + if err := testutils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), dir); err != nil { + t.Fatalf("failed to uncompress test sysfs data: %v", err) + } + + sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + if err != nil { + t.Fatalf("failed to discover test system: %v", err) + } + + cpuClasses := []*cfgapi.CPUClass{{Name: sharedClass}} + for _, c := range claimClasses { + cpuClasses = append(cpuClasses, &cfgapi.CPUClass{Name: c}) + } + + policyOptions := &policyapi.BackendOptions{ + Cache: &mockCache{}, + System: sys, + Config: &cfgapi.Config{ + ReservedResources: cfgapi.Constraints{ + cfgapi.CPU: "750m", + }, + CPUClasses: cpuClasses, + SharedPoolCpuClass: sharedClass, + }, + // See newDRATestPolicy: Start() requires a non-nil WithLock whenever + // p.draPlugin is set. + WithLock: func(f func()) { f() }, + } + + p := New().(*policy) + if err := p.Setup(policyOptions); err != nil { + t.Fatalf("failed to set up test policy: %v", err) + } + if p.cpuClasses == nil { + t.Fatalf("test setup error: p.cpuClasses is nil despite non-empty CPUClasses config") + } + + return p +} + +// findPoolNode returns the pool Node with the given name, failing the test if +// it isn't found. +func findPoolNode(t *testing.T, p *policy, name string) Node { + t.Helper() + for _, n := range p.pools { + if n.Name() == name { + return n + } + } + t.Fatalf("no pool node named %q found", name) + return nil +} + +// ancestorsOf returns node's ancestor chain, closest first, by walking +// Parent(). +func ancestorsOf(n Node) []Node { + var ancestors []Node + for parent := n.Parent(); !parent.IsNil(); parent = parent.Parent() { + ancestors = append(ancestors, parent) + } + return ancestors +} + +func TestSupplyClaimCPUsTreeWide(t *testing.T) { + p := newDRATestPolicy(t) + + // "NUMA node #0" is a leaf under "socket #0", which is a child of "root". + leaf := findPoolNode(t, p, "NUMA node #0") + ancestors := ancestorsOf(leaf) + if len(ancestors) == 0 { + t.Fatalf("expected %q to have ancestors, got none", leaf.Name()) + } + + // Pick a couple of sharable CPUs known to belong to this leaf's supply. + claimed := leaf.FreeSupply().SharableCPUs() + if claimed.Size() < 2 { + t.Fatalf("expected leaf %q to have at least 2 sharable CPUs, got %s", leaf.Name(), claimed) + } + // Take exactly two CPUs from the leaf's sharable set. + claimedList := claimed.List() + cpus := cpuset.New(claimedList[0], claimedList[1]) + + uid := types.UID("claim-uid-1") + + leafSharableBefore := leaf.FreeSupply().SharableCPUs() + leafAllocatableBefore := leaf.FreeSupply().AllocatableSharedCPU() + ancestorSharableBefore := make(map[string]cpuset.CPUSet, len(ancestors)) + for _, a := range ancestors { + ancestorSharableBefore[a.Name()] = a.FreeSupply().SharableCPUs() + } + + leaf.FreeSupply().ClaimCPUs(uid, cpus) + + // The leaf's own sharable set must have shrunk by exactly `cpus`. + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(leafSharableBefore.Difference(cpus)) { + t.Errorf("leaf %q sharable CPUs after claim = %s, want %s", + leaf.Name(), got, leafSharableBefore.Difference(cpus)) + } + + // Every ancestor's sharable set must also have shrunk by `cpus`. + for _, a := range ancestors { + want := ancestorSharableBefore[a.Name()].Difference(cpus) + if got := a.FreeSupply().SharableCPUs(); !got.Equals(want) { + t.Errorf("ancestor %q sharable CPUs after claim = %s, want %s", a.Name(), got, want) + } + } + + // The claimed CPUs must no longer be available for a regular allocation + // from the leaf: AllocatableSharedCPU (milli-CPU) must have dropped by + // exactly 2 full CPUs worth (2000m). + freeAfter := leaf.FreeSupply().AllocatableSharedCPU() + if want := leafAllocatableBefore - 2000; freeAfter != want { + t.Errorf("claimed leaf allocatable shared CPU = %dm, want %dm (before claim: %dm)", + freeAfter, want, leafAllocatableBefore) + } + + // A regular exclusive-CPU allocation request for another container must + // not be able to pick the claimed CPUs, no matter how many full CPUs it + // asks for out of what's still nominally available. + full := freeAfter / 1000 + if full < 1 { + t.Fatalf("expected at least 1 full CPU still allocatable on %q after claim, got %dm", leaf.Name(), freeAfter) + } + req := &request{ + full: full, + container: &mockContainer{}, + } + offer, err := leaf.FreeSupply().GetCPUOffer(req) + if err != nil { + t.Fatalf("GetCPUOffer for %d full CPUs failed unexpectedly: %v", full, err) + } + if offer.Intersection(cpus).Size() != 0 { + t.Errorf("CPU offer %s for another container includes claimed CPUs %s", offer, cpus) + } +} + +// TestSupplyClaimCPUsReservedPartition covers the reserved-CPU case +// poolForCPUs (pools.go) allows for but that ClaimCPUs/UnclaimCPUs used to +// ignore: poolForCPUs matches a pool's static range as +// isolated+reserved+sharable, so a DRA claim can legitimately land on a CPU +// that is part of a pool's reserved partition (the DRA CPU-pick allocator's +// own "allowed" domain, p.allowed, is not required to exclude p.reserved). +// ClaimCPUs must subtract from the reserved cpuset too, or +// AllocatableReservedCPU would keep advertising that CPU's capacity to +// ordinary reserved-type grants even though DRA already owns it exclusively +// (a double-booking gap on the reserved partition). +func TestSupplyClaimCPUsReservedPartition(t *testing.T) { + p := newDRATestPolicy(t) + + var reservedPool Node + for _, n := range p.pools { + if n.GetSupply().ReservedCPUs().Size() > 0 { + reservedPool = n + break + } + } + if reservedPool == nil { + t.Fatalf("test setup error: no pool in the test topology has a non-empty reserved partition") + } + + reservedSupply, ok := reservedPool.FreeSupply().(*supply) + if !ok { + t.Fatalf("pool %q FreeSupply() is not a *supply", reservedPool.Name()) + } + + reserved := reservedSupply.ReservedCPUs() + cpus := cpuset.New(reserved.List()[0]) + uid := types.UID("claim-uid-reserved") + + reservedAllocatableBefore := reservedSupply.AllocatableReservedCPU() + + reservedSupply.ClaimCPUs(uid, cpus) + + if got := reservedSupply.ReservedCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("claimed reserved CPU %s still present in %q reserved set after ClaimCPUs: %s", + cpus, reservedPool.Name(), got) + } + // AllocatableReservedCPU has a special sentinel: it returns -1 (not a + // proportional milliCPU amount) once the reserved cpuset becomes + // entirely empty, rather than when its granted capacity merely drops to + // zero. Claiming the last reserved CPU on this pool hits that sentinel; + // claiming one of several would not. + wantAfterClaim := reservedAllocatableBefore - 1000 + if reserved.Difference(cpus).IsEmpty() { + wantAfterClaim = -1 + } + if got := reservedSupply.AllocatableReservedCPU(); got != wantAfterClaim { + t.Errorf("%q allocatable reserved CPU after claiming 1 reserved CPU = %dm, want %dm", + reservedPool.Name(), got, wantAfterClaim) + } + + reservedSupply.UnclaimCPUs(uid) + + if got := reservedSupply.ReservedCPUs(); !got.Equals(reserved) { + t.Errorf("%q reserved set after UnclaimCPUs = %s, want restored %s", reservedPool.Name(), got, reserved) + } + if got := reservedSupply.AllocatableReservedCPU(); got != reservedAllocatableBefore { + t.Errorf("%q allocatable reserved CPU after UnclaimCPUs = %dm, want restored %dm", + reservedPool.Name(), got, reservedAllocatableBefore) + } +} + +func TestSupplyClaimCPUsIdempotentReplace(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + + uid := types.UID("claim-uid-replace") + cpusA := cpuset.New(sharable[0]) + cpusB := cpuset.New(sharable[1]) + + before := leaf.FreeSupply().SharableCPUs() + + leaf.FreeSupply().ClaimCPUs(uid, cpusA) + leaf.FreeSupply().ClaimCPUs(uid, cpusB) + + // Only cpusB should be subtracted; cpusA must have been restored when the + // second ClaimCPUs call replaced the mark for the same uid. + want := before.Difference(cpusB) + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(want) { + t.Errorf("sharable CPUs after replacing claim = %s, want %s (cpusA=%s should be restored, cpusB=%s subtracted)", + got, want, cpusA, cpusB) + } + + // And ancestors must reflect the same non-stacked (single) subtraction. + for _, a := range ancestorsOf(leaf) { + full := a.FreeSupply().SharableCPUs() + if full.Intersection(cpusA).Size() != cpusA.Size() { + t.Errorf("ancestor %q is missing cpusA=%s (claim replace should have restored it there too): sharable=%s", + a.Name(), cpusA, full) + } + if full.Intersection(cpusB).Size() != 0 { + t.Errorf("ancestor %q still contains cpusB=%s that should be claimed: sharable=%s", + a.Name(), cpusB, full) + } + } +} + +func TestSupplyUnclaimCPUsRestores(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + ancestors := ancestorsOf(leaf) + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + + uid := types.UID("claim-uid-unclaim") + cpus := cpuset.New(sharable[0]) + + leafBefore := leaf.FreeSupply().SharableCPUs() + ancestorBefore := make(map[string]cpuset.CPUSet, len(ancestors)) + for _, a := range ancestors { + ancestorBefore[a.Name()] = a.FreeSupply().SharableCPUs() + } + + leaf.FreeSupply().ClaimCPUs(uid, cpus) + leaf.FreeSupply().UnclaimCPUs(uid) + + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(leafBefore) { + t.Errorf("leaf %q sharable CPUs after unclaim = %s, want restored %s", leaf.Name(), got, leafBefore) + } + for _, a := range ancestors { + if got := a.FreeSupply().SharableCPUs(); !got.Equals(ancestorBefore[a.Name()]) { + t.Errorf("ancestor %q sharable CPUs after unclaim = %s, want restored %s", + a.Name(), got, ancestorBefore[a.Name()]) + } + } +} + +func TestSupplyUnclaimCPUsUnknownUIDNoop(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + before := leaf.FreeSupply().SharableCPUs() + + defer func() { + if r := recover(); r != nil { + t.Fatalf("UnclaimCPUs for an unknown UID must not panic, got: %v", r) + } + }() + + leaf.FreeSupply().UnclaimCPUs(types.UID("never-claimed")) + + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(before) { + t.Errorf("sharable CPUs changed after unclaiming an unknown UID: got %s, want unchanged %s", got, before) + } +} + +func TestSupplyCloneCarriesClaimRefs(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + + uid := types.UID("claim-uid-clone") + cpus := cpuset.New(sharable[0]) + + leaf.FreeSupply().ClaimCPUs(uid, cpus) + + clone := leaf.FreeSupply().Clone() + + // The clone must already reflect the subtraction (it was taken after the + // claim), and — because claimRefs travels with Clone() — unclaiming on + // the clone must be able to restore the CPU, proving the clone knows + // which cpus belong to uid. + beforeUnclaim := clone.SharableCPUs() + if beforeUnclaim.Intersection(cpus).Size() != 0 { + t.Fatalf("clone did not inherit the claimed subtraction: sharable=%s still contains %s", beforeUnclaim, cpus) + } + + clone.UnclaimCPUs(uid) + + afterUnclaim := clone.SharableCPUs() + if afterUnclaim.Intersection(cpus).Size() != cpus.Size() { + t.Errorf("clone.UnclaimCPUs(%s) did not restore %s: sharable=%s -- Clone() must carry claimRefs", + uid, cpus, afterUnclaim) + } + + // The original leaf supply must be unaffected by unclaiming on the clone. + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(cpus).Size() != 0 { + t.Errorf("unclaiming on the clone leaked back into the original supply: sharable=%s still missing %s", got, cpus) + } +} + +func TestSupplyClaimCPUsAncestorNotDoubleSubtracted(t *testing.T) { + p := newDRATestPolicy(t) + + // "NUMA node #0" and "NUMA node #2" are siblings under "socket #0". + leafA := findPoolNode(t, p, "NUMA node #0") + leafB := findPoolNode(t, p, "NUMA node #2") + ancestor := findPoolNode(t, p, "socket #0") + + if leafA.Parent().Name() != ancestor.Name() || leafB.Parent().Name() != ancestor.Name() { + t.Fatalf("expected %q and %q to share parent %q; got parents %q and %q", + leafA.Name(), leafB.Name(), ancestor.Name(), leafA.Parent().Name(), leafB.Parent().Name()) + } + + cpusA := cpuset.New(leafA.FreeSupply().SharableCPUs().List()[0]) + cpusB := cpuset.New(leafB.FreeSupply().SharableCPUs().List()[0]) + if cpusA.Intersection(cpusB).Size() != 0 { + t.Fatalf("test setup error: cpusA and cpusB must be disjoint, got %s and %s", cpusA, cpusB) + } + + ancestorBefore := ancestor.FreeSupply().SharableCPUs() + + leafA.FreeSupply().ClaimCPUs(types.UID("claim-a"), cpusA) + leafB.FreeSupply().ClaimCPUs(types.UID("claim-b"), cpusB) + + want := ancestorBefore.Difference(cpusA).Difference(cpusB) + if got := ancestor.FreeSupply().SharableCPUs(); !got.Equals(want) { + t.Errorf("ancestor %q sharable CPUs after two independent child claims = %s, want %s (exactly one subtraction per claim, no double-subtraction)", + ancestor.Name(), got, want) + } +} diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index e9dc71036..cf1489eb1 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -15,17 +15,20 @@ package topologyaware import ( + "context" "errors" "fmt" "github.com/containers/nri-plugins/pkg/irq" "github.com/containers/nri-plugins/pkg/utils/cpuset" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/apimachinery/pkg/types" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/cpuallocator" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" + "github.com/containers/nri-plugins/pkg/resmgr/dra" "github.com/containers/nri-plugins/pkg/resmgr/events" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" @@ -74,6 +77,60 @@ type policy struct { cpuClasses *cpuclass.Handler // CPU class handler (cpufreq, SST/PCT, etc.) metrics *TopologyAwareMetrics // metrics provided by this policy irqCnt int // last applied [allocations.]irqCnt + + // claimContainerRefs counts, per DRA ResourceClaim UID, how many live + // containers currently reference that claim's CPUs (allocateClaim + // increments, releaseClaim decrements). The pool supply is marked via + // Supply.ClaimCPUs on the first container and unmarked via + // Supply.UnclaimCPUs only once the last referencing container is + // released — this is what makes a multi-container ResourceClaim + // (AllowMultipleAllocations) safe. + claimContainerRefs map[types.UID]int + + // tombstonedDRAClaims holds the allocation results for claims that have + // been unprepared (UnprepareResourceClaims) while at least one container + // was still using them. unprepareDRAClaim defers both the pool unclaim + // and the HP-CPU release until the last container drops; releaseClaim + // drains this map when the refcount reaches zero. + tombstonedDRAClaims map[types.UID][]dra.ResultAlloc + + // claimedCPUsByContainer is, per container ID, the union of every live + // DRA claim's CPUs that container consumes (populated/cleared in + // AllocateResources/ReleaseResources alongside allocateClaim/ + // releaseClaim, and repopulated by reapplyDRAClaims after a restart/ + // Reconfigure). applyGrant/updateSharedAllocations union this in before + // pinning a container's cpuset, so a claim consumer's normal grant + // (computed independently, without regard to its claimed CPUs) doesn't + // end up excluding the CPUs its CDI-injected NRI_CPU env vars claim + // it has. + claimedCPUsByContainer map[string]cpuset.CPUSet + + // draClaimsByContainer keeps the claims consumed by each container + // independently of the plugin's live-claim map, which may lose a claim + // during UnprepareResourceClaims before the container is released. + draClaimsByContainer map[string][]containerClaim + + // draPlugin is the DRA kubelet plugin instance for this driver, or nil + // when DRA is disabled (cfg.DRAEnabled() == false) or Setup() could not + // build one (see buildDRAPlugin in dra.go: missing kube client or node + // name at Setup() time — an empty cpuClasses configuration no longer + // prevents construction; the plugin is built with an empty device set + // instead). AllocateResources + // and ReleaseResources nil-check this field before passing it anywhere a + // claimLister is expected: a nil *dra.Plugin handed to an interface + // parameter would produce a non-nil interface wrapping a nil pointer + // (the typed-nil trap), so callers must guard on p.draPlugin != nil + // themselves rather than relying on claimCPUsFromContainer's internal + // nil check. + draPlugin *dra.Plugin + // draCtxCancel cancels the context draPlugin.Start() was given. Called + // from Stop() to shut the DRA plugin's background goroutines down. nil + // when draPlugin is nil. + draCtxCancel context.CancelFunc + // cdiDir is the directory DRA CDI spec files are written to. Empty + // means the dra package's default (/var/run/cdi). Overridable so tests + // can inject a temporary directory. + cdiDir string } var opt = &cfgapi.Config{} @@ -127,6 +184,15 @@ func (p *policy) Setup(opts *policyapi.BackendOptions) error { return policyError("failed to initialize %s policy: %w", PolicyName, err) } + // Build the DRA plugin, if enabled, once at initial Setup() time. There + // is no DRAEnabled-flip check here — see buildDRAPlugin's doc comment + // (dra.go) for why that check belongs in Reconfigure() instead. + if p.cfg.DRAEnabled() { + if err := p.buildDRAPlugin(opts); err != nil { + return policyError("failed to initialize %s policy: %w", PolicyName, err) + } + } + log.Infof("***** default CPU priority is %s", defaultPrio) return nil @@ -144,6 +210,46 @@ func (p *policy) Description() string { // Start prepares this policy for accepting allocation/release requests. func (p *policy) Start() error { + if err := p.restoreCache(); err != nil { + return policyError("failed to start: %v", err) + } + + // Start the DRA plugin (if built by Setup()) before reapplyDRAClaims: + // draPlugin.Start(ctx) loads the persisted ClaimStore, which + // reapplyDRAClaims reads via LiveClaimsLocked() to re-mark pool + // supplies. Start/PublishResources are not made while holding the + // resmgr lock — they take their own internal WithLock only for the + // specific sections that touch shared Handler/claim state. However, + // once draPlugin.Start(ctx) returns, the kubelet plugin is registered + // and may immediately start serving PrepareResourceClaims/ + // UnprepareResourceClaims RPCs in background goroutines; those mutate + // p.claims under deps.WithLock (= the resmgr write lock). reapplyDRAClaims + // (via LiveClaimsLocked) reads p.claims and therefore must also run under + // that same lock to avoid a concurrent unsynchronized map access. + if p.draPlugin != nil { + ctx, cancel := context.WithCancel(context.Background()) + p.draCtxCancel = cancel + if err := p.draPlugin.Start(ctx); err != nil { + cancel() + return policyError("failed to start DRA plugin: %v", err) + } + if err := p.draPlugin.PublishResources(ctx); err != nil { + cancel() + p.draPlugin.Stop() + return policyError("failed to publish DRA resources: %v", err) + } + + // Re-mark any DRA-claimed CPUs in the freshly rebuilt pool + // supplies. Must run under the resmgr write lock (see above); + // p.options.WithLock is the same closure the DRA plugin itself + // uses (dra.Deps.WithLock), and reapplyDRAClaims/its helpers + // (evictOverlappingGrants, remarkClaimInSupply, reallocateEvicted) + // do not call WithLock themselves, so this cannot deadlock. + p.options.WithLock(func() { + p.reapplyDRAClaims() + }) + } + // Turn coldstart forcibly off if we have movable non-DRAM memory. // Note that although this can change dynamically we only check it // during startup and trust users to either not fiddle with memory @@ -163,6 +269,20 @@ func (p *policy) Start() error { return nil } +// Stop shuts down this policy: it cancels the DRA plugin's context and +// stops the DRA plugin (kubeletplugin registration, background helper +// goroutines), if one was built. A no-op when DRA is disabled +// (draPlugin == nil) — safe to call regardless of whether Start() ran. +func (p *policy) Stop() error { + if p.draCtxCancel != nil { + p.draCtxCancel() + } + if p.draPlugin != nil { + p.draPlugin.Stop() + } + return nil +} + // Sync synchronizes the state of this policy. func (p *policy) Sync(add []cache.Container, del []cache.Container) error { irq.BlockWrites() @@ -243,9 +363,30 @@ func (p *policy) AllocateResources(container cache.Container) error { defer p.commitCpuClasses(container.PrettyName()) defer p.applyIrqAffinity(container.PrettyName()) + defer p.triggerDRARepublish() + + var markedClaims []containerClaim + if p.draPlugin != nil { + for _, cl := range claimCPUsFromContainer(container, p.draPlugin) { + if err := p.allocateClaim(cl.UID, cl.CPUs, cl.ClassCPUs); err != nil { + p.rollbackClaimMarks(container, markedClaims) + return policyError("failed to allocate resources for %s: %v", + container.PrettyName(), err) + } + markedClaims = append(markedClaims, cl) + } + if len(markedClaims) > 0 { + p.setClaimedCPUs(container, markedClaims) + if p.draClaimsByContainer == nil { + p.draClaimsByContainer = make(map[string][]containerClaim) + } + p.draClaimsByContainer[container.GetID()] = markedClaims + } + } err := p.allocateResources(container, "") if err != nil { + p.rollbackClaimMarks(container, markedClaims) return err } @@ -257,6 +398,41 @@ func (p *policy) AllocateResources(container cache.Container) error { return nil } +// setClaimedCPUs unions every claim's CPUs in marked and records the result +// in p.claimedCPUsByContainer, keyed by container's ID. Used by +// AllocateResources after successfully marking every live claim a container +// carries, and by reapplyDRAClaims to repopulate the map after a restart/ +// Reconfigure (see its own comment for why that repopulation alone isn't +// sufficient there). +func (p *policy) setClaimedCPUs(container cache.Container, marked []containerClaim) { + union := cpuset.New() + for _, cl := range marked { + union = union.Union(cl.CPUs) + } + if p.claimedCPUsByContainer == nil { + p.claimedCPUsByContainer = map[string]cpuset.CPUSet{} + } + p.claimedCPUsByContainer[container.GetID()] = union +} + +// rollbackClaimMarks releases every claim mark accumulated so far in an +// AllocateResources call that failed partway through (whether from a later +// claim's own allocateClaim call or from the subsequent normal pool +// allocation), so a partial failure never leaks a claimContainerRefs entry +// (or a stale claimedCPUsByContainer union) for a container that ultimately +// never got its resources allocated. +func (p *policy) rollbackClaimMarks(container cache.Container, marked []containerClaim) { + for _, cl := range marked { + if err := p.releaseClaim(cl.UID, cl.CPUs); err != nil { + log.Errorf("dra: rollback: failed to release claim %s: %v", cl.UID, err) + } + } + if len(marked) > 0 { + delete(p.draClaimsByContainer, container.GetID()) + delete(p.claimedCPUsByContainer, container.GetID()) + } +} + func (p *policy) allocateResources(container cache.Container, poolHint string) error { grant, err := p.allocatePool(container, poolHint) if err != nil { @@ -278,6 +454,21 @@ func (p *policy) ReleaseResources(container cache.Container) error { defer p.commitCpuClasses(container.PrettyName()) defer p.applyIrqAffinity(container.PrettyName()) + defer p.triggerDRARepublish() + + if p.draPlugin != nil { + claims := p.draClaimsByContainer[container.GetID()] + if claims == nil { + claims = claimCPUsFromContainer(container, p.draPlugin) + } + for _, cl := range claims { + if err := p.releaseClaim(cl.UID, cl.CPUs); err != nil { + log.Errorf("failed to release DRA claim for %s: %v", container.PrettyName(), err) + } + } + delete(p.draClaimsByContainer, container.GetID()) + delete(p.claimedCPUsByContainer, container.GetID()) + } if grant, found := p.releasePool(container); found { p.updateSharedAllocations(&grant) @@ -300,6 +491,7 @@ func (p *policy) UpdateResources(container cache.Container) error { defer p.commitCpuClasses(container.PrettyName()) defer p.applyIrqAffinity(container.PrettyName()) + defer p.triggerDRARepublish() grant, found := p.releasePool(container) if !found { @@ -537,6 +729,18 @@ func (p *policy) Reconfigure(newCfg any) error { savedPolicy := *p allocations := savedPolicy.allocations.clone() + // DRA config changes cannot be applied live: buildDRAPlugin only ever + // runs once, from the initial Setup() (see its doc comment in dra.go + // for why this check cannot live there) — Reconfigure() never tears + // down or (re)builds p.draPlugin, so there is no way to safely apply a + // dra.enabled flip or a cpuClass attribute change (which could + // invalidate a live DRA claim) without a restart. Refuse any config + // change outright whenever DRA is (or was) enabled. + if cfg.DRAEnabled() || p.cfg.DRAEnabled() { + return policyError("failed to reconfigure: DRA config changes require a restart " + + "(dra.enabled, cpuClass changes, etc. cannot be applied via live reconfigure)") + } + opt = cfg p.cfg = cfg defaultPrio = cfg.DefaultCPUPriority.Value() @@ -546,6 +750,8 @@ func (p *policy) Reconfigure(newCfg any) error { if err := p.initialize(); err != nil { *p = savedPolicy + opt = p.cfg + defaultPrio = p.cfg.DefaultCPUPriority.Value() return policyError("failed to reconfigure: %v", err) } @@ -566,6 +772,7 @@ func (p *policy) Reconfigure(newCfg any) error { if err := p.restoreAllocations(&allocations); err != nil { *p = savedPolicy opt = p.cfg + defaultPrio = p.cfg.DefaultCPUPriority.Value() return policyError("failed to reconfigure: %v", err) } @@ -739,6 +946,172 @@ func (p *policy) findExistingTopologyLevel(level cfgapi.CPUTopologyLevel) cfgapi return cfgapi.CPUTopologyLevelPackage } +func (p *policy) restoreCache() error { + allocations := p.newAllocations() + if p.cache.GetPolicyEntry(keyAllocations, &allocations) { + if err := p.restoreAllocations(&allocations); err != nil { + return policyError("failed to restore allocations from cache: %v", err) + } + p.allocations.Dump(log.Infof, "restored ") + } + p.saveAllocations() + + return nil +} + +// remarkClaimInSupply marks cpus as claimed by DRA ResourceClaim uid in the +// tightest pool that fully contains them (tree-wide, via +// Supply.ClaimCPUs — see resources.go), without touching +// claimContainerRefs. This is the marking-only counterpart to allocateClaim +// (pools.go): reapplyDRAClaims uses it to restore Supply.claimRefs after +// Start()/Reconfigure() rebuild pool/supply state in initialize(), which +// discards any marks a prior allocateClaim call applied. +// +// Using allocateClaim here instead of this marking-only path would +// double-count in the Reconfigure() case: p.claimContainerRefs is an +// in-process map that Reconfigure() never resets, so containers backing a +// live claim are already reflected in it from the AllocateResources call +// that admitted them. +// +// That reasoning does NOT hold across a process restart: claimContainerRefs +// is a plain in-memory map with no persistence, so it is zero-valued right +// after Start(), even though containers backed by live claims are already +// running. The correct refcount is rebuilt indirectly: pkg/resmgr/nri.go's +// syncWithNRI/Synchronize forces every already-running container through +// ReleaseResources (a no-op here, since the refcount is already 0) followed +// by AllocateResources (which calls allocateClaim and increments the +// refcount) as part of the NRI resync that always follows agent Start(). +// reapplyDRAClaims only has to fix up Supply.claimRefs (pool CPU exclusion) +// for the window between Start() and that resync; claimContainerRefs catches +// up once the resync runs. If that syncWithNRI invariant ever changes (e.g. +// running containers stop being included in both the "allocated" and +// "released" lists), releaseClaim will silently no-op on the eventual real +// ReleaseResources (refcount already 0) and the claimed CPUs will leak out +// of pool capacity permanently, until the next restart. +// +// Also re-applies the physical cpuClass (className) to cpus: initialize() +// (called by both Start() and Reconfigure() before this runs) resets every +// allowed CPU's class back to the shared-pool default +// (resetCpuClass("initialize", p.allowed)), which would otherwise silently +// strip the SST-CP/EPP/governor settings a live DRA claim depends on. +func (p *policy) remarkClaimInSupply(uid types.UID, cpus cpuset.CPUSet, classCPUs map[string]cpuset.CPUSet) error { + if cpus.IsEmpty() { + return policyError("cannot remark DRA claim %s: empty CPU set", uid) + } + + pool, err := p.poolForCPUs(cpus) + if err != nil { + return policyError("cannot remark DRA claim %s (CPUs %s): %v", uid, cpus, err) + } + + pool.FreeSupply().ClaimCPUs(uid, cpus) + + // classCPUs groups cpus by cpuClass (see classifyClaimCPUs): applied per + // subset so a claim spanning more than one class re-applies each class + // only to the CPUs that actually belong to it. Best-effort on the + // restart/reconfigure reapply path: log but do not fail the remark. + if err := p.applyClassCPUs("re-apply", uid, classCPUs); err != nil { + log.Errorf("dra: %v", err) + } + + return nil +} + +// reapplyDRAClaims re-marks pool supplies for every currently live DRA +// claim, restoring the Supply.claimRefs bookkeeping that Start()'s +// restoreCache() and Reconfigure()'s restoreAllocations() lose whenever +// initialize() rebuilds the pool/supply tree from scratch. It is a no-op if +// DRA is disabled (draPlugin == nil). +// +// reapplyDRAClaims runs *after* restoreCache()/restoreAllocations() have +// already reinstated grants (see Start()/Reconfigure()), at a point where +// Supply.claimRefs has just been wiped by initialize() and not yet re-marked. +// reinstateGrants/reallocateResources are therefore unaware of live DRA +// claims while they run: if grant restoration (verbatim reinstatement, or +// its allocatePool-based fallback) happens to hand a regular container CPUs +// that a live claim already owns, that overlap is a real double-booking — +// two workloads pinned to the same physical CPUs. Before marking each +// claim's CPUs here, evict and requeue for reallocation any restored grant +// that overlaps them, exactly like allocateClaim's first-time eviction path +// (evictOverlappingGrants/reallocateEvicted, pools.go) — this does not touch +// claimContainerRefs, so it stays consistent with remarkClaimInSupply's +// marking-only contract. +// +// Caller must hold the resmgr write lock (LiveClaimsLocked's contract). +// Reconfigure() already runs under the caller's lock, so it calls this +// directly. Start() runs unlocked (see its comment), so it must establish +// the lock itself via p.options.WithLock(func() { p.reapplyDRAClaims() }) — +// do not call reapplyDRAClaims from inside a callback that is nested inside +// an *already held* WithLock/resmgr-lock scope, since the lock is not +// reentrant and a second acquisition would deadlock. +func (p *policy) reapplyDRAClaims() { + if p.draPlugin == nil { + return + } + + remarked := false + for uid, allocs := range p.draPlugin.LiveClaimsLocked() { + cpus, classCPUs := classifyClaimCPUs(uid, allocs) + + if cpus.IsEmpty() { + continue + } + + evicted, evictedCpusets := p.evictOverlappingGrants(cpus, fmt.Sprintf("reapplyDRAClaims: claim %s", uid)) + + if err := p.remarkClaimInSupply(uid, cpus, classCPUs); err != nil { + log.Errorf("dra: reapplyDRAClaims: %v", err) + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + log.Errorf("dra: reapplyDRAClaims: failed to restore grants after claim %s could not be remarked: %v", uid, reallocErr) + } + continue + } + remarked = true + + if err := p.reallocateEvicted(evicted, evictedCpusets, cpus, uid); err != nil { + log.Errorf("dra: reapplyDRAClaims: claim %s: evicted %d restored grant(s) overlapping "+ + "claimed CPUs %s but failed to fully reallocate them: %v", uid, len(evicted), cpus, err) + } + } + + // claimedCPUsByContainer is a plain in-memory map with no persistence, + // so it is empty here even though pool accounting above is now + // correct. A prepared claim with no consumer is explicitly unclaimed + // through the DRA plugin's ClaimUnprepare callback, rather than relying + // on a later syncWithNRI resync. restoreCache()'s/Reconfigure()'s own + // restoreAllocations() call already ran applyGrant for these + // containers *before* reapplyDRAClaims was ever reached, without the + // union -- their cpusets need an explicit re-pin here, now, not just + // the map. Re-resolve every live claim's consuming container(s) and + // repopulate the map, then re-pin unconditionally (not gated on + // remarked/updateSharedAllocations's shared-portion check below, which + // skips exactly the common plain-exclusive-CPU claim consumer case). + for _, c := range p.cache.GetContainers() { + claims := claimCPUsFromContainer(c, p.draPlugin) + if len(claims) == 0 { + continue + } + p.setClaimedCPUs(c, claims) + + if grant, ok := p.allocations.getGrant(c.GetID()); ok { + p.applyGrant(grant) + } else if opt.PinCPU { + union := p.claimedCPUsByContainer[c.GetID()] + p.setPreferredCpusetCpus(c, cpuset.New(), union, + fmt.Sprintf(" => re-pinning %s to claimed cpuset %s (no regular grant)", c.PrettyName(), union)) + } + } + + // Re-marking may have subtracted CPUs from one or more pools' sharable + // capacity; any container already pinned (via applyGrant, from before + // the rebuild) to the previous, wider sharable cpuset must be re-pinned + // to the now-reduced set so it cannot keep running on CPUs a live DRA + // claim exclusively owns. + if remarked { + p.updateSharedAllocations(nil) + } +} + func (p *policy) checkColdstartOff() { for _, id := range p.sys.NodeIDs() { node := p.sys.Node(id) diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go b/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go new file mode 100644 index 000000000..8305f91a9 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go @@ -0,0 +1,863 @@ +// Copyright 2026 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "context" + "fmt" + "sync" + "testing" + + corev1 "k8s.io/api/core/v1" + resourceapi "k8s.io/api/resource/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/client-go/kubernetes/fake" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/dra" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// test helpers: a minimal real *dra.Plugin seeded with a live +// claim via PrepareResourceClaims. Building a real Plugin (rather than +// re-deriving the claimLister interface with a fake) is what lets these +// tests exercise the exact typed-nil-trap-prone p.draPlugin field the +// production AllocateResources/ReleaseResources call sites use. ---- + +// fakeDRADeviceLister is a dra.DeviceLister that always returns a fixed list +// of devices, regardless of driverName. +type fakeDRADeviceLister struct { + devices []resourceapi.Device +} + +func (f *fakeDRADeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { + return f.devices, nil +} + +// fakeDRAClaimAllocator is a dra.ClaimAllocator that always "picks" a fixed +// CPUSet and reports every class as HP-eligible. Good enough for +// PrepareResourceClaims to succeed; the pool-accounting side effects this +// package cares about (Supply.ClaimCPUs/UnclaimCPUs) are exercised +// separately via allocateClaim/releaseClaim, not via this allocator. +type fakeDRAClaimAllocator struct { + pick cpuset.CPUSet +} + +func (f *fakeDRAClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { + return f.pick, nil +} +func (f *fakeDRAClaimAllocator) ReleaseHpCpus(_, _ int, _ cpuset.CPUSet) {} +func (f *fakeDRAClaimAllocator) AccountHpCpus(_, _ int, _ cpuset.CPUSet) error { return nil } +func (f *fakeDRAClaimAllocator) IsHPClass(_ string) bool { return true } + +// fakeDRACDIWriter is a dra.CDIWriter that tracks per-UID "written" state +// in memory instead of touching disk. Stateful (rather than a fixed +// false/nil) so that ClaimSpecExists/ListClaims accurately reflect prior +// WriteClaim/RemoveClaim calls: dra.Plugin.Start()'s orphan-claim sweep +// calls ClaimSpecExists for every persisted claim and drops any claim it +// reports as missing. +type fakeDRACDIWriter struct { + written map[types.UID]bool +} + +func (w *fakeDRACDIWriter) WriteClaim(uid types.UID, _ []dra.CDIDevice) error { + if w.written == nil { + w.written = map[types.UID]bool{} + } + w.written[uid] = true + return nil +} +func (w *fakeDRACDIWriter) RemoveClaim(uid types.UID) error { + delete(w.written, uid) + return nil +} +func (w *fakeDRACDIWriter) ClaimSpecExists(uid types.UID) bool { return w.written[uid] } +func (w *fakeDRACDIWriter) ListClaims() ([]types.UID, error) { + uids := make([]types.UID, 0, len(w.written)) + for uid := range w.written { + uids = append(uids, uid) + } + return uids, nil +} + +// fakeDRAClaimStore is a dra.ClaimStore that succeeds without persisting +// anything. +type fakeDRAClaimStore struct{} + +func (*fakeDRAClaimStore) Save(map[types.UID]*dra.ClaimState) error { return nil } +func (*fakeDRAClaimStore) Load() (map[types.UID]*dra.ClaimState, error) { return nil, nil } + +// newTestDRAPlugin builds a real *dra.Plugin backed entirely by fakes, ready +// for PrepareResourceClaims calls. pick is the CPUSet the fake allocator +// hands out for every PickHpCpus call. +func newTestDRAPlugin(t *testing.T, pick cpuset.CPUSet, deviceName string) *dra.Plugin { + t.Helper() + return newTestDRAPluginWithLock(t, pick, deviceName, func(f func()) { f() }) +} + +// newTestDRAPluginWithLock is newTestDRAPlugin with an injectable WithLock, +// letting lock-contract tests share a single non-reentrant stub between the +// DRA plugin's deps.WithLock and the policy's options.WithLock (both are +// backed by the same resmgr write lock in production). +func newTestDRAPluginWithLock(t *testing.T, pick cpuset.CPUSet, deviceName string, withLock func(func())) *dra.Plugin { + t.Helper() + + className := "gold" + device := resourceapi.Device{ + Name: deviceName, + Attributes: map[resourceapi.QualifiedName]resourceapi.DeviceAttribute{ + "nri/cpuClass": {StringValue: &className}, + "nri/packageID": {IntValue: new(int64)}, + "nri/punitID": {IntValue: new(int64)}, + }, + } + + deps := dra.Deps{ + KubeClient: fake.NewClientset(), + NodeName: "test-node", + RegistrarDir: t.TempDir(), + PluginDataDir: t.TempDir(), + ValidateClasses: func() error { return nil }, + ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + DeviceLister: &fakeDRADeviceLister{devices: []resourceapi.Device{device}}, + ClaimAllocator: &fakeDRAClaimAllocator{pick: pick}, + CDIWriter: &fakeDRACDIWriter{}, + ClaimStore: &fakeDRAClaimStore{}, + WithLock: withLock, + ClaimUnprepare: func(_ types.UID, _ []dra.ResultAlloc) {}, + Logger: log, + } + + p, err := dra.New(DRADriverName, deps) + if err != nil { + t.Fatalf("dra.New() failed: %v", err) + } + return p +} + +// seedLiveClaim runs PrepareResourceClaims for a single claim UID against +// plugin, returning the qualified CDI device name the runtime would see on +// the container's CDIDevices (the same string GetCDIDeviceNames() returns). +func seedLiveClaim(t *testing.T, plugin *dra.Plugin, uid types.UID, deviceName string, numCPUs int) string { + t.Helper() + + claim := &resourceapi.ResourceClaim{ + ObjectMeta: metav1.ObjectMeta{UID: uid}, + Status: resourceapi.ResourceClaimStatus{ + Allocation: &resourceapi.AllocationResult{ + Devices: resourceapi.DeviceAllocationResult{ + Results: []resourceapi.DeviceRequestAllocationResult{ + { + Driver: DRADriverName, + Pool: "pool0", + Device: deviceName, + Request: "req0", + ConsumedCapacity: map[resourceapi.QualifiedName]resource.Quantity{ + "nri/cpus": resource.MustParse(fmt.Sprintf("%d", numCPUs)), + }, + }, + }, + }, + }, + }, + } + + result, err := plugin.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if err != nil { + t.Fatalf("PrepareResourceClaims() unexpected error: %v", err) + } + r, ok := result[uid] + if !ok { + t.Fatalf("PrepareResourceClaims() result missing uid %s", uid) + } + if r.Err != nil { + t.Fatalf("PrepareResourceClaims() PrepareResult.Err = %v, want nil", r.Err) + } + if len(r.Devices) != 1 || len(r.Devices[0].CDIDeviceIDs) != 1 { + t.Fatalf("PrepareResourceClaims() result = %+v, want exactly one device with one CDI device ID", r) + } + + return r.Devices[0].CDIDeviceIDs[0] +} + +// TestAllocateResourcesWithTAClaimCallsAllocateClaim verifies that +// AllocateResources recognizes a container carrying a live TA DRA claim's +// CDI device, marks the claimed CPUs in the pool supply (so a subsequent +// regular allocation cannot pick them), and bumps claimContainerRefs. +func TestAllocateResourcesWithTAClaimCallsAllocateClaim(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-alloc-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + + p.draPlugin = plugin + + container := &mockContainer{returnValueForGetID: "c1", cdiDeviceNames: []string{cdiName}} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPUs %s still present in %q sharable set after AllocateResources: %s", claimed, leaf.Name(), got) + } + if got := p.claimContainerRefs[uid]; got != 1 { + t.Errorf("claimContainerRefs[%s] = %d after AllocateResources, want 1", uid, got) + } +} + +// TestReleaseResourcesWithTAClaimCallsReleaseClaim verifies that +// ReleaseResources restores CPUs claimed by allocateClaim once the last +// referencing container is released. +func TestReleaseResourcesWithTAClaimCallsReleaseClaim(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-release-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + + p.draPlugin = plugin + + container := &mockContainer{returnValueForGetID: "c1", cdiDeviceNames: []string{cdiName}} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Fatalf("test setup error: claimed CPUs %s not marked before ReleaseResources", claimed) + } + + if err := p.ReleaseResources(container); err != nil { + t.Fatalf("ReleaseResources() unexpected error: %v", err) + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != claimed.Size() { + t.Errorf("claimed CPUs %s not restored after ReleaseResources: sharable=%s", claimed, got) + } + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs[%s] still present after ReleaseResources released the last container", uid) + } +} + +// TestAllocateResourcesRollsBackClaimOnPoolAllocationFailure verifies that +// AllocateResources rolls back a successful claim-ref mark if the subsequent +// normal pool allocation fails, instead of leaking a claimContainerRefs +// entry (and the corresponding pool supply mark) for a container that never +// actually got its resources allocated. +func TestAllocateResourcesRollsBackClaimOnPoolAllocationFailure(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-rollback-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + + p.draPlugin = plugin + + // An absurdly large exclusive CPU request guarantees the subsequent + // normal pool allocation fails, regardless of topology. + container := &mockContainer{ + returnValueForGetID: "c1", + cdiDeviceNames: []string{cdiName}, + returnValueForGetResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + }, + } + + if err := p.AllocateResources(container); err == nil { + t.Fatal("AllocateResources() expected error from an unsatisfiable resource request, got nil") + } + + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs[%s] still present after AllocateResources failed; claim mark was not rolled back", uid) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != claimed.Size() { + t.Errorf("claimed CPUs %s not restored to pool supply after AllocateResources rollback: sharable=%s", claimed, got) + } +} + +// TestApplyGrantUnionsClaimedCPUsIntoContainerCpuset is the regression test +// for DRA-claimed CPUs missing from a consumer's cpuset: allocateClaim only +// marks pool supply (and, separately, physically paints the claim's +// cpuClass) — it has no way to touch the consuming container's own cgroup +// cpuset. applyGrant is what actually +// calls container.SetCpusetCpus, but until this fix it only ever pinned the +// *normal* grant's own CPUs (Y), silently excluding whatever CPUs (X) the +// container also holds via a live DRA claim — even though the CDI-injected +// NRI_CPU env vars tell the container it has X. This asserts the +// container's actual pinned cpuset is the union of X and Y, and that X was +// not counted towards Y's own (independently-computed) sizing. +func TestApplyGrantUnionsClaimedCPUsIntoContainerCpuset(t *testing.T) { + p := newDRATestPolicy(t) + // applyGrant only calls container.SetCpusetCpus at all when PinCPU is + // enabled; opt and p.cfg alias the same *cfgapi.Config after Setup(). + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimedCPU := sharable[0] + claimed := cpuset.New(claimedCPU) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-cpuset-union-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + container := &mockContainer{ + returnValueForGetID: "c1", + cdiDeviceNames: []string{cdiName}, + returnValueForGetResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("1")}, + }, + } + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + + gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) + } + if !gotCpus.Contains(claimedCPU) { + t.Errorf("container cpuset %s does not include claimed CPU %d (X) — only the normal grant's own CPUs (Y) were pinned", gotCpus, claimedCPU) + } + // The normal 1-CPU exclusive request (Y) must have been sized on its + // own, independent of the claimed CPU: exactly one CPU besides the + // claimed one, not zero (which would mean Y's sizing was inflated by + // counting the already-claimed CPU as if it were Y's own). + exclusiveOnly := gotCpus.Difference(claimed) + if exclusiveOnly.Size() != 1 { + t.Errorf("container cpuset %s minus claimed CPU %s = %s, want exactly 1 CPU for the normal grant (Y), got double-counting or a missing grant", + gotCpus, claimed, exclusiveOnly) + } +} + +// TestAllocateResourcesRollbackClearsClaimedCPUsByContainer verifies that +// rollbackClaimMarks also clears claimedCPUsByContainer's entry for a +// container whose AllocateResources call failed partway through, not just +// release the claims in pool/refcount terms -- otherwise a rolled-back call +// could leave a stale cpuset union in place for a container that no longer +// holds the claim. +func TestAllocateResourcesRollbackClearsClaimedCPUsByContainer(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-rollback-clear-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + container := &mockContainer{ + returnValueForGetID: "rollback-clear-c1", + cdiDeviceNames: []string{cdiName}, + returnValueForGetResourceRequirements: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + Limits: corev1.ResourceList{corev1.ResourceCPU: resource.MustParse("100000")}, + }, + } + + if err := p.AllocateResources(container); err == nil { + t.Fatal("AllocateResources() expected error from an unsatisfiable resource request, got nil") + } + + if _, exists := p.claimedCPUsByContainer[container.GetID()]; exists { + t.Errorf("claimedCPUsByContainer[%s] still present after AllocateResources rollback", container.GetID()) + } +} + +// TestApplyGrantEmptyCpusUsesClaimedCPUsInsteadOfBlanking covers applyGrant's +// cpus.Size() == 0 exit (which otherwise calls container.SetCpusetCpus("")): +// a claim consumer whose *normal* grant happens to compute an empty cpuset +// must not lose access to its claimed CPUs entirely. Uses a +// cpuReserved grant on a leaf pool with no reserved CPUs of its own to +// exercise an empty `cpus` without needing to fully drain a pool's sharable +// capacity. +func TestApplyGrantEmptyCpusUsesClaimedCPUsInsteadOfBlanking(t *testing.T) { + p := newDRATestPolicy(t) + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #1") + if !leaf.GetSupply().ReservedCPUs().IsEmpty() { + t.Fatalf("test setup error: %q has reserved CPUs, want none", leaf.Name()) + } + + container := &mockContainer{returnValueForGetID: "empty-grant-claim-1"} + claimed := cpuset.New(leaf.FreeSupply().SharableCPUs().List()[0]) + p.claimedCPUsByContainer = map[string]cpuset.CPUSet{ + container.GetID(): claimed, + } + + g := newGrant(leaf, container, cpuReserved, "", cpuset.New(), 0, memoryDRAM, nil, 0) + p.applyGrant(g) + + gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) + } + if !gotCpus.Equals(claimed) { + t.Errorf("container cpuset = %s after applyGrant with an empty (reserved) grant, want claimed CPUs %s (must not be blanked to \"\")", gotCpus, claimed) + } +} + +// TestUpdateSharedAllocationsPreservesClaimedCPUsOnRepin covers the +// updateSharedAllocations re-pin path triggered from allocateClaim: +// container A already holds a live claim (claimACPU) plus a plain +// shared-portion grant (no exclusive CPUs). When an unrelated claim for +// container B is allocated, allocateClaim's own updateSharedAllocations(nil) +// call re-pins every other grant -- including A's -- to the pool's now- +// smaller sharable set. That re-pin must still include A's own claimed CPU, +// not just the (shrunk) shared set. +func TestUpdateSharedAllocationsPreservesClaimedCPUsOnRepin(t *testing.T) { + p := newDRATestPolicy(t) + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + + containerA := &mockContainer{returnValueForGetID: "shared-alloc-a"} + claimACPU := sharable[0] + p.claimedCPUsByContainer = map[string]cpuset.CPUSet{ + containerA.GetID(): cpuset.New(claimACPU), + } + grantA := newGrant(leaf, containerA, cpuNormal, "", cpuset.New(), 100, memoryDRAM, nil, 0) + p.allocations.addGrant(grantA) + + claimBCPU := sharable[1] + uidB := types.UID("claim-shared-repin-b") + if err := p.allocateClaim(uidB, cpuset.New(claimBCPU), goldClassCPUs(cpuset.New(claimBCPU))); err != nil { + t.Fatalf("allocateClaim() failed: %v", err) + } + + gotCpus, err := cpuset.Parse(containerA.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container A's cpuset %q: %v", containerA.GetCpusetCpus(), err) + } + if !gotCpus.Contains(claimACPU) { + t.Errorf("container A's cpuset %s lost its own claimed CPU %d after an unrelated claim's updateSharedAllocations re-pin", gotCpus, claimACPU) + } +} + +// TestReapplyDRAClaimsRepinsContainerCpusetWithGrant is the restart-window +// regression test for reapplyDRAClaims's cpuset re-pin: a container +// restored with a regular grant (grantCPU, e.g. by restoreCache() before +// reapplyDRAClaims is ever reached) that also carries a live DRA claim +// (claimedCPU) must end up with both in its cpuset once reapplyDRAClaims +// runs -- not just wait for the next NRI resync's Release+AllocateResources +// to add the claimed CPU in. +func TestReapplyDRAClaimsRepinsContainerCpusetWithGrant(t *testing.T) { + p := newDRATestPolicy(t) + p.cfg.PinCPU = true + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimedCPU := sharable[0] + grantCPU := sharable[1] + claimed := cpuset.New(claimedCPU) + + container := &mockContainer{returnValueForGetID: "reapply-repin-1"} + addTestGrant(t, p, leaf, container, cpuset.New(grantCPU)) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-reapply-repin-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + container.cdiDeviceNames = []string{cdiName} + p.draPlugin = plugin + + mc, ok := p.cache.(*mockCache) + if !ok { + t.Fatalf("test setup error: p.cache is not *mockCache") + } + mc.containers = []cache.Container{container} + + p.reapplyDRAClaims() + + gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + if err != nil { + t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) + } + want := cpuset.New(claimedCPU, grantCPU) + if !gotCpus.Equals(want) { + t.Errorf("container cpuset after reapplyDRAClaims() = %s, want %s (claimed + grant, re-pinned without waiting for the next NRI resync)", gotCpus, want) + } +} + +// TestAllocateResourcesNoCDIDevicesUnaffected verifies that a container +// without any TA CDI device is unaffected by an active (non-nil) draPlugin: +// no claim bookkeeping happens, and regular allocation behaves exactly as it +// did before Step 8. +func TestAllocateResourcesNoCDIDevicesUnaffected(t *testing.T) { + p := newDRATestPolicy(t) + p.draPlugin = newTestDRAPlugin(t, cpuset.New(), "dev0") // no live claims seeded + + container := &mockContainer{returnValueForGetID: "c1"} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() unexpected error: %v", err) + } + if len(p.claimContainerRefs) != 0 { + t.Errorf("claimContainerRefs = %v after AllocateResources on a container with no CDI devices, want empty", p.claimContainerRefs) + } +} + +// TestAllocateResourcesNilDRAPluginNoCrash verifies that AllocateResources +// and ReleaseResources on a policy with DRA disabled (draPlugin == nil, the +// default) neither panic nor attempt any claim lookup. +func TestAllocateResourcesNilDRAPluginNoCrash(t *testing.T) { + p := newDRATestPolicy(t) + if p.draPlugin != nil { + t.Fatalf("test setup error: draPlugin unexpectedly non-nil") + } + + container := &mockContainer{returnValueForGetID: "c1"} + + if err := p.AllocateResources(container); err != nil { + t.Fatalf("AllocateResources() with nil draPlugin: unexpected error: %v", err) + } + if err := p.ReleaseResources(container); err != nil { + t.Fatalf("ReleaseResources() with nil draPlugin: unexpected error: %v", err) + } +} + +// reapplyDRAClaims()/remarkClaimInSupply() and their +// Start()/Reconfigure() wiring. ---- + +// TestStartMarksLiveDRAClaimsInPoolSupply verifies that Start() re-marks +// pool supplies for every claim the (already-loaded) DRA plugin reports as +// live, so a subsequent regular allocation cannot pick those CPUs. This +// stands in for a full ClaimStore-backed reload: p.draPlugin is seeded with +// a live claim via PrepareResourceClaims before Start() runs. +func TestStartMarksLiveDRAClaimsInPoolSupply(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-start-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + if err := p.Start(); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + + free := leaf.FreeSupply() + if got := free.SharableCPUs().Union(free.IsolatedCPUs()).Intersection(claimed); got.Size() != 0 { + t.Errorf("claimed CPUs %s still free in %q supply after Start(): %s", claimed, leaf.Name(), got) + } +} + +// TestStartReappliesDRAClaimsAfterRestoreCache verifies the ordering that +// Start() must observe: restoreCache() alone (which only restores +// previously-cached container allocations, and knows nothing about DRA +// claims) must not mark any claimed CPUs; only the rest of Start() (which +// calls reapplyDRAClaims() after restoreCache() returns) does. +func TestStartReappliesDRAClaimsAfterRestoreCache(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-start-order-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + if err := p.restoreCache(); err != nil { + t.Fatalf("restoreCache() unexpected error: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != claimed.Size() { + t.Fatalf("test setup error: claimed CPUs %s unexpectedly marked by restoreCache() alone (want unmarked at this point): sharable=%s", + claimed, got) + } + + if err := p.Start(); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPUs %s not marked once Start() completed: sharable=%s", claimed, got) + } +} + +// lockContractStub is a WithLock stand-in that panics if invoked while +// already "held", i.e. re-entrantly. Mirrors pkg/resmgr/policy's +// lockContractStub: used here to assert that Start()'s draPlugin.Start(), +// draPlugin.PublishResources(), and reapplyDRAClaims() calls each acquire +// the (shared, non-reentrant) resmgr write lock in strict sequence, never +// nested — the exact bug class of the reapplyDRAClaims/LiveClaimsLocked +// unsynchronized-access race this test guards against. +type lockContractStub struct { + mu sync.Mutex + held bool +} + +func (s *lockContractStub) run(f func()) { + s.mu.Lock() + if s.held { + s.mu.Unlock() + panic("WithLock invoked re-entrantly") + } + s.held = true + s.mu.Unlock() + + defer func() { + s.mu.Lock() + s.held = false + s.mu.Unlock() + }() + + f() +} + +// TestStartReapplyDRAClaimsHoldsWriteLockNotReentrant verifies that Start() +// runs reapplyDRAClaims() (and therefore LiveClaimsLocked(), which reads +// p.claims with no internal synchronization) under the same non-reentrant +// WithLock the DRA plugin's own Start()/PublishResources() use — and never +// nests a second acquisition inside an already-held one, which would +// deadlock. A single lockContractStub is shared between the policy's +// options.WithLock and the DRA plugin's deps.WithLock, exactly as production +// wiring shares one resmgr write lock (m.withWriteLock) between both. +func TestStartReapplyDRAClaimsHoldsWriteLockNotReentrant(t *testing.T) { + stub := &lockContractStub{} + p := newDRATestPolicyWithLock(t, stub.run) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPluginWithLock(t, claimed, "dev0", stub.run) + uid := types.UID("claim-lock-contract-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + var startErr error + panicked := false + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + startErr = p.Start() + }() + + if panicked { + t.Fatalf("Start() panicked: WithLock was invoked re-entrantly") + } + if startErr != nil { + t.Fatalf("Start() unexpected error: %v", startErr) + } + if stub.held { + t.Errorf("resmgr write lock still held after Start() returned") + } + + if got := leaf.FreeSupply().SharableCPUs(); got.Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPUs %s not marked once Start() completed: sharable=%s", claimed, got) + } +} + +// TestClaimContainerRefsRebuiltAfterStartResync verifies the mechanism +// documented on remarkClaimInSupply/reapplyDRAClaims for the restart case: +// p.claimContainerRefs is a plain in-memory map, so it is empty right after +// Start(), even though a container backed by a live DRA claim is already +// running. It is only rebuilt once pkg/resmgr/nri.go's syncWithNRI/ +// Synchronize forces that already-running container through +// ReleaseResources (a no-op, since the refcount is already 0) followed by +// AllocateResources (which increments it) — reproduced here directly via +// p.Sync(add, del) with the same container in both lists, exactly as +// syncWithNRI does for every container discovered in ContainerStateRunning/ +// ContainerStateCreated. +func TestClaimContainerRefsRebuiltAfterStartResync(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 2 { + t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0], sharable[1]) + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-restart-resync-1") + cdiName := seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + if err := p.Start(); err != nil { + t.Fatalf("Start() unexpected error: %v", err) + } + + // Right after Start(), the pool supply already excludes the claimed + // CPUs (reapplyDRAClaims), but claimContainerRefs knows nothing about + // the container yet — it hasn't gone through AllocateResources in this + // process. + if got := p.claimContainerRefs[uid]; got != 0 { + t.Fatalf("test setup error: claimContainerRefs[%s] = %d right after Start(), want 0", uid, got) + } + + container := &mockContainer{returnValueForGetID: "restart-c1", cdiDeviceNames: []string{cdiName}} + + // Mirror syncWithNRI: an already-running container is placed in both + // the "allocated" and "released" lists so Sync() releases (no-op) then + // re-allocates it. + if err := p.Sync([]cache.Container{container}, []cache.Container{container}); err != nil { + t.Fatalf("Sync() unexpected error: %v", err) + } + + if got := p.claimContainerRefs[uid]; got != 1 { + t.Errorf("claimContainerRefs[%s] = %d after Start()+resync Sync(), want 1", uid, got) + } +} + +// TestReapplyDRAClaimsNilDRAPluginNoop verifies that reapplyDRAClaims() is a +// no-op (no panic, no supply changes) when DRA is disabled (draPlugin == +// nil, the default). +func TestReapplyDRAClaimsNilDRAPluginNoop(t *testing.T) { + p := newDRATestPolicy(t) + if p.draPlugin != nil { + t.Fatalf("test setup error: draPlugin unexpectedly non-nil") + } + + leaf := findPoolNode(t, p, "NUMA node #0") + before := leaf.FreeSupply().SharableCPUs() + + p.reapplyDRAClaims() + + if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(before) { + t.Errorf("reapplyDRAClaims() with nil draPlugin changed pool supply: got %s, want unchanged %s", got, before) + } +} + +// TestReapplyDRAClaimsEvictsOverlappingRestoredGrant covers the double- +// booking gap identified in the DRA step 8 review: reapplyDRAClaims() runs +// *after* restoreCache()'s/restoreAllocations()'s grant restoration, at a +// point where Supply.claimRefs has just been wiped by initialize() and not +// yet re-marked. If grant restoration (verbatim reinstatement, or its +// allocatePool-based fallback) handed a regular container CPUs that a live +// DRA claim already owns, that overlap must be detected and evicted here — +// otherwise two workloads end up pinned to the same physical CPUs until the +// next restart. addTestGrant stands in for "restoreCache() already +// reinstated this grant" without needing the full cache/offer machinery. +func TestReapplyDRAClaimsEvictsOverlappingRestoredGrant(t *testing.T) { + p := newDRATestPolicy(t) + + leaf := findPoolNode(t, p, "NUMA node #0") + sharable := leaf.FreeSupply().SharableCPUs().List() + if len(sharable) < 1 { + t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) + } + claimed := cpuset.New(sharable[0]) + + // Simulate restoreCache()/restoreAllocations() having already reinstated + // (or freshly reallocated) a regular grant that happens to overlap the + // CPU a live DRA claim owns — the exact scenario reapplyDRAClaims must + // still be able to correct despite running after grant restoration. + victim := &mockContainer{returnValueForGetID: "reapply-victim"} + addTestGrant(t, p, leaf, victim, claimed) + if _, ok := p.allocations.getGrant("reapply-victim"); !ok { + t.Fatalf("test setup error: victim grant not present before reapplyDRAClaims") + } + + plugin := newTestDRAPlugin(t, claimed, "dev0") + uid := types.UID("claim-reapply-evict-1") + _ = seedLiveClaim(t, plugin, uid, "dev0", claimed.Size()) + p.draPlugin = plugin + + p.reapplyDRAClaims() + + // No grant anywhere may still exclusively hold the claimed CPU: either + // the victim's original grant was released outright, or it was + // reallocated to CPUs that no longer overlap the claim. + for _, g := range p.allocations.grants { + if g.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still exclusively granted to %s after reapplyDRAClaims", + claimed, g.GetContainer().PrettyName()) + } + } + + // And the claimed CPU must not be free for a new regular allocation. + free := leaf.FreeSupply() + if free.SharableCPUs().Union(free.IsolatedCPUs()).Intersection(claimed).Size() != 0 { + t.Errorf("claimed CPU %s still free in %q supply after reapplyDRAClaims", claimed, leaf.Name()) + } + + // The evicted victim must actually have been reallocated a new grant + // (not just released and forgotten) — ample capacity remains on this + // test system for reallocatePool to succeed. + newGrant, ok := p.allocations.getGrant("reapply-victim") + if !ok { + t.Fatalf("victim has no grant after reapplyDRAClaims(); eviction must reallocate, not just release") + } + if newGrant.ExclusiveCPUs().Intersection(claimed).Size() != 0 { + t.Errorf("victim's new grant %s still overlaps claimed CPU %s", newGrant.ExclusiveCPUs(), claimed) + } + + // reapplyDRAClaims() must not touch claimContainerRefs (marking-only + // contract) — even though it evicted/reallocated a container here. + if _, exists := p.claimContainerRefs[uid]; exists { + t.Errorf("claimContainerRefs unexpectedly populated by reapplyDRAClaims (marking-only contract violated)") + } +} diff --git a/config/crd/bases/config.nri_topologyawarepolicies.yaml b/config/crd/bases/config.nri_topologyawarepolicies.yaml index b9143461f..175e85fc1 100644 --- a/config/crd/bases/config.nri_topologyawarepolicies.yaml +++ b/config/crd/bases/config.nri_topologyawarepolicies.yaml @@ -615,6 +615,25 @@ spec: assigned for excusive use to containers which are not annotated to use any CPU class. type: string + dra: + description: |- + DRA controls whether the policy runs a DRA (Dynamic Resource Allocation) + kubelet plugin publishing CPU devices derived from the configured + CPUClasses. If unset, DRA integration is disabled. + properties: + enabled: + default: false + description: |- + Enabled controls whether the DRA kubelet plugin is started and CPU + devices are published for the configured CPUClasses. + type: boolean + sharedCounters: + default: false + description: |- + SharedCounters is reserved for future use (KEP-5941 shared counters). + Setting this to true is currently rejected by validation; leave it unset or false. + type: boolean + type: object instrumentation: description: Config provides runtime configuration for instrumentation. properties: diff --git a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml index b9143461f..175e85fc1 100644 --- a/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml +++ b/deployment/helm/topology-aware/crds/config.nri_topologyawarepolicies.yaml @@ -615,6 +615,25 @@ spec: assigned for excusive use to containers which are not annotated to use any CPU class. type: string + dra: + description: |- + DRA controls whether the policy runs a DRA (Dynamic Resource Allocation) + kubelet plugin publishing CPU devices derived from the configured + CPUClasses. If unset, DRA integration is disabled. + properties: + enabled: + default: false + description: |- + Enabled controls whether the DRA kubelet plugin is started and CPU + devices are published for the configured CPUClasses. + type: boolean + sharedCounters: + default: false + description: |- + SharedCounters is reserved for future use (KEP-5941 shared counters). + Setting this to true is currently rejected by validation; leave it unset or false. + type: boolean + type: object instrumentation: description: Config provides runtime configuration for instrumentation. properties: diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go index 7be7b1153..8c9a0faff 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go @@ -197,6 +197,27 @@ type Config struct { // +optional // +kubebuilder:default={"*"} ControllableInterrupts []string `json:"controllableInterrupts,omitempty"` + // DRA controls whether the policy runs a DRA (Dynamic Resource Allocation) + // kubelet plugin publishing CPU devices derived from the configured + // CPUClasses. If unset, DRA integration is disabled. + // +optional + DRA *TopologyAwareDRA `json:"dra,omitempty"` +} + +// TopologyAwareDRA controls the DRA (Dynamic Resource Allocation) kubelet +// plugin integration for the topology-aware policy. +// +kubebuilder:object:generate=true +type TopologyAwareDRA struct { + // Enabled controls whether the DRA kubelet plugin is started and CPU + // devices are published for the configured CPUClasses. + // +kubebuilder:default=false + // +optional + Enabled bool `json:"enabled,omitempty"` + // SharedCounters is reserved for future use (KEP-5941 shared counters). + // Setting this to true is currently rejected by validation; leave it unset or false. + // +kubebuilder:default=false + // +optional + SharedCounters bool `json:"sharedCounters,omitempty"` } var ( @@ -277,6 +298,25 @@ func (c *Config) Validate() error { return errors.Join(errs...) } +// DRAEnabled returns whether the DRA kubelet plugin integration is enabled. +// It is nil-safe: a nil Config.DRA (the default) means disabled. +func (c *Config) DRAEnabled() bool { + if c == nil || c.DRA == nil { + return false + } + return c.DRA.Enabled +} + +// DRASharedCounters returns whether published DRA CPU devices should use +// shared per-class counters. It is nil-safe: a nil Config.DRA (the default) +// means false. +func (c *Config) DRASharedCounters() bool { + if c == nil || c.DRA == nil { + return false + } + return c.DRA.SharedCounters +} + // GetSchedulingClass returns the named class or nil if it is not defined. func (c *Config) GetSchedulingClass(name string) *SchedulingClass { for _, sc := range c.SchedulingClasses { diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config_test.go b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config_test.go new file mode 100644 index 000000000..fcd6274e9 --- /dev/null +++ b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config_test.go @@ -0,0 +1,45 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "testing" + + "github.com/stretchr/testify/require" +) + +func TestDRAEnabledZeroConfig(t *testing.T) { + c := &Config{} + require.False(t, c.DRAEnabled(), "zero Config (nil DRA) should report DRAEnabled() == false") + require.False(t, c.DRASharedCounters(), "zero Config (nil DRA) should report DRASharedCounters() == false") +} + +func TestDRAEnabledTrue(t *testing.T) { + c := &Config{ + ReservedResources: Constraints{CPU: "750m"}, + DRA: &TopologyAwareDRA{Enabled: true}, + } + require.True(t, c.DRAEnabled()) + require.NoError(t, c.Validate()) +} + +func TestDRASharedCountersTrue(t *testing.T) { + c := &Config{ + ReservedResources: Constraints{CPU: "750m"}, + DRA: &TopologyAwareDRA{SharedCounters: true}, + } + require.True(t, c.DRASharedCounters()) + require.NoError(t, c.Validate()) +} diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go index 73ad92456..c2b2ada52 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/zz_generated.deepcopy.go @@ -93,6 +93,11 @@ func (in *Config) DeepCopyInto(out *Config) { *out = make([]string, len(*in)) copy(*out, *in) } + if in.DRA != nil { + in, out := &in.DRA, &out.DRA + *out = new(TopologyAwareDRA) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Config. @@ -104,3 +109,18 @@ func (in *Config) DeepCopy() *Config { in.DeepCopyInto(out) return out } + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *TopologyAwareDRA) DeepCopyInto(out *TopologyAwareDRA) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TopologyAwareDRA. +func (in *TopologyAwareDRA) DeepCopy() *TopologyAwareDRA { + if in == nil { + return nil + } + out := new(TopologyAwareDRA) + in.DeepCopyInto(out) + return out +} diff --git a/pkg/resmgr/cache/cache.go b/pkg/resmgr/cache/cache.go index f5091999d..677c1e67d 100644 --- a/pkg/resmgr/cache/cache.go +++ b/pkg/resmgr/cache/cache.go @@ -237,6 +237,9 @@ type Container interface { GetMounts() []*Mount // GetDevices returns all the linux devices of the container. GetDevices() []*Device + // GetCDIDeviceNames returns the qualified names of all CDI devices + // injected into this container (nil/empty if there are none). + GetCDIDeviceNames() []string // PrettyName returns the user-friendly $namespace/$pod/$container for the container. PrettyName() string diff --git a/pkg/resmgr/cache/container.go b/pkg/resmgr/cache/container.go index 62b18a28d..5eba1ebe2 100644 --- a/pkg/resmgr/cache/container.go +++ b/pkg/resmgr/cache/container.go @@ -513,6 +513,20 @@ func (c *container) GetDevices() []*Device { return devices } +func (c *container) GetCDIDeviceNames() []string { + cdiDevices := c.Ctr.GetCDIDevices() + if len(cdiDevices) == 0 { + return nil + } + + names := make([]string, 0, len(cdiDevices)) + for _, d := range cdiDevices { + names = append(names, d.GetName()) + } + + return names +} + func (c *container) GetResmgrLabel(key string) (string, bool) { value, ok := c.GetLabel(kubernetes.ResmgrKey(key)) return value, ok diff --git a/pkg/resmgr/cache/container_test.go b/pkg/resmgr/cache/container_test.go index 7331f50af..445d4962f 100644 --- a/pkg/resmgr/cache/container_test.go +++ b/pkg/resmgr/cache/container_test.go @@ -396,6 +396,45 @@ var _ = Describe("Container", func() { Expect(ctrs[0].GetDevices()).To(Equal(devices)) }) + It("can return its CDI device names", func() { + var ( + names = []string{ + "nri.topology-aware.cpu/device=claim-abc-req-dev-0", + "nri.topology-aware.cpu/device=claim-abc-req-dev-1", + } + nriPods = []*nri.PodSandbox{ + makePod(), + } + nriCtrs = []*nri.Container{ + makeCtr( + WithCtrPodID(nriPods[0].GetId()), + WithCtrCDIDevices(names), + ), + } + ) + + _, _, ctrs := makePopulatedCache(nriPods, nriCtrs) + + Expect(ctrs[0].GetCDIDeviceNames()).To(Equal(names)) + }) + + It("returns no CDI device names when there are none", func() { + var ( + nriPods = []*nri.PodSandbox{ + makePod(), + } + nriCtrs = []*nri.Container{ + makeCtr( + WithCtrPodID(nriPods[0].GetId()), + ), + } + ) + + _, _, ctrs := makePopulatedCache(nriPods, nriCtrs) + + Expect(ctrs[0].GetCDIDeviceNames()).To(BeEmpty()) + }) + }) var _ = Describe("Container", func() { @@ -948,6 +987,22 @@ func WithCtrDevices(devices []*nri.LinuxDevice) CtrOption { } } +func WithCtrCDIDevices(names []string) CtrOption { + return func(nriCtr *nri.Container) error { + if names == nil { + nriCtr.CDIDevices = nil + return nil + } + nriCtr.CDIDevices = make([]*nri.CDIDevice, len(names)) + for i, n := range names { + nriCtr.CDIDevices[i] = &nri.CDIDevice{ + Name: n, + } + } + return nil + } +} + func WithCtrLabels(labels map[string]string) CtrOption { return func(nriCtr *nri.Container) error { if labels == nil { diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index df9bbf107..f5dad786f 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -201,6 +201,29 @@ func (h *Handler) IsHPClass(className string) bool { return h.pct.IsHPClass(className) } +// ClassForCPU returns the synthetic class name currently assigned to cpu by +// the most recent UseClass/AssignCPUs call (as tracked in h.cpuClass), or "" +// if cpu is unmanaged or explicitly assigned to no class. Primarily useful +// for tests that need to verify a UseClass call actually changed (or +// restored) a CPU's class, since Handler otherwise exposes no per-CPU class +// query. Nil-safe. +// +// This is a test-only accessor kept on the exported Handler API rather than +// behind an export_test.go shim: its only callers are cmd/plugins/topology- +// aware/policy's tests, a different package, and Go test files are never +// compiled into a package's importable surface across package boundaries — +// an export_test.go in this package would be invisible there. Duplicating +// equivalent instrumentation on the caller side (e.g. inspecting SST mock +// state directly) would require exposing Handler's internal CLOS/class +// mapping some other way, which is a larger change than this narrow, +// clearly-documented read accessor. +func (h *Handler) ClassForCPU(cpu int) string { + if h == nil { + return "" + } + return h.cpuClass[cpu] +} + // Configure (re)applies a configuration spec. Idempotent: may be // called repeatedly with changed classes, turbo-domain mode, or // allowed set. diff --git a/pkg/resmgr/cpuclass/dra.go b/pkg/resmgr/cpuclass/dra.go index 90d4dcf72..82d9d51fd 100644 --- a/pkg/resmgr/cpuclass/dra.go +++ b/pkg/resmgr/cpuclass/dra.go @@ -30,6 +30,23 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" ) +// DRA device attribute/capacity keys shared with pkg/resmgr/dra (which reads +// them back out of the published device list) — defined once here, +// referenced everywhere else, mirroring the DRADriverName convention. +const ( + // AttrCPUClass names the cpuClass a device belongs to. + AttrCPUClass resapi.QualifiedName = "nri/cpuClass" + // AttrPackageID names the CPU package a device's punit lives on. + AttrPackageID resapi.QualifiedName = "nri/packageID" + // AttrPunitID names the SST-TF punit a device represents. + AttrPunitID resapi.QualifiedName = "nri/punitID" + // AttrAllowedCPUs carries the CPUSet string (allowed ∩ punit.CPUs) for a + // device, identifying exactly which CPUs it represents. + AttrAllowedCPUs resapi.QualifiedName = "nri/allowedCPUs" + // CapacityCPUs is the capacity key for the number of CPUs a device grants. + CapacityCPUs resapi.QualifiedName = "nri/cpus" +) + // nonAlphaRe matches runs of characters that are not lowercase letters or digits. // Used by sanitizeBase to replace them with hyphens. var nonAlphaRe = regexp.MustCompile(`[^a-z0-9]+`) @@ -250,9 +267,14 @@ func buildDRADevices( name := deviceName(base, pu.PkgID, pu.PunitID) attrs := map[resapi.QualifiedName]resapi.DeviceAttribute{ - "nri/packageID": intAttr(int64(pu.PkgID)), - "nri/punitID": intAttr(int64(pu.PunitID)), - "nri/cpuClass": strAttr(cc.Name), + AttrPackageID: intAttr(int64(pu.PkgID)), + AttrPunitID: intAttr(int64(pu.PunitID)), + AttrCPUClass: strAttr(cc.Name), + } + // nri/allowedCPUs captures the CPU identity (allowed ∩ punit.CPUs) + // the device represents. + if pu.AllowedCPUs != "" { + attrs[AttrAllowedCPUs] = strAttr(pu.AllowedCPUs) } // nri/pctPriority is only emitted for PCT classes (non-empty PctPriority). // Omitting it for non-PCT classes avoids CEL false-positives on "" values. @@ -265,7 +287,7 @@ func buildDRADevices( Name: name, Attributes: attrs, Capacity: map[resapi.QualifiedName]resapi.DeviceCapacity{ - "nri/cpus": { + CapacityCPUs: { Value: resource.MustParse(capStr), RequestPolicy: &resapi.CapacityRequestPolicy{ Default: kptr.To(resource.MustParse("1")), @@ -281,7 +303,7 @@ func buildDRADevices( NodeAllocatableResources: map[corev1.ResourceName]resapi.NodeAllocatableResource{ corev1.ResourceCPU: { Mapping: &resapi.NodeAllocatableMapping{ - CapacityKey: kptr.To(resapi.QualifiedName("nri/cpus")), + CapacityKey: kptr.To(CapacityCPUs), CapacityMultiplier: kptr.To(resource.MustParse("1")), }, }, @@ -315,3 +337,19 @@ func (h *Handler) DRADevices(_ string) ([]resapi.Device, error) { } return buildDRADevices(h.classes, punits, h.pct.IsHPClass, true), nil } + +// DRADevicesAtMaxCapacity is like DRADevices but sets HPCapacity to +// GuaranteedHpCpus (ignoring hpUsed). Used by Reconfigure to snapshot +// hardware-level capacity for change detection — comparing workload-adjusted +// capacities would produce false negatives when hpUsed coincidentally equals +// the capacity delta introduced by a hardware change. +func (h *Handler) DRADevicesAtMaxCapacity(_ string) ([]resapi.Device, error) { + if h == nil || h.pct == nil { + return []resapi.Device{}, nil + } + punits := h.pct.MaxPunits() + if len(punits) == 0 { + return []resapi.Device{}, nil + } + return buildDRADevices(h.classes, punits, h.pct.IsHPClass, true), nil +} diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index c2825d0b7..68283e293 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -531,8 +531,9 @@ func (a *Allocator) FreeClassCapacity(className string, held cpuset.CPUSet) int type PunitInfo struct { PkgID int PunitID int - HPCapacity int // GuaranteedHpCpus minus non-DRA HP CPUs already in hpUsed, or 0 if the punit is HP-ineligible - NonHPCapacity int // allocatable non-HP CPUs (allowed ∩ punit.CPUs − GuaranteedHpCpus) + HPCapacity int // GuaranteedHpCpus minus non-DRA HP CPUs already in hpUsed, or 0 if the punit is HP-ineligible + NonHPCapacity int // allocatable non-HP CPUs (allowed ∩ punit.CPUs − GuaranteedHpCpus) + AllowedCPUs string // CPUSet string for allowed ∩ punit.CPUs } // Punits returns a snapshot of per-punit DRA-relevant capacity for all @@ -549,6 +550,29 @@ func (a *Allocator) Punits() []PunitInfo { PunitID: pu.PunitID, HPCapacity: a.punitAvailableHPCapacity(i), NonHPCapacity: a.punitNonHPCapacity(i), + AllowedCPUs: pu.CPUs.String(), + } + } + return out +} + +// MaxPunits is like Punits but sets HPCapacity to the full GuaranteedHpCpus +// (not reduced by hpUsed). Used by Handler.DRADevicesAtMaxCapacity for +// hardware-change detection in Reconfigure, where comparing workload-adjusted +// capacities would produce false negatives when hpUsed coincidentally equals +// the capacity delta. +func (a *Allocator) MaxPunits() []PunitInfo { + if !a.Active() { + return nil + } + out := make([]PunitInfo, len(a.punits)) + for i, pu := range a.punits { + out[i] = PunitInfo{ + PkgID: pu.PkgID, + PunitID: pu.PunitID, + HPCapacity: a.punitHPCapacity(i), + NonHPCapacity: a.punitNonHPCapacity(i), + AllowedCPUs: pu.CPUs.String(), } } return out diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index efcfca837..d1ea60b7c 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -1224,8 +1224,8 @@ func TestAllocatorPunits(t *testing.T) { t.Fatalf("Punits() len = %d, want 2", len(pi)) } want := []PunitInfo{ - {PkgID: 0, PunitID: 0, HPCapacity: 2, NonHPCapacity: 2}, - {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3}, + {PkgID: 0, PunitID: 0, HPCapacity: 2, NonHPCapacity: 2, AllowedCPUs: "0-3"}, + {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3, AllowedCPUs: "4-7"}, } for i, w := range want { if pi[i] != w { @@ -1246,8 +1246,8 @@ func TestAllocatorPunits_NonDRAHpUsageReducesCapacity(t *testing.T) { pi := a.Punits() want := []PunitInfo{ - {PkgID: 0, PunitID: 0, HPCapacity: 1, NonHPCapacity: 2}, - {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3}, + {PkgID: 0, PunitID: 0, HPCapacity: 1, NonHPCapacity: 2, AllowedCPUs: "0-3"}, + {PkgID: 0, PunitID: 1, HPCapacity: 1, NonHPCapacity: 3, AllowedCPUs: "4-7"}, } for i, w := range want { if pi[i] != w { diff --git a/pkg/resmgr/dra/deps.go b/pkg/resmgr/dra/deps.go index f4b977a48..f887cf3bb 100644 --- a/pkg/resmgr/dra/deps.go +++ b/pkg/resmgr/dra/deps.go @@ -89,6 +89,10 @@ type ClaimStore interface { Load() (map[types.UID]*ClaimState, error) } +// ClaimUnprepare is called after a prepared claim is removed from plugin +// state, while holding the resmgr lock. +type ClaimUnprepare func(uid types.UID, allocs []ResultAlloc) + // Deps holds the dependencies a policy binary must supply when constructing // a Plugin. type Deps struct { @@ -105,6 +109,12 @@ type Deps struct { // ValidateClasses is a closure that validates the current cpuClass // configuration for DRA compatibility. ValidateClasses func() error + // ValidateCPUsInPool is a closure that reports whether cpus can fit + // within a single allocation leaf pool of the policy. Returns an error + // if the CPUs span multiple leaf pools (or otherwise don't fit within + // one), so that a claim which would later fail at container-creation + // time is instead rejected at Prepare time. Must not be nil. + ValidateCPUsInPool func(cpus cpuset.CPUSet) error // DeviceLister returns the list of DRA devices to publish. DeviceLister DeviceLister // ClaimAllocator provides HP CPU pick/release/account operations. @@ -113,6 +123,8 @@ type Deps struct { CDIWriter CDIWriter // ClaimStore persists and loads claim state via the resmgr cache. ClaimStore ClaimStore + // ClaimUnprepare notifies the policy that a prepared claim was removed. + ClaimUnprepare ClaimUnprepare // WithLock executes f while holding the resmgr write lock. All accesses // to Handler state (ValidateClasses, DRADevices, Prepare, Unprepare, and // RestoreClaims) must run inside WithLock. diff --git a/pkg/resmgr/dra/plugin.go b/pkg/resmgr/dra/plugin.go index b5948255c..af6c355a2 100644 --- a/pkg/resmgr/dra/plugin.go +++ b/pkg/resmgr/dra/plugin.go @@ -42,7 +42,7 @@ var ( errNilAllocation = errors.New("dra plugin: claim has nil Allocation") errMissingConsumedCapacity = errors.New("dra plugin: ConsumedCapacity[nri/cpus] absent or zero") errNonHPNotSupported = errors.New("dra plugin: non-HP CPU class not supported (deferred)") - errMultiPunitNotSupported = errors.New("dra plugin: claim spans multiple punits (leaf pools): unsupported") + errMultiPunitNotSupported = errors.New("dra plugin: claim results on multiple punits not supported") ) // deviceInfo holds the device attributes looked up from the published device list. @@ -80,6 +80,9 @@ func New(driverName string, deps Deps) (*Plugin, error) { if deps.ValidateClasses == nil { return nil, fmt.Errorf("dra plugin: ValidateClasses must not be nil") } + if deps.ValidateCPUsInPool == nil { + return nil, fmt.Errorf("dra plugin: ValidateCPUsInPool must not be nil") + } if deps.DeviceLister == nil { return nil, fmt.Errorf("dra plugin: DeviceLister must not be nil") } @@ -98,6 +101,9 @@ func New(driverName string, deps Deps) (*Plugin, error) { if deps.WithLock == nil { return nil, fmt.Errorf("dra plugin: WithLock must not be nil") } + if deps.ClaimUnprepare == nil { + return nil, fmt.Errorf("dra plugin: ClaimUnprepare must not be nil") + } return &Plugin{ driverName: driverName, deps: deps, @@ -106,6 +112,11 @@ func New(driverName string, deps Deps) (*Plugin, error) { }, nil } +// DriverName returns the plugin's DRA driver name. +func (p *Plugin) DriverName() string { + return p.driverName +} + // shareIDPtr converts a ShareID string to *types.UID. Returns nil if s is "". func shareIDPtr(s string) *types.UID { if s == "" { @@ -126,13 +137,13 @@ func (p *Plugin) deviceIndex() (map[string]deviceInfo, error) { idx := make(map[string]deviceInfo, len(devs)) for _, d := range devs { info := deviceInfo{} - if attr, ok := d.Attributes[resourceapi.QualifiedName("nri/cpuClass")]; ok && attr.StringValue != nil { + if attr, ok := d.Attributes[cpuclass.AttrCPUClass]; ok && attr.StringValue != nil { info.ClassName = *attr.StringValue } - if attr, ok := d.Attributes[resourceapi.QualifiedName("nri/packageID")]; ok && attr.IntValue != nil { + if attr, ok := d.Attributes[cpuclass.AttrPackageID]; ok && attr.IntValue != nil { info.PkgID = int(*attr.IntValue) } - if attr, ok := d.Attributes[resourceapi.QualifiedName("nri/punitID")]; ok && attr.IntValue != nil { + if attr, ok := d.Attributes[cpuclass.AttrPunitID]; ok && attr.IntValue != nil { info.PunitID = int(*attr.IntValue) } idx[d.Name] = info @@ -219,6 +230,8 @@ func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi. heldCPUs := p.allClaimedCPUs() var pickedAllocs []ResultAlloc var cdiDevices []CDIDevice + claimCPUs := cpuset.New() + var punit *deviceInfo for i, r := range filtered { attrs, attrOk := devIdx[r.Device] @@ -231,6 +244,11 @@ func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi. p.rollbackPicks(pickedAllocs) return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: device %q missing nri/cpuClass attribute", r.Device)} } + if punit != nil && (attrs.PkgID != punit.PkgID || attrs.PunitID != punit.PunitID) { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: errMultiPunitNotSupported} + } + punit = &attrs // Reject claims whose results span more than one punit — // the topology-aware consumer requires the union of a @@ -243,7 +261,7 @@ func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi. } } - q, ok := r.ConsumedCapacity[resourceapi.QualifiedName("nri/cpus")] + q, ok := r.ConsumedCapacity[cpuclass.CapacityCPUs] if !ok { p.rollbackPicks(pickedAllocs) return kubeletplugin.PrepareResult{Err: errMissingConsumedCapacity} @@ -265,6 +283,7 @@ func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi. return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: PickHpCpus: %w", pickErr)} } heldCPUs = heldCPUs.Union(picked) + claimCPUs = claimCPUs.Union(picked) // Determine ShareID. shareID := "" @@ -290,6 +309,14 @@ func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi. }) } + // A punit can span multiple leaf pools, so verify the union of + // all result CPUs fits within the policy's allocation domain + // before committing. + if validateErr := p.deps.ValidateCPUsInPool(claimCPUs); validateErr != nil { + p.rollbackPicks(pickedAllocs) + return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: claim CPU set %s is outside supported allocation domain: %w", claimCPUs, validateErr)} + } + if writeErr := p.deps.CDIWriter.WriteClaim(uid, cdiDevices); writeErr != nil { p.rollbackPicks(pickedAllocs) return kubeletplugin.PrepareResult{Err: fmt.Errorf("dra plugin: WriteClaim: %w", writeErr)} @@ -396,47 +423,34 @@ func (p *Plugin) UnprepareResourceClaims(_ context.Context, claims []kubeletplug perUID[uid] = nil continue } - // Release CPUs for each allocation result; parse errors are logged - // but do not block CDI removal or claim deletion. - for _, alloc := range cs.Allocs { - cpus, err := cpuset.Parse(alloc.CPUs) - if err != nil { - p.deps.Logger.Warnf("dra plugin: UnprepareResourceClaims: claim %s device %s: parse CPUs %q: %v (skipping release)", uid, alloc.Device, alloc.CPUs, err) - continue - } - p.deps.ClaimAllocator.ReleaseHpCpus(alloc.PkgID, alloc.PunitID, cpus) + delete(p.claims, uid) + if saveErr := p.deps.ClaimStore.Save(p.claims); saveErr != nil { + p.claims[uid] = cs + perUID[uid] = fmt.Errorf("dra plugin: UnprepareResourceClaims: ClaimStore.Save: %w", saveErr) + continue } + p.deps.ClaimUnprepare(uid, cs.Allocs) // Remove CDI spec unconditionally; log but do not block deletion. if err := p.deps.CDIWriter.RemoveClaim(uid); err != nil { p.deps.Logger.Warnf("dra plugin: UnprepareResourceClaims: claim %s: RemoveClaim: %v", uid, err) } - delete(p.claims, uid) perUID[uid] = nil } - // Persist the updated claims map in a single batch write. - if saveErr := p.deps.ClaimStore.Save(p.claims); saveErr != nil { - p.deps.Logger.Errorf("dra plugin: UnprepareResourceClaims: ClaimStore.Save: %v", saveErr) - } }) return perUID, nil } -// LiveClaimClasses returns a map from className to the number of live claims -// using that class. Each claim is counted once per distinct class it uses. -// Caller must hold the resmgr lock (do not call from inside a WithLock -// callback — the resmgr lock is not reentrant). Used to refuse a Reconfigure -// that would change class-derived attributes while claims are live. -func (p *Plugin) LiveClaimClasses() map[string]int { - result := make(map[string]int) - for _, cs := range p.claims { - // Count each claim once per distinct class it uses. - seen := make(map[string]bool) - for _, alloc := range cs.Allocs { - if alloc.ClassName != "" && !seen[alloc.ClassName] { - result[alloc.ClassName]++ - seen[alloc.ClassName] = true - } - } +// LiveClaimsLocked returns a snapshot of the currently live claims as +// map[types.UID][]ResultAlloc. Caller must hold the resmgr lock (do not call +// from inside a WithLock callback — the resmgr lock is not reentrant). Used +// by the pool-accounting re-apply path (reapplyDRAClaims) after +// Start()/Reconfigure() rebuild policy state. +func (p *Plugin) LiveClaimsLocked() map[types.UID][]ResultAlloc { + result := make(map[types.UID][]ResultAlloc, len(p.claims)) + for uid, cs := range p.claims { + allocs := make([]ResultAlloc, len(cs.Allocs)) + copy(allocs, cs.Allocs) + result[uid] = allocs } return result } diff --git a/pkg/resmgr/dra/plugin_test.go b/pkg/resmgr/dra/plugin_test.go index 4b5047d72..5ad7092f1 100644 --- a/pkg/resmgr/dra/plugin_test.go +++ b/pkg/resmgr/dra/plugin_test.go @@ -56,15 +56,17 @@ func TestNewLogr(t *testing.T) { // validDeps returns a Deps with all required fields populated. func validDeps() Deps { return Deps{ - KubeClient: fake.NewClientset(), - NodeName: "test-node", - ValidateClasses: func() error { return nil }, - DeviceLister: &fixedDeviceLister{}, - ClaimAllocator: &noopClaimAllocator{}, - CDIWriter: &noopCDIWriter{}, - ClaimStore: &noopClaimStore{}, - WithLock: func(f func()) { f() }, - Logger: log.Default(), + KubeClient: fake.NewClientset(), + NodeName: "test-node", + ValidateClasses: func() error { return nil }, + ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + DeviceLister: &fixedDeviceLister{}, + ClaimAllocator: &noopClaimAllocator{}, + CDIWriter: &noopCDIWriter{}, + ClaimStore: &noopClaimStore{}, + WithLock: func(f func()) { f() }, + ClaimUnprepare: func(_ types.UID, _ []ResultAlloc) {}, + Logger: log.Default(), } } @@ -108,6 +110,11 @@ func TestNew_Validation(t *testing.T) { driverName: "test-driver", mutate: func(d *Deps) { d.ValidateClasses = nil }, }, + { + name: "nil ValidateCPUsInPool", + driverName: "test-driver", + mutate: func(d *Deps) { d.ValidateCPUsInPool = nil }, + }, { name: "nil DeviceLister", driverName: "test-driver", @@ -528,17 +535,19 @@ func TestPublishResources_Integration(t *testing.T) { ) deps := Deps{ - KubeClient: fakeClient, - NodeName: "test-node", - RegistrarDir: registrarDir, - PluginDataDir: pluginDataDir, - ValidateClasses: func() error { return nil }, - DeviceLister: &fixedDeviceLister{devices: makeTestDevices(5)}, - ClaimAllocator: &noopClaimAllocator{}, - CDIWriter: &noopCDIWriter{}, - ClaimStore: &noopClaimStore{}, - WithLock: func(f func()) { f() }, - Logger: log.Default(), + KubeClient: fakeClient, + NodeName: "test-node", + RegistrarDir: registrarDir, + PluginDataDir: pluginDataDir, + ValidateClasses: func() error { return nil }, + ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + DeviceLister: &fixedDeviceLister{devices: makeTestDevices(5)}, + ClaimAllocator: &noopClaimAllocator{}, + CDIWriter: &noopCDIWriter{}, + ClaimStore: &noopClaimStore{}, + WithLock: func(f func()) { f() }, + ClaimUnprepare: func(_ types.UID, _ []ResultAlloc) {}, + Logger: log.Default(), } const driverName = "test.driver.io" @@ -1284,10 +1293,8 @@ func TestPrepare_ClaimStoreSaveFailure(t *testing.T) { } } -// TestPrepare_MultiResultTwoPunits verifies that a claim with two results -// spanning different punits is rejected: the topology-aware consumer -// requires the union of a claim's results to fit a single leaf pool, so the -// CPU pick for the first result is rolled back and nothing is written. +// TestPrepare_MultiResultTwoPunits verifies that a claim spanning punit pools +// is rejected before it can be committed. func TestPrepare_MultiResultTwoPunits(t *testing.T) { alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-1"), isHP: true} cdiW := &trackingCDIWriter{} @@ -1326,17 +1333,65 @@ func TestPrepare_MultiResultTwoPunits(t *testing.T) { t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) } r := result[uid] - if r.Err == nil { - t.Fatal("PrepareResult.Err = nil, want multi-punit error") + if !errors.Is(r.Err, errMultiPunitNotSupported) { + t.Fatalf("PrepareResult.Err = %v, want errMultiPunitNotSupported", r.Err) } if len(r.Devices) != 0 { t.Errorf("PrepareResult.Devices len = %d, want 0", len(r.Devices)) } + if len(alloc.releases) != 1 { + t.Errorf("ReleaseHpCpus called %d times, want 1 rollback", len(alloc.releases)) + } if len(cdiW.written) != 0 { t.Errorf("WriteClaim called %d times, want 0", len(cdiW.written)) } + if _, ok := p.claims[uid]; ok { + t.Error("claim stored after multi-punit rejection") + } +} + +// TestPrepare_ClaimCPUsOutsideAllocationDomain verifies that claim CPU picks +// are rejected before CDI/state commit when the policy reports they don't +// fit a single allocation domain (e.g. a punit spanning more than one +// topology leaf pool). +func TestPrepare_ClaimCPUsOutsideAllocationDomain(t *testing.T) { + validateErr := errors.New("spans multiple leaf pools") + alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{} + deps := validDeps() + deps.ClaimAllocator = alloc + deps.CDIWriter = cdiW + deps.ClaimStore = store + deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) + deps.ValidateCPUsInPool = func(_ cpuset.CPUSet) error { return validateErr } + + p, err := New("test-driver", deps) + if err != nil { + t.Fatalf("New() unexpected error: %v", err) + } + + uid := types.UID("uid-invalid-domain") + claim := makeClaim(uid, "test-driver", "pool0", "dev0", "req0", 4) + result, globalErr := p.PrepareResourceClaims(context.Background(), []*resourceapi.ResourceClaim{claim}) + if globalErr != nil { + t.Fatalf("PrepareResourceClaims() unexpected global error: %v", globalErr) + } + r := result[uid] + if !errors.Is(r.Err, validateErr) { + t.Fatalf("PrepareResult.Err = %v, want to wrap validateErr", r.Err) + } if len(alloc.releases) != 1 { - t.Errorf("ReleaseHpCpus called %d times, want 1 (rollback of first pick)", len(alloc.releases)) + t.Errorf("ReleaseHpCpus called %d times, want 1 rollback", len(alloc.releases)) + } + if len(cdiW.written) != 0 { + t.Errorf("WriteClaim called %d times, want 0", len(cdiW.written)) + } + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0", store.saved) + } + if _, ok := p.claims[uid]; ok { + t.Error("claim stored after allocation-domain rejection") } } @@ -1559,9 +1614,6 @@ func TestUnprepare_KnownClaim(t *testing.T) { } else if perErr != nil { t.Errorf("result[uid] = %v, want nil", perErr) } - if len(alloc.releases) != 1 { - t.Errorf("ReleaseHpCpus called %d times, want 1", len(alloc.releases)) - } if len(cdiW.removed) != 1 || cdiW.removed[0] != uid { t.Errorf("RemoveClaim called for %v, want [%v]", cdiW.removed, uid) } @@ -1600,9 +1652,8 @@ func TestUnprepare_UnknownUID(t *testing.T) { if len(alloc.releases) != 0 { t.Errorf("ReleaseHpCpus called %d times, want 0 for unknown UID", len(alloc.releases)) } - // Save must still be called once (batch write even with no-ops). - if store.saved != 1 { - t.Errorf("ClaimStore.Save called %d times, want 1", store.saved) + if store.saved != 0 { + t.Errorf("ClaimStore.Save called %d times, want 0", store.saved) } } @@ -1633,9 +1684,37 @@ func TestUnprepare_CDIRemoveError(t *testing.T) { if _, exists := p.claims[uid]; exists { t.Error("claim still in p.claims after Unprepare despite CDI error") } - // ReleaseHpCpus must have been called once — CPU leak on CDI error goes undetected otherwise. - if len(alloc.releases) != 1 { - t.Errorf("ReleaseHpCpus called %d times, want 1 (must release CPUs even on CDI error)", len(alloc.releases)) + // ReleaseHpCpus is now called by the ClaimUnprepare callback (policy layer), + // not directly by UnprepareResourceClaims, so alloc.releases stays 0 here. +} + +func TestUnprepare_ClaimStoreSaveFailure(t *testing.T) { + saveErr := errors.New("claim store save failed") + alloc := &trackingClaimAllocator{} + cdiW := &trackingCDIWriter{} + store := &trackingClaimStore{saveErr: saveErr} + uid := types.UID("uid-save-err") + claimState := &ClaimState{ + UID: string(uid), + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "4-7", ClassName: "gold"}}, + } + p := preparePlugin(t, alloc, cdiW, store, map[types.UID]*ClaimState{uid: claimState}) + + result, globalErr := p.UnprepareResourceClaims(context.Background(), []kubeletplugin.NamespacedObject{unprepareObj(uid)}) + if globalErr != nil { + t.Fatalf("UnprepareResourceClaims() unexpected global error: %v", globalErr) + } + if !errors.Is(result[uid], saveErr) { + t.Errorf("result[uid] = %v, want to wrap saveErr", result[uid]) + } + if _, exists := p.claims[uid]; !exists { + t.Error("claim removed from memory despite persistence failure") + } + if len(alloc.releases) != 0 { + t.Errorf("ReleaseHpCpus called %d times, want 0 before durable removal", len(alloc.releases)) + } + if len(cdiW.removed) != 0 { + t.Errorf("RemoveClaim called %d times, want 0 before durable removal", len(cdiW.removed)) } } @@ -1695,7 +1774,7 @@ func TestShareIDPtr(t *testing.T) { } } -// ---- Task 9: LiveClaimClasses, RestoreClaimsLocked, Start reconciliation ---- +// ---- Task 9: RestoreClaimsLocked, Start reconciliation ---- // startTestCDIWriter supports per-UID ClaimSpecExists state and a fixed // ListClaims result for Start reconciliation tests. WriteClaim is a no-op. @@ -1739,57 +1818,55 @@ func (s *preloadedClaimStore) Save(claims map[types.UID]*ClaimState) error { return nil } -// TestLiveClaimClasses_Empty verifies that LiveClaimClasses returns an empty +// TestLiveClaimsLocked_Empty verifies that LiveClaimsLocked returns an empty // map when there are no claims. -func TestLiveClaimClasses_Empty(t *testing.T) { +func TestLiveClaimsLocked_Empty(t *testing.T) { p, err := New("test-driver", validDeps()) if err != nil { t.Fatalf("New() unexpected error: %v", err) } - got := p.LiveClaimClasses() + got := p.LiveClaimsLocked() if len(got) != 0 { - t.Errorf("LiveClaimClasses() = %v, want empty map", got) + t.Errorf("LiveClaimsLocked() = %v, want empty map", got) } } -// TestLiveClaimClasses_SameClass verifies that two claims using the same class -// produce a count of 2. -func TestLiveClaimClasses_SameClass(t *testing.T) { +// TestLiveClaimsLocked_Snapshot verifies that LiveClaimsLocked returns a +// snapshot matching p.claims, and that mutating the returned map/slices does +// not corrupt the plugin's internal state (caller holds the resmgr lock, but +// the returned value must still be a defensive copy of the per-claim slice). +func TestLiveClaimsLocked_Snapshot(t *testing.T) { p, err := New("test-driver", validDeps()) if err != nil { t.Fatalf("New() unexpected error: %v", err) } - p.claims[types.UID("a")] = &ClaimState{UID: "a", Allocs: []ResultAlloc{{ClassName: "gold"}}} - p.claims[types.UID("b")] = &ClaimState{UID: "b", Allocs: []ResultAlloc{{ClassName: "gold"}}} - - got := p.LiveClaimClasses() - if got["gold"] != 2 { - t.Errorf("LiveClaimClasses()[gold] = %d, want 2", got["gold"]) - } - if len(got) != 1 { - t.Errorf("LiveClaimClasses() len = %d, want 1", len(got)) + p.claims[types.UID("uid-a")] = &ClaimState{ + UID: "uid-a", + Allocs: []ResultAlloc{{Device: "dev0", PkgID: 0, PunitID: 0, CPUs: "0-3", ClassName: "gold"}}, } -} - -// TestLiveClaimClasses_DifferentClasses verifies that two claims using -// different classes produce two entries in the result map. -func TestLiveClaimClasses_DifferentClasses(t *testing.T) { - p, err := New("test-driver", validDeps()) - if err != nil { - t.Fatalf("New() unexpected error: %v", err) + p.claims[types.UID("uid-b")] = &ClaimState{ + UID: "uid-b", + Allocs: []ResultAlloc{ + {Device: "dev1", PkgID: 0, PunitID: 1, CPUs: "4-5", ClassName: "silver"}, + {Device: "dev2", PkgID: 0, PunitID: 1, CPUs: "6-7", ClassName: "silver"}, + }, } - p.claims[types.UID("a")] = &ClaimState{UID: "a", Allocs: []ResultAlloc{{ClassName: "gold"}}} - p.claims[types.UID("b")] = &ClaimState{UID: "b", Allocs: []ResultAlloc{{ClassName: "silver"}}} - got := p.LiveClaimClasses() + got := p.LiveClaimsLocked() if len(got) != 2 { - t.Errorf("LiveClaimClasses() len = %d, want 2", len(got)) + t.Fatalf("LiveClaimsLocked() len = %d, want 2", len(got)) } - if got["gold"] != 1 { - t.Errorf("LiveClaimClasses()[gold] = %d, want 1", got["gold"]) + if allocs := got[types.UID("uid-a")]; len(allocs) != 1 || allocs[0].ClassName != "gold" { + t.Errorf("LiveClaimsLocked()[uid-a] = %+v, want one gold alloc", allocs) } - if got["silver"] != 1 { - t.Errorf("LiveClaimClasses()[silver] = %d, want 1", got["silver"]) + if allocs := got[types.UID("uid-b")]; len(allocs) != 2 { + t.Errorf("LiveClaimsLocked()[uid-b] len = %d, want 2", len(allocs)) + } + + // Mutating the returned slice must not affect p.claims (defensive copy). + got[types.UID("uid-a")][0].ClassName = "mutated" + if p.claims[types.UID("uid-a")].Allocs[0].ClassName != "gold" { + t.Errorf("LiveClaimsLocked() leaked internal state: p.claims mutated via returned snapshot") } } diff --git a/pkg/resmgr/main/main.go b/pkg/resmgr/main/main.go index b756366a6..127c62dbc 100644 --- a/pkg/resmgr/main/main.go +++ b/pkg/resmgr/main/main.go @@ -88,8 +88,8 @@ func (m *Main) Run() error { }() defer signal.Stop(sigCh) - err := m.mgr.Start() - return err + defer m.mgr.Stop() + return m.mgr.Start() } func (m *Main) ResourceManager() resmgr.ResourceManager { diff --git a/pkg/resmgr/main/main_test.go b/pkg/resmgr/main/main_test.go new file mode 100644 index 000000000..7ae067793 --- /dev/null +++ b/pkg/resmgr/main/main_test.go @@ -0,0 +1,105 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resmgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/events" + policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" +) + +// fakeResourceManager is a minimal ResourceManager implementation that +// records the order in which Start()/Stop() are invoked, so tests can +// assert that Main.Run() reaches Stop() after Start() returns. +type fakeResourceManager struct { + startErr error + calls []string +} + +func (f *fakeResourceManager) Start() error { + f.calls = append(f.calls, "start") + return f.startErr +} + +func (f *fakeResourceManager) Stop() { + f.calls = append(f.calls, "stop") +} + +func (f *fakeResourceManager) SendEvent(interface{}) error { return nil } + +// fakeBackend is a minimal policy.Backend implementation, just enough to +// satisfy Main.Run()'s use of m.policy.Name() (for tracing identity). +type fakeBackend struct{} + +func (fakeBackend) Name() string { return "fake" } +func (fakeBackend) Description() string { return "fake backend for tests" } +func (fakeBackend) Setup(*policyapi.BackendOptions) error { return nil } +func (fakeBackend) Reconfigure(interface{}) error { return nil } +func (fakeBackend) Start() error { return nil } +func (fakeBackend) Stop() error { return nil } +func (fakeBackend) Sync([]cache.Container, []cache.Container) error { return nil } +func (fakeBackend) AllocateResources(cache.Container) error { return nil } +func (fakeBackend) ReleaseResources(cache.Container) error { return nil } +func (fakeBackend) UpdateResources(cache.Container) error { return nil } +func (fakeBackend) HandleEvent(*events.Policy) (bool, error) { return false, nil } +func (fakeBackend) ExportResourceData(cache.Container) map[string]string { + return nil +} +func (fakeBackend) GetTopologyZones() []*policyapi.TopologyZone { return nil } +func (fakeBackend) GetExtendedResources() map[string]*resource.Quantity { + return nil +} + +var _ policyapi.Backend = fakeBackend{} + +// TestMainRunStopsResourceManagerAfterStartReturns verifies that Main.Run() +// calls mgr.Stop() once mgr.Start() returns, so that shutdown reaches the +// active policy's Backend.Stop() (e.g. the DRA plugin) even though nothing +// in the SIGTERM/SIGINT path calls mgr.Stop() directly. +func TestMainRunStopsResourceManagerAfterStartReturns(t *testing.T) { + mgr := &fakeResourceManager{} + m := &Main{ + policy: fakeBackend{}, + mgr: mgr, + } + + err := m.Run() + + require.NoError(t, err) + assert.Equal(t, []string{"start", "stop"}, mgr.calls) +} + +// TestMainRunStopsResourceManagerEvenOnStartError verifies Stop() is still +// reached (and the original Start() error still surfaces) when Start() +// itself fails. +func TestMainRunStopsResourceManagerEvenOnStartError(t *testing.T) { + startErr := assert.AnError + mgr := &fakeResourceManager{startErr: startErr} + m := &Main{ + policy: fakeBackend{}, + mgr: mgr, + } + + err := m.Run() + + assert.ErrorIs(t, err, startErr) + assert.Equal(t, []string{"start", "stop"}, mgr.calls) +} diff --git a/pkg/resmgr/nri.go b/pkg/resmgr/nri.go index 8e2c90384..8ac3b4fac 100644 --- a/pkg/resmgr/nri.go +++ b/pkg/resmgr/nri.go @@ -110,7 +110,7 @@ func (p *nriPlugin) start() error { } func (p *nriPlugin) stop() { - if p == nil { + if p == nil || p.stub == nil { return } @@ -302,6 +302,9 @@ func (p *nriPlugin) Synchronize(ctx context.Context, pods []*api.PodSandbox, con m := p.resmgr + m.Lock() + defer m.Unlock() + allocated, released, err := p.syncWithNRI(pods, containers) if err != nil { nri.Errorf("failed to synchronize with NRI: %v", err) diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index a5fd9af76..ff6f4d952 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -20,6 +20,7 @@ import ( "sort" "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" @@ -56,6 +57,14 @@ type ConstraintSet map[Domain]Constraint type Options struct { // SendEvent is the function for delivering events back to the resource manager. SendEvent SendEventFn + // KubeClientFn returns the shared kubernetes client, or a nil interface + // if none is available (yet). Evaluated by backends at Setup() time. + KubeClientFn func() kubernetes.Interface + // NodeName is the kubernetes node name the resource manager runs on. + NodeName string + // WithLock runs f while holding the resource manager's write lock. + // It is not re-entrant: calling WithLock again from within f deadlocks. + WithLock func(func()) } // BackendOptions describes the options for a policy backend instance @@ -68,6 +77,14 @@ type BackendOptions struct { SendEvent SendEventFn // Config is the policy-specific configuration. Config any + // KubeClientFn returns the shared kubernetes client, or a nil interface + // if none is available (yet). Evaluated by backends at Setup() time. + KubeClientFn func() kubernetes.Interface + // NodeName is the kubernetes node name the resource manager runs on. + NodeName string + // WithLock runs f while holding the resource manager's write lock. + // It is not re-entrant: calling WithLock again from within f deadlocks. + WithLock func(func()) } // CreateFn is the type for functions used to create a policy instance. @@ -102,6 +119,8 @@ type Backend interface { Reconfigure(any) error // Start up and sycnhronizes the policy, using the given cache and resource constraints. Start() error + // Stop shuts down the policy backend, releasing any resources it holds. + Stop() error // Sync synchronizes the policy, allocating/releasing the given containers. Sync([]cache.Container, []cache.Container) error // AllocateResources allocates resources to/for a container. @@ -140,6 +159,8 @@ type Policy interface { ActivePolicy() string // Start starts up policy, prepare for serving resource management requests. Start(any) error + // Stop shuts down the policy, releasing any resources it holds. + Stop() error // Reconfigure the policy. Reconfigure(any) error // Sync synchronizes the state of the active policy. @@ -267,10 +288,13 @@ func (p *policy) Start(cfg any) error { log.Infof("activating '%s' policy...", p.active.Name()) if err := p.active.Setup(&BackendOptions{ - Cache: p.cache, - System: p.system, - SendEvent: p.options.SendEvent, - Config: cfg, + Cache: p.cache, + System: p.system, + SendEvent: p.options.SendEvent, + Config: cfg, + KubeClientFn: p.options.KubeClientFn, + NodeName: p.options.NodeName, + WithLock: p.options.WithLock, }); err != nil { return err } @@ -284,6 +308,14 @@ func (p *policy) Start(cfg any) error { return p.active.Start() } +// Stop shuts down the active policy backend. +func (p *policy) Stop() error { + if p.active == nil { + return nil + } + return p.active.Stop() +} + // Reconfigure the policy. func (p *policy) Reconfigure(cfg any) error { scollect, err := p.newSystemCollector() diff --git a/pkg/resmgr/policy/policy_test.go b/pkg/resmgr/policy/policy_test.go new file mode 100644 index 000000000..7cc3f2c59 --- /dev/null +++ b/pkg/resmgr/policy/policy_test.go @@ -0,0 +1,249 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package policy + +import ( + "path/filepath" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/client-go/kubernetes" + + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/events" +) + +// newTestCache creates a real, temp-dir-backed cache for tests that need a +// non-nil cache.Cache (e.g. because Start() drives metrics collection which +// reads from it). +func newTestCache(t *testing.T) cache.Cache { + t.Helper() + // Use a not-yet-existing subdirectory of t.TempDir(): NewCache + // validates the permissions of a pre-existing CacheDir and rejects + // group-writable ones, but t.TempDir() itself is created 0777&^umask, + // which is group-writable under a permissive umask. Handing NewCache a + // fresh path lets it create the directory itself with the permissions + // it expects. + c, err := cache.NewCache(cache.Options{CacheDir: filepath.Join(t.TempDir(), "cache")}) + require.NoError(t, err) + return c +} + +// mockBackend is a minimal Backend implementation that records calls and +// the options it was set up with, for use by policy-level unit tests. +type mockBackend struct { + setupOpts *BackendOptions + setupErr error + startErr error + stopErr error + reconfigErr error + startCalled bool + stopCalled bool + setupCalled bool + callOrder *[]string + onStart func() + onSetup func(*BackendOptions) +} + +func (m *mockBackend) Name() string { return "mock" } +func (m *mockBackend) Description() string { return "mock backend for tests" } + +func (m *mockBackend) Setup(opts *BackendOptions) error { + m.setupCalled = true + m.setupOpts = opts + if m.callOrder != nil { + *m.callOrder = append(*m.callOrder, "setup") + } + if m.onSetup != nil { + m.onSetup(opts) + } + return m.setupErr +} + +func (m *mockBackend) Reconfigure(interface{}) error { return m.reconfigErr } + +func (m *mockBackend) Start() error { + m.startCalled = true + if m.callOrder != nil { + *m.callOrder = append(*m.callOrder, "start") + } + if m.onStart != nil { + m.onStart() + } + return m.startErr +} + +func (m *mockBackend) Stop() error { + m.stopCalled = true + if m.callOrder != nil { + *m.callOrder = append(*m.callOrder, "stop") + } + return m.stopErr +} + +func (m *mockBackend) Sync([]cache.Container, []cache.Container) error { return nil } +func (m *mockBackend) AllocateResources(cache.Container) error { return nil } +func (m *mockBackend) ReleaseResources(cache.Container) error { return nil } +func (m *mockBackend) UpdateResources(cache.Container) error { return nil } +func (m *mockBackend) HandleEvent(*events.Policy) (bool, error) { return false, nil } +func (m *mockBackend) ExportResourceData(cache.Container) map[string]string { + return nil +} +func (m *mockBackend) GetTopologyZones() []*TopologyZone { return nil } +func (m *mockBackend) GetExtendedResources() map[string]*resource.Quantity { + return nil +} + +var _ Backend = &mockBackend{} + +func TestPolicyStopForwardsToBackend(t *testing.T) { + backend := &mockBackend{} + p, err := NewPolicy(backend, newTestCache(t), &Options{}) + require.NoError(t, err) + + require.NoError(t, p.Start(nil)) + assert.True(t, backend.startCalled) + + require.NoError(t, p.Stop()) + assert.True(t, backend.stopCalled) +} + +func TestPolicyStopNilActiveBackendDoesNotPanic(t *testing.T) { + p := &policy{} + assert.NotPanics(t, func() { + err := p.Stop() + assert.NoError(t, err) + }) +} + +func TestPolicyStartForwardsKubeClientFnNodeNameAndWithLock(t *testing.T) { + backend := &mockBackend{} + wantClient := fakeKubeClient{} + var lockCalls int + withLock := func(f func()) { + lockCalls++ + f() + } + + p, err := NewPolicy(backend, newTestCache(t), &Options{ + KubeClientFn: func() kubernetes.Interface { return wantClient }, + NodeName: "node-under-test", + WithLock: withLock, + }) + require.NoError(t, err) + require.NoError(t, p.Start(nil)) + + require.NotNil(t, backend.setupOpts) + require.NotNil(t, backend.setupOpts.KubeClientFn) + assert.Equal(t, wantClient, backend.setupOpts.KubeClientFn()) + assert.Equal(t, "node-under-test", backend.setupOpts.NodeName) + require.NotNil(t, backend.setupOpts.WithLock) + + // Exercise the forwarded WithLock: it should invoke the callback and + // go through the resmgr-supplied withLock, not some other function. + called := false + backend.setupOpts.WithLock(func() { called = true }) + assert.True(t, called) + assert.Equal(t, 1, lockCalls) +} + +func TestPolicyStartKubeClientFnNilWhenNoClient(t *testing.T) { + backend := &mockBackend{} + p, err := NewPolicy(backend, newTestCache(t), &Options{ + KubeClientFn: func() kubernetes.Interface { return nil }, + }) + require.NoError(t, err) + require.NoError(t, p.Start(nil)) + + require.NotNil(t, backend.setupOpts.KubeClientFn) + // Must be observed as a genuinely nil interface by the backend, not a + // non-nil interface wrapping a typed nil. + assert.Nil(t, backend.setupOpts.KubeClientFn()) +} + +// lockContractStub is a WithLock stand-in that panics if invoked while +// already "held", i.e. re-entrantly. It is used to assert that a backend's +// lifecycle calls made through WithLock never nest. +type lockContractStub struct { + mu sync.Mutex + held bool +} + +func (s *lockContractStub) run(f func()) { + s.mu.Lock() + if s.held { + s.mu.Unlock() + panic("WithLock invoked re-entrantly") + } + s.held = true + s.mu.Unlock() + + defer func() { + s.mu.Lock() + s.held = false + s.mu.Unlock() + }() + + f() +} + +// TestLockContractWithLockNotReentrant asserts that a backend driving two +// logically-sequential operations through WithLock (e.g. a future DRA +// plugin's Start() then PublishResources()) never nests those calls: the +// stub above would panic if it did. +func TestLockContractWithLockNotReentrant(t *testing.T) { + stub := &lockContractStub{} + var order []string + + backend := &mockBackend{ + onStart: func() { + // Simulate a backend that runs two operations under the + // resource manager's write lock, sequentially rather than + // nested. + stub.run(func() { order = append(order, "locked-op-1") }) + stub.run(func() { order = append(order, "locked-op-2") }) + }, + } + + p, err := NewPolicy(backend, newTestCache(t), &Options{WithLock: stub.run}) + require.NoError(t, err) + + assert.NotPanics(t, func() { + require.NoError(t, p.Start(nil)) + }) + assert.Equal(t, []string{"locked-op-1", "locked-op-2"}, order) + assert.False(t, stub.held) +} + +// TestLockContractReentrantCallPanics is a sanity check that the stub +// itself actually detects re-entrancy (guards against a vacuously-true +// contract test above). +func TestLockContractReentrantCallPanics(t *testing.T) { + stub := &lockContractStub{} + assert.Panics(t, func() { + stub.run(func() { + stub.run(func() {}) + }) + }) +} + +// fakeKubeClient is a minimal kubernetes.Interface stand-in used only for +// identity comparison in tests; its methods are never called. +type fakeKubeClient struct { + kubernetes.Interface +} diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index 596ca6dea..e556d933d 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -30,6 +30,7 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/topology" + "k8s.io/client-go/kubernetes" "sigs.k8s.io/yaml" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1" @@ -223,10 +224,32 @@ func (m *resmgr) start(cfg cfgapi.ResmgrConfig) error { func (m *resmgr) Stop() { log.Infof("shutting down...") + // Stop the active policy backend (e.g. the DRA plugin) before + // acquiring the write lock: an in-flight Prepare/AllocateResources + // call may be holding the lock via WithLock, and the backend's + // Stop() must be able to run (and complete) without waiting on it. + // Calling it under the lock would deadlock in that case. + if m.policy != nil { + if err := m.policy.Stop(); err != nil { + log.Warnf("failed to stop policy: %v", err) + } + } + m.Lock() defer m.Unlock() - m.nri.stop() + if m.nri != nil { + m.nri.stop() + } +} + +// withWriteLock runs f while holding the resource manager's write lock. +// It is not re-entrant: calling withWriteLock again from within f deadlocks, +// since the underlying mutex is not recursive. +func (m *resmgr) withWriteLock(f func()) { + m.Lock() + defer m.Unlock() + f() } // setupCache creates a cache and reloads its last saved state if found. @@ -254,7 +277,12 @@ func (m *resmgr) setupPolicy(backend policy.Backend) error { log.Warnf("failed to set active policy: %v", err) } - p, err := policy.NewPolicy(backend, m.cache, &policy.Options{SendEvent: m.SendEvent}) + p, err := policy.NewPolicy(backend, m.cache, &policy.Options{ + SendEvent: m.SendEvent, + KubeClientFn: m.kubeClientFn, + NodeName: m.agent.NodeName(), + WithLock: m.withWriteLock, + }) if err != nil { return resmgrError("failed to create policy %s: %v", backend.Name(), err) } @@ -263,6 +291,19 @@ func (m *resmgr) setupPolicy(backend policy.Backend) error { return nil } +// kubeClientFn returns the resource manager's shared kubernetes client, or +// a nil interface if the agent has none (yet). Wrapping agent.KubeClient() +// directly as a kubernetes.Interface would yield a non-nil interface +// wrapping a typed nil *client.Client in local-config mode; the explicit +// nil check below avoids that trap. +func (m *resmgr) kubeClientFn() kubernetes.Interface { + c := m.agent.KubeClient() + if c == nil { + return nil + } + return c +} + // setupHealthCheck prepares the resource manager for serving health-check requests. func (m *resmgr) setupHealthCheck() { mux := instrumentation.HTTPServer().GetMux() diff --git a/pkg/resmgr/resource_manager_test.go b/pkg/resmgr/resource_manager_test.go new file mode 100644 index 000000000..850a77b93 --- /dev/null +++ b/pkg/resmgr/resource_manager_test.go @@ -0,0 +1,152 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package resmgr + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "k8s.io/apimachinery/pkg/api/resource" + + "github.com/containers/nri-plugins/pkg/agent" + "github.com/containers/nri-plugins/pkg/resmgr/cache" + "github.com/containers/nri-plugins/pkg/resmgr/events" + "github.com/containers/nri-plugins/pkg/resmgr/policy" +) + +// fakePolicy is a minimal policy.Policy implementation that records Stop() +// calls (and, optionally, runs a probe from within Stop()) for use by +// resmgr-level shutdown-ordering tests. +type fakePolicy struct { + stopCalled int + stopErr error + onStop func() +} + +func (f *fakePolicy) ActivePolicy() string { return "fake" } +func (f *fakePolicy) Start(interface{}) error { return nil } +func (f *fakePolicy) Stop() error { + f.stopCalled++ + if f.onStop != nil { + f.onStop() + } + return f.stopErr +} +func (f *fakePolicy) Reconfigure(interface{}) error { return nil } +func (f *fakePolicy) Sync([]cache.Container, []cache.Container) error { return nil } +func (f *fakePolicy) AllocateResources(cache.Container) error { return nil } +func (f *fakePolicy) ReleaseResources(cache.Container) error { return nil } +func (f *fakePolicy) UpdateResources(cache.Container) error { return nil } +func (f *fakePolicy) HandleEvent(*events.Policy) (bool, error) { return false, nil } +func (f *fakePolicy) ExportResourceData(cache.Container) {} +func (f *fakePolicy) GetTopologyZones() []*policy.TopologyZone { return nil } +func (f *fakePolicy) GetExtendedResources() map[string]*resource.Quantity { return nil } + +var _ policy.Policy = &fakePolicy{} + +// TestResmgrStopCallsPolicyStop verifies that resmgr.Stop() reaches the +// active policy's Stop() method (which, in turn, is how Backend.Stop() -- +// e.g. the DRA plugin shutdown -- gets invoked). +func TestResmgrStopCallsPolicyStop(t *testing.T) { + fp := &fakePolicy{} + m := &resmgr{policy: fp} + + m.Stop() + + assert.Equal(t, 1, fp.stopCalled) +} + +// TestResmgrStopCallsPolicyStopBeforeLock verifies that resmgr.Stop() calls +// policy.Stop() before acquiring the resource manager's write lock: an +// in-flight Prepare/AllocateResources call may be holding that lock via +// WithLock, and policy.Stop() must not have to wait for it. +func TestResmgrStopCallsPolicyStopBeforeLock(t *testing.T) { + m := &resmgr{} + fp := &fakePolicy{ + onStop: func() { + // If resmgr.Stop() had already acquired the write lock before + // calling policy.Stop(), this TryLock would fail. + locked := m.TryLock() + require.True(t, locked, "resmgr write lock must not be held while policy.Stop() runs") + m.Unlock() + }, + } + m.policy = fp + + m.Stop() + + assert.Equal(t, 1, fp.stopCalled) +} + +// TestResmgrStopNilPolicyDoesNotPanic verifies that Stop() is safe to call +// when Start() was never called (m.policy is nil). +func TestResmgrStopNilPolicyDoesNotPanic(t *testing.T) { + m := &resmgr{} + assert.NotPanics(t, func() { + m.Stop() + }) +} + +// TestResmgrStopNilNriDoesNotPanic verifies that Stop() remains safe when the +// NRI plugin was never started, so startup failures before m.nri.start() do not +// end up dereferencing a nil stub. +func TestResmgrStopNilNriDoesNotPanic(t *testing.T) { + m := &resmgr{nri: &nriPlugin{}} + assert.NotPanics(t, func() { + m.Stop() + }) +} + +// TestNriPluginStopNilStubDoesNotPanic verifies that shutdown is safe when the +// plugin exists but its stub was never created during a failed startup path. +func TestNriPluginStopNilStubDoesNotPanic(t *testing.T) { + p := &nriPlugin{} + assert.NotPanics(t, func() { + p.stop() + }) +} + +// TestWithWriteLockRunsCallbackUnderLock verifies that withWriteLock +// acquires the resource manager's write lock for the duration of the +// callback. +func TestWithWriteLockRunsCallbackUnderLock(t *testing.T) { + m := &resmgr{} + ran := false + + m.withWriteLock(func() { + ran = true + assert.False(t, m.TryLock(), "lock must be held while the withWriteLock callback runs") + }) + + assert.True(t, ran) + // The lock must have been released once withWriteLock returned. + assert.True(t, m.TryLock()) + m.Unlock() +} + +// TestKubeClientFnNilWhenAgentHasNoClient verifies the typed-nil-safe +// wrapper: in local-config mode agent.KubeClient() returns a nil +// *client.Client, and m.kubeClientFn() must surface that as a genuinely +// nil kubernetes.Interface rather than a non-nil interface wrapping a +// typed nil pointer. +func TestKubeClientFnNilWhenAgentHasNoClient(t *testing.T) { + agt, err := agent.New(agent.TopologyAwareConfigInterface(), agent.WithConfigFile("/nonexistent/config.yaml")) + require.NoError(t, err) + + m := &resmgr{agent: agt} + + assert.Nil(t, m.kubeClientFn()) +} From f9d293b7addca2b37ba712256302b9744af5644e Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 22:27:26 +0300 Subject: [PATCH 08/39] helm/topology-aware: add DRA chart additions Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- .../topology-aware/templates/clusterrole.yaml | 18 ++++ .../topology-aware/templates/daemonset.yaml | 22 +++++ .../topology-aware/templates/deviceclass.yaml | 12 +++ deployment/helm/topology-aware/values.yaml | 8 ++ docs/resource-policy/policy/topology-aware.md | 88 +++++++++++++++++++ 5 files changed, 148 insertions(+) create mode 100644 deployment/helm/topology-aware/templates/deviceclass.yaml diff --git a/deployment/helm/topology-aware/templates/clusterrole.yaml b/deployment/helm/topology-aware/templates/clusterrole.yaml index 3ea31273d..4951c300b 100644 --- a/deployment/helm/topology-aware/templates/clusterrole.yaml +++ b/deployment/helm/topology-aware/templates/clusterrole.yaml @@ -30,3 +30,21 @@ rules: - list - update - delete +{{- if (.Values.config.dra).enabled }} +- apiGroups: + - resource.k8s.io + resources: + - resourceslices + verbs: + - list + - watch + - create + - update + - delete +- apiGroups: + - resource.k8s.io + resources: + - resourceclaims + verbs: + - get +{{- end }} diff --git a/deployment/helm/topology-aware/templates/daemonset.yaml b/deployment/helm/topology-aware/templates/daemonset.yaml index 0302c2bb2..ea5f39b5e 100644 --- a/deployment/helm/topology-aware/templates/daemonset.yaml +++ b/deployment/helm/topology-aware/templates/daemonset.yaml @@ -142,6 +142,14 @@ spec: - name: hostdev mountPath: /host/dev {{- end }} + {{- if (.Values.config.dra).enabled }} + - name: kubelet-plugins + mountPath: /var/lib/kubelet/plugins + - name: kubelet-plugins-registry + mountPath: /var/lib/kubelet/plugins_registry + - name: cdi-spec-dir + mountPath: /var/run/cdi + {{- end }} {{- if .Values.podPriorityClassNodeCritical }} priorityClassName: system-node-critical @@ -176,6 +184,20 @@ spec: path: /dev type: Directory {{- end }} + {{- if (.Values.config.dra).enabled }} + - name: kubelet-plugins + hostPath: + path: /var/lib/kubelet/plugins + type: DirectoryOrCreate + - name: kubelet-plugins-registry + hostPath: + path: /var/lib/kubelet/plugins_registry + type: DirectoryOrCreate + - name: cdi-spec-dir + hostPath: + path: /var/run/cdi + type: DirectoryOrCreate + {{- end }} {{- if .Values.nri.runtime.patchConfig }} - name: containerd-config hostPath: diff --git a/deployment/helm/topology-aware/templates/deviceclass.yaml b/deployment/helm/topology-aware/templates/deviceclass.yaml new file mode 100644 index 000000000..8b765192a --- /dev/null +++ b/deployment/helm/topology-aware/templates/deviceclass.yaml @@ -0,0 +1,12 @@ +{{- if (.Values.config.dra).enabled }} +apiVersion: resource.k8s.io/v1 +kind: DeviceClass +metadata: + name: nri.topology-aware.cpu + labels: + {{- include "nri-plugin.labels" . | nindent 4 }} +spec: + selectors: + - cel: + expression: device.driver == "nri.topology-aware.cpu" +{{- end }} diff --git a/deployment/helm/topology-aware/values.yaml b/deployment/helm/topology-aware/values.yaml index 81c5224ac..807b532c1 100644 --- a/deployment/helm/topology-aware/values.yaml +++ b/deployment/helm/topology-aware/values.yaml @@ -66,6 +66,14 @@ config: prometheusExport: false reportPeriod: 60s samplingRatePerMillion: 0 + # DRA (Dynamic Resource Allocation) kubelet plugin integration. See + # TopologyAwareDRA in + # pkg/apis/config/v1alpha1/resmgr/policy/topologyaware/config.go for what + # these fields control. Requires Kubernetes 1.34+ (resource.k8s.io/v1, + # used by the DeviceClass this chart installs when enabled=true). + dra: + enabled: false + sharedCounters: false # configGroupLabel: config.nri/group diff --git a/docs/resource-policy/policy/topology-aware.md b/docs/resource-policy/policy/topology-aware.md index 8ece885dd..e18d2b2d6 100644 --- a/docs/resource-policy/policy/topology-aware.md +++ b/docs/resource-policy/policy/topology-aware.md @@ -733,6 +733,94 @@ both on startup and when they are returned to the shared pool from exclusive allocation. The CPU assigned for the reserved pool (`kube-system` namespace) will get configured according to the `reserved` CPU class. +### Dynamic Resource Allocation + +The topology-aware policy can optionally register a Kubernetes [Dynamic Resource +Allocation](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-resource-allocation/) +(DRA) kubelet plugin driver that publishes `cpuClasses` (see [Class Based CPU +Tuning](#class-based-cpu-tuning)) as DRA devices. This lets pods request CPUs with a +specific `cpuClass` via a `ResourceClaim` instead of (or in addition to) the +`cpu-class.resource-policy.nri.io` annotation, and lets the scheduler see and account +for `cpuClass`-tagged capacity through `node.status.allocatable`. + +#### Prerequisites + +- Kubernetes 1.34+, with the [KEP-5075](https://github.com/kubernetes/enhancements/issues/5075) + (`DRAConsumableCapacity` / consumable capacity) alpha feature gate enabled on the + API server, kube-scheduler, kube-controller-manager, and kubelet. The `DeviceClass` object below uses the `resource.k8s.io/v1` + API, which requires Kubernetes 1.34+. +- Optionally, Kubernetes 1.37+, with the + [KEP-5517](https://github.com/kubernetes/enhancements/issues/5517) + (`DRANodeAllocatableResources` / node allocatable resources) alpha feature gate + enabled, so the scheduler accounts for `cpuClass`-tagged capacity in its own + in-memory node-allocatable bookkeeping and records the resolved allocation on + `pod.status.nodeAllocatableResourceClaimStatuses[]` at PreBind — **not** on + `node.status.allocatable`, which this feature gate never mutates. Without it, + DRA allocation still works but capacity mirroring is skipped. +- The container runtime must support NRI (as for the rest of this policy). + +#### Enabling + +DRA support is off by default. Enable it via the Helm chart's `config.dra.enabled` +value: + +```console +helm install topology-aware deployment/helm/topology-aware \ + --set config.dra.enabled=true +``` + +This does three things: +- Installs the extra RBAC rules the driver needs (`resourceslices` CRUD, + `resourceclaims: get`). +- Adds the host mounts the driver's kubelet plugin needs + (`/var/lib/kubelet/plugins`, `/var/lib/kubelet/plugins_registry`, `/var/run/cdi`). +- Installs the base `DeviceClass` named `nri.topology-aware.cpu`. + +It also sets `spec.dra.enabled: true` in the `TopologyAwarePolicy` CR, which turns on +the policy's own DRA plugin (kubelet-plugin registration and device publication). + +`config.dra.sharedCounters` (also under `config.dra`, default `false`) enables Model C +publication once the underlying [KEP-5941](https://github.com/kubernetes/enhancements/issues/5941) +shared-counters support is available; leave it `false` otherwise. + +#### Requesting a `cpuClass` via `ResourceClaim` + +Each defined `cpuClass` is published as a DRA device under the +`nri.topology-aware.cpu` `DeviceClass`, with the class's config attributes (such as +`nri/pctPriority`) exposed as device attributes. **Only high-priority PCT classes +are currently published this way** — non-HP classes are filtered out at +publication time, and the driver's `Prepare` rejects claims against them, so +ordinary configured classes without a high-priority tier will not appear as +devices. An eligible HP class can still be hidden from DRA by setting +`cpuClass.dra.publish: false`, which leaves it configurable via the normal +`cpu-class.resource-policy.nri.io` annotation but excludes it from device +publication; `dra.publish` defaults to `true` when unset. A `ResourceClaim` +can select on any of these attributes. For example, to request 2 CPUs from a +high-priority PCT class: + +```yaml +apiVersion: resource.k8s.io/v1 +kind: ResourceClaim +metadata: + name: hp-turbo-cpus +spec: + devices: + requests: + - name: cpus + exactly: + deviceClassName: nri.topology-aware.cpu + capacity: + requests: + nri/cpus: "2" + selectors: + - cel: + expression: | + device.attributes["nri"].pctPriority == "high" +``` + +A pod then references the claim via `resourceClaims` and consumes it in a container's +`resources.claims`, per the standard Kubernetes DRA pod API. + ### IRQ CPU Affinity Tuning Containers eligible for exclusive CPU allocation can be annotated with IRQ From db91a5309bd860042c977bce47079fda7ec8cedd Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 28 Aug 2026 22:30:15 +0300 Subject: [PATCH 09/39] e2e: add DRA e2e test Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- docs/resource-policy/policy/topology-aware.md | 2 +- test/e2e/files/Vagrantfile.in | 2 + test/e2e/playbook/provision.yaml | 33 +++ .../topology-aware/helm-config.yaml.in | 4 + .../n4c16/test20-dra/code.var.sh | 276 ++++++++++++++++++ test/e2e/run.sh | 19 +- test/e2e/run_tests.sh | 6 +- 7 files changed, 336 insertions(+), 6 deletions(-) create mode 100644 test/e2e/policies.test-suite/topology-aware/n4c16/test20-dra/code.var.sh diff --git a/docs/resource-policy/policy/topology-aware.md b/docs/resource-policy/policy/topology-aware.md index e18d2b2d6..7739c6ee6 100644 --- a/docs/resource-policy/policy/topology-aware.md +++ b/docs/resource-policy/policy/topology-aware.md @@ -741,7 +741,7 @@ Allocation](https://kubernetes.io/docs/concepts/scheduling-eviction/dynamic-reso Tuning](#class-based-cpu-tuning)) as DRA devices. This lets pods request CPUs with a specific `cpuClass` via a `ResourceClaim` instead of (or in addition to) the `cpu-class.resource-policy.nri.io` annotation, and lets the scheduler see and account -for `cpuClass`-tagged capacity through `node.status.allocatable`. +for `cpuClass`-tagged capacity when fitting pods onto nodes. #### Prerequisites diff --git a/test/e2e/files/Vagrantfile.in b/test/e2e/files/Vagrantfile.in index f40a314a0..fa940c6cd 100644 --- a/test/e2e/files/Vagrantfile.in +++ b/test/e2e/files/Vagrantfile.in @@ -19,6 +19,7 @@ KERNEL_GETSOURCE = "#{ENV['kernel_getsource']}" KERNEL_CONFIG = "#{ENV['kernel_config']}" K8S_RELEASE = "#{ENV['k8s_release']}" K8S_VERSION = "#{ENV['k8s_version']}" +K8S_FEATURE_GATES = "#{ENV['k8s_feature_gates']}" HELM_RELEASE = "#{ENV['helm_release']}" CRI_RUNTIME = "#{ENV['k8scri']}" CRIO_RELEASE = "1.28.1" @@ -125,6 +126,7 @@ Vagrant.configure("2") do |config| kernel_config: KERNEL_CONFIG, k8s_release: K8S_RELEASE, k8s_version: K8S_VERSION, + k8s_feature_gates: K8S_FEATURE_GATES, helm_release: HELM_RELEASE, cri_runtime: CRI_RUNTIME, containerd_release: CONTAINERD_RELEASE, diff --git a/test/e2e/playbook/provision.yaml b/test/e2e/playbook/provision.yaml index be961c0bf..79a838f58 100644 --- a/test/e2e/playbook/provision.yaml +++ b/test/e2e/playbook/provision.yaml @@ -7,6 +7,8 @@ cri_runtime: "{{ cri_runtime }}" k8s_release: "{{ k8s_release }}" k8s_version: "{{ k8s_version }}" + # Comma-separated "Gate1=true,Gate2=false" list, or empty string. + k8s_feature_gates: "{{ k8s_feature_gates }}" is_containerd: false is_crio: false containerd_release: "{{ containerd_release }}" @@ -501,9 +503,40 @@ daemon_reload: true state: restarted + - name: Write kubeadm --config with feature gates for apiserver/scheduler/kubelet + ansible.builtin.copy: + dest: /root/kubeadm-config.yaml + content: | + apiVersion: kubeadm.k8s.io/v1beta4 + kind: ClusterConfiguration + networking: + podSubnet: "{{ network }}" + apiServer: + extraArgs: + - name: feature-gates + value: "{{ k8s_feature_gates }}" + scheduler: + extraArgs: + - name: feature-gates + value: "{{ k8s_feature_gates }}" + --- + apiVersion: kubelet.config.k8s.io/v1beta1 + kind: KubeletConfiguration + featureGates: + {% for gate in k8s_feature_gates.split(',') if gate.strip() and '=' in gate %} + {{ gate.split('=')[0] }}: {{ gate.split('=')[1] }} + {% endfor %} + when: k8s_feature_gates != "" + - name: Initialize the Kubernetes cluster using kubeadm ansible.builtin.command: cmd: kubeadm init --pod-network-cidr="{{ network }}" + when: k8s_feature_gates == "" + + - name: Initialize the Kubernetes cluster using kubeadm (with feature gates) + ansible.builtin.command: + cmd: kubeadm init --config /root/kubeadm-config.yaml + when: k8s_feature_gates != "" - name: Setup kubeconfig for vagrant user ansible.builtin.command: "{{ item }}" diff --git a/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in b/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in index 894e4dca3..0c9020169 100644 --- a/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in +++ b/test/e2e/policies.test-suite/topology-aware/helm-config.yaml.in @@ -29,6 +29,10 @@ config: $([ -n "$CPU_CLASSES" ] && echo " cpuClasses: $CPU_CLASSES ") + $([ -n "$DRA_ENABLED" ] && echo " + dra: + enabled: $DRA_ENABLED + ") $([ -n "$DEFAULT_EXCLUSIVE_CPUCLASS" ] && echo " defaultExclusiveCPUClass: $DEFAULT_EXCLUSIVE_CPUCLASS ") diff --git a/test/e2e/policies.test-suite/topology-aware/n4c16/test20-dra/code.var.sh b/test/e2e/policies.test-suite/topology-aware/n4c16/test20-dra/code.var.sh new file mode 100644 index 000000000..b76d5252a --- /dev/null +++ b/test/e2e/policies.test-suite/topology-aware/n4c16/test20-dra/code.var.sh @@ -0,0 +1,276 @@ +# test20-dra: DRA (KEP-5075/KEP-5517) integration for the topology-aware +# policy. +# +# Full test: launches the plugin with the SST/cpufreq mock and a +# hp-turbo cpuClass, probes the API server for the KEP-5075/KEP-5517 +# feature gates (skipping cleanly if either is missing), applies a +# ResourceClaim selecting the HP class by its published nri/pctPriority +# attribute plus a pod consuming it, and asserts the claimed CPUs' +# CDI env vars, their CLOS association in the policy log, and the +# scheduler's pod.status.nodeAllocatableResourceClaimStatuses[] record. + +cleanup() { + vm-command "kubectl delete pods --all --now" + vm-command "kubectl delete resourceclaims --all --now" + helm-terminate +} + +# compress-cpulist "8 9 10 12" prints "8-10,12" -- range-collapsing, +# same python3 idiom as test19-cpuclass's expand-cpulist (its inverse). +# Needed because pct.go:786's "associated cpus %s to CLOS %d" log line +# formats its cpuset with +# cpuset.CPUSet.String(), which collapses contiguous ids into ranges, +# so individual/comma-joined ids parsed out of NRI_CPU env vars +# would never match assert-cpu-clos's regex without this. +compress-cpulist() { + local cpus="$1" + + python3 -c ' +import sys +ids = sorted(int(x) for x in sys.argv[1].split()) +ranges = [] +start = prev = None +for i in ids: + if start is None: + start = prev = i + elif i == prev + 1: + prev = i + else: + ranges.append((start, prev)) + start = prev = i +if start is not None: + ranges.append((start, prev)) +print(",".join(str(a) if a == b else "%d-%d" % (a, b) for a, b in ranges)) +' "$cpus" +} + +# wait-node-allocatable-claim-status [timeout=30] [interval=2] +# Polls pod.status.nodeAllocatableResourceClaimStatuses[] until it +# contains an entry with resourceClaimName == , +# listed in .containers, and a .mapping[] entry {name: cpu, quantity: +# }, or fails with command-error on timeout. +wait-node-allocatable-claim-status() { + local pod="$1" claim="$2" ctr="$3" cpus="$4" timeout=${5:-30} interval=${6:-2} elapsed=0 + while [ "$elapsed" -lt "$timeout" ]; do + vm-command "kubectl get pod $pod -o json | jq -c \ + '[(.status.nodeAllocatableResourceClaimStatuses // [])[] | \ + select(.resourceClaimName == \"$claim\" and \ + (.containers // [] | index(\"$ctr\")) and \ + ((.mapping // []) | any(.name == \"cpu\" and .quantity == \"$cpus\")))] | length'" + [ "$COMMAND_OUTPUT" -gt 0 ] 2>/dev/null && return 0 + sleep "$interval" + elapsed=$((elapsed + interval)) + done + vm-command "kubectl get pod $pod -o jsonpath='{.status.nodeAllocatableResourceClaimStatuses}'" + command-error "pod $pod's status.nodeAllocatableResourceClaimStatuses missing entry {resourceClaimName: $claim, containers: [$ctr], mapping: [{name: cpu, quantity: $cpus}]} (got: $COMMAND_OUTPUT)" +} + +# wait-resourceslice-devices [timeout] +# Polls until the DRA driver's published ResourceSlice(s) report at +# least one device, storing a compact JSON array of +# {allowMultipleAllocations, nodeAllocatableResources} objects +# (one per device) in COMMAND_OUTPUT. The driver needs a moment after +# helm-launch to publish its ResourceSlice, so this can't be a single +# one-shot check. +wait-resourceslice-devices() { + local timeout=${1:-30} elapsed=0 + while [ "$elapsed" -lt "$timeout" ]; do + vm-command "kubectl get resourceslices -o json | \ + jq -c '[.items[] | select(.spec.driver == \"nri.topology-aware.cpu\") | (.spec.devices // [])[] | {name, capacity: .capacity[\"nri/cpus\"].value, class: .attributes[\"nri/cpuClass\"].string, priority: .attributes[\"nri/pctPriority\"].string, allowMultipleAllocations, nodeAllocatableResources}]'" + if [ -n "$COMMAND_OUTPUT" ] && [ "$COMMAND_OUTPUT" != "[]" ]; then + return 0 + fi + sleep 1 + elapsed=$((elapsed + 1)) + done + return 1 +} + +OVERRIDE_SYS_CPUFREQ='[{"cpus": "0-15", "base": 2900000, "min": 800000, "max": 3800000}]' +OVERRIDE_SST='{"supported": true, "clos_count": 4, "packages": [{"id": 0, "cpus": "0-7", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}, {"id": 1, "cpus": "8-15", "tf_supported": true, "cp_supported": true, "max_hp_cpus": 2}]}' +OVERRIDE_SST_STATE_DIR="/tmp/nri-pct-mock" + +CPU_CLASSES="[ + { name: hp-turbo, pctPriority: high, pctMinFreq: turbo, pctMaxFreq: turbo }, + { name: shared , minFreq: min, maxFreq: base } ]" +SHARED_CPUCLASS="shared" +DEBUG_LOGGERS="agent cpu cpuclass" +DRA_ENABLED=true + +cleanup + +helm_config=$(COLOCATE_PODS=false \ + DEBUG_LOGGERS="$DEBUG_LOGGERS" \ + CPU_CLASSES="$CPU_CLASSES" \ + SHARED_CPUCLASS="$SHARED_CPUCLASS" \ + DRA_ENABLED="$DRA_ENABLED" \ + EXTRA_ENV_OVERRIDE_SYS_CPUFREQ="$OVERRIDE_SYS_CPUFREQ" \ + EXTRA_ENV_OVERRIDE_SST="$OVERRIDE_SST" \ + EXTRA_ENV_OVERRIDE_SST_STATE_DIR="$OVERRIDE_SST_STATE_DIR" \ + instantiate helm-config.yaml) helm-launch topology-aware + +# +# Feature-gate probe. +# +# If the API server's alpha gates DRAConsumableCapacity/DRANodeAllocatableResources +# are off, it silently strips both fields on write, so they're absent on read back here. +# This requires the plugin to already be running (hence it happens after helm-launch, +# not before), and with the hp-turbo class + SST mock active above so there's +# at least one real device to read back (an empty CPU_CLASSES publishes zero devices). +wait-resourceslice-devices 30 || { + helm-terminate + error "no devices found in any ResourceSlice within timeout (not a feature-gate issue -- check DRA_ENABLED/CPU_CLASSES/SST mock config)" +} + +# Require both fields on the *same* device object, not merely present +# somewhere in the (possibly multi-device) list -- every device this +# driver publishes gets both fields set unconditionally, so checking +# them independently across the whole list would loosely pass even if, +# say, one device kept allowMultipleAllocations while a different +# device kept nodeAllocatableResources; that's not what either gate +# being enabled actually implies. +if jq -e 'any(.[]; .allowMultipleAllocations == true and (.nodeAllocatableResources != null))' \ + >/dev/null 2>&1 <<< "$COMMAND_OUTPUT"; then + echo "DRA feature gates (KEP-5075/KEP-5517) detected as enabled on the API server; continuing." +else + helm-terminate + echo "Test verdict: SKIP (KEP-5075/KEP-5517 feature gate missing)" + exit 0 +fi + +# +# Claim + pod. +# +# A ResourceClaim selecting the HP cpuClass by its published +# nri/pctPriority attribute. +# +# Only "objects create cleanly and the pod reaches Running" is checked +# here + +# Bypassing create() also bypasses its image pre-pull step -- pull the +# pod's image explicitly so a freshly created VM doesn't stall/fail on +# kubectl's own on-demand pull. +vm-command "crictl -i unix://${k8scri_sock} pull quay.io/prometheus/busybox" || + command-error "failed to pre-pull quay.io/prometheus/busybox" + +# Manifests are written to local temp files and copied to the VM with +# vm-put-file, then kubectl apply'd -- this sidesteps vm-command's +# double-quoting/escaping entirely (the YAML below needs literal +# double quotes for its CEL string literals and the "2" capacity +# request). +claim_yaml=$(mktemp) +cat <<'EOF' > "$claim_yaml" +apiVersion: resource.k8s.io/v1 +kind: ResourceClaim +metadata: + name: hp-turbo-cpus +spec: + devices: + requests: + - name: cpus + exactly: + deviceClassName: nri.topology-aware.cpu + capacity: + requests: + nri/cpus: "2" + selectors: + - cel: + expression: | + device.attributes["nri"].pctPriority == "high" +EOF +vm-put-file --cleanup "$claim_yaml" hp-turbo-cpus-claim.yaml +vm-command "kubectl apply -f hp-turbo-cpus-claim.yaml" || + command-error "failed to create ResourceClaim hp-turbo-cpus" + +# The consuming pod. Not routed through create(): create() defaults to +# wait=Ready, which a ResourceClaim never satisfies on its own (it has +# no Ready condition), and guaranteed.yaml.in (the shared template +# create() uses, shared by ~23 other tests) has no +# resourceClaims/resources.claims support -- not modified here. +pod_yaml=$(mktemp) +cat <<'EOF' > "$pod_yaml" +apiVersion: v1 +kind: Pod +metadata: + name: dra-pod0 +spec: + resourceClaims: + - name: cpus + resourceClaimName: hp-turbo-cpus + containers: + - name: dra-pod0c0 + image: quay.io/prometheus/busybox + imagePullPolicy: IfNotPresent + command: + - sh + - -c + - echo dra-pod0c0 $(sleep inf) + resources: + claims: + - name: cpus + requests: + cpu: "1" + memory: "100M" + limits: + cpu: "1" + memory: "100M" + terminationGracePeriodSeconds: 1 +EOF +vm-put-file --cleanup "$pod_yaml" dra-pod0.yaml +vm-command "kubectl apply -f dra-pod0.yaml" || + command-error "failed to create pod dra-pod0" + +vm-command "kubectl wait --for=condition=Ready pod/dra-pod0 --timeout=60s" || + command-error "pod dra-pod0 did not reach Running" + +# +# CLOS association, env vars, allocatable deduction, cleanup. +# + +# Env vars: CDI writes NRI_CLASS/NRI_CPU into the OCI spec at +# container-creation time (pkg/resmgr/dra/cdi.go); they're never +# written back to the pod object, so this must be +# `kubectl exec ... -- env`, not an `-o json` read of the pod/container status. +vm-command "kubectl exec dra-pod0 -c dra-pod0c0 -- env" +env_output="$COMMAND_OUTPUT" + +grep -q '^NRI_CLASS=hp-turbo$' <<< "$env_output" || + command-error "missing/incorrect NRI_CLASS env var in dra-pod0c0 (expected hp-turbo)" + +# Parse the claimed CPU ids out of the NRI_CPU=1 env vars. +claimed_cpus=$(grep -oE '^NRI_CPU[0-9]+=1$' <<< "$env_output" | \ + sed -E 's/^NRI_CPU([0-9]+)=1$/\1/' | sort -n | tr '\n' ' ') +claimed_cpus="${claimed_cpus% }" + +[ -n "$claimed_cpus" ] || + command-error "no NRI_CPU=1 env vars found in dra-pod0c0's environment" + +claimed_cpu_count=$(wc -w <<< "$claimed_cpus") +[ "$claimed_cpu_count" -eq 2 ] || + command-error "expected exactly 2 claimed CPUs via NRI_CPU env vars (claim requested nri/cpus: \"2\"), got $claimed_cpu_count ($claimed_cpus)" + +# The container's actual cpuset must include every claimed CPU id, not +# just the CDI-injected env vars: applyGrant unions the container's +# claimed CPUs into the cpuset it pins, so dra-pod0c0's live cpuset +# (read via Cpus_allowed_list) must be a superset of claimed_cpus. +missing_cpus=$(cpulist-difference "$claimed_cpus" "$(container-cpus dra-pod0 dra-pod0c0)") +[ -z "$missing_cpus" ] || + command-error "claimed CPUs ($claimed_cpus) missing from dra-pod0c0's actual cpuset (missing: $missing_cpus)" + +# CLOS association: the pct.go "associated cpus %s to CLOS %d" line +# -- not the mock's startup-time ConfigureClos line, which would pass +# even if the claim were never allocated. The log line formats the +# cpuset with cpuset.CPUSet.String(), which collapses contiguous ids +# into ranges, so compress the env-derived ids into the same form +# before matching. +assert-cpu-clos "$(compress-cpulist "$claimed_cpus")" "CLOS 0" \ + "Missing CPU association for dra-pod0c0 (expected CLOS 0)" + +# KEP-5517 allocation record: node.status.allocatable.cpu is never +# mutated by DRANodeAllocatableResources. The actual persisted signal +# is pod.status.nodeAllocatableResourceClaimStatuses[], written by the +# scheduler at PreBind. Assert it names the claim, the consuming container, +# and the claimed CPU count. +wait-node-allocatable-claim-status dra-pod0 hp-turbo-cpus dra-pod0c0 "$claimed_cpu_count" 30 2 + +cleanup diff --git a/test/e2e/run.sh b/test/e2e/run.sh index 37b55c798..b6da44951 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -51,6 +51,18 @@ export k8s_release=${k8s_release:-"latest"} k8s_release="${k8s_release#v}" export k8s_version="" +# Comma-separated "Gate1=true,Gate2=false" list passed to +# kubeadm's kube-apiserver/scheduler --feature-gates extraArgs and to +# the kubelet's KubeletConfiguration.featureGates. Unset (default): +# provision.yaml runs its plain "kubeadm init --pod-network-cidr=..." +# exactly as before, no feature-gate plumbing at all. +# +# NOTE for callers enabling DRAConsumableCapacity/DRANodeAllocatableResources +# (KEP-5075/KEP-5517): also pin k8s_release explicitly to >=1.37, e.g. +# k8s_release=1.37 k8s_feature_gates="DRAConsumableCapacity=true,DRANodeAllocatableResources=true" ... +# as DRANodeAllocatableResources requires Kubernetes 1.37+. +export k8s_feature_gates=${k8s_feature_gates:-} + GH_HELM_REPO="helm/helm" export helm_release=${helm_release:-"latest"} @@ -274,9 +286,10 @@ echo " EFI boot = ${efi:-no}" echo " Distro = $distro" echo " Distro image = ${distro_img:-vagrant default}" echo " Kubernetes" -echo " - release = $k8s_release" -echo " - version = $k8s_version" -echo " - Helm = $helm_release" +echo " - release = $k8s_release" +echo " - version = $k8s_version" +echo " - Helm = $helm_release" +echo " - feature gates = ${k8s_feature_gates:-none}" echo " Runtime = $k8scri" echo " Output dir = $OUTPUT_DIR" echo " Test output dir = $TEST_OUTPUT_DIR" diff --git a/test/e2e/run_tests.sh b/test/e2e/run_tests.sh index acb30f7a5..a7508db3b 100755 --- a/test/e2e/run_tests.sh +++ b/test/e2e/run_tests.sh @@ -233,8 +233,10 @@ for POLICY_DIR in "$TESTS_ROOT_DIR"/*; do policy_name="$(basename $POLICY_DIR)" - # Create name for the vm. - export vm_name=$(vm-create-name "$k8scri" "$(basename "$TOPOLOGY_DIR")" ${distro}) + # Create name for the vm. A caller-supplied vm_name (e.g. to + # keep a gated and an ungated VM from fighting over one + # vagrant dir) is preserved rather than clobbered. + export vm_name=${vm_name:-$(vm-create-name "$k8scri" "$(basename "$TOPOLOGY_DIR")" ${distro})} export-and-source-dir "$TOPOLOGY_DIR" # Create ansible inventory file from a template From ed5327be5e83f534cf900278b49b1ff47b8c6d42 Mon Sep 17 00:00:00 2001 From: Ed Bartosh Date: Fri, 4 Sep 2026 16:27:23 +0300 Subject: [PATCH 10/39] e2e: add k8s_log_verbosity var for control-plane/kubelet debug logging Signed-off-by: Ed Bartosh Co-Authored-By: Claude Sonnet 5 --- test/e2e/files/Vagrantfile.in | 2 ++ test/e2e/playbook/provision.yaml | 44 ++++++++++++++++++++++++++++---- test/e2e/run.sh | 7 +++++ 3 files changed, 48 insertions(+), 5 deletions(-) diff --git a/test/e2e/files/Vagrantfile.in b/test/e2e/files/Vagrantfile.in index fa940c6cd..1f622d5b8 100644 --- a/test/e2e/files/Vagrantfile.in +++ b/test/e2e/files/Vagrantfile.in @@ -20,6 +20,7 @@ KERNEL_CONFIG = "#{ENV['kernel_config']}" K8S_RELEASE = "#{ENV['k8s_release']}" K8S_VERSION = "#{ENV['k8s_version']}" K8S_FEATURE_GATES = "#{ENV['k8s_feature_gates']}" +K8S_LOG_VERBOSITY = "#{ENV['k8s_log_verbosity']}" HELM_RELEASE = "#{ENV['helm_release']}" CRI_RUNTIME = "#{ENV['k8scri']}" CRIO_RELEASE = "1.28.1" @@ -127,6 +128,7 @@ Vagrant.configure("2") do |config| k8s_release: K8S_RELEASE, k8s_version: K8S_VERSION, k8s_feature_gates: K8S_FEATURE_GATES, + k8s_log_verbosity: K8S_LOG_VERBOSITY, helm_release: HELM_RELEASE, cri_runtime: CRI_RUNTIME, containerd_release: CONTAINERD_RELEASE, diff --git a/test/e2e/playbook/provision.yaml b/test/e2e/playbook/provision.yaml index 79a838f58..0a540bb8f 100644 --- a/test/e2e/playbook/provision.yaml +++ b/test/e2e/playbook/provision.yaml @@ -9,6 +9,10 @@ k8s_version: "{{ k8s_version }}" # Comma-separated "Gate1=true,Gate2=false" list, or empty string. k8s_feature_gates: "{{ k8s_feature_gates }}" + # klog verbosity level (e.g. "4") for apiserver/scheduler/controller-manager/ + # kubelet, or empty to leave every component at its default verbosity. + # Independent of k8s_feature_gates -- either can be set without the other. + k8s_log_verbosity: "{{ k8s_log_verbosity }}" is_containerd: false is_crio: false containerd_release: "{{ containerd_release }}" @@ -503,7 +507,7 @@ daemon_reload: true state: restarted - - name: Write kubeadm --config with feature gates for apiserver/scheduler/kubelet + - name: Write kubeadm --config with feature gates and/or log verbosity for apiserver/scheduler/controller-manager/kubelet ansible.builtin.copy: dest: /root/kubeadm-config.yaml content: | @@ -513,30 +517,60 @@ podSubnet: "{{ network }}" apiServer: extraArgs: + {% if k8s_feature_gates != "" %} - name: feature-gates value: "{{ k8s_feature_gates }}" + {% endif %} + {% if k8s_log_verbosity != "" %} + - name: v + value: "{{ k8s_log_verbosity }}" + {% endif %} scheduler: extraArgs: + {% if k8s_feature_gates != "" %} - name: feature-gates value: "{{ k8s_feature_gates }}" + {% endif %} + {% if k8s_log_verbosity != "" %} + - name: v + value: "{{ k8s_log_verbosity }}" + {% endif %} + {% if k8s_feature_gates != "" or k8s_log_verbosity != "" %} + controllerManager: + extraArgs: + {% if k8s_feature_gates != "" %} + - name: feature-gates + value: "{{ k8s_feature_gates }}" + {% endif %} + {% if k8s_log_verbosity != "" %} + - name: v + value: "{{ k8s_log_verbosity }}" + {% endif %} + {% endif %} --- apiVersion: kubelet.config.k8s.io/v1beta1 kind: KubeletConfiguration + {% if k8s_feature_gates != "" %} featureGates: {% for gate in k8s_feature_gates.split(',') if gate.strip() and '=' in gate %} {{ gate.split('=')[0] }}: {{ gate.split('=')[1] }} {% endfor %} - when: k8s_feature_gates != "" + {% endif %} + {% if k8s_log_verbosity != "" %} + logging: + verbosity: {{ k8s_log_verbosity }} + {% endif %} + when: k8s_feature_gates != "" or k8s_log_verbosity != "" - name: Initialize the Kubernetes cluster using kubeadm ansible.builtin.command: cmd: kubeadm init --pod-network-cidr="{{ network }}" - when: k8s_feature_gates == "" + when: k8s_feature_gates == "" and k8s_log_verbosity == "" - - name: Initialize the Kubernetes cluster using kubeadm (with feature gates) + - name: Initialize the Kubernetes cluster using kubeadm (with feature gates and/or log verbosity) ansible.builtin.command: cmd: kubeadm init --config /root/kubeadm-config.yaml - when: k8s_feature_gates != "" + when: k8s_feature_gates != "" or k8s_log_verbosity != "" - name: Setup kubeconfig for vagrant user ansible.builtin.command: "{{ item }}" diff --git a/test/e2e/run.sh b/test/e2e/run.sh index b6da44951..fb4d7e15f 100755 --- a/test/e2e/run.sh +++ b/test/e2e/run.sh @@ -63,6 +63,12 @@ export k8s_version="" # as DRANodeAllocatableResources requires Kubernetes 1.37+. export k8s_feature_gates=${k8s_feature_gates:-} +# klog verbosity level (e.g. "4") passed to kube-apiserver's, kube-scheduler's, +# and kube-controller-manager's --v extraArgs, and to the kubelet's +# KubeletConfiguration.logging.verbosity. Unset (default): +# no verbosity plumbing, components log at their normal default level. +export k8s_log_verbosity=${k8s_log_verbosity:-} + GH_HELM_REPO="helm/helm" export helm_release=${helm_release:-"latest"} @@ -290,6 +296,7 @@ echo " - release = $k8s_release" echo " - version = $k8s_version" echo " - Helm = $helm_release" echo " - feature gates = ${k8s_feature_gates:-none}" +echo " - log verbosity = ${k8s_log_verbosity:-none}" echo " Runtime = $k8scri" echo " Output dir = $OUTPUT_DIR" echo " Test output dir = $TEST_OUTPUT_DIR" From e581147569e40d58e5804a9262a27a8c267081b7 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 9 Sep 2026 15:14:42 +0300 Subject: [PATCH 11/39] lib/cpu: implement dense CpuMasks and sparse CpuSets. Define CPUSet and AnyCPUSet interfaces for storing sets of CPUs. Add a dense bitmask-based CpuMask implementation and a sparse CpuSet wrapping the k8s.io cpuset.CPUSet. Add units tests for each. Also, add a benchmark to check how the implementations perform in various operations as the number of maximum and stored CPUs grow. A nil set reads as the empty one. Every operation which does not modify a set takes a nil receiver, and a nil operand, as empty, which is Go's own rule for nil maps and slices and is what the k8s cpuset value type gave its callers for free. Set, Clear and Seal panic instead, saying what to do about it: no method can allocate a set and store it back into the caller's variable, so EmptyIfNil and Clone are how a caller gets one it can modify. Assisted-by: copilot-cli Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/lib/cpu/cpuset-bench_test.go | 666 ++++++ pkg/lib/cpu/cpuset.go | 1040 ++++++++ pkg/lib/cpu/cpuset_test.go | 3857 ++++++++++++++++++++++++++++++ pkg/lib/cpu/doc.go | 172 ++ 4 files changed, 5735 insertions(+) create mode 100644 pkg/lib/cpu/cpuset-bench_test.go create mode 100644 pkg/lib/cpu/cpuset.go create mode 100644 pkg/lib/cpu/cpuset_test.go create mode 100644 pkg/lib/cpu/doc.go diff --git a/pkg/lib/cpu/cpuset-bench_test.go b/pkg/lib/cpu/cpuset-bench_test.go new file mode 100644 index 000000000..2828403d6 --- /dev/null +++ b/pkg/lib/cpu/cpuset-bench_test.go @@ -0,0 +1,666 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package libcpu + +import ( + "flag" + "fmt" + "os" + "testing" + "text/tabwriter" + + "k8s.io/utils/cpuset" +) + +// This file benchmarks our dense [CpuMask] and sparse [CpuSet] against each +// other and against the raw k8s.io/utils/cpuset.CPUSet which CpuSet wraps. +// Every operation is measured for several CPU set sizes and densities, since +// both the number of CPUs in the set and the highest CPU number in it affect +// the implementations differently. +// +// Benchmarks are named //, so +// +// go test -bench 'BenchmarkCPUSet/Contains' +// go test -bench 'BenchmarkCPUSet/.*/1024cpus' +// go test -bench 'BenchmarkCPUSet/.*/CpuMask' +// +// all pick out a useful slice of the matrix. For a single table which shows +// the fastest implementation per operation and scenario, run +// +// CPUSET_BENCH_COMPARE=1 go test -run TestCompareImplementations -v +// +// Note which operations are measured how. New, Parse, Clone, Union, +// Intersection and Difference are called on the concrete type, so the raw +// cpuset.CPUSet is measured as it would be used directly, with no adapter of +// ours in the way, and our own types are measured as their callers now use +// them. See benchDirect below. +// +// The rest are driven through the [CPUSet] interface, which adds the same +// non-inlinable indirection to every implementation. That is what a caller +// taking the interface pays anyway, and it keeps the comparison like for like, +// but it does mean the very cheapest operations are measured with a constant +// overhead included. For those the raw type is reached through [rawCpuSet], +// whose own cost is a single interface call, as it is for the other two. + +// rawCpuSet exposes the raw k8s.io/utils/cpuset.CPUSet through the [CPUSet] +// interface so that it can be benchmarked side by side with our own types. It +// is deliberately as thin as possible: no string or key caching, no seal +// checks. Comparing it against [CpuSet] therefore shows what our wrapper adds +// on top of it, and comparing it against [CpuMask] shows the cost of the +// sparse representation itself. +// +// It is only used for the operations driven through the interface. The ones in +// benchDirect below reach the embedded cpuset.CPUSet instead, so that a result +// which allocates does not also pay for allocating one of these. +// +// The embedded cpuset.CPUSet provides Size, IsEmpty, List, UnsortedList and +// String as is. The rest need adapting, mostly because they take or return +// our CPUSet instead of a cpuset.CPUSet. Those methods assume the other set +// is a *rawCpuSet, which is all the benchmarks below ever pass them. +type rawCpuSet struct { + cpuset.CPUSet +} + +// rawCpuSet should implement CPUSet. +var _ CPUSet = (*rawCpuSet)(nil) + +func newRawCpuSet(cpus ...int) CPUSet { + return &rawCpuSet{CPUSet: cpuset.New(cpus...)} +} + +func parseRawCpuSet(s string) (CPUSet, error) { + cpus, err := cpuset.Parse(s) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrParseFailed, err) + } + return &rawCpuSet{CPUSet: cpus}, nil +} + +func (s *rawCpuSet) Clone() CPUSet { + return &rawCpuSet{CPUSet: s.CPUSet.Clone()} +} + +func (s *rawCpuSet) Set(cpus ...int) { + s.CPUSet = s.CPUSet.Union(cpuset.New(cpus...)) +} + +func (s *rawCpuSet) Clear(cpus ...int) { + s.CPUSet = s.CPUSet.Difference(cpuset.New(cpus...)) +} + +func (s *rawCpuSet) Difference(other CPUSet) CPUSet { + return &rawCpuSet{CPUSet: s.CPUSet.Difference(other.(*rawCpuSet).CPUSet)} +} + +func (s *rawCpuSet) Intersection(other CPUSet) CPUSet { + return &rawCpuSet{CPUSet: s.CPUSet.Intersection(other.(*rawCpuSet).CPUSet)} +} + +func (s *rawCpuSet) Intersects(other CPUSet) bool { + return !s.CPUSet.Intersection(other.(*rawCpuSet).CPUSet).IsEmpty() +} + +func (s *rawCpuSet) Union(others ...CPUSet) CPUSet { + r := s.CPUSet + for _, other := range others { + r = r.Union(other.(*rawCpuSet).CPUSet) + } + return &rawCpuSet{CPUSet: r} +} + +func (s *rawCpuSet) Contains(cpus ...int) bool { + for _, cpu := range cpus { + if !s.CPUSet.Contains(cpu) { + return false + } + } + return true +} + +func (s *rawCpuSet) Equals(other CPUSet) bool { + return s.CPUSet.Equals(other.(*rawCpuSet).CPUSet) +} + +func (s *rawCpuSet) IsSubsetOf(other CPUSet) bool { + return s.CPUSet.IsSubsetOf(other.(*rawCpuSet).CPUSet) +} + +func (s *rawCpuSet) Key() string { + return s.String() +} + +func (s *rawCpuSet) Seal() {} + +func (*rawCpuSet) IsDense() bool { + return false +} + +func (*rawCpuSet) IsSparse() bool { + return true +} + +func (s *rawCpuSet) ForEachCpu(f func(cpu int) bool) { + for _, cpu := range s.UnsortedList() { + if !f(cpu) { + return + } + } +} + +// benchDirect holds the operations of one case which are measured by calling +// them on the concrete type, bound to the data they run on. +// +// Two reasons an operation belongs here rather than in the shared table below. +// +// Clone, Union, Intersection and Difference are not part of CPUSet: each +// implementation returns its own type, which no single interface signature can +// describe. Binding them per implementation is what production code does too, +// so this measures a direct call rather than the type switch an [AnyCPUSet] +// would add on top. +// +// New, Parse and those four also allocate their result, and for the raw +// cpuset.CPUSet the [rawCpuSet] adapter would allocate a second time to wrap it. +// Bound here, the raw implementation is measured unwrapped, which is the honest +// baseline to hold our own types against: it is the code someone would write +// using k8s.io/utils/cpuset directly. +type benchDirect struct { + newSet func() + parseSet func() error + clone func() + union func() + intersection func() + difference func() +} + +// benchImpl is one implementation under test. +type benchImpl struct { + name string + new func(cpus ...int) CPUSet + parse func(s string) (CPUSet, error) + // direct binds the operations measured on the concrete type. cpus and str + // are the case's inputs for New and Parse, a and b its two sets. + direct func(cpus []int, str string, a, b CPUSet) benchDirect +} + +var benchImpls = []benchImpl{ + { + name: "CpuMask", + new: func(cpus ...int) CPUSet { return NewCpuMask(cpus...) }, + parse: func(s string) (CPUSet, error) { return ParseCpuMask(s) }, + direct: func(cpus []int, str string, a, b CPUSet) benchDirect { + x, y := a.(*CpuMask), b.(*CpuMask) + return benchDirect{ + newSet: func() { sinkMask = NewCpuMask(cpus...) }, + parseSet: func() (err error) { sinkMask, err = ParseCpuMask(str); return }, + clone: func() { sinkMask = x.Clone() }, + union: func() { sinkMask = x.Union(y) }, + intersection: func() { sinkMask = x.Intersection(y) }, + difference: func() { sinkMask = x.Difference(y) }, + } + }, + }, + { + name: "CpuSet", + new: func(cpus ...int) CPUSet { return NewCpuSet(cpus...) }, + parse: func(s string) (CPUSet, error) { return ParseCpuSet(s) }, + direct: func(cpus []int, str string, a, b CPUSet) benchDirect { + x, y := a.(*CpuSet), b.(*CpuSet) + return benchDirect{ + newSet: func() { sinkSet = NewCpuSet(cpus...) }, + parseSet: func() (err error) { sinkSet, err = ParseCpuSet(str); return }, + clone: func() { sinkSet = x.Clone() }, + union: func() { sinkSet = x.Union(y) }, + intersection: func() { sinkSet = x.Intersection(y) }, + difference: func() { sinkSet = x.Difference(y) }, + } + }, + }, + { + // The raw k8s type, measured without the rawCpuSet adapter wherever the + // adapter would show up in the result. This is the baseline our own two + // implementations are worth comparing against. + name: "cpuset.CPUSet", + new: newRawCpuSet, + parse: parseRawCpuSet, + direct: func(cpus []int, str string, a, b CPUSet) benchDirect { + x, y := a.(*rawCpuSet).CPUSet, b.(*rawCpuSet).CPUSet + return benchDirect{ + newSet: func() { sinkRaw = cpuset.New(cpus...) }, + parseSet: func() (err error) { sinkRaw, err = cpuset.Parse(str); return }, + clone: func() { sinkRaw = x.Clone() }, + union: func() { sinkRaw = x.Union(y) }, + intersection: func() { sinkRaw = x.Intersection(y) }, + difference: func() { sinkRaw = x.Difference(y) }, + } + }, + }, +} + +// benchScenario describes a CPU set to benchmark with. count CPUs are picked +// stride apart, so count says how much data an operation has to chew through +// and stride how thinly it is spread: the sparse implementations care about +// count only, the dense one about count*stride, the highest CPU in the set. +type benchScenario struct { + name string + count int + stride int + // otherCount is how many CPUs the second operand has; zero means as many + // as the first one. The scenarios which set it are there for the binary + // operations that iterate whichever operand is smaller: with two sets of + // the same size those look no different from a naive implementation. + otherCount int +} + +var benchScenarios = []benchScenario{ + {name: "1cpu", count: 1, stride: 1}, + {name: "8cpus", count: 8, stride: 1}, + {name: "8cpus-spread", count: 8, stride: 128}, + {name: "64cpus", count: 64, stride: 1}, + {name: "64cpus-spread", count: 64, stride: 16}, + {name: "256cpus", count: 256, stride: 1}, + {name: "256cpus-spread", count: 256, stride: 4}, + {name: "1024cpus", count: 1024, stride: 1}, + // asymmetric: a large set against a small one and the other way round + {name: "1024cpus-vs-8", count: 1024, stride: 1, otherCount: 8}, + {name: "8cpus-vs-1024", count: 8, stride: 1, otherCount: 1024}, +} + +// asymmetric reports whether the scenario's two operands differ in size. +func (sc benchScenario) asymmetric() bool { + return sc.otherCount != 0 && sc.otherCount != sc.count +} + +// others is the number of CPUs in the second operand. +func (sc benchScenario) others() int { + if sc.otherCount == 0 { + return sc.count + } + return sc.otherCount +} + +// cpus returns the CPUs of the scenario. +func (sc benchScenario) cpus() []int { + return strided(0, sc.count, sc.stride) +} + +// otherCpus returns a second set of CPUs of the same shape, overlapping the +// first one by half. It is the second operand for the set operations. +func (sc benchScenario) otherCpus() []int { + return strided(max(1, sc.count/2)*sc.stride, sc.others(), sc.stride) +} + +// overlappingCpus returns a set of CPUs of the same shape which is guaranteed +// to share at least one CPU with cpus(). It is otherCpus() for every scenario +// but the single-CPU one, where otherCpus() shifts clear of cpus() entirely +// and would not overlap; there this is cpus() itself. +func (sc benchScenario) overlappingCpus() []int { + return strided(sc.count/2*sc.stride, sc.others(), sc.stride) +} + +// disjointCpus returns a set of CPUs of the same shape which shares no CPU +// with cpus(). It starts just past the last CPU of cpus(), so it is also the +// operand that forces a full scan out of the operations which can bail out as +// soon as they find a CPU in common. +func (sc benchScenario) disjointCpus() []int { + return strided(sc.count*sc.stride, sc.others(), sc.stride) +} + +// strided returns count CPUs starting at first, stride apart. +func strided(first, count, stride int) []int { + cpus := make([]int, count) + for i := range cpus { + cpus[i] = first + i*stride + } + return cpus +} + +// benchCase is everything an operation needs to run: an implementation, and a +// scenario pre-built with it. +type benchCase struct { + impl benchImpl + cpus []int // CPUs in set a + str string // string representation of set a + a, b CPUSet // two sets of the same shape, overlapping by half + direct benchDirect // operations measured on the concrete type + o CPUSet // a set of the same shape guaranteed to overlap a + d CPUSet // a set of the same shape sharing no CPU with a + hi int // highest CPU in a, always present in it + absnt int // lowest CPU not in a +} + +func newBenchCase(impl benchImpl, sc benchScenario) *benchCase { + cpus := sc.cpus() + a := impl.new(cpus...) + b := impl.new(sc.otherCpus()...) + + absent := 0 + for a.Contains(absent) { + absent++ + } + + return &benchCase{ + impl: impl, + cpus: cpus, + str: a.String(), + a: a, + b: b, + o: impl.new(sc.overlappingCpus()...), + d: impl.new(sc.disjointCpus()...), + hi: cpus[len(cpus)-1], + absnt: absent, + direct: impl.direct(cpus, a.String(), a, b), + } +} + +// Sinks for results we do not otherwise use. Without them the compiler is free +// to elide the allocation of a result which never escapes, which it can do for +// some implementations and not others -- the map-based ones came out at zero +// allocations per New. Assigning to a package-level variable makes the result +// escape, as keeping it would in real code. One sink per concrete type, so that +// nothing is boxed into an interface on the way. +var ( + sinkStr string + sinkMask *CpuMask + sinkSet *CpuSet + sinkRaw cpuset.CPUSet +) + +// benchOps are the operations we measure. Every one of them leaves its sets +// unchanged, so that repeated iterations all do the same amount of work. +var benchOps = []struct { + name string + run func(b *testing.B, c *benchCase) +}{ + {"New", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.direct.newSet() + } + }}, + {"Parse", func(b *testing.B, c *benchCase) { + for b.Loop() { + if err := c.direct.parseSet(); err != nil { + b.Fatal(err) + } + } + }}, + {"Clone", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.direct.clone() + } + }}, + // Set adds a CPU which is already in the set, Clear removes one which is + // not, both leaving the set as it was. Note that for a densely packed + // scenario the cleared CPU falls beyond the last word of a CpuMask, which + // CpuMask can reject with a bounds check alone. + {"Set", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.Set(c.hi) + } + }}, + {"Clear", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.Clear(c.absnt) + } + }}, + {"Contains-hit", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.Contains(c.hi) + } + }}, + {"Contains-miss", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.Contains(c.absnt) + } + }}, + {"Size", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.Size() + } + }}, + {"IsEmpty", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.IsEmpty() + } + }}, + {"Union", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.direct.union() + } + }}, + {"Intersection", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.direct.intersection() + } + }}, + // Intersects returns as soon as it finds a CPU in common, so measure both + // the early exit, against a set overlapping a half way in, and the full + // scan, against a set sharing no CPU with it. The miss is the worst case + // and the one directly comparable to Intersection above. + {"Intersects-hit", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.Intersects(c.o) + } + }}, + {"Intersects-miss", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.Intersects(c.d) + } + }}, + {"Difference", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.direct.difference() + } + }}, + // Equals and IsSubsetOf are given a set equal to a, the worst case for + // both: they cannot bail out early. + {"Equals", func(b *testing.B, c *benchCase) { + o := c.impl.new(c.cpus...) + for b.Loop() { + c.a.Equals(o) + } + }}, + {"IsSubsetOf", func(b *testing.B, c *benchCase) { + o := c.impl.new(c.cpus...) + for b.Loop() { + c.a.IsSubsetOf(o) + } + }}, + {"List", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.List() + } + }}, + // Note that these measure repeated calls on an unmodified set, which is + // the case the string and key caches exist for. CpuSet caches String, + // CpuMask caches Key, and the raw cpuset.CPUSet caches neither. + {"String", func(b *testing.B, c *benchCase) { + for b.Loop() { + sinkStr = c.a.String() + } + }}, + {"Key", func(b *testing.B, c *benchCase) { + for b.Loop() { + sinkStr = c.a.Key() + } + }}, + // Both CpuMask and CpuSet cache String, so the op above only ever measures + // a cache hit for them. This one builds a fresh set for every iteration to + // measure the cold path too. Subtract the New row from it to get the cost + // of generating the string itself. + {"String-uncached", func(b *testing.B, c *benchCase) { + for b.Loop() { + sinkStr = c.impl.new(c.cpus...).String() + } + }}, + {"ForEachCpu", func(b *testing.B, c *benchCase) { + for b.Loop() { + c.a.ForEachCpu(func(int) bool { return true }) + } + }}, +} + +// takesSecondOperand reports whether the named operation is measured against +// one of the scenario's second operands, and so whether an asymmetric scenario +// tells us anything a symmetric one does not. Equals and IsSubsetOf are not in +// the list: they deliberately build an operand equal to the first one, so +// their cost does not depend on the scenario's second operand either. +func takesSecondOperand(op string) bool { + switch op { + case "Union", "Intersection", "Difference", "Intersects-hit", "Intersects-miss": + return true + } + return false +} + +// skip reports whether an operation and scenario combination is worth +// measuring. An asymmetric scenario only says something new about the +// operations which look at a second operand; for the rest it would simply +// repeat the row of the symmetric scenario with the same count. +func skip(op string, sc benchScenario) bool { + return sc.asymmetric() && !takesSecondOperand(op) +} + +func BenchmarkCPUSet(b *testing.B) { + for _, op := range benchOps { + b.Run(op.name, func(b *testing.B) { + for _, sc := range benchScenarios { + if skip(op.name, sc) { + continue + } + b.Run(sc.name, func(b *testing.B) { + for _, impl := range benchImpls { + b.Run(impl.name, func(b *testing.B) { + op.run(b, newBenchCase(impl, sc)) + }) + } + }) + } + }) + } +} + +// TestBenchOperands checks the relationships between the operands that the +// benchmark cases rely on. Getting one of these wrong does not fail any +// benchmark, it just silently measures something other than the row's name -- +// which is exactly what happened when overlappingCpus() did not yet exist and +// Intersects-hit used otherCpus(), disjoint from cpus() for a single-CPU set. +func TestBenchOperands(t *testing.T) { + for _, sc := range benchScenarios { + t.Run(sc.name, func(t *testing.T) { + var ( + a = NewCpuMask(sc.cpus()...) + o = NewCpuMask(sc.overlappingCpus()...) + d = NewCpuMask(sc.disjointCpus()...) + ) + + if got := a.Size(); got != sc.count { + t.Errorf("cpus() has %d CPUs, want %d", got, sc.count) + } + if !a.Intersects(o) { + t.Errorf("overlappingCpus() %s does not intersect cpus() %s", o, a) + } + if a.Intersects(d) { + t.Errorf("disjointCpus() %s intersects cpus() %s", d, a) + } + if o.Size() != sc.others() || d.Size() != sc.others() { + t.Errorf("second operands have %d and %d CPUs, want %d each", + o.Size(), d.Size(), sc.others()) + } + if b := NewCpuMask(sc.otherCpus()...); b.Size() != sc.others() { + t.Errorf("otherCpus() has %d CPUs, want %d", b.Size(), sc.others()) + } + + // hi must be in the set and absnt must not, for Contains-hit, + // Contains-miss, Set and Clear to measure what they claim. + for _, impl := range benchImpls { + c := newBenchCase(impl, sc) + if !c.a.Contains(c.hi) { + t.Errorf("%s: a does not contain hi=%d", impl.name, c.hi) + } + if c.a.Contains(c.absnt) { + t.Errorf("%s: a contains absnt=%d", impl.name, c.absnt) + } + } + }) + } +} + +// TestCompareImplementations runs the full benchmark matrix and prints a +// table of ns/op per implementation. The last column names the fastest one +// for each operation and scenario, and how much faster it is than the runner +// up. It is a test rather than a benchmark because it needs to compare +// results against each other. +// +// Because it runs every case, it is opt-in: +// +// CPUSET_BENCH_COMPARE=1 go test -run TestCompareImplementations -v +// +// Each case gets a short run by default, enough to rank the implementations +// but not to trust the absolute numbers. Pass an explicit -benchtime for a +// more accurate, and much slower, table. +func TestCompareImplementations(t *testing.T) { + if os.Getenv("CPUSET_BENCH_COMPARE") == "" { + t.Skip("set CPUSET_BENCH_COMPARE=1 to run the implementation comparison") + } + + if f := flag.Lookup("test.benchtime"); f != nil && f.Value.String() == "1s" { + if err := f.Value.Set("20ms"); err != nil { + t.Fatalf("failed to shorten benchmark time: %v", err) + } + } + + w := tabwriter.NewWriter(os.Stdout, 0, 0, 2, ' ', tabwriter.AlignRight) + defer w.Flush() // nolint:errcheck + + fmt.Fprint(w, "operation\tCPUs\t") // nolint:errcheck + for _, impl := range benchImpls { + fmt.Fprintf(w, "%s\t", impl.name) // nolint:errcheck + } + fmt.Fprint(w, "fastest (vs 2nd)\t\n") // nolint:errcheck + + for _, op := range benchOps { + for _, sc := range benchScenarios { + if skip(op.name, sc) { + continue + } + + var ( + best = benchImpl{} + bestNs = 0.0 + next = 0.0 + ) + + fmt.Fprintf(w, "%s\t%s\t", op.name, sc.name) // nolint:errcheck + + for _, impl := range benchImpls { + c := newBenchCase(impl, sc) + r := testing.Benchmark(func(b *testing.B) { op.run(b, c) }) + ns := float64(r.T.Nanoseconds()) / float64(r.N) + + fmt.Fprintf(w, "%.1f\t", ns) // nolint:errcheck + + switch { + case bestNs == 0 || ns < bestNs: + best, bestNs, next = impl, ns, bestNs + case next == 0 || ns < next: + next = ns + } + } + + fmt.Fprintf(w, "%s (%.1fx)\t\n", best.name, next/bestNs) // nolint:errcheck + } + fmt.Fprint(w, "\t\t\t\t\t\t\n") // nolint:errcheck + } +} diff --git a/pkg/lib/cpu/cpuset.go b/pkg/lib/cpu/cpuset.go new file mode 100644 index 000000000..94b7b8285 --- /dev/null +++ b/pkg/lib/cpu/cpuset.go @@ -0,0 +1,1040 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package libcpu + +import ( + "errors" + "fmt" + "math/bits" + "slices" + "strconv" + "strings" + + "k8s.io/utils/cpuset" +) + +// CPUSet represents an unordered set of CPUs. It is the common +// interface we expect from every data type that represents a set +// of CPUs. We provide two implementations: a dense [CpuMask] and +// a sparse [CpuSet] which just wraps k8s.io/utils/cpuset.CPUSet. +// +// Take this interface to accept any set of CPUs. Note that it does +// not carry the operations which produce a new set: Clone, Union, +// Difference and Intersection are on the implementations, and each +// returns its own type rather than this interface, so that a caller +// holding one does not have to assert on what it already knows. +// Wrap a set in an [AnyCPUSet] to perform those with a set of +// unknown type as the receiver. +type CPUSet interface { + // Set adds the given CPUs to an unsealed set. + Set(cpus ...int) + // Clear removes the given CPUs from an unsealed set. + Clear(cpus ...int) + // Intersects returns true if the two sets have any common CPUs. + Intersects(other CPUSet) bool + // Contains returns true if all the given CPUs are in the set. + Contains(cpus ...int) bool + // Equals returns true if the two sets contain the same CPUs. + Equals(other CPUSet) bool + // Size returns the number of CPUs in the set. + Size() int + // IsEmpty returns true if the set contains no CPUs. + IsEmpty() bool + // IsSubsetOf returns true if all CPUs in this set are also in the other set. + IsSubsetOf(other CPUSet) bool + // List returns the list of all CPUs in the set in increasing order. + List() []int + // UnsortedList returns an unsorted list of all CPUs in the set. + UnsortedList() []int + // String returns a string representation of the set, in a Linux kernel + // cpuset compatible format. + String() string + // Key returns a string usable as a map key for the set. Key() is + // guaranteed to return the same string for two sets of the same + // implementation if and only if the two sets are equal. Note that two + // different implementations may return different keys for sets which + // are equal, and [CpuMask] and [CpuSet] do. + Key() string + // Seal the CPUSet. Any attempt to modify a sealed set will panic. + Seal() + // IsDense returns true if the set implementation is dense. + IsDense() bool + // IsSparse returns true if the set implementation is sparse. + IsSparse() bool + // ForEachCpu calls the given function for each CPU in the set. Iteration + // stops early if the function returns false. + ForEachCpu(f func(cpu int) bool) +} + +var ( + // ErrParseFailed is returned when a CPUSet string cannot be parsed. + ErrParseFailed = errors.New("failed to parse CPU set") +) + +var ( + // EmptyCpuMask is a CpuMask with no CPUs in it, for [CpuMask.EmptyIfNil] to + // answer with. It is sealed: it is shared by everyone who asks, so modifying + // it panics instead of changing what every other caller sees. + EmptyCpuMask = sealedEmptyCpuMask() + + // EmptyCpuSet is the same for [CpuSet.EmptyIfNil]. + EmptyCpuSet = sealedEmptyCpuSet() +) + +func sealedEmptyCpuMask() *CpuMask { + cpus := NewCpuMask() + cpus.Seal() + return cpus +} + +func sealedEmptyCpuSet() *CpuSet { + cpus := NewCpuSet() + cpus.Seal() + return cpus +} + +// CpuMask is a dense implementation of CPUSet. It uses bitmasks +// to store CPUs and is a good choice for large CPU sets with many +// CPUs. +type CpuMask struct { + mask []uint64 + seal bool + size int + str string + key string +} + +// CpuMask should implement CPUSet. +var _ CPUSet = (*CpuMask)(nil) + +// NewCpuMask returns a new CpuMask containing the given CPUs. +func NewCpuMask(cpus ...int) *CpuMask { + words := 0 + if cnt := len(cpus); cnt > 0 { + hi := max(cpus[0], cpus[cnt-1]) + words = hi/64 + 1 + } + mask := make([]uint64, words) + for _, cpu := range cpus { + w, b := cpu/64, cpu&63 + mask = expand(mask, w) + mask[w] |= 1 << b + } + return newCpuMask(mask) +} + +func newCpuMask(mask []uint64) *CpuMask { + return &CpuMask{mask: mask, size: -1} +} + +// ParseCpuMask parses the given string representation of a CPU set +// and returns a corresponding new CpuMask. +func ParseCpuMask(s string) (*CpuMask, error) { + m := NewCpuMask() + if s == "" { + return m, nil + } + + for _, part := range strings.Split(s, ",") { + if !strings.Contains(part, "-") { + cpu, err := strconv.Atoi(part) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrParseFailed, err) + } + m.Set(cpu) + continue + } + + rng := strings.SplitN(part, "-", 2) + min, err := strconv.Atoi(rng[0]) + if err != nil { + return nil, fmt.Errorf("%w: invalid range start %q: %w", + ErrParseFailed, rng[0], err) + } + max, err := strconv.Atoi(rng[1]) + if err != nil { + return nil, fmt.Errorf("%w: invalid range end %q: %w", + ErrParseFailed, rng[1], err) + } + if min > max { + return nil, fmt.Errorf("%w: invalid range %q", ErrParseFailed, part) + } + + for cpu := min; cpu <= max; cpu++ { + m.Set(cpu) + } + } + + return m, nil +} + +// MustParseCpuMask is [ParseCpuMask] for a string which is known to be a CPU +// set: a constant in a test, or a value some other component has already +// validated. It panics if the string does not parse. +func MustParseCpuMask(s string) *CpuMask { + cpus, err := ParseCpuMask(s) + if err != nil { + panic(err) + } + return cpus +} + +// words returns the mask's bitmap words, and none for a nil mask. Every read +// below goes through this, which is what lets a nil mask be read as the empty +// set: ranging over no words yields nothing and len() of them is zero. +func (m *CpuMask) words() []uint64 { + if m == nil { + return nil + } + return m.mask +} + +// isNothing reports whether a set is nothing at all: a nil interface, or a nil +// CpuMask or CpuSet inside one. Such a set reads as the empty set, and this is +// how the operations below tell. +func isNothing(cpus CPUSet) bool { + switch o := cpus.(type) { + case nil: + return true + case *CpuMask: + return o == nil + case *CpuSet: + return o == nil + } + return false +} + +// operandMask resolves a set given to one of the operations below. The second +// result says whether the first is usable as the operand's bitmap: a CpuMask, or +// a set which is nothing at all. Anything else the caller has to read through +// the CPUSet interface. +// +// Note the order. The type assertion comes first because it is the fast path and +// it already answers for a nil CpuMask, whose words() are none; isNothing is a +// type switch, and asking it first cost a couple of nanoseconds on the cheapest +// operations, which is most of what they take. +func operandMask(other CPUSet) (*CpuMask, bool) { + if m, ok := other.(*CpuMask); ok { + return m, true + } + return nil, isNothing(other) +} + +// Clone returns a copy of the mask which is safe to modify: unsealed, and with +// no other owner. A nil mask clones into a new empty one, so +// +// cpus = cpus.Clone() +// +// is the way to get something writable out of a set of unknown provenance, +// whether it is nil, sealed, or shared with whoever handed it over. +// [CpuMask.EmptyIfNil] is the cheaper answer when only nil is in question. +func (m *CpuMask) Clone() *CpuMask { + if m == nil { + return NewCpuMask() + } + return &CpuMask{ + mask: slices.Clone(m.mask), + str: m.str, + key: m.key, + size: m.size, + } +} + +// EmptyIfNil returns the mask, or a new empty one if it is nil. +// +// Reading a nil mask needs no help: every operation which does not modify one +// treats it as the empty set. This is for the ones which do. A nil mask cannot +// be modified, and no method can fix that by itself, since it cannot store a +// mask it just made back into the caller's variable. So assign it first: +// +// cpus = cpus.EmptyIfNil() +// cpus.Set(0, 1) +// +// The new mask is the caller's own, not the shared [EmptyCpuMask], which is +// sealed and would panic on the Set. +// +// This answers for nil and nothing else: a mask which is there comes back as it +// is, sealed included, and modifying that still panics. [CpuMask.Clone] is the +// one which always returns something writable. +func (m *CpuMask) EmptyIfNil() *CpuMask { + if m == nil { + return NewCpuMask() + } + return m +} + +func (m *CpuMask) Set(cpus ...int) { + m.panicIfNilOrSealed() + + for _, cpu := range cpus { + w, b := cpu/64, cpu&63 + m.mask = expand(m.mask, w) + m.mask[w] |= 1 << b + } + + m.invalidateCached() +} + +func (m *CpuMask) Clear(cpus ...int) { + m.panicIfNilOrSealed() + + for _, cpu := range cpus { + w, b := cpu/64, cpu&63 + if w < len(m.mask) { + m.mask[w] &^= 1 << b + } + } + + m.invalidateCached() +} + +func (m *CpuMask) Difference(other CPUSet) *CpuMask { + o, ok := operandMask(other) + if !ok { + o = NewCpuMask(other.UnsortedList()...) + } + + mine, theirs := m.words(), o.words() + r := make([]uint64, len(mine)) + + for w, v := range mine { + if w < len(theirs) { + r[w] = v &^ theirs[w] + } else { + r[w] = v + } + } + + return newCpuMask(r) +} + +func (m *CpuMask) Intersection(other CPUSet) *CpuMask { + o, ok := operandMask(other) + if !ok { + r := NewCpuMask() + for _, cpu := range other.UnsortedList() { + if m.Contains(cpu) { + r.Set(cpu) + } + } + return r + } + + mine, theirs := m.words(), o.words() + r := make([]uint64, min(len(mine), len(theirs))) + for w := range r { + r[w] = mine[w] & theirs[w] + } + + return newCpuMask(r) +} + +func (m *CpuMask) Intersects(other CPUSet) bool { + o, ok := operandMask(other) + if !ok { + for _, cpu := range other.UnsortedList() { + if m.Contains(cpu) { + return true + } + } + return false + } + + if m == nil || o == nil { + return false + } + + c := min(len(m.mask), len(o.mask)) + for w := 0; w < c; w++ { + if m.mask[w]&o.mask[w] != 0 { + return true + } + } + + return false +} + +func (m *CpuMask) Union(others ...CPUSet) *CpuMask { + r := newCpuMask(slices.Clone(m.words())) + + for _, other := range others { + o, ok := operandMask(other) + if !ok { + for _, cpu := range other.UnsortedList() { + r.Set(cpu) + } + continue + } + + theirs := o.words() + r.mask = expand(r.mask, len(theirs)-1) + for w, v := range theirs { + r.mask[w] |= v + } + } + + return r +} + +func (m *CpuMask) Contains(cpus ...int) bool { + mine := m.words() + for _, cpu := range cpus { + w, b := cpu/64, cpu&63 + if w >= len(mine) || (mine[w]&(1<= 0 { + return m.size + } + + size := 0 + for _, v := range m.mask { + size += bits.OnesCount64(v) + } + m.size = size + + return size +} + +func (m *CpuMask) IsEmpty() bool { + for _, v := range m.words() { + if v != 0 { + return false + } + } + + return true +} + +func (m *CpuMask) IsSubsetOf(other CPUSet) bool { + o, ok := operandMask(other) + if !ok { + return other.Contains(m.UnsortedList()...) + } + + if m == nil { + return true + } + if o == nil { + return m.IsEmpty() + } + + c := min(len(m.mask), len(o.mask)) + for w := 0; w < c; w++ { + if m.mask[w]&^o.mask[w] != 0 { + return false + } + } + + if c < len(m.mask) { + for w := c; w < len(m.mask); w++ { + if m.mask[w] != 0 { + return false + } + } + } + + return true +} + +func (m *CpuMask) List() []int { + cpus := make([]int, 0, m.Size()) + + m.ForEachCpu(func(cpu int) bool { + cpus = append(cpus, cpu) + return true + }) + + return cpus +} + +func (m *CpuMask) UnsortedList() []int { + return m.List() +} + +func (m *CpuMask) String() string { + if m == nil { + return "" + } + if m.seal || m.str != "" { + return m.str + } + + var ( + str strings.Builder + rangeStart = -1 + prev = -1 + ) + + flush := func(end int) { + if rangeStart < 0 { + return + } + if str.Len() > 0 { + str.WriteString(",") + } + str.WriteString(strconv.Itoa(rangeStart)) + if end > rangeStart { + str.WriteString("-") + str.WriteString(strconv.Itoa(end)) + } + rangeStart = -1 + } + + m.ForEachCpu(func(cpu int) bool { + if rangeStart >= 0 && cpu == prev+1 { + prev = cpu + return true + } + flush(prev) + rangeStart = cpu + prev = cpu + return true + }) + flush(prev) + + m.str = str.String() + + return m.str +} + +func (m *CpuMask) Key() string { + if m == nil { + return "" + } + if m.seal || m.key != "" { + return m.key + } + + // Trailing all-zero words must not contribute to the key, otherwise + // two equal sets with differently sized masks get different keys. + mask := m.mask + for len(mask) > 0 && mask[len(mask)-1] == 0 { + mask = mask[:len(mask)-1] + } + + buf, sep := strings.Builder{}, "" + for _, w := range mask { + buf.WriteString(sep) + buf.WriteString(strconv.FormatUint(w, 16)) + sep = "-" + } + m.key = buf.String() + + return m.key +} + +func (m *CpuMask) Seal() { + if m == nil { + panic("cannot seal a nil CpuMask: assign cpus = cpus.EmptyIfNil() first") + } + _ = m.Key() + _ = m.String() + _ = m.Size() + m.seal = true +} + +func (*CpuMask) IsDense() bool { + return true +} + +func (*CpuMask) IsSparse() bool { + return false +} + +func (m *CpuMask) ForEachCpu(f func(cpu int) bool) { + for w, v := range m.words() { + for v != 0 { + if !f(w*64 + bits.TrailingZeros64(v)) { + return + } + v &= v - 1 + } + } +} + +func (m *CpuMask) panicIfNilOrSealed() { + if m == nil { + panic("cannot modify a nil CpuMask: assign cpus = cpus.EmptyIfNil() first") + } + if m.seal { + panic("CpuMask is sealed") + } +} + +func (m *CpuMask) invalidateCached() { + m.str = "" + m.key = "" + m.size = -1 +} + +func expand(mask []uint64, w int) []uint64 { + if w < len(mask) { + return mask + } + for len(mask) <= w { + mask = append(mask, 0) + } + return mask +} + +// CpuSet is a sparse implementation of CPUSet. Internally it +// wraps k8s.io/utils/cpuset.CPUSet and is a good choice for +// representing CPU sets which contain a few CPUs. +type CpuSet struct { + cpuset.CPUSet + seal bool + str string +} + +// CpuSet should implement CPUSet. +var _ CPUSet = (*CpuSet)(nil) + +// NewCpuSet returns a new CpuSet containing the given CPUs. +func NewCpuSet(cpus ...int) *CpuSet { + s := &CpuSet{ + CPUSet: cpuset.New(cpus...), + } + return s +} + +// WrapCpuSet returns a CpuSet backed by the given k8s.io/utils/cpuset.CPUSet, +// without copying it. +// +// It is the cheap way in from code which speaks the k8s type -- one allocation +// for the wrapper, against the list-and-rebuild that a NewCpuSet of its members +// costs -- and is worth reaching for where a set arrives from an upstream +// interface. Put a [NewAnyCPUSet] around the result to get the set-producing +// operations as well. +// +// Sharing the set is safe in both directions. cpuset.CPUSet is immutable, and +// Set and Clear here replace the embedded value rather than modify it, so +// neither side disturbs the other. The embedded field stays reachable, so the +// raw set can be had back without a copy too. +func WrapCpuSet(cpus cpuset.CPUSet) *CpuSet { + return &CpuSet{ + CPUSet: cpus, + } +} + +// ParseCpuSet parses the given string representation of a CPU set +// and returns a corresponding new CpuSet. +func ParseCpuSet(s string) (*CpuSet, error) { + cpus, err := cpuset.Parse(s) + if err != nil { + return nil, fmt.Errorf("%w: %w", ErrParseFailed, err) + } + return &CpuSet{ + CPUSet: cpus, + }, nil +} + +// MustParseCpuSet is [MustParseCpuMask] for the sparse implementation. +func MustParseCpuSet(s string) *CpuSet { + cpus, err := ParseCpuSet(s) + if err != nil { + panic(err) + } + return cpus +} + +// emptyCpuSetValue is what a nil set reads as. The k8s type is immutable -- its +// methods all return new sets -- so one shared empty is safe to hand around. +var emptyCpuSetValue = cpuset.New() + +// cpus returns the set's CPUs, and none for a nil set. This is [CpuMask.words] +// for the sparse implementation: every read below goes through it, which is what +// lets a nil set be read as the empty one. +func (s *CpuSet) cpus() cpuset.CPUSet { + if s == nil { + return emptyCpuSetValue + } + return s.CPUSet +} + +// operandCpuSet is [operandMask] for the sparse implementation, in the same +// order and for the same reason. +func operandCpuSet(other CPUSet) (*CpuSet, bool) { + if s, ok := other.(*CpuSet); ok { + return s, true + } + return nil, isNothing(other) +} + +// Clone returns a copy of the set which is safe to modify: unsealed, and with no +// other owner. A nil set clones into a new empty one, as [CpuMask.Clone] does, +// and for the same purpose. +func (s *CpuSet) Clone() *CpuSet { + if s == nil { + return NewCpuSet() + } + return &CpuSet{ + CPUSet: s.CPUSet.Clone(), + } +} + +// EmptyIfNil returns the set, or a new empty one if it is nil. It is +// [CpuMask.EmptyIfNil] for the sparse implementation, with the same purpose and +// the same limit: it answers for nil, not for sealed. Reading a nil set needs no +// help, and [CpuSet.Clone] is the one which always returns something writable. +func (s *CpuSet) EmptyIfNil() *CpuSet { + if s == nil { + return NewCpuSet() + } + return s +} + +func (s *CpuSet) Set(cpus ...int) { + s.panicIfNilOrSealed() + s.CPUSet = s.CPUSet.Union(cpuset.New(cpus...)) + s.str = "" +} + +func (s *CpuSet) Clear(cpus ...int) { + s.panicIfNilOrSealed() + s.CPUSet = s.CPUSet.Difference(cpuset.New(cpus...)) + s.str = "" +} + +func (s *CpuSet) Difference(other CPUSet) *CpuSet { + o, ok := operandCpuSet(other) + if ok { + return &CpuSet{ + CPUSet: s.cpus().Difference(o.cpus()), + } + } + + cpus := make([]int, 0, s.cpus().Size()) + for _, cpu := range s.cpus().UnsortedList() { + if !other.Contains(cpu) { + cpus = append(cpus, cpu) + } + } + return NewCpuSet(cpus...) +} + +func (s *CpuSet) Intersection(other CPUSet) *CpuSet { + o, ok := operandCpuSet(other) + if ok { + return &CpuSet{ + CPUSet: s.cpus().Intersection(o.cpus()), + } + } + + cpus := make([]int, 0, s.cpus().Size()) + for _, cpu := range s.cpus().UnsortedList() { + if other.Contains(cpu) { + cpus = append(cpus, cpu) + } + } + return NewCpuSet(cpus...) +} + +func (s *CpuSet) Intersects(other CPUSet) bool { + if o, ok := operandCpuSet(other); ok { + a, b := s.cpus(), o.cpus() + if b.Size() < a.Size() { + a, b = b, a + } + for _, cpu := range a.UnsortedList() { + if b.Contains(cpu) { + return true + } + } + return false + } + + if other.Size() < s.cpus().Size() { + for _, cpu := range other.UnsortedList() { + if s.cpus().Contains(cpu) { + return true + } + } + return false + } + + for _, cpu := range s.cpus().UnsortedList() { + if other.Contains(cpu) { + return true + } + } + return false +} + +func (s *CpuSet) Union(others ...CPUSet) *CpuSet { + r := s.cpus().Clone() + for _, other := range others { + o, ok := operandCpuSet(other) + if ok { + r = r.Union(o.cpus()) + continue + } + + r = r.Union(cpuset.New(other.UnsortedList()...)) + } + return &CpuSet{CPUSet: r} +} + +func (s *CpuSet) Contains(cpus ...int) bool { + mine := s.cpus() + for _, cpu := range cpus { + if !mine.Contains(cpu) { + return false + } + } + return true +} + +func (s *CpuSet) Equals(other CPUSet) bool { + o, ok := operandCpuSet(other) + if ok { + return s.cpus().Equals(o.cpus()) + } + + return other.Contains(s.UnsortedList()...) && s.Contains(other.UnsortedList()...) +} + +func (s *CpuSet) Size() int { + return s.cpus().Size() +} + +func (s *CpuSet) IsEmpty() bool { + return s.cpus().IsEmpty() +} + +func (s *CpuSet) IsSubsetOf(other CPUSet) bool { + o, ok := operandCpuSet(other) + if ok { + return s.cpus().IsSubsetOf(o.cpus()) + } + + return other.Contains(s.cpus().UnsortedList()...) +} + +func (s *CpuSet) List() []int { + return s.cpus().List() +} + +func (s *CpuSet) UnsortedList() []int { + return s.cpus().UnsortedList() +} + +func (s *CpuSet) String() string { + if s == nil { + return "" + } + if s.seal || s.str != "" { + return s.str + } + + s.str = s.CPUSet.String() + + return s.str +} + +func (s *CpuSet) Key() string { + return s.String() +} + +func (s *CpuSet) Seal() { + if s == nil { + panic("cannot seal a nil CpuSet: assign cpus = cpus.EmptyIfNil() first") + } + _ = s.String() + s.seal = true +} + +func (*CpuSet) IsDense() bool { + return false +} + +func (*CpuSet) IsSparse() bool { + return true +} + +func (m *CpuSet) ForEachCpu(f func(cpu int) bool) { + for _, cpu := range m.UnsortedList() { + if !f(cpu) { + return + } + } +} + +func (s *CpuSet) panicIfNilOrSealed() { + if s == nil { + panic("cannot modify a nil CpuSet: assign cpus = cpus.EmptyIfNil() first") + } + if s.seal { + panic("CpuSet is sealed") + } +} + +// +// Working with a set of unknown implementation +// + +// AsCpuMask returns cpus as a [CpuMask]: cpus itself if it already is one, a new +// dense set holding the same CPUs otherwise. Note that converting a sparse set +// which holds a few high-numbered CPUs allocates a mask large enough to reach +// them. +func AsCpuMask(cpus CPUSet) *CpuMask { + if m, ok := operandMask(cpus); ok { + return m.EmptyIfNil() + } + return NewCpuMask(cpus.UnsortedList()...) +} + +// AsCpuSet returns cpus as a [CpuSet]: cpus itself if it already is one, a new +// sparse set holding the same CPUs otherwise. +func AsCpuSet(cpus CPUSet) *CpuSet { + if s, ok := operandCpuSet(cpus); ok { + return s.EmptyIfNil() + } + return NewCpuSet(cpus.UnsortedList()...) +} + +// AnyCPUSet is a [CPUSet] of unknown implementation with the set-producing +// operations put back: Clone, Union, Difference and Intersection are on the +// implementations, so a caller which only has the interface cannot use one as +// the receiver of those. Wrapping it here makes that possible. +// +// Reach for this only when the implementation genuinely is not known. Holding a +// [CpuMask] or a [CpuSet], call the operations directly and get the same type +// back; that is the common case and it costs nothing. This one pays for a type +// switch and an interface call per operation. +// +// The result of an operation keeps the implementation it was performed on, so a +// wrapped sparse set stays sparse. A set from neither implementation here is +// answered as a [CpuMask], there being no better guess. The embedded CPUSet +// provides everything else, and an AnyCPUSet is itself a CPUSet. +type AnyCPUSet struct { + CPUSet +} + +// NewAnyCPUSet wraps cpus so that the set-producing operations can be performed +// with it as the receiver. +func NewAnyCPUSet(cpus CPUSet) AnyCPUSet { + // Nothing wrapped is the empty set, so that a wrapper reads like the sets it + // wraps. Sealed: it is shared, and has no owner to modify it. + if isNothing(cpus) { + return AnyCPUSet{CPUSet: EmptyCpuMask} + } + return AnyCPUSet{CPUSet: cpus} +} + +// Clone returns a new unsealed copy of the set. +func (a AnyCPUSet) Clone() AnyCPUSet { + switch s := a.CPUSet.(type) { + case *CpuMask: + return AnyCPUSet{CPUSet: s.Clone()} + case *CpuSet: + return AnyCPUSet{CPUSet: s.Clone()} + } + return AnyCPUSet{CPUSet: AsCpuMask(a.CPUSet).Clone()} +} + +// Union returns a new set with all CPUs in this set or in at least one of the +// other sets. +func (a AnyCPUSet) Union(others ...CPUSet) AnyCPUSet { + switch s := a.CPUSet.(type) { + case *CpuMask: + return AnyCPUSet{CPUSet: s.Union(others...)} + case *CpuSet: + return AnyCPUSet{CPUSet: s.Union(others...)} + } + return AnyCPUSet{CPUSet: AsCpuMask(a.CPUSet).Union(others...)} +} + +// Difference returns a new set with all CPUs in this set which are not in the +// other set. +func (a AnyCPUSet) Difference(other CPUSet) AnyCPUSet { + switch s := a.CPUSet.(type) { + case *CpuMask: + return AnyCPUSet{CPUSet: s.Difference(other)} + case *CpuSet: + return AnyCPUSet{CPUSet: s.Difference(other)} + } + return AnyCPUSet{CPUSet: AsCpuMask(a.CPUSet).Difference(other)} +} + +// Intersection returns a new set with the CPUs which are in both sets. +func (a AnyCPUSet) Intersection(other CPUSet) AnyCPUSet { + switch s := a.CPUSet.(type) { + case *CpuMask: + return AnyCPUSet{CPUSet: s.Intersection(other)} + case *CpuSet: + return AnyCPUSet{CPUSet: s.Intersection(other)} + } + return AnyCPUSet{CPUSet: AsCpuMask(a.CPUSet).Intersection(other)} +} + +// AnyCPUSet should implement CPUSet. +var _ CPUSet = AnyCPUSet{} diff --git a/pkg/lib/cpu/cpuset_test.go b/pkg/lib/cpu/cpuset_test.go new file mode 100644 index 000000000..4166a7384 --- /dev/null +++ b/pkg/lib/cpu/cpuset_test.go @@ -0,0 +1,3857 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package libcpu + +import ( + "fmt" + "math/rand" + "slices" + "strings" + "sync" + "testing" + + "k8s.io/utils/cpuset" +) + +// testCPUSet wraps *CpuMask but presents a different concrete type so that +// type assertions to *CpuMask fail, exercising the non-fast-path fallback +// code in Difference, Intersection, Equals, IsSubsetOf, and Union. +type testCPUSet struct { + *CpuMask +} + +var _ CPUSet = (*testCPUSet)(nil) + +func newTestCPUSet(cpus ...int) *testCPUSet { + return &testCPUSet{CpuMask: NewCpuMask(cpus...)} +} + +// cpuRange returns a sorted []int with all integers from lo to hi inclusive. +func cpuRange(lo, hi int) []int { + s := make([]int, hi-lo+1) + for i := range s { + s[i] = lo + i + } + return s +} + +// maskListEqual reports whether two CPUSets contain exactly the same CPUs by +// comparing their sorted lists, avoiding any dependence on Equals itself. +func maskListEqual(a, b CPUSet) bool { + return slices.Equal(a.List(), b.List()) +} + +// ---- TestNewCpuMask ------------------------------------------------------- + +func TestNewCpuMask(t *testing.T) { + tests := []struct { + name string + cpus []int + expected []int + }{ + { + name: "empty", + cpus: []int{}, + expected: []int{}, + }, + { + name: "cpu-0-lowest-bit-word-0", + cpus: []int{0}, + expected: []int{0}, + }, + { + name: "cpu-63-highest-bit-word-0", + cpus: []int{63}, + expected: []int{63}, + }, + { + name: "cpu-64-lowest-bit-word-1", + cpus: []int{64}, + expected: []int{64}, + }, + { + name: "cpu-127-highest-bit-word-1", + cpus: []int{127}, + expected: []int{127}, + }, + { + name: "word-0-all-bits-set", + cpus: cpuRange(0, 63), + expected: cpuRange(0, 63), + }, + { + name: "two-words-all-bits-set", + cpus: cpuRange(0, 127), + expected: cpuRange(0, 127), + }, + { + name: "lowest-and-highest-in-word-0", + cpus: []int{0, 63}, + expected: []int{0, 63}, + }, + { + name: "straddles-word-boundary-63-and-64", + cpus: []int{63, 64}, + expected: []int{63, 64}, + }, + { + name: "cpu-1023-highest-bit-word-15", + cpus: []int{1023}, + expected: []int{1023}, + }, + { + name: "cpu-1024-lowest-bit-word-16", + cpus: []int{1024}, + expected: []int{1024}, + }, + { + name: "lowest-and-highest-across-16-words", + cpus: []int{0, 1023}, + expected: []int{0, 1023}, + }, + { + name: "sparse-word-boundary-cpus", + cpus: []int{0, 64, 128, 512, 960, 1023}, + expected: []int{0, 64, 128, 512, 960, 1023}, + }, + { + name: "large-contiguous-range-512-to-1023", + cpus: cpuRange(512, 1023), + expected: cpuRange(512, 1023), + }, + { + name: "duplicate-cpus-are-deduped", + cpus: []int{0, 0, 63, 63, 64, 64}, + expected: []int{0, 63, 64}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.cpus...) + got := m.List() + if !slices.Equal(got, tc.expected) { + t.Errorf("List() = %v, want %v", got, tc.expected) + } + }) + } +} + +// ---- TestSetAndClear ------------------------------------------------------- + +func TestSetAndClear(t *testing.T) { + type step struct { + set []int + clear []int + } + tests := []struct { + name string + initial []int + steps []step + expected []int + }{ + { + name: "set-cpu-0-on-empty", + steps: []step{{set: []int{0}}}, + expected: []int{0}, + }, + { + name: "set-cpu-63-highest-bit-word-0", + steps: []step{{set: []int{63}}}, + expected: []int{63}, + }, + { + name: "set-cpu-64-first-bit-word-1", + steps: []step{{set: []int{64}}}, + expected: []int{64}, + }, + { + name: "set-cpu-1023", + steps: []step{{set: []int{1023}}}, + expected: []int{1023}, + }, + { + name: "set-cpu-1024", + steps: []step{{set: []int{1024}}}, + expected: []int{1024}, + }, + { + name: "set-is-idempotent", + initial: []int{0}, + steps: []step{{set: []int{0}}}, + expected: []int{0}, + }, + { + name: "set-multiple-across-word-boundaries", + steps: []step{{set: []int{0, 63, 64, 1023}}}, + expected: []int{0, 63, 64, 1023}, + }, + { + name: "clear-middle-cpu", + initial: []int{0, 1, 2}, + steps: []step{{clear: []int{1}}}, + expected: []int{0, 2}, + }, + { + name: "clear-highest-bit-in-word-0", + initial: cpuRange(0, 63), + steps: []step{{clear: []int{63}}}, + expected: cpuRange(0, 62), + }, + { + name: "clear-lowest-bit-in-word-1", + initial: []int{0, 64}, + steps: []step{{clear: []int{64}}}, + expected: []int{0}, + }, + { + name: "clear-cpu-1023", + initial: []int{0, 1023}, + steps: []step{{clear: []int{1023}}}, + expected: []int{0}, + }, + { + name: "clear-nonexistent-is-noop", + initial: []int{0, 2}, + steps: []step{{clear: []int{1}}}, + expected: []int{0, 2}, + }, + { + name: "clear-out-of-range-is-noop", + initial: []int{0}, // mask covers only word 0 + // CPU 128 is word 2, well past the end — must not panic. + steps: []step{{clear: []int{128}}}, + expected: []int{0}, + }, + { + name: "clear-exactly-at-mask-length-boundary", + initial: []int{0, 64}, // mask has 2 words (indices 0 and 1) + // CPU 128 → word 2 == len(mask); must be a no-op, not a panic. + steps: []step{{clear: []int{128}}}, + expected: []int{0, 64}, + }, + { + name: "set-then-clear", + steps: []step{{set: []int{0, 1, 2}}, {clear: []int{1}}}, + expected: []int{0, 2}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.initial...) + for _, s := range tc.steps { + for _, cpu := range s.set { + m.Set(cpu) + } + for _, cpu := range s.clear { + m.Clear(cpu) + } + } + if got := m.List(); !slices.Equal(got, tc.expected) { + t.Errorf("List() = %v, want %v", got, tc.expected) + } + }) + } + + // CpuMask caches String, Key and Size; Set and Clear must invalidate all + // three, not just the one that happens to be read first. + + t.Run("caches-cleared-after-set", func(t *testing.T) { + m := NewCpuMask(0) + primedKey := m.Key() + _, _ = m.String(), m.Size() + + m.Set(1) + + if got := m.String(); got != "0-1" { + t.Errorf("String() not invalidated after Set: got %q, want %q", got, "0-1") + } + if got := m.Key(); got == primedKey { + t.Errorf("Key() not invalidated after Set: still %q", got) + } + if got := m.Size(); got != 2 { + t.Errorf("Size() not invalidated after Set: got %d, want 2", got) + } + if got, want := m.Key(), NewCpuMask(0, 1).Key(); got != want { + t.Errorf("Key() = %q, want %q", got, want) + } + }) + + t.Run("caches-cleared-after-clear", func(t *testing.T) { + m := NewCpuMask(0, 1) + primedKey := m.Key() + _, _ = m.String(), m.Size() + + m.Clear(1) + + if got := m.String(); got != "0" { + t.Errorf("String() not invalidated after Clear: got %q, want %q", got, "0") + } + if got := m.Key(); got == primedKey { + t.Errorf("Key() not invalidated after Clear: still %q", got) + } + if got := m.Size(); got != 1 { + t.Errorf("Size() not invalidated after Clear: got %d, want 1", got) + } + if got, want := m.Key(), NewCpuMask(0).Key(); got != want { + t.Errorf("Key() = %q, want %q", got, want) + } + }) + + t.Run("caches-cleared-when-emptied", func(t *testing.T) { + m := NewCpuMask(0, 1) + _, _, _ = m.String(), m.Key(), m.Size() + + m.Clear(0, 1) + + if got := m.String(); got != "" { + t.Errorf("String() = %q, want %q", got, "") + } + if got := m.Key(); got != "" { + t.Errorf("Key() = %q, want %q", got, "") + } + if got := m.Size(); got != 0 { + t.Errorf("Size() = %d, want 0", got) + } + }) + + // ---- variadic multi-arg Set / Clear calls -------------------------------- + + t.Run("set-no-args-is-noop", func(t *testing.T) { + m := NewCpuMask(5) + m.Set() + if got := m.List(); !slices.Equal(got, []int{5}) { + t.Errorf("Set() no-op: got %v, want [5]", got) + } + }) + + t.Run("clear-no-args-is-noop", func(t *testing.T) { + m := NewCpuMask(5) + m.Clear() + if got := m.List(); !slices.Equal(got, []int{5}) { + t.Errorf("Clear() no-op: got %v, want [5]", got) + } + }) + + t.Run("set-multiple-same-word", func(t *testing.T) { + m := NewCpuMask() + m.Set(0, 1, 2, 3, 63) + want := []int{0, 1, 2, 3, 63} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("set-multiple-across-word-boundaries", func(t *testing.T) { + m := NewCpuMask() + m.Set(0, 63, 64, 127, 128, 1023) + want := []int{0, 63, 64, 127, 128, 1023} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("set-multiple-idempotent", func(t *testing.T) { + m := NewCpuMask(0, 63, 64) + m.Set(0, 63, 64, 64, 63, 0) // duplicates must not double-set + want := []int{0, 63, 64} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("set-multiple-large-range", func(t *testing.T) { + m := NewCpuMask() + args := cpuRange(512, 575) // 64 CPUs filling exactly word 8 + m.Set(args...) + if got := m.List(); !slices.Equal(got, args) { + t.Errorf("got %v, want %v", got, args) + } + }) + + t.Run("clear-multiple-same-word", func(t *testing.T) { + m := NewCpuMask(0, 1, 2, 3, 63) + m.Clear(1, 2, 63) + want := []int{0, 3} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("clear-multiple-across-word-boundaries", func(t *testing.T) { + m := NewCpuMask(0, 63, 64, 127, 128, 1023) + m.Clear(63, 64, 1023) + want := []int{0, 127, 128} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("clear-multiple-some-absent", func(t *testing.T) { + m := NewCpuMask(0, 2, 4) + m.Clear(1, 2, 3) // 1 and 3 are not set — should be no-op for those + want := []int{0, 4} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("clear-multiple-out-of-range-mixed-with-valid", func(t *testing.T) { + m := NewCpuMask(0, 64) + // 512 is beyond the mask; clearing it must not panic and must be a no-op. + m.Clear(64, 512) + want := []int{0} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("clear-all-via-variadic", func(t *testing.T) { + cpus := []int{0, 63, 64, 127, 128, 255, 1023} + m := NewCpuMask(cpus...) + m.Clear(cpus...) + if !m.IsEmpty() { + t.Errorf("expected empty after clearing all CPUs, got %v", m.List()) + } + }) + + t.Run("set-then-clear-multi-arg", func(t *testing.T) { + m := NewCpuMask() + m.Set(0, 63, 64, 127, 512, 1023) + m.Clear(63, 127, 1023) + want := []int{0, 64, 512} + if got := m.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) +} + +// ---- TestClone ------------------------------------------------------------ + +func TestClone(t *testing.T) { + t.Run("clone-of-empty", func(t *testing.T) { + m := NewCpuMask() + c := m.Clone() + if !c.IsEmpty() { + t.Errorf("clone of empty mask is not empty: %v", c.List()) + } + }) + + t.Run("clone-matches-original", func(t *testing.T) { + cpus := []int{0, 63, 64, 127, 512, 1023} + m := NewCpuMask(cpus...) + c := m.Clone() + if !slices.Equal(c.List(), cpus) { + t.Errorf("clone content mismatch: want %v, got %v", cpus, c.List()) + } + }) + + t.Run("clone-is-independent-mutation-does-not-affect-original", func(t *testing.T) { + m := NewCpuMask(0, 1, 2) + c := m.Clone() + c.Set(63) + if m.Contains(63) { + t.Error("mutating clone propagated to original") + } + if !c.Contains(63) { + t.Error("mutation of clone did not take effect") + } + }) + + t.Run("clone-of-sealed-mask-is-not-sealed", func(t *testing.T) { + m := NewCpuMask(0) + m.Seal() + c := m.Clone() + c.Set(1) // must not panic + if !c.Contains(1) { + t.Error("clone of sealed mask should be mutable") + } + }) + + t.Run("clone-of-full-word-mask", func(t *testing.T) { + m := NewCpuMask(cpuRange(0, 63)...) + c := m.Clone() + if !slices.Equal(c.List(), cpuRange(0, 63)) { + t.Errorf("full-word clone mismatch: got %v", c.List()) + } + }) + + t.Run("clone-of-multi-word-high-cpu-mask", func(t *testing.T) { + cpus := cpuRange(960, 1023) + m := NewCpuMask(cpus...) + c := m.Clone() + if !slices.Equal(c.List(), cpus) { + t.Errorf("multi-word clone mismatch: got %v", c.List()) + } + }) +} + +// ---- TestSeal ------------------------------------------------------------- + +func TestSeal(t *testing.T) { + t.Run("set-on-sealed-mask-panics", func(t *testing.T) { + m := NewCpuMask(0) + m.Seal() + defer func() { + if r := recover(); r == nil { + t.Error("expected panic from Set on sealed mask, got none") + } + }() + m.Set(1) + }) + + t.Run("clear-on-sealed-mask-panics", func(t *testing.T) { + m := NewCpuMask(0, 1) + m.Seal() + defer func() { + if r := recover(); r == nil { + t.Error("expected panic from Clear on sealed mask, got none") + } + }() + m.Clear(0) + }) + + t.Run("read-only-operations-work-on-sealed-mask", func(t *testing.T) { + m := NewCpuMask(0, 63, 64, 1023) + m.Seal() + if !m.Contains(63) { + t.Error("Contains should work on sealed mask") + } + if m.Size() != 4 { + t.Errorf("Size should work on sealed mask: got %d, want 4", m.Size()) + } + if m.IsEmpty() { + t.Error("IsEmpty should work on sealed mask") + } + if !slices.Equal(m.List(), []int{0, 63, 64, 1023}) { + t.Errorf("List should work on sealed mask: got %v", m.List()) + } + }) +} + +// ---- TestIsDenseIsSparse --------------------------------------------- + +func TestIsDenseIsSparse(t *testing.T) { + t.Run("cpumask-is-dense-not-sparse", func(t *testing.T) { + m := NewCpuMask(0, 1, 2) + if !m.IsDense() { + t.Error("CpuMask.IsDense() = false, want true") + } + if m.IsSparse() { + t.Error("CpuMask.IsSparse() = true, want false") + } + }) + + t.Run("cpuset-is-sparse-not-dense", func(t *testing.T) { + s := NewCpuSet(0, 1, 2) + if s.IsDense() { + t.Error("CpuSet.IsDense() = true, want false") + } + if !s.IsSparse() { + t.Error("CpuSet.IsSparse() = false, want true") + } + }) +} + +// ---- TestIsEmpty ------------------------------------------------------ + +func TestIsEmpty(t *testing.T) { + tests := []struct { + name string + cpus []int + expected bool + }{ + {name: "empty", cpus: []int{}, expected: true}, + {name: "single-cpu-0", cpus: []int{0}, expected: false}, + {name: "single-cpu-63", cpus: []int{63}, expected: false}, + {name: "single-cpu-64", cpus: []int{64}, expected: false}, + {name: "word-0-all-bits-set", cpus: cpuRange(0, 63), expected: false}, + {name: "single-cpu-1023", cpus: []int{1023}, expected: false}, + {name: "multi-word-sparse", cpus: []int{0, 64, 512, 1023}, expected: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.cpus...) + if got := m.IsEmpty(); got != tc.expected { + t.Errorf("IsEmpty() = %v, want %v", got, tc.expected) + } + }) + } + + t.Run("empty-after-clearing-all-bits", func(t *testing.T) { + m := NewCpuMask(0, 1, 63, 64, 1023) + for _, cpu := range m.List() { + m.Clear(cpu) + } + if !m.IsEmpty() { + t.Errorf("mask should be empty after clearing all bits, got %v", m.List()) + } + }) + + t.Run("empty-with-trailing-zero-words", func(t *testing.T) { + // Set then clear a high CPU to leave trailing zero words in the mask array. + m := NewCpuMask(0, 1023) + m.Clear(1023) + m.Clear(0) + if !m.IsEmpty() { + t.Errorf("mask with only zero words should report empty, got %v", m.List()) + } + }) +} + +// ---- TestSize --------------------------------------------------------- + +func TestSize(t *testing.T) { + tests := []struct { + name string + cpus []int + expected int + }{ + {name: "empty", cpus: []int{}, expected: 0}, + {name: "single-cpu-0", cpus: []int{0}, expected: 1}, + {name: "single-cpu-63-highest-word-0", cpus: []int{63}, expected: 1}, + {name: "single-cpu-64-lowest-word-1", cpus: []int{64}, expected: 1}, + {name: "single-cpu-127-highest-word-1", cpus: []int{127}, expected: 1}, + {name: "word-0-all-64-bits", cpus: cpuRange(0, 63), expected: 64}, + {name: "two-words-all-128-bits", cpus: cpuRange(0, 127), expected: 128}, + {name: "single-cpu-1023", cpus: []int{1023}, expected: 1}, + {name: "single-cpu-1024", cpus: []int{1024}, expected: 1}, + {name: "large-range-512-to-1023", cpus: cpuRange(512, 1023), expected: 512}, + {name: "sparse-boundary-cpus", cpus: []int{0, 63, 64, 127, 512, 1023}, expected: 6}, + {name: "all-cpus-0-to-1023", cpus: cpuRange(0, 1023), expected: 1024}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.cpus...) + if got := m.Size(); got != tc.expected { + t.Errorf("Size() = %d, want %d", got, tc.expected) + } + }) + } +} + +// ---- TestContains ----------------------------------------------------- + +func TestContains(t *testing.T) { + tests := []struct { + name string + cpus []int + check int + expected bool + }{ + {name: "empty-mask-check-cpu-0", cpus: []int{}, check: 0, expected: false}, + {name: "cpu-0-present", cpus: []int{0}, check: 0, expected: true}, + {name: "cpu-0-absent", cpus: []int{1}, check: 0, expected: false}, + {name: "cpu-63-present", cpus: []int{63}, check: 63, expected: true}, + {name: "cpu-63-absent", cpus: []int{62}, check: 63, expected: false}, + {name: "cpu-64-present", cpus: []int{64}, check: 64, expected: true}, + {name: "cpu-64-absent-only-63-set", cpus: []int{63}, check: 64, expected: false}, + {name: "cpu-63-absent-only-64-set", cpus: []int{64}, check: 63, expected: false}, + {name: "word-boundary-63-and-64-check-63", cpus: []int{63, 64}, check: 63, expected: true}, + {name: "word-boundary-63-and-64-check-64", cpus: []int{63, 64}, check: 64, expected: true}, + {name: "cpu-1023-present", cpus: []int{1023}, check: 1023, expected: true}, + {name: "cpu-1023-absent", cpus: []int{1022}, check: 1023, expected: false}, + {name: "check-beyond-mask-range", cpus: []int{0}, check: 1023, expected: false}, + {name: "middle-of-full-word", cpus: cpuRange(0, 63), check: 32, expected: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.cpus...) + if got := m.Contains(tc.check); got != tc.expected { + t.Errorf("Contains(%d) = %v, want %v", tc.check, got, tc.expected) + } + }) + } + + // ---- variadic multi-arg / zero-arg Contains calls ------------------------ + + t.Run("contains-no-args-is-vacuously-true", func(t *testing.T) { + m := NewCpuMask(0, 1, 2) + if !m.Contains() { + t.Error("Contains() with no args should be true") + } + if got := NewCpuMask().Contains(); !got { + t.Error("Contains() on empty mask with no args should be true") + } + }) + + t.Run("contains-multiple-all-present", func(t *testing.T) { + m := NewCpuMask(0, 63, 64, 1023) + if !m.Contains(0, 63, 64, 1023) { + t.Error("Contains(0, 63, 64, 1023) = false, want true") + } + }) + + t.Run("contains-multiple-one-missing", func(t *testing.T) { + m := NewCpuMask(0, 63, 64) + if m.Contains(0, 63, 64, 1023) { + t.Error("Contains(0, 63, 64, 1023) = true, want false (1023 absent)") + } + }) + + t.Run("contains-multiple-duplicates", func(t *testing.T) { + m := NewCpuMask(0, 64) + if !m.Contains(0, 0, 64, 64) { + t.Error("Contains with duplicate CPUs = false, want true") + } + }) +} + +// ---- TestString ------------------------------------------------------- + +func TestString(t *testing.T) { + tests := []struct { + name string + cpus []int + expected string + }{ + {name: "empty", cpus: []int{}, expected: ""}, + {name: "single-cpu-0", cpus: []int{0}, expected: "0"}, + {name: "single-cpu-63", cpus: []int{63}, expected: "63"}, + {name: "single-cpu-64", cpus: []int{64}, expected: "64"}, + {name: "single-cpu-1023", cpus: []int{1023}, expected: "1023"}, + {name: "single-cpu-1024", cpus: []int{1024}, expected: "1024"}, + {name: "full-word-0", cpus: cpuRange(0, 63), expected: "0-63"}, + {name: "straddles-word-boundary-63-64", cpus: []int{63, 64}, expected: "63-64"}, + {name: "two-full-words-0-to-127", cpus: cpuRange(0, 127), expected: "0-127"}, + {name: "lowest-and-highest-of-two-words", cpus: []int{0, 63, 64, 127}, expected: "0,63-64,127"}, + {name: "sparse-word-boundary-cpus", cpus: []int{0, 64, 128, 192}, expected: "0,64,128,192"}, + {name: "low-and-very-high", cpus: []int{0, 1023}, expected: "0,1023"}, + {name: "large-contiguous-range-512-to-1023", cpus: cpuRange(512, 1023), expected: "512-1023"}, + {name: "mixed-ranges-across-words", cpus: []int{0, 1, 2, 64, 65, 128}, expected: "0-2,64-65,128"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.cpus...) + got := m.String() + if got != tc.expected { + t.Errorf("String() = %q, want %q", got, tc.expected) + } + // Second call must return the cached value unchanged. + if got2 := m.String(); got2 != got { + t.Errorf("cached String() = %q, first was %q", got2, got) + } + }) + } +} + +// ---- TestListAndUnsortedList ----------------------------------------------- + +func TestListAndUnsortedList(t *testing.T) { + tests := []struct { + name string + cpus []int + expected []int + }{ + {name: "empty", cpus: []int{}, expected: []int{}}, + {name: "single-cpu-0", cpus: []int{0}, expected: []int{0}}, + {name: "out-of-order-input-sorted-output", cpus: []int{3, 1, 2, 0}, expected: []int{0, 1, 2, 3}}, + {name: "multi-word-sparse", cpus: []int{0, 64, 127, 512, 1023}, expected: []int{0, 64, 127, 512, 1023}}, + {name: "full-word-0", cpus: cpuRange(0, 63), expected: cpuRange(0, 63)}, + {name: "large-range-960-to-1023", cpus: cpuRange(960, 1023), expected: cpuRange(960, 1023)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.cpus...) + if got := m.List(); !slices.Equal(got, tc.expected) { + t.Errorf("List() = %v, want %v", got, tc.expected) + } + // UnsortedList only promises the same elements, not an order, so + // sort before comparing. (CpuMask happens to return them sorted, + // but nothing in the CPUSet contract requires that.) + unsorted := m.UnsortedList() + slices.Sort(unsorted) + if !slices.Equal(unsorted, tc.expected) { + t.Errorf("UnsortedList() (sorted) = %v, want %v", unsorted, tc.expected) + } + }) + } +} + +// ---- TestForEachCpu ------------------------------------------------------- + +func TestForEachCpu(t *testing.T) { + t.Run("empty-mask-f-never-called", func(t *testing.T) { + m := NewCpuMask() + called := false + m.ForEachCpu(func(_ int) bool { + called = true + return true + }) + if called { + t.Error("ForEachCpu called f on empty mask") + } + }) + + t.Run("single-cpu", func(t *testing.T) { + m := NewCpuMask(42) + var visited []int + m.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + if !slices.Equal(visited, []int{42}) { + t.Errorf("expected [42], got %v", visited) + } + }) + + t.Run("ascending-order", func(t *testing.T) { + want := []int{0, 7, 63, 64, 127, 512, 1023} + m := NewCpuMask(want...) + var visited []int + m.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + if !slices.Equal(visited, want) { + t.Errorf("expected %v, got %v", want, visited) + } + }) + + t.Run("full-word-0-visits-all-64", func(t *testing.T) { + m := NewCpuMask(cpuRange(0, 63)...) + var visited []int + m.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + if !slices.Equal(visited, cpuRange(0, 63)) { + t.Errorf("expected CPUs 0-63, got %v", visited) + } + }) + + t.Run("sparse-bits-within-one-word", func(t *testing.T) { + // Several CPUs spread across a single word: iteration must skip the + // zero bits between them and visit each set bit exactly once. + want := []int{0, 16, 32, 48} + m := NewCpuMask(want...) + var visited []int + m.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + if !slices.Equal(visited, want) { + t.Errorf("expected %v, got %v", want, visited) + } + }) + + t.Run("early-termination-stops-iteration", func(t *testing.T) { + m := NewCpuMask(0, 63, 64, 1023) + var visited []int + m.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return len(visited) < 2 // stop after two CPUs + }) + if !slices.Equal(visited, []int{0, 63}) { + t.Errorf("expected [0, 63], got %v", visited) + } + }) + + t.Run("high-cpu-numbers-960-to-1023", func(t *testing.T) { + want := cpuRange(960, 1023) + m := NewCpuMask(want...) + var visited []int + m.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + if !slices.Equal(visited, want) { + t.Errorf("expected CPUs 960-1023, got %v", visited) + } + }) + + t.Run("word-boundary-highest-bits", func(t *testing.T) { + // Highest bit of each of the first four words. + want := []int{63, 127, 191, 255} + m := NewCpuMask(want...) + var visited []int + m.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + if !slices.Equal(visited, want) { + t.Errorf("expected %v, got %v", want, visited) + } + }) +} + +// ---- TestDifference --------------------------------------------------- + +func TestDifference(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected []int + }{ + // *CpuMask fast path + { + name: "mask: empty-minus-empty", + a: []int{}, + b: NewCpuMask(), + expected: []int{}, + }, + { + name: "mask: empty-minus-nonempty", + a: []int{}, + b: NewCpuMask(0, 1, 2), + expected: []int{}, + }, + { + name: "mask: nonempty-minus-empty", + a: []int{0, 1, 2}, + b: NewCpuMask(), + expected: []int{0, 1, 2}, + }, + { + name: "mask: a-minus-a-equals-empty", + a: []int{0, 1, 2}, + b: NewCpuMask(0, 1, 2), + expected: []int{}, + }, + { + name: "mask: disjoint-sets", + a: []int{0, 2, 4}, + b: NewCpuMask(1, 3, 5), + expected: []int{0, 2, 4}, + }, + { + name: "mask: superset-minus-subset", + a: []int{0, 1, 2, 3}, + b: NewCpuMask(1, 2), + expected: []int{0, 3}, + }, + { + name: "mask: subset-minus-superset-equals-empty", + a: []int{1, 2}, + b: NewCpuMask(0, 1, 2, 3), + expected: []int{}, + }, + { + name: "mask: a-longer-than-b-extra-a-words-kept", + a: []int{0, 64, 128, 1023}, + b: NewCpuMask(64), + expected: []int{0, 128, 1023}, + }, + { + name: "mask: b-longer-than-a-extra-b-words-ignored", + a: []int{0, 64}, + b: NewCpuMask(64, 128, 1023), + expected: []int{0}, + }, + { + name: "mask: full-word-0-minus-word-1-leaves-word-0-intact", + a: cpuRange(0, 63), + b: NewCpuMask(cpuRange(64, 127)...), + expected: cpuRange(0, 63), + }, + { + name: "mask: high-cpus-partial-difference", + a: cpuRange(960, 1023), + b: NewCpuMask(cpuRange(992, 1023)...), + expected: cpuRange(960, 991), + }, + // testCPUSet fallback path + { + name: "non-mask: basic-difference", + a: []int{0, 1, 2, 3}, + b: newTestCPUSet(1, 2), + expected: []int{0, 3}, + }, + { + name: "non-mask: empty-a", + a: []int{}, + b: newTestCPUSet(0, 1), + expected: []int{}, + }, + { + name: "non-mask: empty-b", + a: []int{0, 1}, + b: newTestCPUSet(), + expected: []int{0, 1}, + }, + { + name: "non-mask: high-cpus", + a: cpuRange(512, 1023), + b: newTestCPUSet(cpuRange(768, 1023)...), + expected: cpuRange(512, 767), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuMask(tc.a...) + got := a.Difference(tc.b) + exp := NewCpuMask(tc.expected...) + if !maskListEqual(got, exp) { + t.Errorf("Difference() = %v, want %v", got.List(), tc.expected) + } + }) + } +} + +// ---- TestEquals ------------------------------------------------------- + +func TestEquals(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected bool + }{ + // *CpuMask fast path + { + name: "mask: both-empty", + a: []int{}, + b: NewCpuMask(), + expected: true, + }, + { + name: "mask: same-single-cpu-0", + a: []int{0}, + b: NewCpuMask(0), + expected: true, + }, + { + name: "mask: different-single-cpu", + a: []int{0}, + b: NewCpuMask(1), + expected: false, + }, + { + name: "mask: one-empty-one-not", + a: []int{0}, + b: NewCpuMask(), + expected: false, + }, + { + name: "mask: same-full-word-0", + a: cpuRange(0, 63), + b: NewCpuMask(cpuRange(0, 63)...), + expected: true, + }, + { + name: "mask: same-multi-word", + a: []int{0, 64, 128, 1023}, + b: NewCpuMask(0, 64, 128, 1023), + expected: true, + }, + { + name: "mask: different-multi-word", + a: []int{0, 64}, + b: NewCpuMask(0, 128), + expected: false, + }, + { + name: "mask: high-cpus-equal", + a: cpuRange(960, 1023), + b: NewCpuMask(cpuRange(960, 1023)...), + expected: true, + }, + { + name: "mask: high-cpus-differ-by-one", + a: cpuRange(960, 1022), + b: NewCpuMask(cpuRange(960, 1023)...), + expected: false, + }, + { + // m (built from a) has fewer words than b; b's extra high word is + // all zero, so the sets are still equal. Exercises the + // `case c < len(other.mask)` branch in the fast path with a true + // outcome. + name: "mask: other-has-extra-all-zero-word-still-equal", + a: []int{0}, + b: func() CPUSet { + o := NewCpuMask(0, 1024) + o.Clear(1024) + return o + }(), + expected: true, + }, + { + // Same as above, but b's extra high word has a bit set, so the + // sets differ. Exercises the `case c < len(other.mask)` branch + // with a false outcome. + name: "mask: other-has-extra-nonzero-word-not-equal", + a: []int{0}, + b: NewCpuMask(0, 1024), + expected: false, + }, + // testCPUSet fallback path + { + name: "non-mask: equal", + a: []int{0, 1, 2}, + b: newTestCPUSet(0, 1, 2), + expected: true, + }, + { + name: "non-mask: not-equal-different-cpu", + a: []int{0, 1}, + b: newTestCPUSet(0, 2), + expected: false, + }, + { + name: "non-mask: not-equal-different-size", + a: []int{0, 1, 2}, + b: newTestCPUSet(0, 1), + expected: false, + }, + { + name: "non-mask: both-empty", + a: []int{}, + b: newTestCPUSet(), + expected: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuMask(tc.a...) + if got := a.Equals(tc.b); got != tc.expected { + t.Errorf("Equals() = %v, want %v", got, tc.expected) + } + }) + } + + t.Run("mask: trailing-zero-words-equal-shorter-mask", func(t *testing.T) { + // Set CPU 64 then clear it, leaving a trailing zero word in the array. + a := NewCpuMask(0, 64) + a.Clear(64) // a.mask = [0x1, 0x0] + b := NewCpuMask(0) // b.mask = [0x1] + if !a.Equals(b) { + t.Error("mask with trailing zero word should equal shorter mask with same bits") + } + if !b.Equals(a) { + t.Error("equality should be symmetric for trailing-zero case") + } + }) +} + +// ---- TestIntersection ------------------------------------------------- + +func TestIntersection(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected []int + }{ + // *CpuMask fast path + { + name: "mask: both-empty", + a: []int{}, + b: NewCpuMask(), + expected: []int{}, + }, + { + name: "mask: one-empty", + a: []int{0, 1, 2}, + b: NewCpuMask(), + expected: []int{}, + }, + { + name: "mask: no-overlap", + a: []int{0, 2}, + b: NewCpuMask(1, 3), + expected: []int{}, + }, + { + name: "mask: full-overlap-single-word", + a: cpuRange(0, 63), + b: NewCpuMask(cpuRange(0, 63)...), + expected: cpuRange(0, 63), + }, + { + name: "mask: partial-overlap-single-word", + a: []int{0, 1, 2}, + b: NewCpuMask(1, 2, 3), + expected: []int{1, 2}, + }, + { + name: "mask: a-shorter-than-b-extra-b-words-ignored", + a: []int{0}, + b: NewCpuMask(0, 64), + expected: []int{0}, + }, + { + name: "mask: b-shorter-than-a-extra-a-words-dropped", + a: []int{0, 64}, + b: NewCpuMask(0), + expected: []int{0}, + }, + { + name: "mask: multi-word-overlap", + a: []int{0, 64, 128, 512}, + b: NewCpuMask(64, 128, 256, 512), + expected: []int{64, 128, 512}, + }, + { + name: "mask: word-boundary-63-and-64", + a: []int{63, 64}, + b: NewCpuMask(63, 64), + expected: []int{63, 64}, + }, + { + name: "mask: high-cpus-partial-overlap", + a: cpuRange(512, 1023), + b: NewCpuMask(cpuRange(960, 1023)...), + expected: cpuRange(960, 1023), + }, + // testCPUSet fallback path + { + name: "non-mask: partial-overlap", + a: []int{0, 1, 2}, + b: newTestCPUSet(1, 2, 3), + expected: []int{1, 2}, + }, + { + name: "non-mask: empty-a", + a: []int{}, + b: newTestCPUSet(0, 1), + expected: []int{}, + }, + { + name: "non-mask: empty-b", + a: []int{0, 1}, + b: newTestCPUSet(), + expected: []int{}, + }, + { + name: "non-mask: high-cpus", + a: cpuRange(512, 1023), + b: newTestCPUSet(cpuRange(960, 1023)...), + expected: cpuRange(960, 1023), + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuMask(tc.a...) + got := a.Intersection(tc.b) + exp := NewCpuMask(tc.expected...) + if !maskListEqual(got, exp) { + t.Errorf("Intersection() = %v, want %v", got.List(), tc.expected) + } + }) + } +} + +// ---- TestIntersects --------------------------------------------------- + +func TestIntersects(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected bool + }{ + // *CpuMask fast path + {name: "mask: both-empty", a: []int{}, b: NewCpuMask(), expected: false}, + {name: "mask: a-empty", a: []int{}, b: NewCpuMask(0, 1), expected: false}, + {name: "mask: b-empty", a: []int{0, 1}, b: NewCpuMask(), expected: false}, + {name: "mask: identical", a: []int{0, 1}, b: NewCpuMask(0, 1), expected: true}, + {name: "mask: single-common-cpu", a: []int{0, 2}, b: NewCpuMask(2, 4), expected: true}, + {name: "mask: no-overlap-same-word", a: []int{0, 2}, b: NewCpuMask(1, 3), expected: false}, + {name: "mask: overlap-only-in-word-0", a: []int{0, 64}, b: NewCpuMask(0, 128), expected: true}, + {name: "mask: overlap-only-in-high-word", a: []int{0, 1023}, b: NewCpuMask(1, 1023), expected: true}, + {name: "mask: word-boundary-63-vs-64", a: []int{63}, b: NewCpuMask(64), expected: false}, + {name: "mask: word-boundary-63-and-64", a: []int{63, 64}, b: NewCpuMask(64), expected: true}, + {name: "mask: disjoint-words-entirely", a: cpuRange(0, 63), b: NewCpuMask(cpuRange(64, 127)...), expected: false}, + // a is shorter than b: b's extra high words must not be consulted + {name: "mask: a-shorter-no-overlap", a: []int{0}, b: NewCpuMask(64, 1023), expected: false}, + {name: "mask: a-shorter-with-overlap", a: []int{0}, b: NewCpuMask(0, 1023), expected: true}, + // a is longer than b: a's extra high words must not be consulted + {name: "mask: a-longer-no-overlap", a: []int{64, 1023}, b: NewCpuMask(0), expected: false}, + {name: "mask: a-longer-with-overlap", a: []int{0, 1023}, b: NewCpuMask(0), expected: true}, + {name: "mask: high-cpus-partial-overlap", a: cpuRange(960, 1000), b: NewCpuMask(cpuRange(1000, 1023)...), expected: true}, + {name: "mask: high-cpus-no-overlap", a: cpuRange(960, 999), b: NewCpuMask(cpuRange(1000, 1023)...), expected: false}, + // testCPUSet fallback path + {name: "non-mask: overlap", a: []int{0, 1, 2}, b: newTestCPUSet(2, 3), expected: true}, + {name: "non-mask: no-overlap", a: []int{0, 2}, b: newTestCPUSet(1, 3), expected: false}, + {name: "non-mask: b-empty", a: []int{0, 1}, b: newTestCPUSet(), expected: false}, + {name: "non-mask: a-empty", a: []int{}, b: newTestCPUSet(0, 1), expected: false}, + {name: "non-mask: high-cpus-overlap", a: cpuRange(512, 1023), b: newTestCPUSet(1023), expected: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuMask(tc.a...) + if got := a.Intersects(tc.b); got != tc.expected { + t.Errorf("Intersects() = %v, want %v", got, tc.expected) + } + // Intersects must agree with Intersection being non-empty. + if got := !a.Intersection(tc.b).IsEmpty(); got != tc.expected { + t.Errorf("Intersection().IsEmpty() implies %v, want %v", got, tc.expected) + } + }) + } + + t.Run("mask: trailing-zero-words-do-not-imply-overlap", func(t *testing.T) { + // a keeps a trailing zero word after clearing its only high CPU. + a := NewCpuMask(0, 64) + a.Clear(64) + if a.Intersects(NewCpuMask(64)) { + t.Error("mask with a trailing zero word should not intersect CPU 64") + } + if !a.Intersects(NewCpuMask(0)) { + t.Error("mask should still intersect CPU 0") + } + }) +} + +// ---- TestKey ---------------------------------------------------------- + +func TestKey(t *testing.T) { + t.Run("equal-sets-share-a-key", func(t *testing.T) { + tests := []struct { + name string + cpus []int + }{ + {name: "empty", cpus: []int{}}, + {name: "single-cpu-0", cpus: []int{0}}, + {name: "single-cpu-63", cpus: []int{63}}, + {name: "single-cpu-64", cpus: []int{64}}, + {name: "full-word-0", cpus: cpuRange(0, 63)}, + {name: "straddles-word-boundary", cpus: []int{63, 64}}, + {name: "multi-word-sparse", cpus: []int{0, 64, 512, 1023}}, + {name: "high-cpus", cpus: cpuRange(960, 1023)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a, b := NewCpuMask(tc.cpus...), NewCpuMask(tc.cpus...) + if a.Key() != b.Key() { + t.Errorf("equal sets have different keys: %q vs %q", a.Key(), b.Key()) + } + // The key must be stable across calls. + if first, second := a.Key(), a.Key(); first != second { + t.Errorf("Key() not stable: %q then %q", first, second) + } + }) + } + }) + + t.Run("different-sets-have-different-keys", func(t *testing.T) { + tests := []struct { + name string + a, b []int + }{ + {name: "adjacent-cpus", a: []int{0}, b: []int{1}}, + {name: "across-word-boundary", a: []int{63}, b: []int{64}}, + {name: "subset", a: []int{0, 1}, b: []int{0, 1, 2}}, + {name: "different-high-word", a: []int{0, 64}, b: []int{0, 128}}, + {name: "empty-vs-nonempty", a: []int{}, b: []int{0}}, + {name: "high-cpus-differ-by-one", a: cpuRange(960, 1022), b: cpuRange(960, 1023)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a, b := NewCpuMask(tc.a...), NewCpuMask(tc.b...) + if a.Key() == b.Key() { + t.Errorf("different sets share key %q", a.Key()) + } + }) + } + }) + + t.Run("key-ignores-trailing-zero-words", func(t *testing.T) { + // Equal sets whose masks have a different number of words must still + // produce the same key. + grown := NewCpuMask(0, 1023) + grown.Clear(1023) + if got, want := grown.Key(), NewCpuMask(0).Key(); got != want { + t.Errorf("Key() = %q, want %q", got, want) + } + + // Difference allocates len(receiver.mask) words and does not trim. + diffed := NewCpuMask(0, 1023).Difference(NewCpuMask(1023)) + if got, want := diffed.Key(), NewCpuMask(0).Key(); got != want { + t.Errorf("Key() after Difference = %q, want %q", got, want) + } + + // An all-zero mask must key the same as a never-grown empty one. + emptied := NewCpuMask(1023) + emptied.Clear(1023) + if got, want := emptied.Key(), NewCpuMask().Key(); got != want { + t.Errorf("emptied Key() = %q, want %q", got, want) + } + }) + + t.Run("empty-key-is-empty-string", func(t *testing.T) { + if got := NewCpuMask().Key(); got != "" { + t.Errorf("Key() = %q, want %q", got, "") + } + }) +} + +// ---- TestIsSubsetOf --------------------------------------------------- + +func TestIsSubsetOf(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected bool + }{ + // *CpuMask fast path + { + name: "mask: empty-is-subset-of-empty", + a: []int{}, + b: NewCpuMask(), + expected: true, + }, + { + name: "mask: empty-is-subset-of-nonempty", + a: []int{}, + b: NewCpuMask(0, 1), + expected: true, + }, + { + name: "mask: nonempty-is-not-subset-of-empty", + a: []int{0}, + b: NewCpuMask(), + expected: false, + }, + { + name: "mask: set-is-subset-of-itself", + a: []int{0, 1, 2}, + b: NewCpuMask(0, 1, 2), + expected: true, + }, + { + name: "mask: proper-subset", + a: []int{0, 1}, + b: NewCpuMask(0, 1, 2), + expected: true, + }, + { + name: "mask: not-subset-has-extra-cpu", + a: []int{0, 3}, + b: NewCpuMask(0, 1, 2), + expected: false, + }, + { + name: "mask: word-0-all-bits-subset-of-0-to-127", + a: cpuRange(0, 63), + b: NewCpuMask(cpuRange(0, 127)...), + expected: true, + }, + { + name: "mask: word-0-not-subset-of-word-1", + a: cpuRange(0, 63), + b: NewCpuMask(cpuRange(64, 127)...), + expected: false, + }, + { + name: "mask: a-longer-extra-nonzero-word-not-subset", + a: []int{0, 128}, + b: NewCpuMask(0, 64), + expected: false, + }, + { + name: "mask: high-cpus-proper-subset", + a: cpuRange(992, 1023), + b: NewCpuMask(cpuRange(960, 1023)...), + expected: true, + }, + { + name: "mask: high-cpu-outside-superset-not-subset", + a: []int{0, 1023}, + b: NewCpuMask(cpuRange(960, 1023)...), + expected: false, + }, + // testCPUSet fallback path + { + name: "non-mask: proper-subset", + a: []int{0, 1}, + b: newTestCPUSet(0, 1, 2), + expected: true, + }, + { + name: "non-mask: not-subset", + a: []int{0, 3}, + b: newTestCPUSet(0, 1, 2), + expected: false, + }, + { + name: "non-mask: empty-is-subset", + a: []int{}, + b: newTestCPUSet(0, 1), + expected: true, + }, + { + name: "non-mask: high-cpus-subset", + a: cpuRange(992, 1023), + b: newTestCPUSet(cpuRange(960, 1023)...), + expected: true, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuMask(tc.a...) + if got := a.IsSubsetOf(tc.b); got != tc.expected { + t.Errorf("IsSubsetOf() = %v, want %v", got, tc.expected) + } + }) + } + + t.Run("mask: a-longer-with-trailing-zero-words-is-still-subset", func(t *testing.T) { + // a has trailing zero words; it should still report as a subset. + a := NewCpuMask(0, 64) + a.Clear(64) // a.mask = [0x1, 0x0] + b := NewCpuMask(0, 1) + if !a.IsSubsetOf(b) { + t.Error("mask with trailing zero words should still be recognised as a subset") + } + }) +} + +// ---- TestUnion -------------------------------------------------------- + +func TestUnion(t *testing.T) { + tests := []struct { + name string + a []int + others []CPUSet + expected []int + }{ + // *CpuMask fast path + { + name: "mask: empty-union-empty", + a: []int{}, + others: []CPUSet{NewCpuMask()}, + expected: []int{}, + }, + { + name: "mask: a-union-empty-returns-a", + a: []int{0, 1}, + others: []CPUSet{NewCpuMask()}, + expected: []int{0, 1}, + }, + { + name: "mask: empty-union-b-returns-b", + a: []int{}, + others: []CPUSet{NewCpuMask(0, 1)}, + expected: []int{0, 1}, + }, + { + name: "mask: disjoint-single-word", + a: []int{0}, + others: []CPUSet{NewCpuMask(1)}, + expected: []int{0, 1}, + }, + { + name: "mask: overlapping", + a: []int{0, 1}, + others: []CPUSet{NewCpuMask(1, 2)}, + expected: []int{0, 1, 2}, + }, + { + name: "mask: a-shorter-than-b-includes-b-words", + a: []int{0}, + others: []CPUSet{NewCpuMask(64, 128)}, + expected: []int{0, 64, 128}, + }, + { + name: "mask: a-longer-than-b-keeps-extra-a-words", + a: []int{0, 64, 128}, + others: []CPUSet{NewCpuMask(0)}, + expected: []int{0, 64, 128}, + }, + { + name: "mask: word-boundary-63-and-64", + a: []int{63}, + others: []CPUSet{NewCpuMask(64)}, + expected: []int{63, 64}, + }, + { + name: "mask: multiple-others", + a: []int{0}, + others: []CPUSet{NewCpuMask(64), NewCpuMask(128)}, + expected: []int{0, 64, 128}, + }, + { + name: "mask: large-range-union", + a: cpuRange(512, 767), + others: []CPUSet{NewCpuMask(cpuRange(768, 1023)...)}, + expected: cpuRange(512, 1023), + }, + // zero-argument union must return a copy of m + { + name: "mask: no-others-returns-copy-of-a", + a: []int{0, 63, 64, 1023}, + others: nil, + expected: []int{0, 63, 64, 1023}, + }, + { + name: "mask: no-others-empty-a-returns-empty", + a: []int{}, + others: nil, + expected: []int{}, + }, + // testCPUSet fallback path — m's CPUs must be included + { + name: "non-mask: a-union-b", + a: []int{0, 1}, + others: []CPUSet{newTestCPUSet(2, 3)}, + expected: []int{0, 1, 2, 3}, + }, + { + name: "non-mask: overlapping", + a: []int{0, 1}, + others: []CPUSet{newTestCPUSet(1, 2)}, + expected: []int{0, 1, 2}, + }, + { + name: "non-mask: empty-b-returns-a", + a: []int{0, 1}, + others: []CPUSet{newTestCPUSet()}, + expected: []int{0, 1}, + }, + { + name: "non-mask: empty-a-union-b-returns-b", + a: []int{}, + others: []CPUSet{newTestCPUSet(0, 1)}, + expected: []int{0, 1}, + }, + { + name: "non-mask: high-cpus", + a: []int{0}, + others: []CPUSet{newTestCPUSet(1023)}, + expected: []int{0, 1023}, + }, + // mixed CpuMask and testCPUSet in others + { + name: "mixed: cpumask-and-non-mask-others", + a: []int{0}, + others: []CPUSet{NewCpuMask(64), newTestCPUSet(128)}, + expected: []int{0, 64, 128}, + }, + { + name: "mixed: non-mask-then-cpumask-others", + a: []int{0}, + others: []CPUSet{newTestCPUSet(64), NewCpuMask(128)}, + expected: []int{0, 64, 128}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuMask(tc.a...) + got := a.Union(tc.others...) + exp := NewCpuMask(tc.expected...) + if !maskListEqual(got, exp) { + t.Errorf("Union() = %v, want %v", got.List(), tc.expected) + } + }) + } + + // The zero-argument case is documented as returning a copy, so the result + // must not share storage with the receiver. + t.Run("mask: no-others-result-is-independent-of-a", func(t *testing.T) { + a := NewCpuMask(0, 63, 64) + got := a.Union() + got.Set(1023) + if a.Contains(1023) { + t.Error("mutating the Union() result propagated to the receiver") + } + if !got.Contains(1023) { + t.Error("mutating the Union() result had no effect") + } + }) + + // Union must never modify its operands either. + t.Run("mask: operands-are-not-modified", func(t *testing.T) { + a := NewCpuMask(0, 1) + b := NewCpuMask(64) + _ = a.Union(b) + if got, want := a.List(), []int{0, 1}; !slices.Equal(got, want) { + t.Errorf("receiver modified: %v, want %v", got, want) + } + if got, want := b.List(), []int{64}; !slices.Equal(got, want) { + t.Errorf("argument modified: %v, want %v", got, want) + } + }) +} + +// ---- TestParseCpuMask ------------------------------------------------- + +func TestParseCpuMask(t *testing.T) { + tests := []struct { + name string + input string + expected string + err bool + }{ + {name: "single-high-cpu-1024", input: "1024", expected: "1024"}, + {name: "large-contiguous-range-512-to-1023", input: "512-1023", expected: "512-1023"}, + {name: "two-non-adjacent-ranges", input: "0-63,128-191", expected: "0-63,128-191"}, + {name: "word-boundary-range-63-to-64", input: "63-64", expected: "63-64"}, + {name: "degenerate-range-0-to-0", input: "0-0", expected: "0"}, + {name: "low-and-high-boundary", input: "0,1023", expected: "0,1023"}, + {name: "full-word-0", input: "0-63", expected: "0-63"}, + {name: "duplicate-cpus-deduped", input: "0,0,1,1,63,63", expected: "0-1,63"}, + {name: "unordered-cpus-sorted-in-output", input: "63,0,127,64", expected: "0,63-64,127"}, + {name: "all-cpus-0-to-1023", input: "0-1023", expected: "0-1023"}, + {name: "empty-string-returns-empty-mask", input: "", expected: ""}, + // error cases + {name: "error-non-numeric", input: "abc", err: true}, + {name: "error-reversed-range", input: "5-3", err: true}, + {name: "error-bad-range-min", input: "a-3", err: true}, + {name: "error-bad-range-max", input: "3-a", err: true}, + {name: "error-empty-part-from-double-comma", input: "1,,2", err: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + m, err := ParseCpuMask(tc.input) + if tc.err { + if err == nil { + t.Errorf("ParseCpuMask(%q) expected error, got nil (mask=%v)", tc.input, m.List()) + } + return + } + if err != nil { + t.Errorf("ParseCpuMask(%q) unexpected error: %v", tc.input, err) + return + } + if got := m.String(); got != tc.expected { + t.Errorf("ParseCpuMask(%q).String() = %q, want %q", tc.input, got, tc.expected) + } + }) + } +} + +// =========================================================================== +// CpuSet tests +// +// Binary-operation test cases are labelled "cpuset:" (fast path: both +// operands are *CpuSet, delegating to k8s cpuset methods) and "cpumask:" +// (fallback path: second operand is *CpuMask, exercising the iteration-based +// fallback in Difference, Intersection, Equals, IsSubsetOf, and Union). +// =========================================================================== + +// ---- TestNewCpuSet -------------------------------------------------------- + +func TestNewCpuSet(t *testing.T) { + tests := []struct { + name string + cpus []int + expected []int + }{ + {name: "empty", cpus: []int{}, expected: []int{}}, + {name: "single-cpu-0", cpus: []int{0}, expected: []int{0}}, + {name: "single-cpu-63", cpus: []int{63}, expected: []int{63}}, + {name: "single-cpu-64", cpus: []int{64}, expected: []int{64}}, + {name: "word-0-all-bits", cpus: cpuRange(0, 63), expected: cpuRange(0, 63)}, + {name: "straddles-word-boundary", cpus: []int{63, 64}, expected: []int{63, 64}}, + {name: "cpu-1023", cpus: []int{1023}, expected: []int{1023}}, + {name: "lowest-and-highest-multi-word", cpus: []int{0, 1023}, expected: []int{0, 1023}}, + {name: "sparse-word-boundaries", cpus: []int{0, 64, 128, 512, 1023}, expected: []int{0, 64, 128, 512, 1023}}, + {name: "large-contiguous-range", cpus: cpuRange(512, 1023), expected: cpuRange(512, 1023)}, + {name: "duplicate-cpus-deduped", cpus: []int{0, 0, 63, 63, 64, 64}, expected: []int{0, 63, 64}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := NewCpuSet(tc.cpus...) + if got := s.List(); !slices.Equal(got, tc.expected) { + t.Errorf("List() = %v, want %v", got, tc.expected) + } + }) + } +} + +// ---- TestParseCpuSet ------------------------------------------------- + +func TestParseCpuSet(t *testing.T) { + tests := []struct { + name string + input string + expected string + err bool + }{ + {name: "empty-string-returns-empty-set", input: "", expected: ""}, + {name: "single-cpu-0", input: "0", expected: "0"}, + {name: "range-and-single", input: "0-3,5", expected: "0-3,5"}, + {name: "word-boundary-range", input: "63-64", expected: "63-64"}, + {name: "high-cpu-1023", input: "1023", expected: "1023"}, + // error cases + {name: "error-non-numeric", input: "abc", err: true}, + {name: "error-bad-range", input: "3-a", err: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s, err := ParseCpuSet(tc.input) + if tc.err { + if err == nil { + t.Errorf("ParseCpuSet(%q) expected error, got nil (set=%v)", tc.input, s.List()) + } + return + } + if err != nil { + t.Errorf("ParseCpuSet(%q) unexpected error: %v", tc.input, err) + return + } + if got := s.String(); got != tc.expected { + t.Errorf("ParseCpuSet(%q).String() = %q, want %q", tc.input, got, tc.expected) + } + }) + } +} + +// ---- TestParsersAgree ----------------------------------------------------- + +// ParseCpuMask and ParseCpuSet are siblings with the same job, so they must +// accept and reject the same input and yield the same CPUs. It is easy to +// break that by touching only one of them. +func TestParsersAgree(t *testing.T) { + inputs := []string{ + // accepted + "", "0", "0-3", "0-3,5", "63-64", "1023", "1024", + "0,0,1", "63,0,127,64", "0-0", "0-1023", "512-1023", + // accepted, but only because strconv.Atoi is lenient + "+1", "00", + // rejected + "abc", "5-3", "a-3", "3-a", "1,,2", ",", "-", "0-", "-1", + "0--3", "1-2-3", "0x10", "1_0", "99999999999999999999", + " 0-3", "0-3 ", "0, 1", "0 - 3", + } + for _, in := range inputs { + t.Run(fmt.Sprintf("%q", in), func(t *testing.T) { + m, merr := ParseCpuMask(in) + s, serr := ParseCpuSet(in) + + if (merr != nil) != (serr != nil) { + t.Fatalf("parsers disagree on validity: ParseCpuMask err=%v, ParseCpuSet err=%v", + merr, serr) + } + if merr != nil { + // Both rejected it. Neither may return a usable value + // alongside the error. + if m != nil { + t.Errorf("ParseCpuMask returned non-nil mask %v with an error", m) + } + if s != nil { + t.Errorf("ParseCpuSet returned non-nil set %v with an error", s) + } + return + } + if !m.Equals(s) { + t.Errorf("parsers produced different sets: mask %q vs set %q", + m.String(), s.String()) + } + if got, want := m.String(), s.String(); got != want { + t.Errorf("String() differs: mask %q vs set %q", got, want) + } + }) + } +} + +// ---- TestCpuSetSetAndClear ------------------------------------------------ + +func TestCpuSetSetAndClear(t *testing.T) { + type step struct { + set []int + clear []int + } + tests := []struct { + name string + initial []int + steps []step + expected []int + }{ + { + name: "set-cpu-0-on-empty", + steps: []step{{set: []int{0}}}, + expected: []int{0}, + }, + { + name: "set-cpu-63", + steps: []step{{set: []int{63}}}, + expected: []int{63}, + }, + { + name: "set-cpu-64-crosses-word", + steps: []step{{set: []int{64}}}, + expected: []int{64}, + }, + { + name: "set-cpu-1023", + steps: []step{{set: []int{1023}}}, + expected: []int{1023}, + }, + { + name: "set-is-idempotent", + initial: []int{0}, + steps: []step{{set: []int{0}}}, + expected: []int{0}, + }, + { + name: "set-multiple-cpus-in-one-call", + steps: []step{{set: []int{0, 63, 64, 1023}}}, + expected: []int{0, 63, 64, 1023}, + }, + { + name: "clear-middle-cpu", + initial: []int{0, 1, 2}, + steps: []step{{clear: []int{1}}}, + expected: []int{0, 2}, + }, + { + name: "clear-multiple-cpus-in-one-call", + initial: []int{0, 1, 2, 3}, + steps: []step{{clear: []int{1, 2}}}, + expected: []int{0, 3}, + }, + { + name: "clear-nonexistent-is-noop", + initial: []int{0, 2}, + steps: []step{{clear: []int{1}}}, + expected: []int{0, 2}, + }, + { + name: "clear-out-of-range-is-noop", + initial: []int{0}, + steps: []step{{clear: []int{1023}}}, + expected: []int{0}, + }, + { + name: "clear-cpu-1023", + initial: []int{0, 1023}, + steps: []step{{clear: []int{1023}}}, + expected: []int{0}, + }, + { + name: "set-then-clear", + steps: []step{{set: []int{0, 1, 2}}, {clear: []int{1}}}, + expected: []int{0, 2}, + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := NewCpuSet(tc.initial...) + for _, step := range tc.steps { + if len(step.set) > 0 { + s.Set(step.set...) + } + if len(step.clear) > 0 { + s.Clear(step.clear...) + } + } + if got := s.List(); !slices.Equal(got, tc.expected) { + t.Errorf("List() = %v, want %v", got, tc.expected) + } + }) + } + + // CpuSet caches String (and Key, which delegates to it); Set and Clear + // must invalidate it. Size is not cached, it comes straight from the + // underlying k8s cpuset. + + t.Run("caches-cleared-after-set", func(t *testing.T) { + s := NewCpuSet(0) + _, _ = s.String(), s.Key() + + s.Set(1) + + if got := s.String(); got != "0-1" { + t.Errorf("String() after Set: got %q, want %q", got, "0-1") + } + if got := s.Key(); got != "0-1" { + t.Errorf("Key() after Set: got %q, want %q", got, "0-1") + } + if got := s.Size(); got != 2 { + t.Errorf("Size() after Set: got %d, want 2", got) + } + }) + + t.Run("caches-cleared-after-clear", func(t *testing.T) { + s := NewCpuSet(0, 1) + _, _ = s.String(), s.Key() + + s.Clear(1) + + if got := s.String(); got != "0" { + t.Errorf("String() after Clear: got %q, want %q", got, "0") + } + if got := s.Key(); got != "0" { + t.Errorf("Key() after Clear: got %q, want %q", got, "0") + } + if got := s.Size(); got != 1 { + t.Errorf("Size() after Clear: got %d, want 1", got) + } + }) + + t.Run("caches-cleared-when-emptied", func(t *testing.T) { + s := NewCpuSet(0, 1) + _, _ = s.String(), s.Key() + + s.Clear(0, 1) + + if got := s.String(); got != "" { + t.Errorf("String() = %q, want %q", got, "") + } + if got := s.Key(); got != "" { + t.Errorf("Key() = %q, want %q", got, "") + } + if got := s.Size(); got != 0 { + t.Errorf("Size() = %d, want 0", got) + } + }) + + // ---- variadic zero-arg / multi-arg Set / Clear calls --------------------- + + t.Run("set-no-args-is-noop", func(t *testing.T) { + s := NewCpuSet(5) + s.Set() + if got := s.List(); !slices.Equal(got, []int{5}) { + t.Errorf("Set() no-op: got %v, want [5]", got) + } + }) + + t.Run("clear-no-args-is-noop", func(t *testing.T) { + s := NewCpuSet(5) + s.Clear() + if got := s.List(); !slices.Equal(got, []int{5}) { + t.Errorf("Clear() no-op: got %v, want [5]", got) + } + }) + + t.Run("set-multiple-across-word-boundaries", func(t *testing.T) { + s := NewCpuSet() + s.Set(0, 63, 64, 127, 128, 1023) + want := []int{0, 63, 64, 127, 128, 1023} + if got := s.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) + + t.Run("clear-multiple-across-word-boundaries", func(t *testing.T) { + s := NewCpuSet(0, 63, 64, 127, 128, 1023) + s.Clear(63, 64, 1023) + want := []int{0, 127, 128} + if got := s.List(); !slices.Equal(got, want) { + t.Errorf("got %v, want %v", got, want) + } + }) +} + +// ---- TestCpuSetSeal -------------------------------------------------- + +func TestCpuSetSeal(t *testing.T) { + t.Run("set-on-sealed-set-panics", func(t *testing.T) { + s := NewCpuSet(0) + s.Seal() + defer func() { + if r := recover(); r == nil { + t.Error("expected panic from Set on sealed set, got none") + } + }() + s.Set(1) + }) + + t.Run("clear-on-sealed-set-panics", func(t *testing.T) { + s := NewCpuSet(0, 1) + s.Seal() + defer func() { + if r := recover(); r == nil { + t.Error("expected panic from Clear on sealed set, got none") + } + }() + s.Clear(0) + }) + + t.Run("read-only-operations-work-on-sealed-set", func(t *testing.T) { + s := NewCpuSet(0, 63, 64, 1023) + s.Seal() + if !s.Contains(63) { + t.Error("Contains should work on sealed set") + } + if s.Size() != 4 { + t.Errorf("Size should work on sealed set: got %d, want 4", s.Size()) + } + if s.IsEmpty() { + t.Error("IsEmpty should work on sealed set") + } + if !slices.Equal(s.List(), []int{0, 63, 64, 1023}) { + t.Errorf("List should work on sealed set: got %v", s.List()) + } + }) +} + +// ---- TestCpuSetClone ------------------------------------------------------ + +func TestCpuSetClone(t *testing.T) { + t.Run("clone-of-empty", func(t *testing.T) { + s := NewCpuSet() + if c := s.Clone(); !c.IsEmpty() { + t.Errorf("clone of empty is not empty: %v", c.List()) + } + }) + + t.Run("clone-matches-original", func(t *testing.T) { + cpus := []int{0, 63, 64, 512, 1023} + s := NewCpuSet(cpus...) + if c := s.Clone(); !slices.Equal(c.List(), cpus) { + t.Errorf("clone content mismatch: want %v, got %v", cpus, c.List()) + } + }) + + t.Run("clone-is-independent", func(t *testing.T) { + s := NewCpuSet(0, 1, 2) + c := s.Clone() + c.Set(63) + if s.Contains(63) { + t.Error("mutating clone propagated to original") + } + if !c.Contains(63) { + t.Error("mutation on clone did not take effect") + } + }) + + t.Run("clone-of-large-mask", func(t *testing.T) { + cpus := cpuRange(512, 1023) + s := NewCpuSet(cpus...) + if c := s.Clone(); !slices.Equal(c.List(), cpus) { + t.Errorf("large clone mismatch: got %v", c.List()) + } + }) +} + +// ---- TestCpuSetIsEmpty ---------------------------------------------------- + +func TestCpuSetIsEmpty(t *testing.T) { + tests := []struct { + name string + cpus []int + expected bool + }{ + {name: "empty", cpus: []int{}, expected: true}, + {name: "single-cpu-0", cpus: []int{0}, expected: false}, + {name: "single-cpu-63", cpus: []int{63}, expected: false}, + {name: "single-cpu-64", cpus: []int{64}, expected: false}, + {name: "word-0-all-bits", cpus: cpuRange(0, 63), expected: false}, + {name: "cpu-1023", cpus: []int{1023}, expected: false}, + {name: "multi-word-sparse", cpus: []int{0, 64, 512, 1023}, expected: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := NewCpuSet(tc.cpus...) + if got := s.IsEmpty(); got != tc.expected { + t.Errorf("IsEmpty() = %v, want %v", got, tc.expected) + } + }) + } + + t.Run("empty-after-clearing-all", func(t *testing.T) { + s := NewCpuSet(0, 63, 64, 1023) + s.Clear(s.List()...) + if !s.IsEmpty() { + t.Errorf("should be empty after clearing all, got %v", s.List()) + } + }) +} + +// ---- TestCpuSetSize ------------------------------------------------------- + +func TestCpuSetSize(t *testing.T) { + tests := []struct { + name string + cpus []int + expected int + }{ + {name: "empty", cpus: []int{}, expected: 0}, + {name: "single-cpu-0", cpus: []int{0}, expected: 1}, + {name: "single-cpu-63", cpus: []int{63}, expected: 1}, + {name: "single-cpu-64", cpus: []int{64}, expected: 1}, + {name: "word-0-all-64-bits", cpus: cpuRange(0, 63), expected: 64}, + {name: "two-words-all-128-bits", cpus: cpuRange(0, 127), expected: 128}, + {name: "single-cpu-1023", cpus: []int{1023}, expected: 1}, + {name: "large-range-512-to-1023", cpus: cpuRange(512, 1023), expected: 512}, + {name: "sparse-boundary-cpus", cpus: []int{0, 63, 64, 127, 512, 1023}, expected: 6}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := NewCpuSet(tc.cpus...) + if got := s.Size(); got != tc.expected { + t.Errorf("Size() = %d, want %d", got, tc.expected) + } + }) + } +} + +// ---- TestCpuSetContains --------------------------------------------------- + +func TestCpuSetContains(t *testing.T) { + tests := []struct { + name string + cpus []int + check int + expected bool + }{ + {name: "empty-check-0", cpus: []int{}, check: 0, expected: false}, + {name: "cpu-0-present", cpus: []int{0}, check: 0, expected: true}, + {name: "cpu-0-absent", cpus: []int{1}, check: 0, expected: false}, + {name: "cpu-63-present", cpus: []int{63}, check: 63, expected: true}, + {name: "cpu-64-present", cpus: []int{64}, check: 64, expected: true}, + {name: "cpu-63-absent-only-64-set", cpus: []int{64}, check: 63, expected: false}, + {name: "cpu-64-absent-only-63-set", cpus: []int{63}, check: 64, expected: false}, + {name: "word-boundary-check-63", cpus: []int{63, 64}, check: 63, expected: true}, + {name: "word-boundary-check-64", cpus: []int{63, 64}, check: 64, expected: true}, + {name: "cpu-1023-present", cpus: []int{1023}, check: 1023, expected: true}, + {name: "cpu-1023-absent", cpus: []int{1022}, check: 1023, expected: false}, + {name: "check-beyond-set", cpus: []int{0}, check: 1023, expected: false}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := NewCpuSet(tc.cpus...) + if got := s.Contains(tc.check); got != tc.expected { + t.Errorf("Contains(%d) = %v, want %v", tc.check, got, tc.expected) + } + }) + } + + // ---- variadic multi-arg / zero-arg Contains calls ------------------------ + + t.Run("contains-no-args-is-vacuously-true", func(t *testing.T) { + s := NewCpuSet(0, 1, 2) + if !s.Contains() { + t.Error("Contains() with no args should be true") + } + if got := NewCpuSet().Contains(); !got { + t.Error("Contains() on empty set with no args should be true") + } + }) + + t.Run("contains-multiple-all-present", func(t *testing.T) { + s := NewCpuSet(0, 63, 64, 1023) + if !s.Contains(0, 63, 64, 1023) { + t.Error("Contains(0, 63, 64, 1023) = false, want true") + } + }) + + t.Run("contains-multiple-one-missing", func(t *testing.T) { + s := NewCpuSet(0, 63, 64) + if s.Contains(0, 63, 64, 1023) { + t.Error("Contains(0, 63, 64, 1023) = true, want false (1023 absent)") + } + }) + + t.Run("contains-multiple-duplicates", func(t *testing.T) { + s := NewCpuSet(0, 64) + if !s.Contains(0, 0, 64, 64) { + t.Error("Contains with duplicate CPUs = false, want true") + } + }) +} + +// ---- TestCpuSetString ----------------------------------------------------- + +func TestCpuSetString(t *testing.T) { + tests := []struct { + name string + cpus []int + expected string + }{ + {name: "empty", cpus: []int{}, expected: ""}, + {name: "single-cpu-0", cpus: []int{0}, expected: "0"}, + {name: "single-cpu-63", cpus: []int{63}, expected: "63"}, + {name: "single-cpu-64", cpus: []int{64}, expected: "64"}, + {name: "full-word-0", cpus: cpuRange(0, 63), expected: "0-63"}, + {name: "straddles-word-boundary", cpus: []int{63, 64}, expected: "63-64"}, + {name: "two-full-words", cpus: cpuRange(0, 127), expected: "0-127"}, + {name: "lowest-and-highest-of-two-words", cpus: []int{0, 63, 64, 127}, expected: "0,63-64,127"}, + {name: "sparse-word-boundaries", cpus: []int{0, 64, 128, 192}, expected: "0,64,128,192"}, + {name: "low-and-very-high", cpus: []int{0, 1023}, expected: "0,1023"}, + {name: "large-range", cpus: cpuRange(512, 1023), expected: "512-1023"}, + {name: "complex", cpus: []int{0, 1, 2, 4, 5, 7}, expected: "0-2,4-5,7"}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := NewCpuSet(tc.cpus...) + got := s.String() + if got != tc.expected { + t.Errorf("String() = %q, want %q", got, tc.expected) + } + if got2 := s.String(); got2 != got { + t.Errorf("cached String() differs: %q vs %q", got2, got) + } + }) + } +} + +// ---- TestCpuSetList ------------------------------------------------------- + +func TestCpuSetList(t *testing.T) { + tests := []struct { + name string + cpus []int + expected []int + }{ + {name: "empty", cpus: []int{}, expected: []int{}}, + {name: "single", cpus: []int{0}, expected: []int{0}}, + {name: "multi-word-sparse", cpus: []int{0, 64, 127, 512, 1023}, expected: []int{0, 64, 127, 512, 1023}}, + {name: "full-word-0", cpus: cpuRange(0, 63), expected: cpuRange(0, 63)}, + {name: "large-range", cpus: cpuRange(960, 1023), expected: cpuRange(960, 1023)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + s := NewCpuSet(tc.cpus...) + if got := s.List(); !slices.Equal(got, tc.expected) { + t.Errorf("List() = %v, want %v", got, tc.expected) + } + // UnsortedList must contain exactly the same elements. + unsorted := s.UnsortedList() + slices.Sort(unsorted) + if !slices.Equal(unsorted, tc.expected) { + t.Errorf("UnsortedList() (sorted) = %v, want %v", unsorted, tc.expected) + } + }) + } +} + +// ---- TestCpuSetForEachCpu -------------------------------------------- + +func TestCpuSetForEachCpu(t *testing.T) { + // CpuSet.ForEachCpu iterates over UnsortedList(), which (unlike + // CpuMask's bitmask-driven iteration) does not guarantee any + // particular order, so these tests only assert on the set of visited + // CPUs and call counts, not on ordering. + + t.Run("empty-set-f-never-called", func(t *testing.T) { + s := NewCpuSet() + called := false + s.ForEachCpu(func(_ int) bool { + called = true + return true + }) + if called { + t.Error("ForEachCpu called f on empty set") + } + }) + + t.Run("single-cpu", func(t *testing.T) { + s := NewCpuSet(42) + var visited []int + s.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + if !slices.Equal(visited, []int{42}) { + t.Errorf("expected [42], got %v", visited) + } + }) + + t.Run("multiple-cpus-visits-all-exactly-once", func(t *testing.T) { + want := []int{0, 7, 63, 64, 127, 512, 1023} + s := NewCpuSet(want...) + var visited []int + s.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + slices.Sort(visited) + if !slices.Equal(visited, want) { + t.Errorf("expected %v (in any order), got %v", want, visited) + } + }) + + t.Run("full-word-0-visits-all-64", func(t *testing.T) { + want := cpuRange(0, 63) + s := NewCpuSet(want...) + var visited []int + s.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + slices.Sort(visited) + if !slices.Equal(visited, want) { + t.Errorf("expected CPUs 0-63, got %v", visited) + } + }) + + t.Run("high-cpu-numbers-960-to-1023", func(t *testing.T) { + want := cpuRange(960, 1023) + s := NewCpuSet(want...) + var visited []int + s.ForEachCpu(func(cpu int) bool { + visited = append(visited, cpu) + return true + }) + slices.Sort(visited) + if !slices.Equal(visited, want) { + t.Errorf("expected CPUs 960-1023, got %v", visited) + } + }) + + t.Run("early-termination-stops-iteration", func(t *testing.T) { + s := NewCpuSet(0, 63, 64, 1023) + calls := 0 + s.ForEachCpu(func(_ int) bool { + calls++ + return false // stop after the very first CPU + }) + if calls != 1 { + t.Errorf("expected f to be called exactly once, got %d calls", calls) + } + }) + + t.Run("partial-termination-stops-after-n", func(t *testing.T) { + s := NewCpuSet(0, 1, 2, 3, 4) + calls := 0 + s.ForEachCpu(func(_ int) bool { + calls++ + return calls < 3 // stop after three CPUs + }) + if calls != 3 { + t.Errorf("expected f to be called exactly 3 times, got %d calls", calls) + } + }) +} + +// ---- TestCpuSetDifference ------------------------------------------------- + +func TestCpuSetDifference(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected []int + }{ + // *CpuSet fast path + {name: "cpuset: empty-minus-empty", a: []int{}, b: NewCpuSet(), expected: []int{}}, + {name: "cpuset: empty-minus-nonempty", a: []int{}, b: NewCpuSet(0, 1), expected: []int{}}, + {name: "cpuset: nonempty-minus-empty", a: []int{0, 1}, b: NewCpuSet(), expected: []int{0, 1}}, + {name: "cpuset: a-minus-a", a: []int{0, 1, 2}, b: NewCpuSet(0, 1, 2), expected: []int{}}, + {name: "cpuset: disjoint", a: []int{0, 2}, b: NewCpuSet(1, 3), expected: []int{0, 2}}, + {name: "cpuset: superset-minus-subset", a: []int{0, 1, 2, 3}, b: NewCpuSet(1, 2), expected: []int{0, 3}}, + {name: "cpuset: subset-minus-superset", a: []int{1, 2}, b: NewCpuSet(0, 1, 2, 3), expected: []int{}}, + {name: "cpuset: high-cpus", a: cpuRange(960, 1023), b: NewCpuSet(cpuRange(992, 1023)...), expected: cpuRange(960, 991)}, + // *CpuMask fallback path + {name: "cpumask: basic", a: []int{0, 1, 2, 3}, b: NewCpuMask(1, 2), expected: []int{0, 3}}, + {name: "cpumask: empty-a", a: []int{}, b: NewCpuMask(0, 1), expected: []int{}}, + {name: "cpumask: empty-b", a: []int{0, 1}, b: NewCpuMask(), expected: []int{0, 1}}, + {name: "cpumask: disjoint", a: []int{0, 2}, b: NewCpuMask(1, 3), expected: []int{0, 2}}, + {name: "cpumask: high-cpus", a: cpuRange(512, 1023), b: NewCpuMask(cpuRange(768, 1023)...), expected: cpuRange(512, 767)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuSet(tc.a...) + got := a.Difference(tc.b) + if !maskListEqual(got, NewCpuSet(tc.expected...)) { + t.Errorf("Difference() = %v, want %v", got.List(), tc.expected) + } + }) + } +} + +// ---- TestCpuSetEquals ----------------------------------------------------- + +func TestCpuSetEquals(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected bool + }{ + // *CpuSet fast path + {name: "cpuset: both-empty", a: []int{}, b: NewCpuSet(), expected: true}, + {name: "cpuset: same-single", a: []int{0}, b: NewCpuSet(0), expected: true}, + {name: "cpuset: different-single", a: []int{0}, b: NewCpuSet(1), expected: false}, + {name: "cpuset: one-empty-one-not", a: []int{0}, b: NewCpuSet(), expected: false}, + {name: "cpuset: same-full-word-0", a: cpuRange(0, 63), b: NewCpuSet(cpuRange(0, 63)...), expected: true}, + {name: "cpuset: same-multi-word", a: []int{0, 64, 128, 1023}, b: NewCpuSet(0, 64, 128, 1023), expected: true}, + {name: "cpuset: different-multi-word", a: []int{0, 64}, b: NewCpuSet(0, 128), expected: false}, + {name: "cpuset: high-cpus-equal", a: cpuRange(960, 1023), b: NewCpuSet(cpuRange(960, 1023)...), expected: true}, + {name: "cpuset: high-cpus-differ", a: cpuRange(960, 1022), b: NewCpuSet(cpuRange(960, 1023)...), expected: false}, + // *CpuMask fallback path + {name: "cpumask: equal", a: []int{0, 1, 2}, b: NewCpuMask(0, 1, 2), expected: true}, + {name: "cpumask: not-equal", a: []int{0, 1}, b: NewCpuMask(0, 2), expected: false}, + {name: "cpumask: size-mismatch", a: []int{0, 1, 2}, b: NewCpuMask(0, 1), expected: false}, + {name: "cpumask: both-empty", a: []int{}, b: NewCpuMask(), expected: true}, + {name: "cpumask: high-cpus-equal", a: cpuRange(512, 1023), b: NewCpuMask(cpuRange(512, 1023)...), expected: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuSet(tc.a...) + if got := a.Equals(tc.b); got != tc.expected { + t.Errorf("Equals() = %v, want %v", got, tc.expected) + } + }) + } +} + +// ---- TestCpuSetIntersection ----------------------------------------------- + +func TestCpuSetIntersection(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected []int + }{ + // *CpuSet fast path + {name: "cpuset: both-empty", a: []int{}, b: NewCpuSet(), expected: []int{}}, + {name: "cpuset: one-empty", a: []int{0, 1}, b: NewCpuSet(), expected: []int{}}, + {name: "cpuset: no-overlap", a: []int{0, 2}, b: NewCpuSet(1, 3), expected: []int{}}, + {name: "cpuset: full-overlap", a: cpuRange(0, 63), b: NewCpuSet(cpuRange(0, 63)...), expected: cpuRange(0, 63)}, + {name: "cpuset: partial-overlap", a: []int{0, 1, 2}, b: NewCpuSet(1, 2, 3), expected: []int{1, 2}}, + {name: "cpuset: word-boundary", a: []int{63, 64}, b: NewCpuSet(63, 64), expected: []int{63, 64}}, + {name: "cpuset: high-cpus", a: cpuRange(512, 1023), b: NewCpuSet(cpuRange(960, 1023)...), expected: cpuRange(960, 1023)}, + // *CpuMask fallback path + {name: "cpumask: partial-overlap", a: []int{0, 1, 2}, b: NewCpuMask(1, 2, 3), expected: []int{1, 2}}, + {name: "cpumask: no-overlap", a: []int{0, 2}, b: NewCpuMask(1, 3), expected: []int{}}, + {name: "cpumask: empty-b", a: []int{0, 1}, b: NewCpuMask(), expected: []int{}}, + {name: "cpumask: high-cpus", a: cpuRange(512, 1023), b: NewCpuMask(cpuRange(960, 1023)...), expected: cpuRange(960, 1023)}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuSet(tc.a...) + got := a.Intersection(tc.b) + if !maskListEqual(got, NewCpuSet(tc.expected...)) { + t.Errorf("Intersection() = %v, want %v", got.List(), tc.expected) + } + }) + } +} + +// ---- TestCpuSetIsSubsetOf ------------------------------------------------- + +func TestCpuSetIsSubsetOf(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected bool + }{ + // *CpuSet fast path + {name: "cpuset: empty-subset-empty", a: []int{}, b: NewCpuSet(), expected: true}, + {name: "cpuset: empty-subset-nonempty", a: []int{}, b: NewCpuSet(0, 1), expected: true}, + {name: "cpuset: nonempty-not-subset-of-empty", a: []int{0}, b: NewCpuSet(), expected: false}, + {name: "cpuset: equal-sets", a: []int{0, 1, 2}, b: NewCpuSet(0, 1, 2), expected: true}, + {name: "cpuset: proper-subset", a: []int{0, 1}, b: NewCpuSet(0, 1, 2), expected: true}, + {name: "cpuset: not-subset", a: []int{0, 3}, b: NewCpuSet(0, 1, 2), expected: false}, + {name: "cpuset: word-0-subset-of-0-127", a: cpuRange(0, 63), b: NewCpuSet(cpuRange(0, 127)...), expected: true}, + {name: "cpuset: word-0-not-subset-of-word-1", a: cpuRange(0, 63), b: NewCpuSet(cpuRange(64, 127)...), expected: false}, + {name: "cpuset: high-cpus-subset", a: cpuRange(992, 1023), b: NewCpuSet(cpuRange(960, 1023)...), expected: true}, + {name: "cpuset: high-cpu-not-in-superset", a: []int{0, 1023}, b: NewCpuSet(cpuRange(960, 1023)...), expected: false}, + // *CpuMask fallback path + {name: "cpumask: proper-subset", a: []int{0, 1}, b: NewCpuMask(0, 1, 2), expected: true}, + {name: "cpumask: not-subset", a: []int{0, 3}, b: NewCpuMask(0, 1, 2), expected: false}, + {name: "cpumask: empty-subset", a: []int{}, b: NewCpuMask(0, 1), expected: true}, + {name: "cpumask: high-cpus-subset", a: cpuRange(992, 1023), b: NewCpuMask(cpuRange(960, 1023)...), expected: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuSet(tc.a...) + if got := a.IsSubsetOf(tc.b); got != tc.expected { + t.Errorf("IsSubsetOf() = %v, want %v", got, tc.expected) + } + }) + } +} + +// ---- TestCpuSetUnion ------------------------------------------------------ + +func TestCpuSetUnion(t *testing.T) { + tests := []struct { + name string + a []int + others []CPUSet + expected []int + }{ + // *CpuSet fast path + {name: "cpuset: empty-union-empty", a: []int{}, others: []CPUSet{NewCpuSet()}, expected: []int{}}, + {name: "cpuset: a-union-empty", a: []int{0, 1}, others: []CPUSet{NewCpuSet()}, expected: []int{0, 1}}, + {name: "cpuset: empty-union-b", a: []int{}, others: []CPUSet{NewCpuSet(0, 1)}, expected: []int{0, 1}}, + {name: "cpuset: disjoint", a: []int{0}, others: []CPUSet{NewCpuSet(1)}, expected: []int{0, 1}}, + {name: "cpuset: overlapping", a: []int{0, 1}, others: []CPUSet{NewCpuSet(1, 2)}, expected: []int{0, 1, 2}}, + {name: "cpuset: word-boundary", a: []int{63}, others: []CPUSet{NewCpuSet(64)}, expected: []int{63, 64}}, + {name: "cpuset: multiple-others", a: []int{0}, others: []CPUSet{NewCpuSet(64), NewCpuSet(128)}, expected: []int{0, 64, 128}}, + {name: "cpuset: large-range", a: cpuRange(512, 767), others: []CPUSet{NewCpuSet(cpuRange(768, 1023)...)}, expected: cpuRange(512, 1023)}, + // no-args union must return a copy of a + {name: "cpuset: no-others-returns-copy", a: []int{0, 63, 64, 1023}, others: nil, expected: []int{0, 63, 64, 1023}}, + {name: "cpuset: no-others-empty", a: []int{}, others: nil, expected: []int{}}, + // *CpuMask fallback path + {name: "cpumask: a-union-b", a: []int{0, 1}, others: []CPUSet{NewCpuMask(2, 3)}, expected: []int{0, 1, 2, 3}}, + {name: "cpumask: overlapping", a: []int{0, 1}, others: []CPUSet{NewCpuMask(1, 2)}, expected: []int{0, 1, 2}}, + {name: "cpumask: empty-b", a: []int{0, 1}, others: []CPUSet{NewCpuMask()}, expected: []int{0, 1}}, + {name: "cpumask: empty-a", a: []int{}, others: []CPUSet{NewCpuMask(0, 1)}, expected: []int{0, 1}}, + {name: "cpumask: high-cpus", a: []int{0}, others: []CPUSet{NewCpuMask(1023)}, expected: []int{0, 1023}}, + // mixed CpuSet and CpuMask in others + {name: "mixed: cpuset-and-cpumask", a: []int{0}, others: []CPUSet{NewCpuSet(64), NewCpuMask(128)}, expected: []int{0, 64, 128}}, + {name: "mixed: cpumask-and-cpuset", a: []int{0}, others: []CPUSet{NewCpuMask(64), NewCpuSet(128)}, expected: []int{0, 64, 128}}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuSet(tc.a...) + got := a.Union(tc.others...) + if !maskListEqual(got, NewCpuSet(tc.expected...)) { + t.Errorf("Union() = %v, want %v", got.List(), tc.expected) + } + }) + } + + t.Run("cpuset: no-others-result-is-independent-of-a", func(t *testing.T) { + a := NewCpuSet(0, 63, 64) + got := a.Union() + got.Set(1023) + if a.Contains(1023) { + t.Error("mutating the Union() result propagated to the receiver") + } + if !got.Contains(1023) { + t.Error("mutating the Union() result had no effect") + } + }) + + t.Run("cpuset: operands-are-not-modified", func(t *testing.T) { + a := NewCpuSet(0, 1) + b := NewCpuSet(64) + _ = a.Union(b) + if got, want := a.List(), []int{0, 1}; !slices.Equal(got, want) { + t.Errorf("receiver modified: %v, want %v", got, want) + } + if got, want := b.List(), []int{64}; !slices.Equal(got, want) { + t.Errorf("argument modified: %v, want %v", got, want) + } + }) +} + +// ---- TestCpuSetIntersects ------------------------------------------------- + +func TestCpuSetIntersects(t *testing.T) { + tests := []struct { + name string + a []int + b CPUSet + expected bool + }{ + // *CpuSet operand + {name: "cpuset: both-empty", a: []int{}, b: NewCpuSet(), expected: false}, + {name: "cpuset: a-empty", a: []int{}, b: NewCpuSet(0, 1), expected: false}, + {name: "cpuset: b-empty", a: []int{0, 1}, b: NewCpuSet(), expected: false}, + {name: "cpuset: identical", a: []int{0, 1}, b: NewCpuSet(0, 1), expected: true}, + {name: "cpuset: single-common-cpu", a: []int{0, 2}, b: NewCpuSet(2, 4), expected: true}, + {name: "cpuset: no-overlap", a: []int{0, 2}, b: NewCpuSet(1, 3), expected: false}, + {name: "cpuset: word-boundary-63-vs-64", a: []int{63}, b: NewCpuSet(64), expected: false}, + {name: "cpuset: word-boundary-63-and-64", a: []int{63, 64}, b: NewCpuSet(64), expected: true}, + {name: "cpuset: high-cpus-overlap", a: cpuRange(960, 1000), b: NewCpuSet(cpuRange(1000, 1023)...), expected: true}, + {name: "cpuset: high-cpus-no-overlap", a: cpuRange(960, 999), b: NewCpuSet(cpuRange(1000, 1023)...), expected: false}, + // *CpuMask operand + {name: "cpumask: overlap", a: []int{0, 1, 2}, b: NewCpuMask(2, 3), expected: true}, + {name: "cpumask: no-overlap", a: []int{0, 2}, b: NewCpuMask(1, 3), expected: false}, + {name: "cpumask: b-empty", a: []int{0, 1}, b: NewCpuMask(), expected: false}, + {name: "cpumask: a-empty", a: []int{}, b: NewCpuMask(0, 1), expected: false}, + {name: "cpumask: overlap-in-high-word-only", a: []int{0, 1023}, b: NewCpuMask(1, 1023), expected: true}, + {name: "cpumask: high-cpus-overlap", a: cpuRange(512, 1023), b: NewCpuMask(1023), expected: true}, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + a := NewCpuSet(tc.a...) + if got := a.Intersects(tc.b); got != tc.expected { + t.Errorf("Intersects() = %v, want %v", got, tc.expected) + } + if got := !a.Intersection(tc.b).IsEmpty(); got != tc.expected { + t.Errorf("Intersection().IsEmpty() implies %v, want %v", got, tc.expected) + } + }) + } +} + +// ---- TestCpuSetKey -------------------------------------------------------- + +func TestCpuSetKey(t *testing.T) { + // CpuSet.Key delegates to String, so the key is the cpuset string. + t.Run("key-is-the-cpuset-string", func(t *testing.T) { + for _, cpus := range [][]int{ + {}, {0}, {63}, {64}, {63, 64}, {0, 64, 512, 1023}, cpuRange(0, 63), + } { + s := NewCpuSet(cpus...) + if got, want := s.Key(), s.String(); got != want { + t.Errorf("%v: Key() = %q, want %q", cpus, got, want) + } + } + }) + + t.Run("equal-sets-share-a-key", func(t *testing.T) { + for _, cpus := range [][]int{ + {}, {0}, {63, 64}, {0, 64, 512, 1023}, cpuRange(960, 1023), + } { + a, b := NewCpuSet(cpus...), NewCpuSet(cpus...) + if a.Key() != b.Key() { + t.Errorf("%v: equal sets have different keys: %q vs %q", cpus, a.Key(), b.Key()) + } + } + }) + + t.Run("different-sets-have-different-keys", func(t *testing.T) { + for _, tc := range []struct{ a, b []int }{ + {[]int{0}, []int{1}}, + {[]int{63}, []int{64}}, + {[]int{0, 1}, []int{0, 1, 2}}, + {[]int{}, []int{0}}, + } { + a, b := NewCpuSet(tc.a...), NewCpuSet(tc.b...) + if a.Key() == b.Key() { + t.Errorf("%v and %v share key %q", tc.a, tc.b, a.Key()) + } + } + }) + + t.Run("empty-key-is-empty-string", func(t *testing.T) { + if got := NewCpuSet().Key(); got != "" { + t.Errorf("Key() = %q, want %q", got, "") + } + }) +} + +// =========================================================================== +// Cross-cutting tests +// +// The tables above exercise one method at a time. The tests below cut across +// methods and implementations: +// +// - cross-implementation dispatch, in *both* directions for every pair of +// implementations. The fast paths type-assert to their own concrete type +// and fall back to a generic loop otherwise, so a bug can hide in one +// direction while the other stays correct. +// - Seal semantics: a sealed set materializes its lazily cached values, so +// that concurrent readers never write. Run with -race. +// - the mask sizing heuristic in NewCpuMask. +// - randomized cross-checks against an independent oracle. +// =========================================================================== + +// setCtor builds a CPUSet of one particular implementation. +type setCtor struct { + name string + new func(...int) CPUSet +} + +// ctors covers every real implementation. rawCpuSet from cpuset-bench_test.go +// is deliberately excluded: it is a benchmark-only shim which assumes that the +// other set is a *rawCpuSet and panics otherwise. +var ctors = []setCtor{ + {"CpuMask", func(cpus ...int) CPUSet { return NewCpuMask(cpus...) }}, + {"CpuSet", func(cpus ...int) CPUSet { return NewCpuSet(cpus...) }}, + {"testCPUSet", func(cpus ...int) CPUSet { return newTestCPUSet(cpus...) }}, +} + +// setPair is an input pair for the cross-implementation tests. Between them +// they cover empty and non-empty operands, equal sets, both subset directions, +// disjoint sets, differences confined to a high word, and operands whose masks +// end up with different word counts. +type setPair struct { + name string + a []int + b []int +} + +var setPairs = []setPair{ + {"both-empty", nil, nil}, + {"empty-vs-one", nil, []int{0}}, + {"one-vs-empty", []int{0}, nil}, + {"equal-single", []int{0}, []int{0}}, + {"equal-multi-word", []int{0, 64, 128, 1023}, []int{0, 64, 128, 1023}}, + {"receiver-subset", []int{0, 1}, []int{0, 1, 2}}, + {"receiver-superset", []int{0, 1, 2}, []int{0, 1}}, + {"receiver-subset-high", []int{960}, []int{960, 1023}}, + {"receiver-subset-word-boundary", cpuRange(0, 62), cpuRange(0, 63)}, + {"disjoint-low", []int{0, 2}, []int{1, 3}}, + {"disjoint-words", []int{0, 64}, []int{128, 192}}, + {"overlap-high-word-only", []int{0, 1023}, []int{1023}}, + {"differ-across-words", []int{0, 64}, []int{0, 128}}, + {"full-word-vs-subset", cpuRange(0, 63), cpuRange(0, 31)}, +} + +// normalize returns the sorted, de-duplicated CPUs of s, never nil. +func normalize(s []int) []int { + out := slices.Compact(slices.Sorted(slices.Values(s))) + if out == nil { + return []int{} + } + return out +} + +// wantEquals etc. compute the expected results directly from the input +// slices, independently of any CPUSet method. +func wantEquals(a, b []int) bool { return slices.Equal(normalize(a), normalize(b)) } + +func wantSubsetOf(a, b []int) bool { + nb := normalize(b) + for _, c := range normalize(a) { + if !slices.Contains(nb, c) { + return false + } + } + return true +} + +func wantIntersects(a, b []int) bool { + nb := normalize(b) + for _, c := range normalize(a) { + if slices.Contains(nb, c) { + return true + } + } + return false +} + +func wantUnion(a, b []int) []int { return normalize(append(slices.Clone(a), b...)) } + +func wantIntersection(a, b []int) []int { + nb, out := normalize(b), []int{} + for _, c := range normalize(a) { + if slices.Contains(nb, c) { + out = append(out, c) + } + } + return out +} + +func wantDifference(a, b []int) []int { + nb, out := normalize(b), []int{} + for _, c := range normalize(a) { + if !slices.Contains(nb, c) { + out = append(out, c) + } + } + return out +} + +// forEachPairing runs fn for every (pair, receiver impl, argument impl) +// combination. +func forEachPairing(t *testing.T, fn func(t *testing.T, p setPair, a, b CPUSet)) { + t.Helper() + for _, p := range setPairs { + for _, ac := range ctors { + for _, bc := range ctors { + name := fmt.Sprintf("%s/%s.op(%s)", p.name, ac.name, bc.name) + t.Run(name, func(t *testing.T) { + fn(t, p, ac.new(p.a...), bc.new(p.b...)) + }) + } + } + } +} + +func TestCrossImplEquals(t *testing.T) { + forEachPairing(t, func(t *testing.T, p setPair, a, b CPUSet) { + want := wantEquals(p.a, p.b) + if got := a.Equals(b); got != want { + t.Errorf("Equals() = %v, want %v", got, want) + } + // Equality is symmetric regardless of which implementation is the + // receiver. + if got := b.Equals(a); got != want { + t.Errorf("reversed Equals() = %v, want %v", got, want) + } + }) +} + +func TestCrossImplIsSubsetOf(t *testing.T) { + forEachPairing(t, func(t *testing.T, p setPair, a, b CPUSet) { + if got, want := a.IsSubsetOf(b), wantSubsetOf(p.a, p.b); got != want { + t.Errorf("IsSubsetOf() = %v, want %v", got, want) + } + if got, want := b.IsSubsetOf(a), wantSubsetOf(p.b, p.a); got != want { + t.Errorf("reversed IsSubsetOf() = %v, want %v", got, want) + } + }) +} + +func TestCrossImplIntersects(t *testing.T) { + forEachPairing(t, func(t *testing.T, p setPair, a, b CPUSet) { + want := wantIntersects(p.a, p.b) + if got := a.Intersects(b); got != want { + t.Errorf("Intersects() = %v, want %v", got, want) + } + // Intersects is symmetric. + if got := b.Intersects(a); got != want { + t.Errorf("reversed Intersects() = %v, want %v", got, want) + } + // and must agree with Intersection() + if got := !NewAnyCPUSet(a).Intersection(b).IsEmpty(); got != want { + t.Errorf("Intersects()=%v disagrees with Intersection()=%s", want, NewAnyCPUSet(a).Intersection(b)) + } + }) +} + +func TestCrossImplUnion(t *testing.T) { + forEachPairing(t, func(t *testing.T, p setPair, a, b CPUSet) { + want := wantUnion(p.a, p.b) + if got := NewAnyCPUSet(a).Union(b).List(); !slices.Equal(got, want) { + t.Errorf("Union() = %v, want %v", got, want) + } + if got := NewAnyCPUSet(b).Union(a).List(); !slices.Equal(got, want) { + t.Errorf("reversed Union() = %v, want %v", got, want) + } + // the operands must not be modified + if got := a.List(); !slices.Equal(got, normalize(p.a)) { + t.Errorf("Union() modified receiver: %v", got) + } + if got := b.List(); !slices.Equal(got, normalize(p.b)) { + t.Errorf("Union() modified argument: %v", got) + } + }) +} + +func TestCrossImplIntersection(t *testing.T) { + forEachPairing(t, func(t *testing.T, p setPair, a, b CPUSet) { + if got, want := NewAnyCPUSet(a).Intersection(b).List(), wantIntersection(p.a, p.b); !slices.Equal(got, want) { + t.Errorf("Intersection() = %v, want %v", got, want) + } + if got, want := NewAnyCPUSet(b).Intersection(a).List(), wantIntersection(p.b, p.a); !slices.Equal(got, want) { + t.Errorf("reversed Intersection() = %v, want %v", got, want) + } + }) +} + +func TestCrossImplDifference(t *testing.T) { + forEachPairing(t, func(t *testing.T, p setPair, a, b CPUSet) { + if got, want := NewAnyCPUSet(a).Difference(b).List(), wantDifference(p.a, p.b); !slices.Equal(got, want) { + t.Errorf("Difference() = %v, want %v", got, want) + } + if got, want := NewAnyCPUSet(b).Difference(a).List(), wantDifference(p.b, p.a); !slices.Equal(got, want) { + t.Errorf("reversed Difference() = %v, want %v", got, want) + } + }) +} + +// ---- Key ------------------------------------------------------------------ + +// NewCpuMask must build the same mask from the same CPUs however they are +// ordered, mask word count included: it estimates that count from the first +// and last element and lets expand() grow it, so an unsorted argument takes a +// different route to the same result. The cross-implementation keying recipe in +// doc.go relies on this, since CpuSet.UnsortedList returns CPUs in no +// particular order. +func TestNewCpuMaskIsOrderIndependent(t *testing.T) { + rng := rand.New(rand.NewSource(3)) + for _, cpus := range [][]int{ + {}, {0}, {1023}, {0, 1023}, {7, 1023, 3}, {63, 64}, + {0, 64, 512, 1023}, {5, 5, 5}, cpuRange(0, 63), cpuRange(960, 1023), + } { + t.Run(fmt.Sprint(cpus), func(t *testing.T) { + ref := NewCpuMask(cpus...) + for range 200 { + p := slices.Clone(cpus) + rng.Shuffle(len(p), func(a, b int) { p[a], p[b] = p[b], p[a] }) + + m := NewCpuMask(p...) + if len(m.mask) != len(ref.mask) { + t.Fatalf("%v: %d mask words, want %d", p, len(m.mask), len(ref.mask)) + } + if !slices.Equal(m.mask, ref.mask) { + t.Fatalf("%v: mask words differ from %v", p, cpus) + } + if m.Key() != ref.Key() { + t.Fatalf("%v: Key() = %q, want %q", p, m.Key(), ref.Key()) + } + } + }) + } +} + +// maskLikeKey is the recipe doc.go gives for a key which is comparable across +// implementations. +func maskLikeKey(s CPUSet) string { + if m, ok := s.(*CpuMask); ok { + return m.Key() + } + return NewCpuMask(s.UnsortedList()...).Key() +} + +// Key is only comparable within one implementation, so a CpuMask and a CpuSet +// holding the same CPUs have different keys. Going through the CpuMask form +// restores the "equal keys exactly for equal sets" property across them. +func TestMaskLikeKeyIsComparableAcrossImplementations(t *testing.T) { + // the mismatch the recipe exists for + if m, s := NewCpuMask(0, 5), NewCpuSet(0, 5); m.Key() == s.Key() { + t.Errorf("expected CpuMask and CpuSet keys to differ, both are %q", m.Key()) + } + + for _, p := range setPairs { + t.Run(p.name, func(t *testing.T) { + equal := NewCpuMask(p.a...).Equals(NewCpuMask(p.b...)) + for _, ac := range ctors { + for _, bc := range ctors { + k1 := maskLikeKey(ac.new(p.a...)) + k2 := maskLikeKey(bc.new(p.b...)) + if (k1 == k2) != equal { + t.Errorf("%s/%s: keys %q and %q match=%v, want %v", + ac.name, bc.name, k1, k2, k1 == k2, equal) + } + } + } + }) + } +} + +// Key must be equal exactly when the sets are equal, within one +// implementation. TestKey and TestCpuSetKey cover the per-implementation +// details; this checks the invariant holds across the whole input matrix. +func TestKeyMatchesEquality(t *testing.T) { + for _, c := range ctors { + t.Run(c.name, func(t *testing.T) { + for _, p := range setPairs { + a, b := c.new(p.a...), c.new(p.b...) + sameKey, equal := a.Key() == b.Key(), a.Equals(b) + if sameKey != equal { + t.Errorf("%s: Key() equality %v disagrees with Equals() %v (%q vs %q)", + p.name, sameKey, equal, a.Key(), b.Key()) + } + } + }) + } +} + +// ---- Seal ----------------------------------------------------------------- + +var sealCases = [][]int{ + nil, // empty: String() and Key() are "", which is also + {0}, // the "not cached yet" sentinel + {0, 1, 2}, + {5, 64, 1023}, + {63, 64}, // word boundary + cpuRange(0, 63), +} + +// Sealing must not change any observable value. +func TestSealPreservesValues(t *testing.T) { + for _, c := range ctors { + for _, cpus := range sealCases { + t.Run(fmt.Sprintf("%s/%v", c.name, cpus), func(t *testing.T) { + unsealed := c.new(cpus...) + wStr, wKey, wSize := unsealed.String(), unsealed.Key(), unsealed.Size() + + sealed := c.new(cpus...) + sealed.Seal() + + if got := sealed.String(); got != wStr { + t.Errorf("String() = %q, want %q", got, wStr) + } + if got := sealed.Key(); got != wKey { + t.Errorf("Key() = %q, want %q", got, wKey) + } + if got := sealed.Size(); got != wSize { + t.Errorf("Size() = %d, want %d", got, wSize) + } + if got := sealed.List(); !slices.Equal(got, normalize(cpus)) { + t.Errorf("List() = %v, want %v", got, normalize(cpus)) + } + }) + } + } +} + +// Sealing must also work when the caches were already populated. +func TestSealAfterCachesPrimed(t *testing.T) { + for _, c := range ctors { + t.Run(c.name, func(t *testing.T) { + s := c.new(3, 7) + wStr, wKey, wSize := s.String(), s.Key(), s.Size() + s.Seal() + if s.String() != wStr || s.Key() != wKey || s.Size() != wSize { + t.Errorf("after Seal: (%q,%q,%d), want (%q,%q,%d)", + s.String(), s.Key(), s.Size(), wStr, wKey, wSize) + } + }) + } +} + +// A sealed set materializes its caches, so concurrent readers never write to +// it. This only fails under -race. +func TestSealedConcurrentReadsAreRaceFree(t *testing.T) { + for _, c := range ctors { + for _, cpus := range sealCases { + t.Run(fmt.Sprintf("%s/%v", c.name, cpus), func(t *testing.T) { + s := c.new(cpus...) + wStr, wKey, wSize := s.String(), s.Key(), s.Size() + s.Seal() + + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for range 200 { + if got := s.Size(); got != wSize { + t.Errorf("Size() = %d, want %d", got, wSize) + } + if got := s.String(); got != wStr { + t.Errorf("String() = %q, want %q", got, wStr) + } + if got := s.Key(); got != wKey { + t.Errorf("Key() = %q, want %q", got, wKey) + } + } + }() + } + wg.Wait() + }) + } + } +} + +// A set that was emptied before sealing is the awkward case: "" is a valid +// String() and Key() for it, so the caches cannot be recognized as populated +// by their value alone. +func TestSealEmptiedSet(t *testing.T) { + m := NewCpuMask(0, 1) + m.Clear(0, 1) + m.Seal() + + var wg sync.WaitGroup + for range 8 { + wg.Add(1) + go func() { + defer wg.Done() + for range 200 { + if got := m.String(); got != "" { + t.Errorf("String() = %q, want %q", got, "") + } + if got := m.Key(); got != "" { + t.Errorf("Key() = %q, want %q", got, "") + } + if got := m.Size(); got != 0 { + t.Errorf("Size() = %d, want 0", got) + } + } + }() + } + wg.Wait() +} + +// Clone returns an unsealed copy: it must be writable, must not disturb the +// original, and must be safe to share once sealed again. +func TestCloneOfSealedSet(t *testing.T) { + for _, c := range ctors { + t.Run(c.name, func(t *testing.T) { + orig := c.new(0, 5) + orig.Seal() + + clone := NewAnyCPUSet(orig).Clone() + clone.Set(9) + + if got, want := clone.List(), []int{0, 5, 9}; !slices.Equal(got, want) { + t.Errorf("clone.List() = %v, want %v", got, want) + } + if got, want := orig.List(), []int{0, 5}; !slices.Equal(got, want) { + t.Errorf("original modified: List() = %v, want %v", got, want) + } + // the clone's caches must have been invalidated by Set + if got, want := clone.String(), "0,5,9"; got != want { + t.Errorf("clone.String() = %q, want %q", got, want) + } + + // re-sealing the clone makes it shareable again + clone.Seal() + wStr, wKey, wSize := clone.String(), clone.Key(), clone.Size() + var wg sync.WaitGroup + for range 4 { + wg.Add(1) + go func() { + defer wg.Done() + for range 200 { + if clone.String() != wStr || clone.Key() != wKey || clone.Size() != wSize { + t.Error("resealed clone returned inconsistent values") + } + } + }() + } + wg.Wait() + }) + } +} + +// ---- NewCpuMask sizing ---------------------------------------------------- + +// neededWords is the number of mask words the given CPUs actually require. +func neededWords(cpus []int) int { + if len(cpus) == 0 { + return 0 + } + hi := cpus[0] + for _, c := range cpus { + if c > hi { + hi = c + } + } + return hi/64 + 1 +} + +// NewCpuMask estimates the mask size from the two endpoints of the input, +// which is exact for sorted input. expand() covers any shortfall, so this is +// about avoiding reallocation, not correctness. +func TestNewCpuMaskSizingIsExactForSortedInput(t *testing.T) { + for _, tc := range []struct { + name string + cpus []int + }{ + {"empty", nil}, + {"single-0", []int{0}}, + {"single-1023", []int{1023}}, + {"dense-0-7", cpuRange(0, 7)}, + {"dense-0-63", cpuRange(0, 63)}, + {"dense-0-64", cpuRange(0, 64)}, + {"dense-0-79", cpuRange(0, 79)}, + {"dense-0-255", cpuRange(0, 255)}, + {"dense-512-575", cpuRange(512, 575)}, + {"sparse-0-and-1023", []int{0, 1023}}, + {"sparse-ascending", []int{0, 100, 500, 900}}, + } { + t.Run(tc.name, func(t *testing.T) { + m := NewCpuMask(tc.cpus...) + if got, want := len(m.mask), neededWords(tc.cpus); got != want { + t.Errorf("mask has %d words, want exactly %d", got, want) + } + }) + } +} + +// Unsorted or duplicate-bearing input may mis-size the initial mask, but the +// result must still be correct. +func TestNewCpuMaskUnsortedAndDuplicateInput(t *testing.T) { + for _, cpus := range [][]int{ + {5, 5, 5}, + {0, 1023, 0}, + {7, 1023, 3}, // max in the middle: the estimate falls short + {100, 1, 5000}, // max last + {5000, 1, 100}, // max first + {63, 0, 64}, + } { + t.Run(fmt.Sprint(cpus), func(t *testing.T) { + m := NewCpuMask(cpus...) + if got, want := m.List(), normalize(cpus); !slices.Equal(got, want) { + t.Errorf("List() = %v, want %v", got, want) + } + if got, want := m.Size(), len(normalize(cpus)); got != want { + t.Errorf("Size() = %d, want %d", got, want) + } + if got, want := len(m.mask), neededWords(cpus); got < want { + t.Errorf("mask has %d words, need at least %d", got, want) + } + }) + } +} + +// Randomized cross-check: after any sequence of Set and Clear, the cached +// String, Key and Size must agree with the set's contents. +func TestRandomizedMutationKeepsCachesConsistent(t *testing.T) { + rng := rand.New(rand.NewSource(1)) + m := NewCpuMask() + tracked := map[int]bool{} + + for i := range 3000 { + cpu := rng.Intn(300) + if rng.Intn(2) == 0 { + m.Set(cpu) + tracked[cpu] = true + } else { + m.Clear(cpu) + delete(tracked, cpu) + } + + if rng.Intn(4) == 0 { + // prime the caches part way through + _, _, _ = m.Size(), m.String(), m.Key() + } + + want := make([]int, 0, len(tracked)) + for c := range tracked { + want = append(want, c) + } + slices.Sort(want) + + if got := m.List(); !slices.Equal(got, want) { + t.Fatalf("iteration %d: List() = %v, want %v", i, got, want) + } + if got := m.Size(); got != len(want) { + t.Fatalf("iteration %d: Size() = %d, want %d", i, got, len(want)) + } + rt, err := ParseCpuMask(m.String()) + if err != nil { + t.Fatalf("iteration %d: ParseCpuMask(%q): %v", i, m.String(), err) + } + if !rt.Equals(m) { + t.Fatalf("iteration %d: round trip of %q gave %q", i, m.String(), rt.String()) + } + } +} + +// The two implementations must be interchangeable: the same inputs and +// operations must produce the same CPUs whichever one is used. +func TestImplementationsAgreeOnRandomOperations(t *testing.T) { + rng := rand.New(rand.NewSource(42)) + + randomCpus := func() []int { + n := rng.Intn(20) + cpus := make([]int, n) + for i := range cpus { + cpus[i] = rng.Intn(200) + } + return cpus + } + + for i := range 500 { + x, y := randomCpus(), randomCpus() + + mx, my := NewCpuMask(x...), NewCpuMask(y...) + sx, sy := NewCpuSet(x...), NewCpuSet(y...) + + for _, op := range []struct { + name string + fn func(a, b CPUSet) CPUSet + }{ + {"Union", func(a, b CPUSet) CPUSet { return NewAnyCPUSet(a).Union(b) }}, + {"Intersection", func(a, b CPUSet) CPUSet { return NewAnyCPUSet(a).Intersection(b) }}, + {"Difference", func(a, b CPUSet) CPUSet { return NewAnyCPUSet(a).Difference(b) }}, + } { + gotMask := op.fn(mx, my).List() + gotSet := op.fn(sx, sy).List() + if !slices.Equal(gotMask, gotSet) { + t.Fatalf("iteration %d: %s: CpuMask gave %v, CpuSet gave %v (a=%v b=%v)", + i, op.name, gotMask, gotSet, x, y) + } + // and mixing implementations must agree too + if mixed := op.fn(mx, sy).List(); !slices.Equal(mixed, gotMask) { + t.Fatalf("iteration %d: %s: CpuMask.op(CpuSet) gave %v, want %v", + i, op.name, mixed, gotMask) + } + if mixed := op.fn(sx, my).List(); !slices.Equal(mixed, gotSet) { + t.Fatalf("iteration %d: %s: CpuSet.op(CpuMask) gave %v, want %v", + i, op.name, mixed, gotSet) + } + } + + for _, op := range []struct { + name string + fn func(a, b CPUSet) bool + }{ + {"Equals", func(a, b CPUSet) bool { return a.Equals(b) }}, + {"IsSubsetOf", func(a, b CPUSet) bool { return a.IsSubsetOf(b) }}, + {"Intersects", func(a, b CPUSet) bool { return a.Intersects(b) }}, + } { + want := op.fn(mx, my) + for _, c := range []struct { + name string + a, b CPUSet + }{ + {"CpuSet/CpuSet", sx, sy}, + {"CpuMask/CpuSet", mx, sy}, + {"CpuSet/CpuMask", sx, my}, + } { + if got := op.fn(c.a, c.b); got != want { + t.Fatalf("iteration %d: %s on %s = %v, want %v (a=%v b=%v)", + i, op.name, c.name, got, want, x, y) + } + } + } + } +} + +// AsCpuMask and AsCpuSet hand back the set they were given when it already is +// of the wanted type, and convert only when it is not. +func TestAsCpuMaskAndAsCpuSet(t *testing.T) { + var ( + cpus = []int{0, 5, 70} + mask = NewCpuMask(cpus...) + set = NewCpuSet(cpus...) + other = newTestCPUSet(cpus...) + ) + + t.Run("AsCpuMask", func(t *testing.T) { + if got := AsCpuMask(mask); got != mask { + t.Error("AsCpuMask did not return the CpuMask it was given") + } + for _, from := range []struct { + name string + cpus CPUSet + }{{"CpuSet", set}, {"testCPUSet", other}} { + got := AsCpuMask(from.cpus) + if !got.IsDense() { + t.Errorf("AsCpuMask(%s) is not dense", from.name) + } + if !got.Equals(mask) { + t.Errorf("AsCpuMask(%s) = %s, want %s", from.name, got, mask) + } + } + }) + + t.Run("AsCpuSet", func(t *testing.T) { + if got := AsCpuSet(set); got != set { + t.Error("AsCpuSet did not return the CpuSet it was given") + } + for _, from := range []struct { + name string + cpus CPUSet + }{{"CpuMask", mask}, {"testCPUSet", other}} { + got := AsCpuSet(from.cpus) + if !got.IsSparse() { + t.Errorf("AsCpuSet(%s) is not sparse", from.name) + } + if !got.Equals(set) { + t.Errorf("AsCpuSet(%s) = %s, want %s", from.name, got, set) + } + } + }) +} + +// An operation through an AnyCPUSet keeps the implementation it was performed +// on, so that wrapping a sparse set does not quietly turn it dense. A set from +// neither implementation here has no representation to keep and comes back as a +// CpuMask. +func TestAnyCPUSetKeepsImplementation(t *testing.T) { + other := NewCpuMask(1) + + for _, tc := range []struct { + name string + cpus CPUSet + dense bool + }{ + {"CpuMask", NewCpuMask(0, 1), true}, + {"CpuSet", NewCpuSet(0, 1), false}, + {"testCPUSet", newTestCPUSet(0, 1), true}, + } { + t.Run(tc.name, func(t *testing.T) { + a := NewAnyCPUSet(tc.cpus) + for name, got := range map[string]AnyCPUSet{ + "Clone": a.Clone(), + "Union": a.Union(other), + "Difference": a.Difference(other), + "Intersection": a.Intersection(other), + } { + if got.IsDense() != tc.dense { + t.Errorf("%s: IsDense() = %v, want %v", + name, got.IsDense(), tc.dense) + } + } + }) + } +} + +// WrapCpuSet takes a k8s.io/utils/cpuset.CPUSet over without copying it. The +// two then share a set neither of them modifies in place, so what matters is +// that changing one leaves the other alone. +func TestWrapCpuSet(t *testing.T) { + raw := cpuset.New(0, 5, 70) + + t.Run("holds-the-same-cpus", func(t *testing.T) { + s := WrapCpuSet(raw) + if got, want := s.List(), []int{0, 5, 70}; !slices.Equal(got, want) { + t.Errorf("List() = %v, want %v", got, want) + } + if !s.IsSparse() { + t.Error("a wrapped cpuset.CPUSet is not sparse") + } + if got, want := s.String(), raw.String(); got != want { + t.Errorf("String() = %q, want %q", got, want) + } + }) + + t.Run("wrapper-is-unsealed", func(t *testing.T) { + s := WrapCpuSet(raw) + s.Set(9) // must not panic + if !s.Contains(9) { + t.Error("Set() on a wrapped set did not take effect") + } + }) + + t.Run("mutating-the-wrapper-leaves-the-raw-set-alone", func(t *testing.T) { + s := WrapCpuSet(raw) + s.Set(9) + s.Clear(0) + + if got, want := raw.List(), []int{0, 5, 70}; !slices.Equal(got, want) { + t.Errorf("the raw set changed to %v, want %v", got, want) + } + }) + + t.Run("composes-with-AnyCPUSet", func(t *testing.T) { + got := NewAnyCPUSet(WrapCpuSet(raw)).Difference(NewCpuMask(5)) + if want := []int{0, 70}; !slices.Equal(got.List(), want) { + t.Errorf("Difference() = %v, want %v", got.List(), want) + } + if !got.IsSparse() { + t.Error("the result of an operation on a wrapped set is not sparse") + } + }) +} + +func TestEmptyIfNil(t *testing.T) { + t.Run("a nil mask reads as empty", func(t *testing.T) { + var m *CpuMask + + // the point of the helper: this is a method call on a nil pointer, which + // is legal, and everything after it is a call on a real set + cpus := m.EmptyIfNil() + if cpus == nil { + t.Fatal("EmptyIfNil returned nil") + } + if !cpus.IsEmpty() || cpus.Size() != 0 || cpus.String() != "" { + t.Errorf("expected an empty mask, got %q", cpus) + } + if cpus.Union(NewCpuMask(1)).String() != "1" { + t.Error("the empty mask does not work as an operand") + } + }) + + t.Run("a real mask is itself", func(t *testing.T) { + m := NewCpuMask(1, 2) + if got := m.EmptyIfNil(); got != m { + t.Errorf("EmptyIfNil returned %q, not the mask it was called on", got) + } + }) + + t.Run("a nil set reads as empty", func(t *testing.T) { + var s *CpuSet + + cpus := s.EmptyIfNil() + if cpus == nil { + t.Fatal("EmptyIfNil returned nil") + } + if !cpus.IsEmpty() || cpus.Size() != 0 { + t.Errorf("expected an empty set, got %q", cpus) + } + }) + + t.Run("a real set is itself", func(t *testing.T) { + s := NewCpuSet(1, 2) + if got := s.EmptyIfNil(); got != s { + t.Errorf("EmptyIfNil returned %q, not the set it was called on", got) + } + }) + + t.Run("the empty sets are sealed", func(t *testing.T) { + for _, tc := range []struct { + name string + mutate func() + }{ + {"EmptyCpuMask", func() { EmptyCpuMask.Set(0) }}, + {"EmptyCpuSet", func() { EmptyCpuSet.Set(0) }}, + } { + func() { + defer func() { + if recover() == nil { + t.Errorf("modifying %s did not panic", tc.name) + } + }() + tc.mutate() + }() + } + }) +} + +func TestMustParse(t *testing.T) { + t.Run("a good string parses", func(t *testing.T) { + if got := MustParseCpuMask("0-3,7").String(); got != "0-3,7" { + t.Errorf("MustParseCpuMask gave %q", got) + } + if got := MustParseCpuSet("0-3,7").String(); got != "0-3,7" { + t.Errorf("MustParseCpuSet gave %q", got) + } + }) + + for _, tc := range []struct { + name string + parse func() + }{ + {"MustParseCpuMask", func() { MustParseCpuMask("nonsense") }}, + {"MustParseCpuSet", func() { MustParseCpuSet("nonsense") }}, + } { + t.Run(tc.name+" panics on a bad string", func(t *testing.T) { + defer func() { + if recover() == nil { + t.Errorf("%s did not panic", tc.name) + } + }() + tc.parse() + }) + } +} + +// TestNilReadsAsEmpty checks the rule the package promises: a nil set is the +// empty set for everything which does not modify it, whether it is the receiver +// or an operand, and modifying one is an error which says what to do instead. +func TestNilReadsAsEmpty(t *testing.T) { + var ( + nilMask *CpuMask + nilSet *CpuSet + none CPUSet // a nil interface, not a nil set inside one + ) + + t.Run("reading a nil mask", func(t *testing.T) { + if got := nilMask.Size(); got != 0 { + t.Errorf("Size() = %d", got) + } + if !nilMask.IsEmpty() { + t.Error("IsEmpty() is false") + } + if got := nilMask.String(); got != "" { + t.Errorf("String() = %q", got) + } + if got := nilMask.Key(); got != "" { + t.Errorf("Key() = %q", got) + } + if got := nilMask.List(); len(got) != 0 { + t.Errorf("List() = %v", got) + } + if got := nilMask.UnsortedList(); len(got) != 0 { + t.Errorf("UnsortedList() = %v", got) + } + if nilMask.Contains(0) { + t.Error("Contains(0) is true") + } + if !nilMask.IsDense() || nilMask.IsSparse() { + t.Error("a nil mask is not dense") + } + nilMask.ForEachCpu(func(int) bool { + t.Error("ForEachCpu called f") + return false + }) + }) + + t.Run("reading a nil set", func(t *testing.T) { + if got := nilSet.Size(); got != 0 { + t.Errorf("Size() = %d", got) + } + if !nilSet.IsEmpty() { + t.Error("IsEmpty() is false") + } + if got := nilSet.String(); got != "" { + t.Errorf("String() = %q", got) + } + if got := nilSet.Key(); got != "" { + t.Errorf("Key() = %q", got) + } + if got := nilSet.List(); len(got) != 0 { + t.Errorf("List() = %v", got) + } + if got := nilSet.UnsortedList(); len(got) != 0 { + t.Errorf("UnsortedList() = %v", got) + } + if nilSet.Contains(0) { + t.Error("Contains(0) is true") + } + if nilSet.IsDense() || !nilSet.IsSparse() { + t.Error("a nil set is not sparse") + } + nilSet.ForEachCpu(func(int) bool { + t.Error("ForEachCpu called f") + return false + }) + }) + + t.Run("set algebra on a nil receiver", func(t *testing.T) { + one := NewCpuMask(1) + if got := nilMask.Union(one).String(); got != "1" { + t.Errorf("Union = %q", got) + } + if got := nilMask.Intersection(one).String(); got != "" { + t.Errorf("Intersection = %q", got) + } + if got := nilMask.Difference(one).String(); got != "" { + t.Errorf("Difference = %q", got) + } + if nilMask.Intersects(one) { + t.Error("Intersects is true") + } + if nilMask.Equals(one) { + t.Error("Equals a non-empty set") + } + if !nilMask.IsSubsetOf(one) { + t.Error("the empty set is a subset of everything") + } + + oneSparse := NewCpuSet(1) + if got := nilSet.Union(oneSparse).String(); got != "1" { + t.Errorf("sparse Union = %q", got) + } + if got := nilSet.Intersection(oneSparse).String(); got != "" { + t.Errorf("sparse Intersection = %q", got) + } + if got := nilSet.Difference(oneSparse).String(); got != "" { + t.Errorf("sparse Difference = %q", got) + } + if nilSet.Intersects(oneSparse) { + t.Error("sparse Intersects is true") + } + if !nilSet.IsSubsetOf(oneSparse) { + t.Error("the empty sparse set is a subset of everything") + } + }) + + t.Run("nothing as an operand", func(t *testing.T) { + // three ways to say nothing, each of which used to panic + for name, nothing := range map[string]CPUSet{ + "a nil interface": none, + "a nil mask": nilMask, + "a nil set": nilSet, + } { + t.Run(name, func(t *testing.T) { + m := NewCpuMask(1, 2) + if got := m.Union(nothing).String(); got != "1-2" { + t.Errorf("mask Union = %q", got) + } + if got := m.Difference(nothing).String(); got != "1-2" { + t.Errorf("mask Difference = %q", got) + } + if got := m.Intersection(nothing).String(); got != "" { + t.Errorf("mask Intersection = %q", got) + } + if m.Intersects(nothing) { + t.Error("mask Intersects nothing") + } + if m.Equals(nothing) { + t.Error("mask Equals nothing") + } + if m.IsSubsetOf(nothing) { + t.Error("mask is a subset of nothing") + } + if !NewCpuMask().Equals(nothing) { + t.Error("an empty mask does not equal nothing") + } + + s := NewCpuSet(1, 2) + if got := s.Union(nothing).String(); got != "1-2" { + t.Errorf("set Union = %q", got) + } + if got := s.Difference(nothing).String(); got != "1-2" { + t.Errorf("set Difference = %q", got) + } + if got := s.Intersection(nothing).String(); got != "" { + t.Errorf("set Intersection = %q", got) + } + if s.Intersects(nothing) { + t.Error("set Intersects nothing") + } + if s.Equals(nothing) { + t.Error("set Equals nothing") + } + if s.IsSubsetOf(nothing) { + t.Error("set is a subset of nothing") + } + }) + } + }) + + t.Run("nothing converted", func(t *testing.T) { + for name, nothing := range map[string]CPUSet{ + "a nil interface": none, + "a nil mask": nilMask, + "a nil set": nilSet, + } { + t.Run(name, func(t *testing.T) { + if m := AsCpuMask(nothing); m == nil || !m.IsEmpty() { + t.Errorf("AsCpuMask gave %v", m) + } + if s := AsCpuSet(nothing); s == nil || !s.IsEmpty() { + t.Errorf("AsCpuSet gave %v", s) + } + if a := NewAnyCPUSet(nothing); !a.IsEmpty() || a.Size() != 0 { + t.Errorf("NewAnyCPUSet gave %v", a) + } + }) + } + }) + + t.Run("a clone of nothing is writable", func(t *testing.T) { + m := nilMask.Clone() + m.Set(3) + if got := m.String(); got != "3" { + t.Errorf("mask clone = %q", got) + } + s := nilSet.Clone() + s.Set(3) + if got := s.String(); got != "3" { + t.Errorf("set clone = %q", got) + } + }) + + t.Run("EmptyIfNil is writable", func(t *testing.T) { + m := nilMask.EmptyIfNil() + m.Set(4) + if got := m.String(); got != "4" { + t.Errorf("mask = %q", got) + } + s := nilSet.EmptyIfNil() + s.Set(4) + if got := s.String(); got != "4" { + t.Errorf("set = %q", got) + } + // and a set which is there comes back untouched + real := NewCpuMask(5) + if got := real.EmptyIfNil(); got != real { + t.Error("EmptyIfNil replaced a set which was there") + } + realSparse := NewCpuSet(5) + if got := realSparse.EmptyIfNil(); got != realSparse { + t.Error("EmptyIfNil replaced a sparse set which was there") + } + }) + + // EmptyIfNil answers for nil and nothing else, which is the reason Clone + // exists beside it: a sealed set comes back sealed and still cannot be + // modified, while a clone of one always can. + t.Run("EmptyIfNil does not unseal, Clone does", func(t *testing.T) { + sealedMask := NewCpuMask(0, 5) + sealedMask.Seal() + sealedSet := NewCpuSet(0, 5) + sealedSet.Seal() + + for name, modify := range map[string]func(){ + "mask": func() { sealedMask.EmptyIfNil().Set(9) }, + "set": func() { sealedSet.EmptyIfNil().Set(9) }, + } { + t.Run(name+" stays sealed", func(t *testing.T) { + defer func() { + if recover() == nil { + t.Error("modifying it through EmptyIfNil did not panic") + } + }() + modify() + }) + } + + if clone := sealedMask.Clone(); func() bool { + clone.Set(9) + return !clone.Contains(9) + }() { + t.Error("a clone of a sealed mask did not take the CPU") + } + if clone := sealedSet.Clone(); func() bool { + clone.Set(9) + return !clone.Contains(9) + }() { + t.Error("a clone of a sealed set did not take the CPU") + } + }) + + t.Run("modifying nothing says what to do", func(t *testing.T) { + for name, modify := range map[string]func(){ + "mask Set": func() { nilMask.Set(0) }, + "mask Clear": func() { nilMask.Clear(0) }, + "mask Seal": func() { nilMask.Seal() }, + "set Set": func() { nilSet.Set(0) }, + "set Clear": func() { nilSet.Clear(0) }, + "set Seal": func() { nilSet.Seal() }, + } { + t.Run(name, func(t *testing.T) { + defer func() { + r := recover() + if r == nil { + t.Fatal("did not panic") + } + if msg, ok := r.(string); !ok || + !strings.Contains(msg, "EmptyIfNil") { + t.Errorf("panic does not say what to do: %v", r) + } + }() + modify() + }) + } + }) +} diff --git a/pkg/lib/cpu/doc.go b/pkg/lib/cpu/doc.go new file mode 100644 index 000000000..1423d859a --- /dev/null +++ b/pkg/lib/cpu/doc.go @@ -0,0 +1,172 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package libcpu provides set types for CPU ids. +// +// [CPUSet] is the interface. There are two implementations of it, and which +// one fits depends on the sets you keep: +// +// - [CpuMask] is dense: a bitmask with one bit per CPU. Set algebra runs a +// word at a time and comparisons take a few nanoseconds whatever the size, +// which makes it the right choice for sets that are large, long lived, or +// combined often. +// - [CpuSet] is sparse: it wraps k8s.io/utils/cpuset.CPUSet, so a map. It +// suits small sets, and code which mostly passes cpuset strings around and +// rarely does set algebra. +// +// cpuset-bench_test.go measures the two against each other and against the +// bare k8s.io/utils/cpuset.CPUSet. For a table of the whole matrix run +// +// CPUSET_BENCH_COMPARE=1 go test -run TestCompareImplementations -v +// +// Summarised: CpuMask wins almost everything, for large sets by two or three +// orders of magnitude, and trails only marginally on Size and IsEmpty. +// +// # The interface, and what is deliberately not in it +// +// [CPUSet] carries the operations which answer a question about a set: Contains, +// Equals, Size, Intersects, List, String and the rest. Take it to accept any set +// of CPUs. +// +// The operations which produce a new set -- Clone, Union, Difference, +// Intersection -- are not in it. They are on [CpuMask] and [CpuSet], and each +// returns its own type. Go has no covariant returns, so a method returning +// *CpuMask cannot satisfy an interface method declared to return CPUSet; a +// single interface therefore cannot describe them without forcing every caller +// which knows what it holds to assert on the result. Since a caller almost +// always does know, they are left off, following the usual advice to accept +// interfaces and return concrete types. +// +// The cost is that a caller holding only a CPUSet cannot use it as the receiver +// of those four. Wrap it in an [AnyCPUSet] when that happens -- rarely, and +// visibly, at the cost of a type switch per operation. [AsCpuMask] and +// [AsCpuSet] convert a set of unknown implementation to a known one. +// +// # Coming in from k8s.io/utils/cpuset +// +// [WrapCpuSet] takes a cpuset.CPUSet over as a [CpuSet] without copying it, for +// one allocation and no dependence on how many CPUs are in it. Listing the +// members and rebuilding is the alternative, and it is not close: at a hundred +// or so CPUs wrapping is two orders of magnitude cheaper. Use it wherever a set +// arrives from an upstream interface, and [NewAnyCPUSet] on top of it if the set +// algebra is needed as well. +// +// Going back out is free: the embedded field of a [CpuSet] is the cpuset.CPUSet +// itself. +// +// # Nil sets +// +// A nil *CpuMask or *CpuSet is the empty set for everything which does not +// modify it. Size, IsEmpty, String, Key, List, Contains, ForEachCpu and the set +// algebra all read one as empty, whether it is the receiver or an operand, so a +// set which came from a map with no such key, or a struct field nobody assigned, +// needs no guarding: +// +// free := byNode[id] // nothing there +// fmt.Println(free.Size()) // 0 +// fmt.Println(all.Difference(free)) // all of them +// +// This is Go's own rule for nil maps and slices: readable, not writable. The +// operations which do modify a set -- Set, Clear and Seal -- panic on a nil one, +// because no method can allocate a set and store it back into the caller's +// variable. Assign it first. There are two idioms for that, and which one you +// want depends on what you know about the set: +// +// cpus = cpus.EmptyIfNil() // nothing becomes an empty set, the rest is kept +// cpus = cpus.Clone() // always your own set, unsealed, and a copy +// +// EmptyIfNil only answers for nil. It hands back the set itself when there is +// one, which costs nothing, but a *sealed* set comes back sealed and the Set +// after it still panics. Reach for it when the set is one you own and know is +// unsealed, for instance a field of your own you may not have filled in yet. +// +// Clone answers for both, at the price of a copy: nil or not, sealed or not, what +// comes back is an unsealed set with no other owner. Reach for it for a set of +// unknown provenance -- anything the hardware package hands out is sealed -- and +// whenever you are about to modify something you were given rather than made. +// +// # Concurrency +// +// Nothing here is safe for concurrent use without external synchronisation, +// and the reason is less obvious than it looks: reading is not enough to make +// it safe. String, Key and Size fill their caches on first use, so a call that +// only reads a set can still write to it. +// +// [CPUSet.Seal] is the way out. It marks a set immutable, so that Set and +// Clear panic from then on, and it materialises every cached value up front. A +// sealed set can be read from any number of goroutines. So for a set that is +// built once and then shared: +// +// cpus := libcpu.NewCpuMask(ids...) +// cpus.Seal() +// // ...safe to hand out now +// +// Clone deliberately returns an *unsealed* copy, since cloning is how you +// obtain a set you are allowed to modify. A clone of a sealed set is therefore +// not safe to share until it has been sealed again. +// +// # Gotchas +// +// Keys are per implementation. [CPUSet.Key] returns equal strings for equal +// sets of the same implementation only. CpuMask keys on its hex mask words and +// CpuSet on the cpuset string, so the set {0,5} keys as "21" through the one +// and "0,5" through the other. Never mix implementations in a single keyed +// map. +// +// If you do need one key for both, take the CpuMask form of it: +// +// func key(s libcpu.CPUSet) string { +// if m, ok := s.(*libcpu.CpuMask); ok { +// return m.Key() // already the right form, and cached +// } +// return libcpu.NewCpuMask(s.UnsortedList()...).Key() +// } +// +// That is deterministic, even though CpuSet.UnsortedList returns CPUs in no +// particular order: NewCpuMask builds the same mask from the same CPUs however +// they are ordered, and CpuMask.Key is a function of the mask alone. +// +// It is not cheap though, and nothing caches it: for a set of a thousand CPUs +// it costs some tens of microseconds and a few dozen allocations, against about +// a nanosecond and none for either Key. Key it once and keep the string if you +// need it more than once. +// +// A CpuMask costs memory in proportion to its highest CPU id, not to how many +// CPUs it holds. One holding only CPU 1023 takes 16 words, as much as one +// holding all of 0-1023. A sparse set with high ids is the case where CpuSet +// may be the cheaper representation. +// +// Mixing implementations works, but it is slower. Every binary operation has a +// fast path for its own type and a fallback for everything else, and the +// fallback crosses the interface once per CPU. Mixing is supported so that the +// results are correct, not because it is fast: prefer a single implementation +// throughout any one data structure. +// +// CPU ids must not be negative. A negative id is not rejected, it aliases: +// NewCpuMask(-1) yields the set {63}. Others, Set(-64) among them, panic. The +// parsers do reject negative input. +// +// [CPUSet.UnsortedList] means what its name says. CpuMask happens to return +// CPUs in increasing order and CpuSet does not; rely on neither. Use +// [CPUSet.List] when the order matters, or [CPUSet.ForEachCpu] to walk a +// CpuMask without allocating a slice at all. +// +// [CPUSet.Contains] is variadic and means "all of", which makes Contains() +// with no arguments true. Equals depends on that. +// +// [ParseCpuMask] and [ParseCpuSet] both take the Linux cpuset list format +// ("0-3,8") and agree on what counts as valid, down to the leniency they +// inherit from strconv.Atoi: "+1" parses as 1 and "00" as 0. Neither accepts +// surrounding whitespace. +package libcpu From 33e6fd9551d103ff33ed27bb3d89ce07a0cefe23 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 9 Sep 2026 18:21:35 +0300 Subject: [PATCH 12/39] lib/hardware: discover CPU and memory topology. Add pkg/lib/hardware, a leaner replacement for pkg/sysfs. Discover reads a machine once and returns an immutable Machine which is safe to share. Reading goes through an io/fs.FS rooted at the host root, so WithRoot points discovery at a mounted host filesystem and WithFS substitutes a recorded or synthetic one. Handles are concrete, interned and nil-safe: a lookup for absent hardware answers Valid() == false rather than returning a typed nil whose methods then panic, as pkg/sysfs does. CPU sets are libcpu.CpuMask, sealed, so they are safe to share and panic if modified. A Zone is a named set of CPUs at one Level. Zones deliberately do not form a tree: whether a cluster sits inside a NUMA node or spans several is a property of the machine, and which levels are worth nesting differs per caller. Zones(level) is complete, and SameZones says when two levels cut the machine the same way, which is what the policies ask by hand today. TopologyIndex flattens a Machine into a coordinate lookup table for callers which consult it per allocation. The convenience layer holds the groupings several consumers had each grown their own version of: cache groups, logical clusters, thread rounding and closest-node queries. Unit tests cover the readers with fstest.MapFS, and discovery against the recorded sysfs trees plus synthetic ones, asserting invariants rather than fixed numbers. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/lib/hardware/build.go | 450 +++++++++++++++++++ pkg/lib/hardware/cache.go | 228 ++++++++++ pkg/lib/hardware/convenience.go | 418 ++++++++++++++++++ pkg/lib/hardware/cpu.go | 259 +++++++++++ pkg/lib/hardware/derived_test.go | 494 +++++++++++++++++++++ pkg/lib/hardware/discover.go | 665 +++++++++++++++++++++++++++++ pkg/lib/hardware/discover_test.go | 557 ++++++++++++++++++++++++ pkg/lib/hardware/doc.go | 85 ++++ pkg/lib/hardware/machine.go | 239 +++++++++++ pkg/lib/hardware/memory.go | 262 ++++++++++++ pkg/lib/hardware/overrides.go | 145 +++++++ pkg/lib/hardware/overrides_test.go | 280 ++++++++++++ pkg/lib/hardware/read.go | 256 +++++++++++ pkg/lib/hardware/read_test.go | 494 +++++++++++++++++++++ pkg/lib/hardware/topology.go | 459 ++++++++++++++++++++ pkg/lib/hardware/zone.go | 204 +++++++++ 16 files changed, 5495 insertions(+) create mode 100644 pkg/lib/hardware/build.go create mode 100644 pkg/lib/hardware/cache.go create mode 100644 pkg/lib/hardware/convenience.go create mode 100644 pkg/lib/hardware/cpu.go create mode 100644 pkg/lib/hardware/derived_test.go create mode 100644 pkg/lib/hardware/discover.go create mode 100644 pkg/lib/hardware/discover_test.go create mode 100644 pkg/lib/hardware/doc.go create mode 100644 pkg/lib/hardware/machine.go create mode 100644 pkg/lib/hardware/memory.go create mode 100644 pkg/lib/hardware/overrides.go create mode 100644 pkg/lib/hardware/overrides_test.go create mode 100644 pkg/lib/hardware/read.go create mode 100644 pkg/lib/hardware/read_test.go create mode 100644 pkg/lib/hardware/topology.go create mode 100644 pkg/lib/hardware/zone.go diff --git a/pkg/lib/hardware/build.go b/pkg/lib/hardware/build.go new file mode 100644 index 000000000..9f3f5aace --- /dev/null +++ b/pkg/lib/hardware/build.go @@ -0,0 +1,450 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "os" + "slices" + "strconv" + "strings" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// +// Memory kinds +// + +// classifyMemory works out what sort of memory each NUMA node holds. +// +// The kernel does not say, so this is inference, and the same inference +// pkg/sysfs makes: a node with CPUs of its own holds ordinary DRAM, and a node +// with memory but no CPUs holds something special. Which special sort is +// guessed from size, since high-bandwidth memory is smaller than a DRAM node +// and persistent memory is larger. +// +// Unlike pkg/sysfs this does not fail when the guess cannot be made. A machine +// with only CPU-less nodes, or with no DRAM to compare against, gets +// [MemoryKindUnknown] and stays usable; refusing to describe the machine at all +// is worse than admitting to not knowing. +func (d *discovery) classifyMemory() error { + m := d.m + + var ( + dramTotal int64 + dramNodes int64 + special []*MemoryNode + ) + + for _, node := range m.nodes { + switch { + case !node.cpus.IsEmpty(): + // CPUs of its own: ordinary memory, whether it has any or not + node.kind = MemoryKindDRAM + if node.capacity > 0 { + dramTotal += node.capacity + dramNodes++ + } + case node.capacity == 0: + // no CPUs and no memory: nothing to classify, and nothing which + // would want to allocate from it either + node.kind = MemoryKindUnknown + default: + special = append(special, node) + } + } + + if len(special) == 0 { + return nil + } + + if dramNodes == 0 { + // Nothing to compare against. Leave them unknown rather than guessing. + return nil + } + + average := dramTotal / dramNodes + for _, node := range special { + if node.capacity < average { + node.kind = MemoryKindHBM + } else { + node.kind = MemoryKindPMEM + } + } + + return nil +} + +// +// Zones +// + +// buildZones collects the zones of every level the machine has. +// +// There is no tree. Zones at one level are a set of CPU sets, and nothing here +// decides which level contains which or which level is worth keeping: those are +// questions the hardware does not answer and different callers answer +// differently. [Machine.Zones] lists them, [Machine.SameZones] says when two +// levels cut the machine the same way, and containment is a query. +func (d *discovery) buildZones() error { + m := d.m + + for _, level := range allLevels { + zones := d.zonesAt(level) + if len(zones) == 0 { + continue + } + m.zones[level] = zones + m.levels = append(m.levels, level) + } + + d.indexZonesByCPU() + + return nil +} + +// zonesAt collects the zones of one level. +func (d *discovery) zonesAt(level Level) []*Zone { + m := d.m + + // group holds the CPUs of each zone, keyed by a value which is unique at + // this level, and the coordinates that key stands for. + type group struct { + cpus *libcpu.CpuMask + id ID + at Coordinates + cache *Cache + } + groups := map[ID]*group{} + + add := func(key ID, id ID, at Coordinates, cache *Cache, cpus ...int) { + g, ok := groups[key] + if !ok { + g = &group{cpus: libcpu.NewCpuMask(), id: id, at: at, cache: cache} + groups[key] = g + } + g.cpus.Set(cpus...) + } + + switch level { + case LevelPackage, LevelDie, LevelCluster, LevelCore: + for _, id := range m.cpuIDs { + c := m.cpus[id] + if !c.online { + continue + } + key, zid, ok := zoneKeyOf(c, level) + if !ok { + continue + } + add(key, zid, c.coordinates(), nil, c.id) + } + + case LevelNUMANode: + for _, id := range m.nodeIDs { + node := m.nodes[id] + if node.cpus.IsEmpty() { + // memory only: a MemoryNode, but no set of CPUs to be a zone of + continue + } + at := Coordinates{ + CPU: unknownID, Package: unknownID, Die: unknownID, + Cluster: unknownID, MemoryNode: id, Core: unknownID, + } + add(id, id, at, nil, node.cpus.UnsortedList()...) + } + + case LevelL3Cache, LevelL2Cache: + want := cacheLevelOf(level) + for key, cache := range m.caches { + // only unified caches make a zone: a level with separate data and + // instruction caches has two per group of CPUs, which would be two + // zones holding the same CPUs + if key.Level != want || key.Kind != UnifiedCache || cache.cpus.IsEmpty() { + continue + } + at := Coordinates{ + CPU: unknownID, Package: unknownID, Die: unknownID, + Cluster: unknownID, MemoryNode: unknownID, Core: unknownID, + } + add(cache.id, cache.id, at, cache, cache.cpus.UnsortedList()...) + } + + case LevelThread: + for _, id := range m.cpuIDs { + add(id, id, m.cpus[id].coordinates(), nil, id) + } + } + + keys := make([]ID, 0, len(groups)) + for key := range groups { + keys = append(keys, key) + } + slices.Sort(keys) + + zones := make([]*Zone, 0, len(keys)) + for _, key := range keys { + g := groups[key] + g.cpus.Seal() + + z := &Zone{ + m: m, valid: true, level: level, id: g.id, cpus: g.cpus, + pkg: g.at.Package, die: g.at.Die, cluster: g.at.Cluster, + node: g.at.MemoryNode, cache: g.cache, + } + // A zone whose CPUs span several of something is in none of them. + d.narrowCoordinates(z) + z.name = z.zoneName() + + if z.cache != nil { + z.cache.zone = z + } + zones = append(zones, z) + } + + return zones +} + +// narrowCoordinates drops any coordinate of a zone which is not the same for +// every CPU in it, since a zone spanning two packages is in neither. +func (d *discovery) narrowCoordinates(z *Zone) { + first := true + z.cpus.ForEachCpu(func(id int) bool { + c, ok := d.m.cpus[id] + if !ok { + return true + } + at := c.coordinates() + if first { + z.pkg, z.die = at.Package, at.Die + z.cluster, z.node = at.Cluster, at.MemoryNode + first = false + return true + } + if z.pkg != at.Package { + z.pkg = unknownID + } + if z.die != at.Die { + z.die = unknownID + } + if z.cluster != at.Cluster { + z.cluster = unknownID + } + if z.node != at.MemoryNode { + z.node = unknownID + } + return true + }) + + // a die or cluster id only means something together with its package + if z.pkg == unknownID { + z.die, z.cluster = unknownID, unknownID + } + if z.die == unknownID { + z.cluster = unknownID + } +} + +// zoneName names a zone from its coordinates, so that the name says where it is +// without depending on a tree. +func (z *Zone) zoneName() string { + name := levelNames[z.level] + "#" + strconv.Itoa(z.id) + + switch z.level { + case LevelDie, LevelCore: + if z.pkg != unknownID { + name = "package#" + strconv.Itoa(z.pkg) + "/" + name + } + case LevelCluster: + if z.pkg != unknownID && z.die != unknownID { + name = "package#" + strconv.Itoa(z.pkg) + + "/die#" + strconv.Itoa(z.die) + "/" + name + } + } + + return name +} + +// zoneKeyOf returns a key which is unique for a CPU's zone at a level, the id +// that zone reports, and whether the machine reports one at all. +// +// Package, die, cluster and core ids are numbered within their parent, so a +// bare id would collide across packages. The key combines them; the id stays +// the kernel's. +func zoneKeyOf(c *CPU, level Level) (key ID, id ID, ok bool) { + switch level { + case LevelPackage: + return c.pkg, c.pkg, c.pkg != unknownID + case LevelDie: + return c.pkg<<20 | c.die, c.die, + c.pkg != unknownID && c.die != unknownID + case LevelCluster: + return c.pkg<<40 | c.die<<20 | c.cluster, c.cluster, + c.pkg != unknownID && c.die != unknownID && c.cluster != unknownID + case LevelCore: + return c.pkg<<20 | c.core, c.core, + c.pkg != unknownID && c.core != unknownID + } + return unknownID, unknownID, false +} + +// indexZonesByCPU records which zone at each level holds each CPU and each NUMA +// node, so that CPU.Zone and MemoryNode.Zone are an array index, not a search. +func (d *discovery) indexZonesByCPU() { + m := d.m + + for _, level := range m.levels { + for _, z := range m.zones[level] { + z.cpus.ForEachCpu(func(id int) bool { + if c, ok := m.cpus[id]; ok { + c.zones[level] = z + } + return true + }) + } + } + + for _, id := range m.nodeIDs { + node := m.nodes[id] + if node.cpus.IsEmpty() { + continue + } + for _, level := range m.levels { + for _, z := range m.zones[level] { + if !node.cpus.IsSubsetOf(z.cpus) { + continue + } + have := node.zones[level] + if have == nil || z.cpus.Size() < have.cpus.Size() { + node.zones[level] = z + } + } + } + } +} + +// cacheLevelOf returns the cache level a zone level stands for, or 0. +func cacheLevelOf(level Level) int { + switch level { + case LevelL2Cache: + return 2 + case LevelL3Cache: + return 3 + } + return 0 +} + +// +// Environment overrides +// + +// The variables [WithEnvOverrides] reads. Their contents are what pkg/sysfs +// accepts, so that an end-to-end test which sets them keeps working across the +// move. +const ( + envCoreCPUs = "OVERRIDE_SYS_CORE_CPUS" + envAtomCPUs = "OVERRIDE_SYS_ATOM_CPUS" + envCaches = "OVERRIDE_SYS_CACHES" + envCPUFreq = "OVERRIDE_SYS_CPUFREQ" +) + +// WithEnvOverrides applies the OVERRIDE_SYS_* environment variables, which +// substitute core kinds, cache layout and CPU frequencies for the ones the +// machine reports. +// +// They exist for the end-to-end tests. Those install real plugin binaries into +// a freshly provisioned qemu virtual machine, with a real CRI runtime and a real +// Kubernetes cluster, and then check what the plugins actually do. Some of the +// hardware worth testing against is not something qemu emulates -- hybrid +// cores, a particular cache layout, cpufreq at all -- so a test sets these to +// make the plugin see hardware the VM does not have, and the feature gets +// exercised at least that far. +// +// That is what they are for. Inside this repository's own tests [WithFS] is the +// better tool: a recorded or synthetic topology needs no environment variable +// and no virtual machine. +func WithEnvOverrides() Option { + return func(o *options) error { + return o.applyEnvOverrides() + } +} + +// applyEnvOverrides reads the OVERRIDE_SYS_* variables into the options. +func (o *options) applyEnvOverrides() error { + for kind, name := range map[CoreKind]string{ + PerformanceCore: envCoreCPUs, + EfficientCore: envAtomCPUs, + } { + value := os.Getenv(name) + if value == "" { + continue + } + cpus, err := libcpu.ParseCpuMask(value) + if err != nil { + return newOverrideError(name, value, err) + } + cpus.Seal() + if o.kinds == nil { + o.kinds = map[CoreKind]*libcpu.CpuMask{} + } + o.kinds[kind] = cpus + } + + if value := os.Getenv(envCaches); value != "" { + caches, err := parseCacheOverrides(value) + if err != nil { + return newOverrideError(envCaches, value, err) + } + o.caches = caches + } + + if value := os.Getenv(envCPUFreq); value != "" { + freq, err := parseFreqOverrides(value) + if err != nil { + return newOverrideError(envCPUFreq, value, err) + } + o.freq = freq + } + + return nil +} + +// newOverrideError says which variable was wrong, since a mistake in one is +// otherwise hard to place. +func newOverrideError(name, value string, err error) error { + return &overrideError{name: name, value: value, err: err} +} + +// overrideError is a malformed OVERRIDE_SYS_* variable. +type overrideError struct { + name string + value string + err error +} + +// Error implements error. +func (e *overrideError) Error() string { + return "bad " + e.name + "=" + strconv.Quote(e.value) + ": " + e.err.Error() +} + +// Unwrap returns the underlying parse failure. +func (e *overrideError) Unwrap() error { + return e.err +} + +// trimmed is strings.TrimSpace, named for what the overrides need it for. +func trimmed(s string) string { + return strings.TrimSpace(s) +} diff --git a/pkg/lib/hardware/cache.go b/pkg/lib/hardware/cache.go new file mode 100644 index 000000000..f8c79e243 --- /dev/null +++ b/pkg/lib/hardware/cache.go @@ -0,0 +1,228 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "fmt" + "slices" + "strconv" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// Cache is one CPU cache, and the CPUs which share it. It is a handle into +// the [Machine] which produced it: never nil, and safe to use even when it +// refers to no cache, in which case it reports Valid() == false. +// +// A cache the machine reports once is one Cache here, so two CPUs sharing an L3 +// get the same handle for it and comparing handles is a valid identity test. +// [Cache.Key] is for when a string is wanted instead, for a log line or a map +// keyed by something other than the handle. +type Cache struct { + m *Machine + valid bool + + id ID + level int + kind CacheKind + size int64 + cpus *libcpu.CpuMask + zone *Zone +} + +// invalidCache is what a lookup for a cache the machine does not have returns. +// As in pkg/sysfs its methods answer with zero values rather than panicking. +var invalidCache = &Cache{} + +// Valid reports whether this handle refers to a cache the machine has. +func (c *Cache) Valid() bool { + return c.valid +} + +// ID returns the id of this cache, as the kernel numbers it. Ids are unique +// within a level and kind, not across the machine; see [Cache.Key]. +func (c *Cache) ID() ID { + return c.id +} + +// Level returns which level of cache this is: 1 for L1, 2 for L2, and so on. +func (c *Cache) Level() int { + return c.level +} + +// Kind returns whether this cache holds data, instructions, or both. +func (c *Cache) Kind() CacheKind { + return c.kind +} + +// Size returns the size of this cache in bytes, or 0 if the machine does not +// say. +func (c *Cache) Size() int64 { + return c.size +} + +// CPUs returns the CPUs sharing this cache. The set is sealed. +func (c *Cache) CPUs() *libcpu.CpuMask { + if c.cpus == nil { + return emptyCPUs + } + return c.cpus +} + +// CacheID returns the full coordinates of this cache, for looking it up in a +// [TopologyIndex] or keying a map by it. Prefer this to assembling a [CacheID] +// by hand: all three fields are needed, and forgetting the kind silently names a +// different cache. +func (c *Cache) CacheID() CacheID { + if !c.valid { + return CacheID{Level: 0, Kind: UnifiedCache, ID: unknownID} + } + return CacheID{Level: c.level, Kind: c.kind, ID: c.id} +} + +// Key returns a string which identifies this cache within the machine, unlike +// [Cache.ID] which is only unique within a level and kind. It is usable as a +// map key. +func (c *Cache) Key() string { + if !c.valid { + return "" + } + return "L" + strconv.Itoa(c.level) + c.kind.suffix() + "#" + strconv.Itoa(c.id) +} + +// Zone returns this cache as a zone, or an invalid zone if the cache is at a +// level which does not group CPUs usefully. +func (c *Cache) Zone() *Zone { + if c.zone == nil { + return invalidZone + } + return c.zone +} + +// String returns something like "L3#0 (unified, 32M)". +func (c *Cache) String() string { + if !c.valid { + return "L?#?" + } + return fmt.Sprintf("%s (%s, %s)", c.Key(), c.kind, sizeString(c.size)) +} + +// CacheKind is what a cache holds. +type CacheKind int + +const ( + // DataCache holds data only. + DataCache CacheKind = iota + // InstructionCache holds instructions only. + InstructionCache + // UnifiedCache holds both. + UnifiedCache +) + +// String returns "data", "instruction" or "unified". +func (k CacheKind) String() string { + switch k { + case DataCache: + return "data" + case InstructionCache: + return "instruction" + case UnifiedCache: + return "unified" + } + return "unknown cache kind" +} + +// suffix distinguishes a data from an instruction cache in a [Cache.Key], +// since the two can share a level and an id. A unified cache needs none. +func (k CacheKind) suffix() string { + switch k { + case DataCache: + return "d" + case InstructionCache: + return "i" + } + return "" +} + +// parseCacheKind returns the kind the kernel names in a cache's type attribute. +func parseCacheKind(s string) (CacheKind, error) { + switch s { + case "Data": + return DataCache, nil + case "Instruction": + return InstructionCache, nil + case "Unified": + return UnifiedCache, nil + } + return UnifiedCache, fmt.Errorf("unknown cache type %q", s) +} + +// compareCaches orders caches by level, then kind, then id, which is the order +// everything here hands them out in. +func compareCaches(a, b *Cache) int { + if a.level != b.level { + return a.level - b.level + } + if a.kind != b.kind { + return int(a.kind) - int(b.kind) + } + return a.id - b.id +} + +// sortCaches orders a CPU's caches the way discovery hands them out: level +// first, then kind. +func sortCaches(caches []*Cache) { + slices.SortStableFunc(caches, compareCaches) +} + +// parseSize parses a cache size as the kernel writes it: a number, optionally +// followed by a unit of K, M or G. +func parseSize(s string) (int64, error) { + if s == "" { + return 0, nil + } + + mult := int64(1) + switch s[len(s)-1] { + case 'K', 'k': + mult, s = 1<<10, s[:len(s)-1] + case 'M', 'm': + mult, s = 1<<20, s[:len(s)-1] + case 'G', 'g': + mult, s = 1<<30, s[:len(s)-1] + } + + n, err := strconv.ParseInt(s, 10, 64) + if err != nil { + return 0, err + } + + return n * mult, nil +} + +// sizeString renders a size the way the kernel writes one. +func sizeString(size int64) string { + switch { + case size == 0: + return "0" + case size%(1<<30) == 0: + return strconv.FormatInt(size/(1<<30), 10) + "G" + case size%(1<<20) == 0: + return strconv.FormatInt(size/(1<<20), 10) + "M" + case size%(1<<10) == 0: + return strconv.FormatInt(size/(1<<10), 10) + "K" + } + return strconv.FormatInt(size, 10) +} diff --git a/pkg/lib/hardware/convenience.go b/pkg/lib/hardware/convenience.go new file mode 100644 index 000000000..08ff5aa83 --- /dev/null +++ b/pkg/lib/hardware/convenience.go @@ -0,0 +1,418 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +// Everything below is derived from what the rest of the package already +// exposes: it adds no state and reads nothing the core did not read. It is here +// because several callers had each grown their own version of it. +// +// Keeping it in one file, written only against the exported API above, is what +// keeps the core small. Nothing here may reach into unexported fields. + +import ( + "slices" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// +// Containment +// + +// ZoneOf returns the smallest zone at the given level which holds all of cpus, +// or an invalid zone if none does. +func ZoneOf(m *Machine, level Level, cpus libcpu.CPUSet) *Zone { + best := invalidZone + for _, z := range m.Zones(level) { + if !cpus.IsSubsetOf(z.cpus) { + continue + } + if !best.valid || z.cpus.Size() < best.cpus.Size() { + best = z + } + } + return best +} + +// ZonesWithin returns the zones at the given level all of whose CPUs are in +// cpus. A zone only partly covered is left out, which is what a caller +// subdividing a set of CPUs it may allocate from wants: a half-covered cache is +// not a cache it can hand out whole. +func ZonesWithin(m *Machine, level Level, cpus libcpu.CPUSet) []*Zone { + var out []*Zone + for _, z := range m.Zones(level) { + if z.cpus.IsSubsetOf(cpus) { + out = append(out, z) + } + } + return out +} + +// ZonesOverlapping returns the zones at the given level which have any CPU in +// cpus. A zone only partly covered is included, which is what a caller asking +// what a set of CPUs touches wants: a die with one CPU in the set is still a +// die it has to account for. +// +// This and [ZonesWithin] differ only in that predicate, and choosing the wrong +// one is easy to do quietly. "The caches I can allocate" is Within; "the dies I +// have to reprogram" is Overlapping. +func ZonesOverlapping(m *Machine, level Level, cpus libcpu.CPUSet) []*Zone { + var out []*Zone + for _, z := range m.Zones(level) { + if z.cpus.Intersects(cpus) { + out = append(out, z) + } + } + return out +} + +// +// Threads and cores +// + +// AllThreads returns cpus together with every other CPU sharing a core with +// one of them, i.e. cpus rounded up to whole cores. +func AllThreads(m *Machine, cpus libcpu.CPUSet) *libcpu.CpuMask { + all := libcpu.NewCpuMask() + cpus.ForEachCpu(func(id int) bool { + if c := m.CPU(id); c.Valid() && !c.Threads().IsEmpty() { + all.Set(c.Threads().UnsortedList()...) + } else { + all.Set(id) + } + return true + }) + return sealed(all) +} + +// SingleThreadPerCore returns the subset of cpus holding only the +// lowest-numbered CPU of each core it covers. +func SingleThreadPerCore(m *Machine, cpus libcpu.CPUSet) *libcpu.CpuMask { + var ( + out = libcpu.NewCpuMask() + done = libcpu.NewCpuMask() + ) + + // List, not UnsortedList: which thread of a core is kept has to be the + // lowest-numbered one, and that needs a defined order. + for _, id := range cpus.List() { + if done.Contains(id) { + continue + } + out.Set(id) + done.Set(id) + if c := m.CPU(id); c.Valid() { + done.Set(c.Threads().UnsortedList()...) + } + } + + return sealed(out) +} + +// +// Caches +// + +// CPUsSharingCache returns cpus together with every other CPU sharing a cache +// at the given level with one of them, i.e. cpus rounded up to whole cache +// groups. +func CPUsSharingCache(m *Machine, level int, cpus libcpu.CPUSet) *libcpu.CpuMask { + all := libcpu.NewCpuMask() + cpus.ForEachCpu(func(id int) bool { + if all.Contains(id) { + return true + } + if cache := m.CPU(id).Cache(level); cache.Valid() { + all.Set(cache.CPUs().UnsortedList()...) + } else { + all.Set(id) + } + return true + }) + return sealed(all) +} + +// CacheGrouping is one cache level, and the caches at that level which group +// CPUs in a way worth allocating along. The Groups are what [CacheGroups] +// returns for the Level. +type CacheGrouping struct { + Level int + Groups []*Cache +} + +// GroupingCacheLevels returns every cache level whose caches group CPUs in a way +// no coarser or finer level of the topology already offers, coarsest level first. +// It is empty for a machine where no cache level says anything of its own. +// +// There can be more than one such level, which is why this does not answer with +// a single one. A machine whose core types differ can be grouped at a different +// level in each region: the cores of one kind sharing a cache the other kind has +// no access to, while the other kind is grouped by a cache further out. A caller +// which wants one grouping for the whole machine takes the first, the coarsest; +// one which handles such a machine properly walks them all and asks which groups +// hold the CPUs it is placing. +// +// A level with a single group is reported like any other. One group still says +// which CPUs belong together, which is the whole question here, and on a machine +// with two kinds of core there may be exactly one group per kind. +func GroupingCacheLevels(m *Machine) []CacheGrouping { + var groupings []CacheGrouping + + // From the largest caches inwards, so that the coarsest grouping comes first: + // a machine whose L3 groups usefully should be allocated along its L3 before + // an L2 which is finer than anything a caller wants to keep intact. + levels := m.CacheLevels() + for i := len(levels) - 1; i >= 0; i-- { + if groups := CacheGroups(m, levels[i]); len(groups) > 0 { + groupings = append(groupings, CacheGrouping{ + Level: levels[i], + Groups: groups, + }) + } + } + + return groupings +} + +// CacheGroups returns the caches at the given level which group CPUs +// non-trivially, ordered by package, die, NUMA node and lowest CPU. A cache +// shared by exactly one core, or by a whole die or package, is left out: it +// duplicates a grouping the topology already offers. +// +// A cache which covers the same CPUs as a cluster or a NUMA node is reported, +// even though that level names the same group. Those levels are not always +// there, and a caller which does not consult them would otherwise be told +// nothing about a grouping which is real. +// +// This is the set of groups a CPU allocator should prefer to keep intact, for +// the levels [GroupingCacheLevels] reports. +func CacheGroups(m *Machine, level int) []*Cache { + var groups []*Cache + + for _, cache := range m.Caches(level) { + cpus := cache.CPUs() + + // A cache shared by one CPU, or by exactly the threads of one core, says + // no more than the core level does. One covering a whole die or package + // says no more than those do. + switch { + case cpus.Size() <= 1: + continue + case sameAsSomeZone(m, LevelCore, cpus): + continue + case sameAsSomeZone(m, LevelDie, cpus): + continue + case sameAsSomeZone(m, LevelPackage, cpus): + continue + } + + groups = append(groups, cache) + } + + // Ordered by where they sit, then by lowest CPU, so that a caller walking + // them in order walks the machine in order. + slices.SortFunc(groups, func(a, b *Cache) int { + x, y := groupCoordinates(m, a), groupCoordinates(m, b) + if x.Package != y.Package { + return x.Package - y.Package + } + if x.Die != y.Die { + return x.Die - y.Die + } + if x.MemoryNode != y.MemoryNode { + return x.MemoryNode - y.MemoryNode + } + return a.CPUs().List()[0] - b.CPUs().List()[0] + }) + + return groups +} + +// +// Clusters +// + +// SingleCoreClusters says what [LogicalClusters] does with a cluster which holds +// nothing but the threads of one core. +// +// Some machines report every core as its own cluster, which makes the cluster +// level say no more than the core level does. Such a cluster is never reported as +// it stands; this is how a caller says whether it wants those CPUs grouped +// somewhere or not reported at all. Which is right depends on what the clusters +// are being used for, so there is no default. +type SingleCoreClusters bool + +const ( + // OmitSingleCoreClusters leaves them out, so that what remains describes + // real sharing. A die whose every cluster is one core yields nothing. + OmitSingleCoreClusters SingleCoreClusters = false + // MergeSingleCoreClusters gathers them into a single cluster. A die whose + // every cluster is one core yields that one merged cluster. + // + // This is what pkg/sysfs reported. A caller which groups CPUs by cluster and + // would rather group these somewhere than drop them wants this. + MergeSingleCoreClusters SingleCoreClusters = true +) + +// LogicalClusters returns the CPUs of each cluster of the given die, ordered by +// cluster id, with the single-core clusters treated as single says. +// +// A merged cluster spans several of the machine's real clusters, so there is no +// [Zone] to return for it and these are plain CPU sets. The cluster id of any of +// them, merged or not, is the cluster id of its lowest-numbered CPU: +// +// id := m.CPU(cpus.List()[0]).ClusterID() +// +// which is what the ordering is by. +func LogicalClusters(m *Machine, pkg, die ID, single SingleCoreClusters) []*libcpu.CpuMask { + var ( + want = DieID{Package: pkg, Die: die} + clusters []*libcpu.CpuMask + merged = libcpu.NewCpuMask() + ) + + for _, z := range m.Zones(LevelCluster) { + if z.DieID() != want { + continue + } + if sameAsSomeZone(m, LevelCore, z.CPUs()) { + if single == MergeSingleCoreClusters { + merged.Set(z.CPUs().UnsortedList()...) + } + continue + } + clusters = append(clusters, z.CPUs()) + } + + if !merged.IsEmpty() { + clusters = append(clusters, sealed(merged)) + } + + slices.SortFunc(clusters, func(a, b *libcpu.CpuMask) int { + return m.CPU(a.List()[0]).ClusterID() - m.CPU(b.List()[0]).ClusterID() + }) + + return clusters +} + +// +// NUMA nodes +// + +// MemoryNodeDistanceGroup is a set of NUMA nodes all equally far from some +// other node. +type MemoryNodeDistanceGroup struct { + // Distance is how far the nodes are, as the kernel reports it. + Distance int + // Nodes are the nodes at that distance, in increasing order of id. + Nodes []ID +} + +// ClosestMemoryNodes returns the NUMA nodes other than from which satisfy +// match, grouped by how far they are and ordered nearest first. A nil match +// accepts every node. +// +// A caller looking for the nearest node with ordinary memory reads the first +// group; one willing to look further walks on. +func ClosestMemoryNodes(m *Machine, from ID, match func(*MemoryNode) bool) []MemoryNodeDistanceGroup { + origin := m.MemoryNode(from) + if !origin.Valid() { + return nil + } + + byDistance := map[int][]ID{} + for _, node := range m.MemoryNodes() { + if node.ID() == from { + continue + } + if match != nil && !match(node) { + continue + } + d := origin.Distance(node.ID()) + if d < 0 { + continue + } + byDistance[d] = append(byDistance[d], node.ID()) + } + + distances := make([]int, 0, len(byDistance)) + for d := range byDistance { + distances = append(distances, d) + } + slices.Sort(distances) + + groups := make([]MemoryNodeDistanceGroup, 0, len(distances)) + for _, d := range distances { + nodes := byDistance[d] + slices.Sort(nodes) + groups = append(groups, MemoryNodeDistanceGroup{Distance: d, Nodes: nodes}) + } + + return groups +} + +// MemoryNodesFor returns the ids of the NUMA nodes any of whose CPUs are in +// cpus, i.e. the memory local to those CPUs. +func MemoryNodesFor(m *Machine, cpus libcpu.CPUSet) []ID { + var out []ID + for _, node := range m.MemoryNodes() { + if node.CPUs().Intersects(cpus) { + out = append(out, node.ID()) + } + } + return out +} + +// MemoryNodesOfKind returns the ids of the NUMA nodes holding the given kind of +// memory. +func MemoryNodesOfKind(m *Machine, kind MemoryKind) []ID { + var out []ID + for _, node := range m.MemoryNodes() { + if node.Kind() == kind { + out = append(out, node.ID()) + } + } + return out +} + +// +// shared helpers +// + +// sameAsSomeZone reports whether cpus is exactly the CPUs of one of the zones at +// a level. It is how a derived grouping tells whether it is saying anything the +// topology does not already say. +func sameAsSomeZone(m *Machine, level Level, cpus libcpu.CPUSet) bool { + for _, z := range m.Zones(level) { + if z.CPUs().Equals(cpus) { + return true + } + } + return false +} + +// groupCoordinates returns where a cache group sits, taken from one of its CPUs. +// Every CPU of a group is in the same package and die; a group which straddles +// either would have been left out for matching no zone. +func groupCoordinates(m *Machine, cache *Cache) Coordinates { + cpus := cache.CPUs().List() + if len(cpus) == 0 { + return Coordinates{ + CPU: unknownID, Package: unknownID, Die: unknownID, + Cluster: unknownID, MemoryNode: unknownID, Core: unknownID, + } + } + return m.TopologyIndex().CoordinatesOf(cpus[0]) +} diff --git a/pkg/lib/hardware/cpu.go b/pkg/lib/hardware/cpu.go new file mode 100644 index 000000000..af739d122 --- /dev/null +++ b/pkg/lib/hardware/cpu.go @@ -0,0 +1,259 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "strconv" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// CPU is one CPU as the kernel counts them, i.e. a hardware thread. It is a +// handle into the [Machine] which produced it: never nil, and safe to use even +// when it refers to no CPU, in which case it reports Valid() == false. +type CPU struct { + m *Machine + dir string // where it was read from, for the write paths + id ID + valid bool + + online bool + isolated bool + kind CoreKind + + pkg ID + die ID + cluster ID + node ID + core ID + + threads *libcpu.CpuMask + freq Freq + caches []*Cache + + // zone at each level, indexed by Level, filled in when the zones are built + zones [numLevels]*Zone +} + +// invalidCPU is what a lookup for a CPU the machine does not have returns. Its +// methods answer with zero values rather than panicking. +var invalidCPU = &CPU{ + pkg: unknownID, die: unknownID, cluster: unknownID, + node: unknownID, core: unknownID, +} + +// Valid reports whether this handle refers to a CPU the machine has. +func (c *CPU) Valid() bool { + return c.valid +} + +// ID returns the id of this CPU. +func (c *CPU) ID() ID { + return c.id +} + +// Online reports whether this CPU is online. Only an online CPU has a known +// place in the topology: the zones and ids of an offline CPU are not known and +// read as invalid. +func (c *CPU) Online() bool { + return c.online +} + +// Isolated reports whether the kernel was told to isolate this CPU. +func (c *CPU) Isolated() bool { + return c.isolated +} + +// Kind returns whether this is a performance or an efficiency core. On a +// machine whose cores are all alike it is [PerformanceCore]. +func (c *CPU) Kind() CoreKind { + return c.kind +} + +// Zone returns the zone at the given level which contains this CPU, or an +// invalid zone if the machine has no zones at that level. +func (c *CPU) Zone(level Level) *Zone { + if level < 0 || int(level) >= numLevels || c.zones[level] == nil { + return invalidZone + } + return c.zones[level] +} + +// PackageID returns the id of the package this CPU is in, or -1 if unknown. It +// is shorthand for c.Zone(LevelPackage).ID(). +func (c *CPU) PackageID() ID { + return c.pkg +} + +// DieID returns the id of the die this CPU is in, or -1 if unknown. +func (c *CPU) DieID() ID { + return c.die +} + +// ClusterID returns the id of the cluster this CPU is in, or -1 if unknown. +func (c *CPU) ClusterID() ID { + return c.cluster +} + +// NodeID returns the id of the NUMA node this CPU is in, or -1 if unknown. +func (c *CPU) NodeID() ID { + return c.node +} + +// CoreID returns the id of the core this CPU is a thread of, or -1 if unknown. +func (c *CPU) CoreID() ID { + return c.core +} + +// Threads returns all the CPUs of this CPU's core, including this one. On a +// machine without simultaneous multithreading that is this CPU alone. The set +// is sealed. +func (c *CPU) Threads() *libcpu.CpuMask { + if c.threads == nil { + return emptyCPUs + } + return c.threads +} + +// MemoryNode returns the NUMA node this CPU belongs to. +func (c *CPU) MemoryNode() *MemoryNode { + if c.m == nil { + return invalidMemoryNode + } + return c.m.MemoryNode(c.node) +} + +// Caches returns the caches this CPU uses, lowest level first. +func (c *CPU) Caches() []*Cache { + return c.caches +} + +// Cache returns this CPU's cache at the given level, or an invalid cache if it +// has none. Where a CPU has both a data and an instruction cache at a level, +// this returns the data one; use [CPU.Caches] to see all of them. +func (c *CPU) Cache(level int) *Cache { + for _, cache := range c.caches { + if cache.level == level { + return cache + } + if cache.level > level { + break + } + } + return invalidCache +} + +// coordinates is where this CPU sits, as one value. +func (c *CPU) coordinates() Coordinates { + return Coordinates{ + CPU: c.id, Package: c.pkg, Die: c.die, Cluster: c.cluster, + MemoryNode: c.node, Core: c.core, Kind: c.kind, + } +} + +// Freq returns what is known about this CPU's clock frequency. Absent cpufreq +// support the zero value is returned. +func (c *CPU) Freq() Freq { + return c.freq +} + +// String returns "cpu#". +func (c *CPU) String() string { + if !c.valid { + return "cpu#?" + } + return "cpu#" + strconv.Itoa(c.id) +} + +// CoreKind classifies a core by the role the hardware intends it for. +type CoreKind int + +const ( + // PerformanceCore is a P-core, and the kind of every core on a machine + // which does not distinguish. + PerformanceCore CoreKind = iota + // EfficientCore is an E-core. + EfficientCore +) + +// String returns "P-core" or "E-core". +func (k CoreKind) String() string { + switch k { + case PerformanceCore: + return "P-core" + case EfficientCore: + return "E-core" + } + return "unknown core kind" +} + +// Freq is what is known about a CPU's clock frequency, in kHz. A zero field +// means the machine did not report that value. +type Freq struct { + // Base is the frequency the CPU is nominally clocked at. + Base uint64 + // Min is the lowest frequency it can be scaled to. + Min uint64 + // Max is the highest, including any turbo range. + Max uint64 + // EPP is the energy performance preference the governor is set to. + EPP EPP +} + +// EPP is an energy performance preference: how the governor is told to trade +// power against performance. Lower values prefer performance. +type EPP int + +const ( + // EPPPerformance prefers performance unconditionally. + EPPPerformance EPP = iota + // EPPBalancePerformance prefers performance but will save power. + EPPBalancePerformance + // EPPBalancePower prefers saving power but will perform. + EPPBalancePower + // EPPPower prefers saving power unconditionally. + EPPPower + // EPPUnknown is reported when the machine does not say. + EPPUnknown +) + +// String returns the kernel's name for the preference, or "" for +// [EPPUnknown]. +func (e EPP) String() string { + if e < 0 || int(e) >= len(eppNames) { + return "" + } + return eppNames[e] +} + +// eppNames are the kernel's names for the preferences, indexed by [EPP]. +// EPPUnknown is last and has no name. +var eppNames = [...]string{ + EPPPerformance: "performance", + EPPBalancePerformance: "balance_performance", + EPPBalancePower: "balance_power", + EPPPower: "power", + EPPUnknown: "", +} + +// ParseEPP returns the preference the kernel calls s, or [EPPUnknown]. +func ParseEPP(s string) EPP { + for epp, name := range eppNames { + if name != "" && name == s { + return EPP(epp) + } + } + return EPPUnknown +} diff --git a/pkg/lib/hardware/derived_test.go b/pkg/lib/hardware/derived_test.go new file mode 100644 index 000000000..5ba6ab255 --- /dev/null +++ b/pkg/lib/hardware/derived_test.go @@ -0,0 +1,494 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "slices" + "testing" + "testing/fstest" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// TestTopologyIndexAgreesWithZones checks that the flattened index answers the +// same thing the zones do. It is a lookup table over them, so any disagreement +// is a bug in the flattening. +func TestTopologyIndexAgreesWithZones(t *testing.T) { + for _, name := range recordedTrees { + t.Run(name, func(t *testing.T) { + m := openRecorded(t, name) + x := m.TopologyIndex() + if x == nil { + t.Fatal("TopologyIndex() is nil") + } + + // the whole-machine sets are the machine's own + if !x.AllCPUs().Equals(m.PresentCPUs()) { + t.Errorf("AllCPUs %s != PresentCPUs %s", x.AllCPUs(), m.PresentCPUs()) + } + if !x.OnlineCPUs().Equals(m.OnlineCPUs()) { + t.Error("OnlineCPUs disagrees with the machine") + } + if !x.OfflineCPUs().Equals(m.OfflineCPUs()) { + t.Error("OfflineCPUs disagrees with the machine") + } + if !x.IsolatedCPUs().Equals(m.IsolatedCPUs()) { + t.Error("IsolatedCPUs disagrees with the machine") + } + + // every coordinate the index enumerates has the zone's CPUs + for _, z := range m.Zones(LevelPackage) { + if got := x.PackageCPUs(z.ID()); !got.Equals(z.CPUs()) { + t.Errorf("PackageCPUs(%d) = %s, want %s", z.ID(), got, z.CPUs()) + } + } + for _, z := range m.Zones(LevelDie) { + if got := x.DieCPUs(z.DieID()); !got.Equals(z.CPUs()) { + t.Errorf("DieCPUs(%s) = %s, want %s", z.DieID(), got, z.CPUs()) + } + } + for _, z := range m.Zones(LevelCluster) { + if got := x.ClusterCPUs(z.ClusterID()); !got.Equals(z.CPUs()) { + t.Errorf("ClusterCPUs(%s) = %s, want %s", + z.ClusterID(), got, z.CPUs()) + } + } + for _, n := range m.MemoryNodes() { + if got := x.MemoryNodeCPUs(n.ID()); !got.Equals(n.CPUs()) { + t.Errorf("MemoryNodeCPUs(%d) = %s, want %s", n.ID(), got, n.CPUs()) + } + } + for _, c := range m.Caches(0) { + // c.CacheID(), not a hand-built one: the kind is part of the + // coordinate and leaving it out names a different cache + if got := x.CacheCPUs(c.CacheID()); !got.Equals(c.CPUs()) { + t.Errorf("CacheCPUs(%s) = %s, want %s", + c.CacheID(), got, c.CPUs()) + } + } + + // the enumerations cover exactly what the machine has + if got, want := len(x.PackageIDs()), len(m.Zones(LevelPackage)); got != want { + t.Errorf("PackageIDs has %d entries, want %d", got, want) + } + if got, want := len(x.DieIDs()), len(m.Zones(LevelDie)); got != want { + t.Errorf("DieIDs has %d entries, want %d", got, want) + } + if got, want := len(x.MemoryNodeIDs()), len(m.MemoryNodes()); got != want { + t.Errorf("MemoryNodeIDs has %d entries, want %d", got, want) + } + if got, want := len(x.CacheIDs()), len(m.Caches(0)); got != want { + t.Errorf("CacheIDs has %d entries, want %d", got, want) + } + + // filtering by package keeps only that package's dies and cores + for _, pkg := range x.PackageIDs() { + for _, die := range x.DieIDs(pkg) { + if die.Package != pkg { + t.Errorf("DieIDs(%d) returned %s", pkg, die) + } + } + for _, core := range x.CoreIDs(pkg) { + if core.Package != pkg { + t.Errorf("CoreIDs(%d) returned %s", pkg, core) + } + } + } + + // thread siblings agree with the CPUs' own view + for _, id := range m.CPUIDs() { + if !m.CPU(id).Online() { + continue + } + if got := x.ThreadsOf(id); !got.Equals(m.CPU(id).Threads()) { + t.Errorf("ThreadsOf(%d) = %s, want %s", + id, got, m.CPU(id).Threads()) + } + } + + // coordinates round-trip back to the right zones + for _, id := range m.CPUIDs() { + c := m.CPU(id) + if !c.Online() { + continue + } + at := x.CoordinatesOf(id) + if at.CPU != id { + t.Errorf("CoordinatesOf(%d).CPU = %d", id, at.CPU) + } + if at.Package != c.PackageID() || at.Core != c.CoreID() || + at.MemoryNode != c.NodeID() || at.Kind != c.Kind() { + t.Errorf("CoordinatesOf(%d) = %s disagrees with the CPU", id, at) + } + if !x.PackageCPUs(at.Package).Contains(id) { + t.Errorf("cpu%d is not in its own package's CPUs", id) + } + } + + // an absent CPU answers with unknown coordinates, not a panic + at := x.CoordinatesOf(1 << 20) + if at.CPU != unknownID || at.Package != unknownID { + t.Errorf("CoordinatesOf(absent) = %s, want all unknown", at) + } + // so do absent coordinates + if !x.PackageCPUs(1 << 20).IsEmpty() { + t.Error("PackageCPUs of an absent package is not empty") + } + if !x.DieCPUs(DieID{Package: 1 << 20}).IsEmpty() { + t.Error("DieCPUs of an absent die is not empty") + } + + // asking twice gives the same table + if m.TopologyIndex() != x { + t.Error("TopologyIndex() built a second table") + } + + t.Logf("\n%s", x) + }) + } +} + +// TestDerivedSets checks the set-to-set helpers against the topology they are +// derived from. +func TestDerivedSets(t *testing.T) { + for _, name := range recordedTrees { + t.Run(name, func(t *testing.T) { + m := openRecorded(t, name) + online := m.OnlineCPUs() + + // AllThreads only ever grows a set, and always to whole cores + for _, cpus := range sampleSets(m) { + all := AllThreads(m, cpus) + if !cpus.IsSubsetOf(all) { + t.Errorf("AllThreads(%s) = %s dropped CPUs", cpus, all) + } + assertSealedNamed(t, "AllThreads result", all) + if again := AllThreads(m, all); !again.Equals(all) { + t.Errorf("AllThreads is not idempotent on %s", cpus) + } + all.ForEachCpu(func(id int) bool { + if c := m.CPU(id); c.Online() && !c.Threads().IsSubsetOf(all) { + t.Errorf("AllThreads(%s) = %s is missing a sibling of cpu%d", + cpus, all, id) + } + return true + }) + } + + // SingleThreadPerCore only ever shrinks, and keeps one CPU per core + for _, cpus := range sampleSets(m) { + one := SingleThreadPerCore(m, cpus) + if !one.IsSubsetOf(cpus) { + t.Errorf("SingleThreadPerCore(%s) = %s added CPUs", cpus, one) + } + assertSealedNamed(t, "SingleThreadPerCore result", one) + if again := SingleThreadPerCore(m, one); !again.Equals(one) { + t.Errorf("SingleThreadPerCore is not idempotent on %s", cpus) + } + seen := libcpu.NewCpuMask() + one.ForEachCpu(func(id int) bool { + if seen.Contains(id) { + t.Errorf("SingleThreadPerCore(%s) kept two threads of a core", + cpus) + } + seen.Set(m.CPU(id).Threads().UnsortedList()...) + return true + }) + } + + // CPUsSharingCache grows to whole cache groups + for _, level := range m.CacheLevels() { + for _, cpus := range sampleSets(m) { + all := CPUsSharingCache(m, level, cpus) + if !cpus.IsSubsetOf(all) { + t.Errorf("CPUsSharingCache(%d, %s) dropped CPUs", level, cpus) + } + assertSealedNamed(t, "CPUsSharingCache result", all) + } + } + + // grouping cache levels come coarsest first, and the groups at each + // are non-trivial, disjoint, and within the machine + coarser := 0 + for i, grouping := range GroupingCacheLevels(m) { + t.Logf("grouping cache level %d, %d groups", + grouping.Level, len(grouping.Groups)) + + if i > 0 && grouping.Level >= coarser { + t.Errorf("cache level %d follows %d, not coarsest first", + grouping.Level, coarser) + } + coarser = grouping.Level + + if len(grouping.Groups) == 0 { + t.Errorf("cache level %d reported with no groups", grouping.Level) + } + + // groups of one level partition, groups of different levels nest + seen := emptyCPUs.Union() + for _, g := range grouping.Groups { + if g.Level() != grouping.Level { + t.Errorf("cache group %s is not at level %d", + g.Key(), grouping.Level) + } + if g.CPUs().Size() <= 1 { + t.Errorf("cache group %s has %d CPUs", g.Key(), g.CPUs().Size()) + } + if g.CPUs().Intersects(seen) { + t.Errorf("cache group %s overlaps another", g.Key()) + } + seen = seen.Union(g.CPUs()) + if sameAsSomeZone(m, LevelCore, g.CPUs()) { + t.Errorf("cache group %s is just a core", g.Key()) + } + if sameAsSomeZone(m, LevelPackage, g.CPUs()) { + t.Errorf("cache group %s is just a package", g.Key()) + } + } + } + + // logical clusters, for every die the machine has + for _, die := range m.TopologyIndex().DieIDs() { + dieCPUs := m.TopologyIndex().DieCPUs(die) + + // omitted: nothing reported is just a core, and every cluster + // reported is one of the die's own + for _, cpus := range LogicalClusters(m, die.Package, die.Die, + OmitSingleCoreClusters) { + if !cpus.IsSubsetOf(dieCPUs) { + t.Errorf("LogicalClusters(%s) returned %s, not on the die", + die, cpus) + } + if sameAsSomeZone(m, LevelCore, cpus) { + t.Errorf("logical cluster %s is just a core", cpus) + } + } + + // merged: the same clusters plus at most one more, and together + // they cover no more than the die + var ( + omit = LogicalClusters(m, die.Package, die.Die, OmitSingleCoreClusters) + merge = LogicalClusters(m, die.Package, die.Die, MergeSingleCoreClusters) + seen = emptyCPUs.Union() + ) + if len(merge) != len(omit) && len(merge) != len(omit)+1 { + t.Errorf("%s: %d merged clusters, want %d or %d", + die, len(merge), len(omit), len(omit)+1) + } + for _, cpus := range merge { + if cpus.Intersects(seen) { + t.Errorf("%s: merged cluster %s overlaps another", die, cpus) + } + seen = seen.Union(cpus) + if !cpus.IsSubsetOf(dieCPUs) { + t.Errorf("%s: merged cluster %s is not on the die", die, cpus) + } + } + + // the ids the ordering promises are increasing + last := -1 + for _, cpus := range merge { + id := m.CPU(cpus.List()[0]).ClusterID() + if id <= last { + t.Errorf("%s: cluster ids are not increasing: %d after %d", + die, id, last) + } + last = id + } + } + + // closest nodes: nearest first, never the node itself + for _, node := range m.MemoryNodes() { + groups := ClosestMemoryNodes(m, node.ID(), nil) + last := -1 + for _, g := range groups { + if g.Distance <= last { + t.Errorf("node#%d: distances are not increasing: %d after %d", + node.ID(), g.Distance, last) + } + last = g.Distance + if slices.Contains(g.Nodes, node.ID()) { + t.Errorf("node#%d is among its own closest nodes", node.ID()) + } + } + + // a match which accepts nothing yields nothing + if got := ClosestMemoryNodes(m, node.ID(), func(*MemoryNode) bool { + return false + }); len(got) != 0 { + t.Errorf("node#%d: a false match returned %d groups", node.ID(), len(got)) + } + } + if got := ClosestMemoryNodes(m, 1<<20, nil); got != nil { + t.Error("ClosestMemoryNodes of an absent node returned groups") + } + + // nodes local to a set of CPUs + if got := MemoryNodesFor(m, online); len(got) == 0 { + t.Error("no NUMA nodes are local to the online CPUs") + } + if got := MemoryNodesFor(m, emptyCPUs); len(got) != 0 { + t.Errorf("MemoryNodesFor(empty) returned %v", got) + } + + // every node is of exactly one kind + total := 0 + for _, kind := range []MemoryKind{ + MemoryKindDRAM, MemoryKindPMEM, MemoryKindHBM, MemoryKindUnknown, + } { + total += len(MemoryNodesOfKind(m, kind)) + } + if total != len(m.MemoryNodes()) { + t.Errorf("kinds cover %d nodes, want %d", total, len(m.MemoryNodes())) + } + }) + } +} + +// sampleSets returns a spread of CPU sets to exercise a set-to-set helper with: +// nothing, one CPU, one core, one node, one package, everything, and a set which +// deliberately cuts across cores. +func sampleSets(m *Machine) []*libcpu.CpuMask { + sets := []*libcpu.CpuMask{emptyCPUs, m.OnlineCPUs()} + + ids := m.OnlineCPUs().List() + if len(ids) == 0 { + return sets + } + + sets = append(sets, sealed(libcpu.NewCpuMask(ids[0]))) + sets = append(sets, m.CPU(ids[0]).Threads()) + + if nodes := m.MemoryNodes(); len(nodes) > 0 { + sets = append(sets, nodes[0].CPUs()) + } + if pkgs := m.Zones(LevelPackage); len(pkgs) > 0 { + sets = append(sets, pkgs[0].CPUs()) + } + + // every other online CPU, which splits cores wherever they are paired + ragged := libcpu.NewCpuMask() + for i := 0; i < len(ids); i += 2 { + ragged.Set(ids[i]) + } + sets = append(sets, sealed(ragged)) + + return sets +} + +// hybridCacheFS is a machine whose two kinds of core are grouped at two +// different cache levels: CPUs 0-3 share an L3 the other four have no access to, +// and CPUs 4-7 share an L2 the first four do not. Each level therefore holds +// exactly one group. +// +// No recorded tree looks like this, and it is the shape which says why +// [GroupingCacheLevels] answers with every level rather than with one. +func hybridCacheFS() fstest.MapFS { + fsys := fstest.MapFS{ + "proc/meminfo": file("MemTotal: 1048576 kB\n"), + } + + fsys["sys/devices/system/cpu/possible"] = file("0-7\n") + fsys["sys/devices/system/cpu/present"] = file("0-7\n") + fsys["sys/devices/system/cpu/online"] = file("0-7\n") + + for i := 0; i < 8; i++ { + dir := "sys/devices/system/cpu/cpu" + itoa(i) + fsys[dir+"/topology/physical_package_id"] = file("0\n") + fsys[dir+"/topology/core_id"] = file(itoa(i) + "\n") + fsys[dir+"/topology/core_cpus_list"] = file(itoa(i) + "\n") + + if i < 4 { + // a private L2, and an L3 shared by these four only + fsys[dir+"/cache/index0/level"] = file("2\n") + fsys[dir+"/cache/index0/type"] = file("Unified\n") + fsys[dir+"/cache/index0/id"] = file(itoa(i) + "\n") + fsys[dir+"/cache/index0/shared_cpu_list"] = file(itoa(i) + "\n") + fsys[dir+"/cache/index0/size"] = file("2048K\n") + fsys[dir+"/cache/index1/level"] = file("3\n") + fsys[dir+"/cache/index1/type"] = file("Unified\n") + fsys[dir+"/cache/index1/id"] = file("0\n") + fsys[dir+"/cache/index1/shared_cpu_list"] = file("0-3\n") + fsys[dir+"/cache/index1/size"] = file("16384K\n") + } else { + // one L2 for the four of them, and no L3 at all + fsys[dir+"/cache/index0/level"] = file("2\n") + fsys[dir+"/cache/index0/type"] = file("Unified\n") + fsys[dir+"/cache/index0/id"] = file("4\n") + fsys[dir+"/cache/index0/shared_cpu_list"] = file("4-7\n") + fsys[dir+"/cache/index0/size"] = file("4096K\n") + } + } + + return fsys +} + +func TestGroupingCacheLevels(t *testing.T) { + t.Run("two levels, one group each", func(t *testing.T) { + m, err := Discover(WithFS(hybridCacheFS())) + if err != nil { + t.Fatalf("failed to discover the test machine: %v", err) + } + + groupings := GroupingCacheLevels(m) + if len(groupings) != 2 { + t.Fatalf("expected two grouping levels, got %v", groupings) + } + + // coarsest first, so the L3 of CPUs 0-3 comes before the L2 of 4-7 + for i, want := range []struct { + level int + cpus string + }{ + {level: 3, cpus: "0-3"}, + {level: 2, cpus: "4-7"}, + } { + got := groupings[i] + if got.Level != want.level { + t.Errorf("grouping %d: expected level %d, got %d", + i, want.level, got.Level) + } + if len(got.Groups) != 1 { + t.Fatalf("grouping %d: expected one group, got %v", i, got.Groups) + } + if cpus := got.Groups[0].CPUs().String(); cpus != want.cpus { + t.Errorf("grouping %d: expected CPUs %s, got %s", i, want.cpus, cpus) + } + } + }) + + t.Run("sample1", func(t *testing.T) { + m := openRecorded(t, "sample1") + + // The one recorded machine with a cache grouping of its own: its two + // E-core modules share an L2 each, and nothing else at any level says + // anything a coarser or finer level does not. + groupings := GroupingCacheLevels(m) + if len(groupings) != 1 { + t.Fatalf("expected one grouping level, got %v", groupings) + } + if groupings[0].Level != 2 { + t.Errorf("expected level 2, got %d", groupings[0].Level) + } + if len(groupings[0].Groups) != 2 { + t.Fatalf("expected two groups, got %v", groupings[0].Groups) + } + for _, g := range groupings[0].Groups { + if g.CPUs().Size() != 4 { + t.Errorf("expected a group of 4 CPUs, got %s", g.CPUs()) + } + } + }) +} diff --git a/pkg/lib/hardware/discover.go b/pkg/lib/hardware/discover.go new file mode 100644 index 000000000..c148c2f56 --- /dev/null +++ b/pkg/lib/hardware/discover.go @@ -0,0 +1,665 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "fmt" + "io/fs" + "path" + "slices" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// Where things are, relative to the host root. +const ( + sysCPUDir = "sys/devices/system/cpu" + sysNodeDir = "sys/devices/system/node" + procMemDir = "proc" +) + +// Discover reads the topology of a machine and returns it. +func Discover(opts ...Option) (*Machine, error) { + o := &options{} + for _, apply := range opts { + if err := apply(o); err != nil { + return nil, err + } + } + if o.fsys == nil { + o.fsys = HostFS("/") + } + + d := &discovery{ + m: &Machine{ + fsys: o.fsys, + cpus: map[ID]*CPU{}, + nodes: map[ID]*MemoryNode{}, + caches: map[CacheID]*Cache{}, + kinds: map[CoreKind]*libcpu.CpuMask{}, + zones: map[Level][]*Zone{}, + }, + opts: o, + } + + for _, step := range []struct { + what string + run func() error + }{ + {"CPUs", d.discoverCPUs}, + {"core kinds", d.discoverCoreKinds}, + {"NUMA nodes", d.discoverMemoryNodes}, + {"memory kinds", d.classifyMemory}, + {"zones", d.buildZones}, + } { + if err := step.run(); err != nil { + return nil, fmt.Errorf("failed to discover %s: %w", step.what, err) + } + } + + d.m.index = d.m.buildIndex() + + return d.m, nil +} + +// discovery is the state of one Discover call. +type discovery struct { + m *Machine + opts *options +} + +// fsys is the filesystem being read. +func (d *discovery) fsys() fs.FS { + return d.m.fsys +} + +// +// CPUs +// + +// discoverCPUs reads the CPU sets the kernel publishes for the machine as a +// whole, then every cpuN directory. +func (d *discovery) discoverCPUs() error { + m := d.m + + // The four machine-wide sets. Only "present" is essential; a kernel which + // does not publish one of the others leaves it empty rather than failing, + // except that "online" falls back to "present" because too much depends on + // knowing which CPUs are usable. + m.possible = d.readCPUsOrEmpty(path.Join(sysCPUDir, "possible")) + m.present = d.readCPUsOrEmpty(path.Join(sysCPUDir, "present")) + m.online = d.readCPUsOrEmpty(path.Join(sysCPUDir, "online")) + m.isolated = d.readCPUsOrEmpty(path.Join(sysCPUDir, "isolated")) + + if m.online.IsEmpty() && !m.present.IsEmpty() { + m.online = m.present + } + if m.possible.IsEmpty() { + m.possible = m.present + } + + names, ids, err := globIDs(d.fsys(), path.Join(sysCPUDir, "cpu[0-9]*")) + if err != nil { + return err + } + if len(names) == 0 { + return fmt.Errorf("no CPUs found under %s", sysCPUDir) + } + + for i, name := range names { + cpu, err := d.discoverCPU(name, ids[i]) + if err != nil { + return fmt.Errorf("cpu%d: %w", ids[i], err) + } + m.cpus[cpu.id] = cpu + m.cpuIDs = append(m.cpuIDs, cpu.id) + } + + slices.Sort(m.cpuIDs) + + // A kernel which publishes no "present" is old enough that we should just + // believe the directories instead. + if m.present.IsEmpty() { + present := libcpu.NewCpuMask(m.cpuIDs...) + present.Seal() + m.present = present + if m.online.IsEmpty() { + m.online = present + } + } + + return nil +} + +// discoverCPU reads one cpuN directory. +// +// An offline CPU publishes no topology at all, so its ids stay unknown. That is +// not an error: the machine is expected to have offline CPUs. +func (d *discovery) discoverCPU(dir string, id ID) (*CPU, error) { + c := &CPU{ + m: d.m, + id: id, + valid: true, + dir: dir, + online: d.m.online.Contains(id), + isolated: d.m.isolated.Contains(id), + pkg: unknownID, + die: unknownID, + cluster: unknownID, + node: unknownID, + core: unknownID, + } + + if c.online { + if err := d.readCPUTopology(c); err != nil { + return nil, err + } + } + + c.freq = d.readCPUFreq(c) + + caches, err := d.discoverCPUCaches(c) + if err != nil { + return nil, err + } + c.caches = caches + + return c, nil +} + +// readCPUTopology reads the topology/ ids of an online CPU. The package and core +// ids are required; the rest of the machine cannot be assembled without them. +// A die or cluster the kernel does not report stays unknown. +func (d *discovery) readCPUTopology(c *CPU) error { + topo := path.Join(c.dir, "topology") + + pkg, err := readInt(d.fsys(), path.Join(topo, "physical_package_id")) + if err != nil { + return fmt.Errorf("no package id: %w", err) + } + c.pkg = pkg + + core, err := readInt(d.fsys(), path.Join(topo, "core_id")) + if err != nil { + return fmt.Errorf("no core id: %w", err) + } + c.core = core + + if die, err := readInt(d.fsys(), path.Join(topo, "die_id")); err == nil { + c.die = die + } + if cluster, err := readInt(d.fsys(), path.Join(topo, "cluster_id")); err == nil { + c.cluster = cluster + } + + // core_cpus_list is the current name, thread_siblings_list the old one. + threads, err := readCPUs(d.fsys(), path.Join(topo, "core_cpus_list")) + if err != nil { + threads, err = readCPUs(d.fsys(), path.Join(topo, "thread_siblings_list")) + if err != nil { + return fmt.Errorf("no thread siblings: %w", err) + } + } + c.threads = threads + + // Which NUMA node a CPU is in shows up as a nodeN symlink in its directory. + // A kernel built without NUMA has none, and everything is in node 0. + if nodes, err := glob(d.fsys(), path.Join(c.dir, "node[0-9]*")); err == nil { + if len(nodes) == 1 { + if node, ok := trailingID(nodes[0]); ok { + c.node = node + } + } + } + if c.node == unknownID { + c.node = 0 + } + + // A die or a cluster the kernel does not name is still one of each: treat the + // package as holding a single one, so that the level exists uniformly and is + // simply uninformative. [Machine.SameZones] is then how a caller finds out + // that it says nothing -- on such a machine the die and cluster levels hold + // the same zones as each other and as the package. + if c.die == unknownID { + c.die = 0 + } + if c.cluster == unknownID { + c.cluster = 0 + } + + return nil +} + +// readCPUFreq reads what cpufreq says about a CPU. Absent cpufreq support, or +// with the values overridden, the zero value or the override stands. +func (d *discovery) readCPUFreq(c *CPU) Freq { + if freq, ok := d.opts.freq[c.id]; ok { + return freq + } + + dir := path.Join(c.dir, "cpufreq") + freq := Freq{EPP: EPPUnknown} + + if base, err := readUint64(d.fsys(), path.Join(dir, "base_frequency")); err == nil { + freq.Base = base + } + if min, err := readUint64(d.fsys(), path.Join(dir, "cpuinfo_min_freq")); err == nil { + freq.Min = min + } + if max, err := readUint64(d.fsys(), path.Join(dir, "cpuinfo_max_freq")); err == nil { + freq.Max = max + } + if epp, err := readFile(d.fsys(), path.Join(dir, "energy_performance_preference")); err == nil { + freq.EPP = ParseEPP(epp) + } + + return freq +} + +// +// Caches +// + +// discoverCPUCaches reads the caches of one CPU, lowest level first. +// +// A cache shared by several CPUs is read once and shared: the second CPU to +// mention it gets the same *Cache. Its identity is (level, kind, id), because +// the kernel numbers caches within a level and kind rather than across the +// machine. +func (d *discovery) discoverCPUCaches(c *CPU) ([]*Cache, error) { + if caches, ok := d.opts.caches[c.id]; ok { + return d.internCaches(caches), nil + } + + names, _, err := globIDs(d.fsys(), path.Join(c.dir, "cache", "index[0-9]*")) + if err != nil { + return nil, err + } + + caches := make([]*Cache, 0, len(names)) + for _, name := range names { + cache, err := d.readCache(name) + if err != nil { + return nil, fmt.Errorf("%s: %w", name, err) + } + caches = append(caches, cache) + } + + // Ordered by level, then kind, so that Cache(level) and the last-level + // helpers can rely on it. globIDs already gives index order, which is + // normally the same thing, but nothing promises that. + slices.SortStableFunc(caches, func(a, b *Cache) int { + if a.level != b.level { + return a.level - b.level + } + return int(a.kind) - int(b.kind) + }) + + return caches, nil +} + +// readCache reads one cache/indexN directory, returning the shared instance if +// this cache has been seen already. +func (d *discovery) readCache(dir string) (*Cache, error) { + level, err := readInt(d.fsys(), path.Join(dir, "level")) + if err != nil { + return nil, fmt.Errorf("no level: %w", err) + } + + kindStr, err := readFile(d.fsys(), path.Join(dir, "type")) + if err != nil { + return nil, fmt.Errorf("no type: %w", err) + } + kind, err := parseCacheKind(kindStr) + if err != nil { + return nil, err + } + + // A cache without an id is one the kernel does not number. Fall back to the + // lowest CPU sharing it, which is unique per cache within a level and kind. + id, err := readInt(d.fsys(), path.Join(dir, "id")) + haveID := err == nil + + cpus, err := readCPUs(d.fsys(), path.Join(dir, "shared_cpu_list")) + if err != nil { + return nil, fmt.Errorf("no shared CPUs: %w", err) + } + if !haveID { + if cpus.IsEmpty() { + return nil, fmt.Errorf("no id and no shared CPUs") + } + id = cpus.List()[0] + } + + key := CacheID{Level: level, Kind: kind, ID: id} + if have, ok := d.m.caches[key]; ok { + return have, nil + } + + size := int64(0) + if str, err := readFile(d.fsys(), path.Join(dir, "size")); err == nil { + if size, err = parseSize(str); err != nil { + return nil, fmt.Errorf("bad size %q: %w", str, err) + } + } + + cache := &Cache{ + m: d.m, + valid: true, + id: id, + level: level, + kind: kind, + size: size, + cpus: cpus, + } + d.m.caches[key] = cache + + return cache, nil +} + +// internCaches turns overridden cache descriptions into shared instances. +func (d *discovery) internCaches(caches []*Cache) []*Cache { + out := make([]*Cache, 0, len(caches)) + for _, c := range caches { + key := CacheID{Level: c.level, Kind: c.kind, ID: c.id} + have, ok := d.m.caches[key] + if !ok { + c.m, c.valid = d.m, true + d.m.caches[key] = c + have = c + } + out = append(out, have) + } + return out +} + +// +// Core kinds +// + +// discoverCoreKinds works out which CPUs are performance cores and which are +// efficiency cores. +// +// The kernel exposes this as two lists, one per kind. A machine whose cores are +// all alike has neither, and everything is a performance core. A machine which +// names only one kind has the rest inferred, since there are only two. +func (d *discovery) discoverCoreKinds() error { + m := d.m + + if len(d.opts.kinds) > 0 { + for kind, cpus := range d.opts.kinds { + m.kinds[kind] = cpus + } + } else { + for kind, dir := range coreKindDirs { + cpus, err := readCPUs(d.fsys(), dir) + if err != nil || cpus.IsEmpty() { + continue + } + m.kinds[kind] = cpus + } + } + + switch len(m.kinds) { + case 0: + m.kinds[PerformanceCore] = m.online + + case 1: + for kind, cpus := range m.kinds { + // Round the named kind up to whole cores, and let the other kind be + // whatever is left. A partial list is a list of cores, not threads. + named := m.allThreads(cpus) + if named.Equals(m.online) { + break + } + rest := m.online.Difference(named) + + named.Seal() + other := libcpu.NewCpuMask(rest.UnsortedList()...) + other.Seal() + + m.kinds[kind] = named + m.kinds[otherCoreKind(kind)] = other + break + } + } + + return d.checkCoreKinds() +} + +// checkCoreKinds rejects a core kind split which cannot be true: kinds have to +// be whole cores, a core cannot be of two kinds, and every online CPU has to +// have one. +func (d *discovery) checkCoreKinds() error { + m := d.m + + seen := libcpu.NewCpuMask() + for kind, cpus := range m.kinds { + if missing := m.allThreads(cpus).Difference(cpus); !missing.IsEmpty() { + return fmt.Errorf("%s CPUs (%s) are missing thread siblings (%s)", + kind, cpus, missing) + } + if overlap := cpus.Intersection(seen); !overlap.IsEmpty() { + return fmt.Errorf("%s CPUs (%s) overlap another kind (%s)", + kind, cpus, overlap) + } + seen = libcpu.NewCpuMask(seen.Union(cpus).UnsortedList()...) + } + + if missing := m.online.Difference(seen); !missing.IsEmpty() { + return fmt.Errorf("CPUs %s are of no known core kind", missing) + } + + for kind, cpus := range m.kinds { + cpus.ForEachCpu(func(id int) bool { + if c, ok := m.cpus[id]; ok { + c.kind = kind + } + return true + }) + } + + return nil +} + +// +// Memory nodes +// + +// discoverMemoryNodes reads the NUMA nodes. +// +// A kernel built without NUMA has no node directories at all, in which case +// there is one node holding every online CPU and all of the memory. +func (d *discovery) discoverMemoryNodes() error { + m := d.m + + names, ids, err := globIDs(d.fsys(), path.Join(sysNodeDir, "node[0-9]*")) + if err != nil { + return err + } + + if len(names) == 0 { + node := &MemoryNode{ + m: m, + valid: true, + id: 0, + cpus: m.online, + distance: []int{localDistance}, + normal: true, + kind: MemoryKindDRAM, + meminfo: path.Join(procMemDir, "meminfo"), + } + + // Its capacity is the machine's own, there being no node to ask. Read it + // here as the per-node case below does: a node whose capacity is unknown + // reads as a node with no memory, which is worse than not discovering. + info, err := readMemInfo(d.fsys(), node.meminfo, node.id) + if err != nil { + return fmt.Errorf("no NUMA nodes and no %s: %w", node.meminfo, err) + } + node.capacity = info.Total + + m.nodes[0] = node + m.nodeIDs = []ID{0} + return nil + } + + normal := d.readCPUsOrEmpty(path.Join(sysNodeDir, "has_normal_memory")) + + for i, name := range names { + node := &MemoryNode{ + m: m, + valid: true, + id: ids[i], + dir: name, + meminfo: path.Join(name, "meminfo"), + kind: MemoryKindUnknown, + normal: normal.Contains(ids[i]), + } + + if node.cpus, err = readCPUs(d.fsys(), path.Join(name, "cpulist")); err != nil { + return fmt.Errorf("node%d: no CPU list: %w", ids[i], err) + } + if node.distance, err = readInts(d.fsys(), path.Join(name, "distance"), " "); err != nil { + return fmt.Errorf("node%d: no distance vector: %w", ids[i], err) + } + + info, err := readMemInfo(d.fsys(), node.meminfo, ids[i]) + if err != nil { + return fmt.Errorf("node%d: %w", ids[i], err) + } + node.capacity = info.Total + + m.nodes[node.id] = node + m.nodeIDs = append(m.nodeIDs, node.id) + } + + slices.Sort(m.nodeIDs) + + d.symmetrizeDistances() + + return nil +} + +// symmetrizeDistances averages a NUMA distance matrix which is not symmetric. +// +// Distance is a cost, and a cost which differs by direction breaks every caller +// which treats it as one. Rather than let that surface far away, average the two +// and say so. +func (d *discovery) symmetrizeDistances() { + m := d.m + + for _, i := range m.nodeIDs { + for _, j := range m.nodeIDs { + if i >= j { + continue + } + a, b := m.nodes[i], m.nodes[j] + if j >= len(a.distance) || i >= len(b.distance) { + continue + } + if a.distance[j] == b.distance[i] { + continue + } + avg := (a.distance[j] + b.distance[i]) / 2 + a.distance[j], b.distance[i] = avg, avg + } + } +} + +// +// Options +// + +// Option configures [Discover]. +type Option func(*options) error + +// options is the accumulated configuration of a [Discover] call. +type options struct { + fsys fs.FS + kinds map[CoreKind]*libcpu.CpuMask + caches map[ID][]*Cache + freq map[ID]Freq +} + +// WithRoot reads the topology below root instead of "/", for a host filesystem +// mounted somewhere else. It is shorthand for WithFS(HostFS(root)). +func WithRoot(root string) Option { + return func(o *options) error { + o.fsys = HostFS(root) + return nil + } +} + +// WithFS reads the topology through fsys instead of the real filesystem. Paths +// are relative to the host root, so a "sys" and a "proc" directory are both +// expected within it. Discovery only reads, so a read-only fs.FS is enough. +func WithFS(fsys fs.FS) Option { + return func(o *options) error { + if fsys == nil { + return fmt.Errorf("WithFS: nil filesystem") + } + o.fsys = fsys + return nil + } +} + +// +// helpers +// + +// unknownID is the id of a coordinate the machine does not report. +const unknownID = -1 + +// localDistance is the NUMA distance from a node to itself. +const localDistance = 10 + +// coreKindDirs is where the kernel lists the CPUs of each core kind. +var coreKindDirs = map[CoreKind]string{ + PerformanceCore: "sys/devices/cpu_core/cpus", + EfficientCore: "sys/devices/cpu_atom/cpus", +} + +// otherCoreKind returns the kind which is not this one. There are two. +func otherCoreKind(kind CoreKind) CoreKind { + if kind == PerformanceCore { + return EfficientCore + } + return PerformanceCore +} + +// readCPUsOrEmpty reads a CPU list, treating anything unreadable as empty. It is +// for the attributes whose absence means "none", not "broken". +func (d *discovery) readCPUsOrEmpty(name string) *libcpu.CpuMask { + if cpus, err := readCPUs(d.fsys(), name); err == nil { + return cpus + } + return emptyCPUs +} + +// allThreads rounds a set of CPUs up to whole cores. It is on Machine rather +// than in convenience.go because discovery needs it before a Machine is finished. +func (m *Machine) allThreads(cpus libcpu.CPUSet) *libcpu.CpuMask { + all := libcpu.NewCpuMask() + cpus.ForEachCpu(func(id int) bool { + if c, ok := m.cpus[id]; ok && c.threads != nil { + all.Set(c.threads.UnsortedList()...) + } else { + all.Set(id) + } + return true + }) + return all +} diff --git a/pkg/lib/hardware/discover_test.go b/pkg/lib/hardware/discover_test.go new file mode 100644 index 000000000..b832bcc62 --- /dev/null +++ b/pkg/lib/hardware/discover_test.go @@ -0,0 +1,557 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "os" + "path/filepath" + "testing" + "testing/fstest" +) + +// recordedTrees are the sysfs trees pkg/sysfs keeps for its own tests, which +// test-setup.sh unpacks into pkg/sysfs/testdata. Discovery has to make sense of +// every one of them. +var recordedTrees = []string{"sample1", "sample2"} + +// openRecorded returns a Machine discovered from one recorded tree, skipping the +// test if the trees have not been unpacked. +func openRecorded(t *testing.T, name string) *Machine { + t.Helper() + + root, err := filepath.Abs(filepath.Join("..", "..", "sysfs", "testdata", name)) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(root, "sys")); err != nil { + t.Skipf("recorded tree %s is not unpacked: run pkg/sysfs/test-setup.sh", name) + } + + m, err := Discover(WithRoot(root)) + if err != nil { + t.Fatalf("Discover(%s): %v", name, err) + } + + return m +} + +// TestDiscoverRecorded checks the invariants which have to hold for any machine, +// against the recorded trees. It does not assert particular numbers: those +// belong in the differential test against pkg/sysfs, which has the reference +// answers. +func TestDiscoverRecorded(t *testing.T) { + for _, name := range recordedTrees { + t.Run(name, func(t *testing.T) { + m := openRecorded(t, name) + checkMachineInvariants(t, m) + + t.Logf("%s: %d CPUs (%s), %d nodes, levels %v", + name, m.PresentCPUs().Size(), m.PresentCPUs(), + len(m.MemoryNodes()), m.Levels()) + for _, level := range m.Levels() { + zones := m.Zones(level) + same := "" + for _, other := range m.Levels() { + if other != level && m.SameZones(level, other) { + same += " =" + other.String() + } + } + t.Logf(" %-8s %3d zones%s", level, len(zones), same) + } + }) + } +} + +// checkMachineInvariants asserts the things which must be true of any discovered +// machine, whatever the hardware. +func checkMachineInvariants(t *testing.T, m *Machine) { + t.Helper() + + if m.PresentCPUs().IsEmpty() { + t.Fatal("no CPUs present") + } + + // the machine-wide sets have to nest + if !m.OnlineCPUs().IsSubsetOf(m.PresentCPUs()) { + t.Errorf("online %s is not within present %s", m.OnlineCPUs(), m.PresentCPUs()) + } + if !m.PresentCPUs().IsSubsetOf(m.PossibleCPUs()) { + t.Errorf("present %s is not within possible %s", + m.PresentCPUs(), m.PossibleCPUs()) + } + if got := m.OnlineCPUs().Union(m.OfflineCPUs()); !got.Equals(m.PresentCPUs()) { + t.Errorf("online+offline %s != present %s", got, m.PresentCPUs()) + } + if m.OnlineCPUs().Intersects(m.OfflineCPUs()) { + t.Error("a CPU is both online and offline") + } + + // every set handed out has to be sealed + for _, tc := range []struct { + what string + cpus interface{ Set(...int) } + }{ + {"PossibleCPUs", m.PossibleCPUs()}, + {"PresentCPUs", m.PresentCPUs()}, + {"OnlineCPUs", m.OnlineCPUs()}, + {"IsolatedCPUs", m.IsolatedCPUs()}, + {"OfflineCPUs", m.OfflineCPUs()}, + } { + assertSealedNamed(t, tc.what, tc.cpus) + } + + // a CPU which is not there answers rather than panicking + absent := m.CPU(1 << 20) + if absent == nil { + t.Fatal("CPU() returned nil") + } + if absent.Valid() { + t.Error("a CPU which is not there reports Valid()") + } + _, _, _ = absent.PackageID(), absent.Threads(), absent.String() + if absent.Zone(LevelPackage).Valid() { + t.Error("an absent CPU has a valid package zone") + } + if absent.MemoryNode().Valid() { + t.Error("an absent CPU has a valid memory node") + } + + checkZones(t, m) + checkContainment(t, m) + checkCPUs(t, m) + checkMemoryNodes(t, m) +} + +// checkZones asserts what has to be true of the zones of every level: they are +// valid and sealed, they hold only CPUs the machine has, and the zones of one +// level do not overlap each other. +// +// There is no tree to check. That is the point: nothing here has an opinion +// about which level contains which. +func checkZones(t *testing.T, m *Machine) { + t.Helper() + + for _, level := range m.Levels() { + zones := m.Zones(level) + if len(zones) == 0 { + t.Errorf("level %s is listed but has no zones", level) + continue + } + + seen := emptyCPUs.Union() + seenID := map[ID]bool{} + for _, z := range zones { + if !z.Valid() { + t.Errorf("level %s has an invalid zone", level) + continue + } + if z.Level() != level { + t.Errorf("zone %s is listed at %s but reports %s", + z.Name(), level, z.Level()) + } + assertSealedNamed(t, "zone "+z.Name(), z.CPUs()) + + if !z.CPUs().IsSubsetOf(m.PresentCPUs()) { + t.Errorf("zone %s holds CPUs the machine does not have: %s", + z.Name(), z.CPUs().Difference(m.PresentCPUs())) + } + if z.CPUs().Intersects(seen) { + t.Errorf("zone %s overlaps another zone at level %s: %s", + z.Name(), level, z.CPUs().Intersection(seen)) + } + seen = seen.Union(z.CPUs()) + + // ids are unique within a level only where the kernel numbers them + // per machine; for dies, clusters and cores they repeat per package, + // which is why there is no Zone(level, id) lookup + if z.Level() == LevelPackage || z.Level() == LevelNUMANode { + if seenID[z.ID()] { + t.Errorf("level %s has two zones with id %d", level, z.ID()) + } + seenID[z.ID()] = true + } + } + } + + // a level the machine does not have answers empty rather than panicking + for _, level := range allLevels { + if len(m.Zones(level)) > 0 { + continue + } + for _, other := range allLevels { + if m.SameZones(level, other) { + t.Errorf("empty level %s reports the same zones as %s", level, other) + } + } + } + + // SameZones has to agree with the zones themselves + for _, a := range m.Levels() { + if !m.SameZones(a, a) { + t.Errorf("SameZones(%s, %s) is false", a, a) + } + for _, b := range m.Levels() { + if m.SameZones(a, b) != m.SameZones(b, a) { + t.Errorf("SameZones is not symmetric for %s and %s", a, b) + } + } + } +} + +// checkContainment exercises the containment queries against the zones. +func checkContainment(t *testing.T, m *Machine) { + t.Helper() + + for _, level := range m.Levels() { + for _, z := range m.Zones(level) { + // a zone is within itself, and overlaps itself + within := ZonesWithin(m, level, z.CPUs()) + if !containsZone(within, z) { + t.Errorf("ZonesWithin(%s, %s) does not include %s", + level, z.CPUs(), z.Name()) + } + over := ZonesOverlapping(m, level, z.CPUs()) + if !containsZone(over, z) { + t.Errorf("ZonesOverlapping(%s, %s) does not include %s", + level, z.CPUs(), z.Name()) + } + // overlapping is the weaker predicate, so it can only be larger + if len(over) < len(within) { + t.Errorf("%s: overlapping (%d) is smaller than within (%d)", + z.Name(), len(over), len(within)) + } + if got := ZoneOf(m, level, z.CPUs()); got != z && got.CPUs().Size() > z.CPUs().Size() { + t.Errorf("ZoneOf(%s, %s) = %s, want %s or smaller", + level, z.CPUs(), got.Name(), z.Name()) + } + } + + // nothing is within an empty set, and nothing overlaps it + if got := ZonesWithin(m, level, emptyCPUs); len(got) != 0 { + t.Errorf("ZonesWithin(%s, empty) returned %d zones", level, len(got)) + } + if got := ZonesOverlapping(m, level, emptyCPUs); len(got) != 0 { + t.Errorf("ZonesOverlapping(%s, empty) returned %d zones", level, len(got)) + } + // everything is within the whole machine + if got, want := len(ZonesWithin(m, level, m.PresentCPUs())), len(m.Zones(level)); got != want { + t.Errorf("ZonesWithin(%s, all) returned %d zones, want %d", level, got, want) + } + } +} + +func containsZone(zones []*Zone, want *Zone) bool { + for _, z := range zones { + if z == want { + return true + } + } + return false +} + +// checkCPUs asserts what has to be true of every CPU. +func checkCPUs(t *testing.T, m *Machine) { + t.Helper() + + kinds := emptyCPUs.Union() + for _, kind := range m.CoreKinds() { + cpus := m.CoreKindCPUs(kind) + if cpus.IsEmpty() { + t.Errorf("core kind %s is listed but has no CPUs", kind) + } + if kinds.Intersects(cpus) { + t.Errorf("core kind %s overlaps another", kind) + } + kinds = kinds.Union(cpus) + } + if missing := m.OnlineCPUs().Difference(kinds); !missing.IsEmpty() { + t.Errorf("online CPUs %s are of no core kind", missing) + } + + for _, id := range m.CPUIDs() { + c := m.CPU(id) + if !c.Valid() { + t.Errorf("cpu%d is listed but not valid", id) + continue + } + if c.ID() != id { + t.Errorf("cpu%d reports id %d", id, c.ID()) + } + if c.Online() != m.OnlineCPUs().Contains(id) { + t.Errorf("cpu%d Online()=%v disagrees with the online set", id, c.Online()) + } + if c.Isolated() != m.IsolatedCPUs().Contains(id) { + t.Errorf("cpu%d Isolated()=%v disagrees with the isolated set", id, + c.Isolated()) + } + + if !c.Online() { + // an offline CPU has no topology, and must say so rather than + // answering with a plausible lie + continue + } + + if !c.Threads().Contains(id) { + t.Errorf("cpu%d is not among its own threads %s", id, c.Threads()) + } + assertSealedNamed(t, c.String()+" threads", c.Threads()) + + // its zones have to contain it, and agree with its ids + for _, level := range m.Levels() { + z := c.Zone(level) + if !z.Valid() { + t.Errorf("cpu%d has no zone at level %s", id, level) + continue + } + if !z.CPUs().Contains(id) { + t.Errorf("cpu%d is not in its own %s zone %s", id, level, z.Name()) + } + } + if pkg := c.Zone(LevelPackage); pkg.Valid() && pkg.ID() != c.PackageID() { + t.Errorf("cpu%d package zone is #%d but PackageID() is %d", + id, pkg.ID(), c.PackageID()) + } + + if node := c.MemoryNode(); node.Valid() && node.ID() != c.NodeID() { + t.Errorf("cpu%d node is #%d but NodeID() is %d", + id, node.ID(), c.NodeID()) + } + + // caches come out lowest level first, and each holds this CPU + last := 0 + for _, cache := range c.Caches() { + if cache.Level() < last { + t.Errorf("cpu%d caches are out of order: %d after %d", + id, cache.Level(), last) + } + last = cache.Level() + if !cache.CPUs().Contains(id) { + t.Errorf("cpu%d is not among the CPUs of its own cache %s", + id, cache.Key()) + } + } + } +} + +// checkMemoryNodes asserts what has to be true of every NUMA node. +func checkMemoryNodes(t *testing.T, m *Machine) { + t.Helper() + + nodes := m.MemoryNodes() + if len(nodes) == 0 { + t.Fatal("no NUMA nodes") + } + + for _, n := range nodes { + if !n.Valid() { + t.Errorf("node#%d is listed but not valid", n.ID()) + continue + } + assertSealedNamed(t, n.String()+" CPUs", n.CPUs()) + + // the distance vector covers every node, and is shortest to itself + for _, other := range nodes { + d := n.Distance(other.ID()) + if d < 0 { + t.Errorf("node#%d has no distance to node#%d", n.ID(), other.ID()) + continue + } + if other.ID() == n.ID() { + continue + } + if self := n.Distance(n.ID()); d < self { + t.Errorf("node#%d is closer to node#%d (%d) than to itself (%d)", + n.ID(), other.ID(), d, self) + } + // symmetrized during discovery + if back := other.Distance(n.ID()); d != back { + t.Errorf("distance node#%d->node#%d is %d but back is %d", + n.ID(), other.ID(), d, back) + } + } + + if n.Distance(1<<20) != unknownID { + t.Errorf("node#%d gave a distance to a node which does not exist", n.ID()) + } + + // its CPUs agree with the CPUs' own view + n.CPUs().ForEachCpu(func(id int) bool { + if got := m.CPU(id).NodeID(); got != n.ID() { + t.Errorf("cpu%d is in node#%d's CPU list but reports node#%d", + id, n.ID(), got) + } + return true + }) + + if n.HasMemory() != (n.Capacity() > 0) { + t.Errorf("node#%d HasMemory()=%v with capacity %d", + n.ID(), n.HasMemory(), n.Capacity()) + } + } + + // a node which is not there answers rather than panicking + absent := m.MemoryNode(1 << 20) + if absent.Valid() { + t.Error("a node which is not there reports Valid()") + } + _, _, _ = absent.Capacity(), absent.CPUs(), absent.String() +} + +// TestDiscoverSynthetic runs discovery over topologies built by hand, for the +// shapes the recorded trees do not have. +func TestDiscoverSynthetic(t *testing.T) { + t.Run("no-numa", func(t *testing.T) { + // A kernel built without NUMA has no node directories at all. Everything + // has to land in one node holding every CPU. + m, err := Discover(WithFS(syntheticFS(2, false))) + if err != nil { + t.Fatalf("Discover: %v", err) + } + checkMachineInvariants(t, m) + + if got := len(m.MemoryNodes()); got != 1 { + t.Fatalf("got %d nodes, want 1", got) + } + node := m.MemoryNode(0) + if !node.CPUs().Equals(m.OnlineCPUs()) { + t.Errorf("the single node holds %s, want %s", node.CPUs(), m.OnlineCPUs()) + } + + // ...and all of the memory. There is no node to ask for a capacity here, + // so it comes from the machine's own meminfo. Left unread it is zero, and + // a node with no memory gets refused everything by whoever allocates from + // it, which is how a machine without NUMA fails as a whole. + if got, want := node.Capacity(), int64(1048576*1024); got != want { + t.Errorf("the single node has capacity %d, want %d", got, want) + } + if !node.HasMemory() { + t.Error("the single node reports no memory") + } + if !node.HasNormalMemory() { + t.Error("the single node reports no normal memory") + } + + // and Usage, which reads the same file live, agrees with it + usage, err := node.Usage() + if err != nil { + t.Fatalf("Usage: %v", err) + } + if usage.Total != node.Capacity() { + t.Errorf("Usage().Total = %d but Capacity() = %d", + usage.Total, node.Capacity()) + } + }) + + t.Run("no-numa-no-meminfo", func(t *testing.T) { + // Without node directories the machine's own meminfo is the only source of + // a capacity, so failing to read it has to fail discovery rather than yield + // a machine which looks like it has no memory at all. + fsys := syntheticFS(2, false) + delete(fsys, "proc/meminfo") + + if _, err := Discover(WithFS(fsys)); err == nil { + t.Fatal("discovering a NUMA-less machine with no meminfo succeeded") + } + }) + + t.Run("with-numa", func(t *testing.T) { + m, err := Discover(WithFS(syntheticFS(4, true))) + if err != nil { + t.Fatalf("Discover: %v", err) + } + checkMachineInvariants(t, m) + }) + + t.Run("no-cpus", func(t *testing.T) { + // Nothing to describe. This has to fail rather than return an empty + // machine which every caller then has to check for. + _, err := Discover(WithFS(fstest.MapFS{"proc/meminfo": file("MemTotal: 1 kB\n")})) + if err == nil { + t.Fatal("discovering a machine with no CPUs succeeded") + } + }) + + t.Run("nil-fs", func(t *testing.T) { + if _, err := Discover(WithFS(nil)); err == nil { + t.Fatal("WithFS(nil) was accepted") + } + }) +} + +// syntheticFS builds a minimal single-package machine with n CPUs, no +// hyperthreads and one L2 cache each, optionally with a NUMA node. +func syntheticFS(n int, numa bool) fstest.MapFS { + fsys := fstest.MapFS{ + "proc/meminfo": file("MemTotal: 1048576 kB\nMemFree: 524288 kB\n"), + } + + last := itoa(n - 1) + fsys["sys/devices/system/cpu/possible"] = file("0-" + last + "\n") + fsys["sys/devices/system/cpu/present"] = file("0-" + last + "\n") + fsys["sys/devices/system/cpu/online"] = file("0-" + last + "\n") + fsys["sys/devices/system/cpu/isolated"] = file("\n") + + for i := 0; i < n; i++ { + dir := "sys/devices/system/cpu/cpu" + itoa(i) + fsys[dir+"/topology/physical_package_id"] = file("0\n") + fsys[dir+"/topology/core_id"] = file(itoa(i) + "\n") + fsys[dir+"/topology/core_cpus_list"] = file(itoa(i) + "\n") + fsys[dir+"/cache/index0/level"] = file("2\n") + fsys[dir+"/cache/index0/type"] = file("Unified\n") + fsys[dir+"/cache/index0/id"] = file(itoa(i) + "\n") + fsys[dir+"/cache/index0/shared_cpu_list"] = file(itoa(i) + "\n") + fsys[dir+"/cache/index0/size"] = file("1024K\n") + if numa { + fsys[dir+"/node0/x"] = file("") + } + } + + if numa { + fsys["sys/devices/system/node/has_normal_memory"] = file("0\n") + fsys["sys/devices/system/node/node0/cpulist"] = file("0-" + last + "\n") + fsys["sys/devices/system/node/node0/distance"] = file("10\n") + fsys["sys/devices/system/node/node0/meminfo"] = + file("Node 0 MemTotal: 1048576 kB\nNode 0 MemFree: 524288 kB\n") + } + + return fsys +} + +// +// helpers +// + +func itoa(i int) string { + if i == 0 { + return "0" + } + digits := "" + for i > 0 { + digits = string(rune('0'+i%10)) + digits + i /= 10 + } + return digits +} + +// assertSealedNamed checks that a set cannot be modified, saying which set it is. +func assertSealedNamed(t *testing.T, what string, cpus interface{ Set(...int) }) { + t.Helper() + defer func() { + if recover() == nil { + t.Errorf("%s is not sealed", what) + } + }() + cpus.Set(0) +} diff --git a/pkg/lib/hardware/doc.go b/pkg/lib/hardware/doc.go new file mode 100644 index 000000000..77826fb10 --- /dev/null +++ b/pkg/lib/hardware/doc.go @@ -0,0 +1,85 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package hardware discovers the CPU and memory topology of a machine. +// +// [Discover] reads the topology once and returns an immutable [Machine]. +// Nothing in a Machine changes afterwards, so a single instance can be +// discovered at startup and shared by everything that needs it. +// +// # Zones +// +// A [Zone] is a set of CPUs which share a piece of hardware: a package, a die, +// a cluster, a NUMA node, a cache, a core. Each is tagged with the [Level] it +// represents, and [Machine.Zones] returns all of them at one level: all the +// packages, or all the L3 caches. [Machine.Levels] says which levels a machine +// has at all. +// +// The zones are flat, not a tree. Which one contains which is a property of the +// hardware rather than of the [Level] constants, so containment is a query: +// [ZoneOf] for the zone at a level holding a set of CPUs, [ZonesWithin] and +// [ZonesOverlapping] for the zones inside or straddling one. A zone which needs +// naming rather than searching for is addressed by its coordinates, since the +// kernel numbers dies and cores within their package; [TopologyIndex] is the +// lookup table for those. +// +// This is deliberately one abstraction rather than an accessor family per +// level. Adding a level costs a constant, not a new set of methods, and a +// consumer which wants the topology as a hierarchy builds one over the levels +// it cares about, instead of walking one this package guessed at. +// +// # Handles +// +// [CPU], [MemoryNode], [Cache] and [Zone] are handles into the Machine which +// returned them. They are never nil, and every method on them is safe to call +// on a handle for something which does not exist; such a handle reports +// Valid() == false and otherwise answers with zero values. Looking up a CPU +// which is not present is therefore not an error to be checked at every call +// site, only where it matters. +// +// There is one handle per piece of hardware, so asking a Machine twice for the +// same CPU, node, cache or zone gives back the same pointer. Handles are +// therefore comparable, and usable as map keys: code which builds a structure +// of its own over the zones can key that structure by *Zone. +// +// # Sets +// +// CPU sets are [libcpu.CpuMask]. Every set a Machine hands out is sealed, so it +// is safe to read from several goroutines and will panic if modified: Clone it +// first if you need to change one. Sets of ids which are not CPUs -- packages, +// NUMA nodes, caches -- are plain sorted []ID. +// +// Functions here take sets as the [libcpu.CPUSet] interface, so that either +// implementation can be passed in, and return the concrete [libcpu.CpuMask] +// they built. +// +// # Filesystem +// +// Discovery reads through an [fs.FS] rooted at the host root, so +// "sys/devices/system/cpu/online" and "proc/meminfo" are both reachable. By +// default that is the real filesystem at "/"; [WithRoot] points it elsewhere, +// for a host filesystem mounted inside a container, and [WithFS] substitutes +// any fs.FS, which is how the tests run against recorded topologies and +// synthetic ones. +// +// # Scope +// +// This package reads topology. Discovery never writes anything, and it does not +// cover Intel Speed Select, cpufreq control, or uncore frequency: those are +// separate concerns with their own state and their own failure modes. +// +// The one concession to writing is [WriterFS]: a caller which reads a topology +// through an fs.FS and then wants to write to the same tree can supply one, and +// nothing here requires it. +package hardware diff --git a/pkg/lib/hardware/machine.go b/pkg/lib/hardware/machine.go new file mode 100644 index 000000000..28cda529f --- /dev/null +++ b/pkg/lib/hardware/machine.go @@ -0,0 +1,239 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "io/fs" + "slices" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// ID identifies a hardware element within its kind: a CPU, a package, a die, a +// NUMA node, a cache. It is an alias for int, so a []ID and a []int are the +// same type. +type ID = int + +// Machine is the CPU and memory topology of a machine, as discovered once by +// [Discover]. It is immutable and safe for concurrent use. +// +// discover.go builds one; doc.go says what the handles it hands out promise. +type Machine struct { + fsys fs.FS + + cpus map[ID]*CPU + cpuIDs []ID + + nodes map[ID]*MemoryNode + nodeIDs []ID + + caches map[CacheID]*Cache + + zones map[Level][]*Zone + levels []Level + + possible *libcpu.CpuMask + present *libcpu.CpuMask + online *libcpu.CpuMask + isolated *libcpu.CpuMask + kinds map[CoreKind]*libcpu.CpuMask + + index *TopologyIndex +} + +// Zones returns every zone at the given level, ordered by id, or nothing if the +// machine has no zones at that level. +// +// A level with a single zone is still a level: a machine with one package has +// one package zone. Nothing is left out for being uninformative, which is a +// judgement callers make for themselves; [Machine.SameZones] and the length of +// this are what they make it from. +func (m *Machine) Zones(level Level) []*Zone { + return m.zones[level] +} + +// There is deliberately no Zone(level, id) lookup: the kernel numbers dies, +// clusters and cores within their package, so an id alone does not name one. +// [TopologyIndex] addresses those by their full coordinates. + +// SameZones reports whether two levels cut the machine the same way, i.e. +// whether their zones hold exactly the same sets of CPUs. +// +// It is how to ask the questions the policies ask today by hand: whether a +// machine's clusters are just its cores, or whether its dies are just its +// packages. A level with no zones is the same as no other level. +func (m *Machine) SameZones(a, b Level) bool { + za, zb := m.zones[a], m.zones[b] + if len(za) == 0 || len(zb) == 0 || len(za) != len(zb) { + return false + } + + for _, x := range za { + found := false + for _, y := range zb { + if x.cpus.Equals(y.cpus) { + found = true + break + } + } + if !found { + return false + } + } + + return true +} + +// Levels returns the levels which the machine actually has zones at, outermost +// first. +func (m *Machine) Levels() []Level { + return m.levels +} + +// CPU returns a handle for the CPU with the given id. The handle is never nil; +// if the CPU is not present it reports Valid() == false. +func (m *Machine) CPU(id ID) *CPU { + if c, ok := m.cpus[id]; ok { + return c + } + return invalidCPU +} + +// CPUIDs returns the ids of all present CPUs, in increasing order. +func (m *Machine) CPUIDs() []ID { + return m.cpuIDs +} + +// PresentCPUs returns the CPUs the machine has, whether they are online or not. +func (m *Machine) PresentCPUs() *libcpu.CpuMask { + return m.present +} + +// PossibleCPUs returns the CPUs the kernel has reserved room for, a superset of +// [Machine.PresentCPUs] on a machine which supports CPU hotplug. +func (m *Machine) PossibleCPUs() *libcpu.CpuMask { + return m.possible +} + +// OnlineCPUs returns the CPUs which are online. +func (m *Machine) OnlineCPUs() *libcpu.CpuMask { + return m.online +} + +// OfflineCPUs returns the CPUs which are present but not online. +func (m *Machine) OfflineCPUs() *libcpu.CpuMask { + return sealed(m.present.Difference(m.online)) +} + +// IsolatedCPUs returns the CPUs the kernel was told to isolate. +func (m *Machine) IsolatedCPUs() *libcpu.CpuMask { + return m.isolated +} + +// CoreKinds returns the core kinds the machine has, or a single +// [PerformanceCore] on a machine whose cores are all alike. +func (m *Machine) CoreKinds() []CoreKind { + kinds := make([]CoreKind, 0, len(m.kinds)) + for kind := range m.kinds { + kinds = append(kinds, kind) + } + slices.Sort(kinds) + return kinds +} + +// CoreKindCPUs returns the CPUs of the given core kind. +func (m *Machine) CoreKindCPUs(kind CoreKind) *libcpu.CpuMask { + if cpus, ok := m.kinds[kind]; ok { + return cpus + } + return emptyCPUs +} + +// MemoryNode returns a handle for the NUMA node with the given id. The handle +// is never nil; if there is no such node it reports Valid() == false. +func (m *Machine) MemoryNode(id ID) *MemoryNode { + if n, ok := m.nodes[id]; ok { + return n + } + return invalidMemoryNode +} + +// MemoryNodes returns all NUMA nodes, ordered by id. A machine built without +// NUMA support has a single node, holding every CPU and all of the memory. +func (m *Machine) MemoryNodes() []*MemoryNode { + nodes := make([]*MemoryNode, 0, len(m.nodeIDs)) + for _, id := range m.nodeIDs { + nodes = append(nodes, m.nodes[id]) + } + return nodes +} + +// MemoryNodeIDs returns the ids of all NUMA nodes, in increasing order. +func (m *Machine) MemoryNodeIDs() []ID { + return m.nodeIDs +} + +// Caches returns every cache at the given level, ordered by id, or nothing if +// the machine has no caches at that level. Level 0 returns all caches. +func (m *Machine) Caches(level int) []*Cache { + caches := make([]*Cache, 0, len(m.caches)) + for _, c := range m.caches { + if level == 0 || c.level == level { + caches = append(caches, c) + } + } + slices.SortFunc(caches, compareCaches) + return caches +} + +// CacheLevels returns the cache levels the machine has, lowest first. +func (m *Machine) CacheLevels() []int { + var levels []int + for _, c := range m.caches { + if !slices.Contains(levels, c.level) { + levels = append(levels, c.level) + } + } + slices.Sort(levels) + return levels +} + +// FS returns the filesystem this machine was discovered from, rooted at the host +// root. A caller which wants to read something discovery does not, or write to +// an attribute of hardware it found here, should go through this rather than the +// real filesystem: it is the only thing which knows where the tree actually is. +// Writing needs it to be a [WriterFS], which the default filesystem is. +func (m *Machine) FS() fs.FS { + return m.fsys +} + +// +// Shared zero values +// + +// emptyCPUs is the set returned for a coordinate the machine does not have. It +// is sealed, so handing the same one out everywhere is safe. +var emptyCPUs = func() *libcpu.CpuMask { + cpus := libcpu.NewCpuMask() + cpus.Seal() + return cpus +}() + +// sealed seals a set before it is handed out, so that nothing can modify a set +// this package has published. +func sealed(cpus *libcpu.CpuMask) *libcpu.CpuMask { + cpus.Seal() + return cpus +} diff --git a/pkg/lib/hardware/memory.go b/pkg/lib/hardware/memory.go new file mode 100644 index 000000000..abffbf754 --- /dev/null +++ b/pkg/lib/hardware/memory.go @@ -0,0 +1,262 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "fmt" + "io/fs" + "strconv" + "strings" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// MemoryNode is a NUMA node: some memory, and the CPUs which are closest to +// it. A node may have no memory, and it may have no CPUs; a machine with +// high-bandwidth or persistent memory typically has nodes of the latter kind. +// +// It is a handle into the [Machine] which produced it: never nil, and safe to +// use even when it refers to no node, in which case it reports Valid() == +// false. +type MemoryNode struct { + m *Machine + dir string // where it was read from, "" when the kernel has no NUMA + valid bool + + id ID + kind MemoryKind + cpus *libcpu.CpuMask + capacity int64 + normal bool + distance []int + + meminfo string // where Usage() reads from + zones [numLevels]*Zone +} + +// invalidMemoryNode is what a lookup for a node the machine does not have +// returns. Its methods answer with zero values rather than panicking. +var invalidMemoryNode = &MemoryNode{id: unknownID, kind: MemoryKindUnknown} + +// Valid reports whether this handle refers to a node the machine has. +func (n *MemoryNode) Valid() bool { + return n.valid +} + +// ID returns the id of this node. +func (n *MemoryNode) ID() ID { + return n.id +} + +// Kind returns what sort of memory this node has. +func (n *MemoryNode) Kind() MemoryKind { + return n.kind +} + +// CPUs returns the CPUs closest to this node's memory, which is empty for a +// node holding only memory. The set is sealed. +func (n *MemoryNode) CPUs() *libcpu.CpuMask { + if n.cpus == nil { + return emptyCPUs + } + return n.cpus +} + +// Capacity returns how much memory this node has, in bytes, as read once +// during discovery. It is 0 for a node with no memory of its own. +func (n *MemoryNode) Capacity() int64 { + return n.capacity +} + +// HasMemory reports whether this node has any memory, i.e. whether +// [MemoryNode.Capacity] is above zero. +func (n *MemoryNode) HasMemory() bool { + return n.capacity > 0 +} + +// HasNormalMemory reports whether some of this node's memory is in a zone the +// kernel will satisfy ordinary allocations from. +func (n *MemoryNode) HasNormalMemory() bool { + return n.normal +} + +// Usage reads how much of this node's memory is in use now. Unlike everything +// else here it goes to the machine on every call, because the answer changes; +// [MemoryNode.Capacity] does not and is not re-read. +func (n *MemoryNode) Usage() (MemInfo, error) { + if !n.valid || n.m == nil { + return MemInfo{}, fmt.Errorf("no such NUMA node") + } + return readMemInfo(n.m.fsys, n.meminfo, n.id) +} + +// Distances returns the cost of reaching every NUMA node from this one, indexed +// by node id, as the kernel reports it. +func (n *MemoryNode) Distances() []int { + return n.distance +} + +// Distance returns the cost of reaching node to from this one, or -1 if there +// is no such node. +func (n *MemoryNode) Distance(to ID) int { + if to < 0 || to >= len(n.distance) { + return unknownID + } + return n.distance[to] +} + +// Zone returns the zone at the given level which contains this node's CPUs, or +// an invalid zone if it has none or spans more than one. +func (n *MemoryNode) Zone(level Level) *Zone { + if level < 0 || int(level) >= numLevels || n.zones[level] == nil { + return invalidZone + } + return n.zones[level] +} + +// PackageID returns the id of the package this node belongs to, or -1 if it has +// no CPUs. It is shorthand for n.Zone(LevelPackage).ID(). +func (n *MemoryNode) PackageID() ID { + return n.Zone(LevelPackage).ID() +} + +// DieID returns the id of the die this node belongs to, or -1 if it has no +// CPUs. +func (n *MemoryNode) DieID() ID { + return n.Zone(LevelDie).ID() +} + +// String returns "node#". +func (n *MemoryNode) String() string { + if !n.valid { + return "node#?" + } + return "node#" + strconv.Itoa(n.id) +} + +// MemoryKind is the sort of memory a [MemoryNode] holds. +// +// The kernel does not report this, so it is inferred: a node with CPUs of its +// own holds ordinary DRAM, and one without is told apart by how much memory it +// has relative to the DRAM nodes. That heuristic can be wrong, which is why +// [MemoryKindUnknown] exists rather than a guess being forced. +type MemoryKind int + +const ( + // MemoryKindDRAM is ordinary system memory. + MemoryKindDRAM MemoryKind = iota + // MemoryKindPMEM is persistent memory: larger and slower than DRAM. + MemoryKindPMEM + // MemoryKindHBM is high-bandwidth memory: smaller and faster than DRAM. + MemoryKindHBM + // MemoryKindUnknown is reported when the kind could not be determined. + MemoryKindUnknown +) + +// String returns "DRAM", "PMEM", "HBM" or "unknown". +func (k MemoryKind) String() string { + switch k { + case MemoryKindDRAM: + return "DRAM" + case MemoryKindPMEM: + return "PMEM" + case MemoryKindHBM: + return "HBM" + } + return "unknown" +} + +// ParseMemoryKind returns the kind named by s, or an error. +func ParseMemoryKind(s string) (MemoryKind, error) { + switch strings.ToUpper(s) { + case "DRAM": + return MemoryKindDRAM, nil + case "PMEM": + return MemoryKindPMEM, nil + case "HBM": + return MemoryKindHBM, nil + case "UNKNOWN": + return MemoryKindUnknown, nil + } + return MemoryKindUnknown, fmt.Errorf("unknown memory kind %q", s) +} + +// readMemInfo reads a meminfo file, either a NUMA node's or /proc/meminfo. +// +// A node's is "Node 0 MemTotal: 32768 kB" and /proc's is "MemTotal: 32768 kB", +// so the two differ only in a two-field prefix. Everything is reported in kB and +// returned in bytes. +func readMemInfo(fsys fs.FS, name string, id ID) (MemInfo, error) { + blob, err := readFile(fsys, name) + if err != nil { + return MemInfo{}, err + } + + info := MemInfo{} + for _, line := range strings.Split(blob, "\n") { + fields := strings.Fields(line) + // drop the "Node " prefix a per-node meminfo has + if len(fields) > 2 && fields[0] == "Node" { + fields = fields[2:] + } + if len(fields) < 2 { + continue + } + + dst := (*int64)(nil) + switch fields[0] { + case "MemTotal:": + dst = &info.Total + case "MemFree:": + dst = &info.Free + default: + continue + } + + value, err := strconv.ParseInt(fields[1], 10, 64) + if err != nil { + return MemInfo{}, fmt.Errorf("%s: bad %s %q: %w", + name, fields[0], fields[1], err) + } + if len(fields) > 2 && fields[2] == "kB" { + value *= 1024 + } + *dst = value + } + + // Some kernel and hardware combinations have been seen reporting more free + // than total memory. Callers compute usage from the difference and go badly + // wrong on a negative, so refuse it here where it can still be explained. + if info.Free > info.Total { + return MemInfo{}, fmt.Errorf( + "%s: node #%d reports more free (%d) than total (%d) memory; "+ + "this is a kernel bug, try a newer kernel", + name, id, info.Free, info.Total) + } + + info.Used = info.Total - info.Free + + return info, nil +} + +// MemInfo is how much memory is present and how much of it is in use, in bytes. +type MemInfo struct { + // Total is how much memory there is. + Total int64 + // Free is how much of it is unused. + Free int64 + // Used is Total less Free. + Used int64 +} diff --git a/pkg/lib/hardware/overrides.go b/pkg/lib/hardware/overrides.go new file mode 100644 index 000000000..fcd463891 --- /dev/null +++ b/pkg/lib/hardware/overrides.go @@ -0,0 +1,145 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "encoding/json" + "fmt" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// The shapes of the OVERRIDE_SYS_CACHES and OVERRIDE_SYS_CPUFREQ variables. +// They are what pkg/sysfs accepts, so that an end-to-end test which sets them +// keeps working across the move. See [WithEnvOverrides] for why they exist. + +// cacheOverride describes caches to report instead of the ones the machine has. +// Every cpuset listed gets a cache, numbered from zero within its level and +// kind. +// +// [{"cpusets": ["0-15,32-47", "16-31,48-63"], "level": 3, "size": "128M"}] +type cacheOverride struct { + Cpusets []string `json:"cpusets"` + Level int `json:"level"` + Kind string `json:"kind"` + Size string `json:"size"` +} + +// freqOverride describes frequencies to report instead of the ones cpufreq +// gives, for a virtual machine which has no cpufreq at all. +type freqOverride struct { + Cpus string `json:"cpus"` + Base uint64 `json:"base"` + Min uint64 `json:"min"` + Max uint64 `json:"max"` +} + +// parseCacheOverrides turns the JSON of OVERRIDE_SYS_CACHES into the caches of +// each CPU, ordered by level and kind as discovery orders the real ones. +func parseCacheOverrides(blob string) (map[ID][]*Cache, error) { + var overrides []cacheOverride + if err := json.Unmarshal([]byte(blob), &overrides); err != nil { + return nil, err + } + + // ids are handed out per level and kind, as the kernel numbers them + next := map[CacheID]ID{} + byCPU := map[ID][]*Cache{} + + for _, o := range overrides { + kind, err := parseOverrideCacheKind(o.Kind) + if err != nil { + return nil, err + } + + level := o.Level + if level <= 0 { + level = 1 + } + + size, err := parseSize(trimmed(o.Size)) + if err != nil { + return nil, fmt.Errorf("bad size %q: %w", o.Size, err) + } + + for _, str := range o.Cpusets { + cpus, err := libcpu.ParseCpuMask(trimmed(str)) + if err != nil { + return nil, fmt.Errorf("bad cpuset %q: %w", str, err) + } + cpus.Seal() + + key := CacheID{Level: level, ID: int(kind)} + cache := &Cache{ + valid: true, + id: next[key], + level: level, + kind: kind, + size: size, + cpus: cpus, + } + next[key]++ + + cpus.ForEachCpu(func(id int) bool { + byCPU[id] = append(byCPU[id], cache) + return true + }) + } + } + + for _, caches := range byCPU { + sortCaches(caches) + } + + return byCPU, nil +} + +// parseOverrideCacheKind accepts the short and long spellings an override may +// use, and treats an unspecified kind as unified. +func parseOverrideCacheKind(s string) (CacheKind, error) { + switch trimmed(s) { + case "d", "data", "Data": + return DataCache, nil + case "i", "instruction", "Instruction": + return InstructionCache, nil + case "u", "unified", "Unified", "": + return UnifiedCache, nil + } + return UnifiedCache, fmt.Errorf("unknown cache kind %q", s) +} + +// parseFreqOverrides turns the JSON of OVERRIDE_SYS_CPUFREQ into the +// frequencies of each CPU. +func parseFreqOverrides(blob string) (map[ID]Freq, error) { + var overrides []freqOverride + if err := json.Unmarshal([]byte(blob), &overrides); err != nil { + return nil, err + } + + byCPU := map[ID]Freq{} + for _, o := range overrides { + cpus, err := libcpu.ParseCpuMask(trimmed(o.Cpus)) + if err != nil { + return nil, fmt.Errorf("bad CPU list %q: %w", o.Cpus, err) + } + freq := Freq{Base: o.Base, Min: o.Min, Max: o.Max, EPP: EPPUnknown} + cpus.ForEachCpu(func(id int) bool { + byCPU[id] = freq + return true + }) + } + + return byCPU, nil +} diff --git a/pkg/lib/hardware/overrides_test.go b/pkg/lib/hardware/overrides_test.go new file mode 100644 index 000000000..cf8f8a7a8 --- /dev/null +++ b/pkg/lib/hardware/overrides_test.go @@ -0,0 +1,280 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "errors" + "slices" + "testing" +) + +// The e2e tests set these on a deployed plugin, so a change in what they accept +// is a change in behaviour for something outside this repository's tests. + +func TestParseCacheOverrides(t *testing.T) { + t.Run("two-l3-caches", func(t *testing.T) { + // the shape pkg/sysfs documents: two 128M L3 caches, one per socket + byCPU, err := parseCacheOverrides( + `[{"cpusets": ["0-3,8-11", "4-7,12-15"], "level": 3, "size": "128M"}]`) + if err != nil { + t.Fatalf("parseCacheOverrides: %v", err) + } + + // ids are handed out per level and kind, from zero + first, second := byCPU[0], byCPU[4] + if len(first) != 1 || len(second) != 1 { + t.Fatalf("cpu0 has %d caches, cpu4 has %d, want 1 each", + len(first), len(second)) + } + if first[0].id != 0 || second[0].id != 1 { + t.Errorf("ids are %d and %d, want 0 and 1", first[0].id, second[0].id) + } + if first[0] == second[0] { + t.Error("the two cpusets share one cache") + } + + // every CPU of a cpuset gets the same shared instance + for _, id := range []int{0, 1, 2, 3, 8, 9, 10, 11} { + if len(byCPU[id]) != 1 || byCPU[id][0] != first[0] { + t.Errorf("cpu%d does not share cpu0's cache", id) + } + } + + if got, want := first[0].size, int64(128<<20); got != want { + t.Errorf("size = %d, want %d", got, want) + } + if got := first[0].level; got != 3 { + t.Errorf("level = %d, want 3", got) + } + if got := first[0].kind; got != UnifiedCache { + t.Errorf("kind = %s, want unified: an unspecified kind is unified", got) + } + if got, want := first[0].cpus.List(), []int{0, 1, 2, 3, 8, 9, 10, 11}; !slices.Equal(got, want) { + t.Errorf("CPUs = %v, want %v", got, want) + } + }) + + t.Run("kinds-and-ordering", func(t *testing.T) { + // a CPU's caches come out ordered by level then kind, as discovery + // orders the real ones + byCPU, err := parseCacheOverrides(`[ + {"cpusets": ["0"], "level": 3, "size": "8M"}, + {"cpusets": ["0"], "level": 1, "kind": "i", "size": "32k"}, + {"cpusets": ["0"], "level": 1, "kind": "data", "size": "48k"}, + {"cpusets": ["0"], "level": 2, "kind": "unified", "size": "2M"} + ]`) + if err != nil { + t.Fatalf("parseCacheOverrides: %v", err) + } + + caches := byCPU[0] + if len(caches) != 4 { + t.Fatalf("cpu0 has %d caches, want 4", len(caches)) + } + + want := []struct { + level int + kind CacheKind + size int64 + }{ + {1, DataCache, 48 << 10}, + {1, InstructionCache, 32 << 10}, + {2, UnifiedCache, 2 << 20}, + {3, UnifiedCache, 8 << 20}, + } + for i, w := range want { + got := caches[i] + if got.level != w.level || got.kind != w.kind || got.size != w.size { + t.Errorf("cache %d is L%d %s %d, want L%d %s %d", + i, got.level, got.kind, got.size, w.level, w.kind, w.size) + } + } + + // a data and an instruction cache at one level are numbered separately + if caches[0].id != 0 || caches[1].id != 0 { + t.Errorf("L1d and L1i have ids %d and %d, want 0 each", + caches[0].id, caches[1].id) + } + }) + + t.Run("defaults", func(t *testing.T) { + // no level means level 1, as pkg/sysfs does + byCPU, err := parseCacheOverrides(`[{"cpusets": ["0"], "size": "1M"}]`) + if err != nil { + t.Fatalf("parseCacheOverrides: %v", err) + } + if got := byCPU[0][0].level; got != 1 { + t.Errorf("level = %d, want 1 when unspecified", got) + } + }) + + t.Run("bad-input", func(t *testing.T) { + for _, tc := range []struct{ name, json string }{ + {"not-json", `not json`}, + {"bad-cpuset", `[{"cpusets": ["nope"], "level": 3}]`}, + {"bad-kind", `[{"cpusets": ["0"], "kind": "sideways"}]`}, + {"bad-size", `[{"cpusets": ["0"], "size": "big"}]`}, + } { + t.Run(tc.name, func(t *testing.T) { + if _, err := parseCacheOverrides(tc.json); err == nil { + t.Errorf("parseCacheOverrides(%s) succeeded", tc.json) + } + }) + } + }) +} + +func TestParseFreqOverrides(t *testing.T) { + byCPU, err := parseFreqOverrides( + `[{"cpus": "0-3", "base": 2400000, "min": 800000, "max": 3600000}, + {"cpus": "4-7", "base": 1800000, "min": 800000, "max": 2400000}]`) + if err != nil { + t.Fatalf("parseFreqOverrides: %v", err) + } + + for _, id := range []int{0, 1, 2, 3} { + if got := byCPU[id]; got.Base != 2400000 || got.Min != 800000 || got.Max != 3600000 { + t.Errorf("cpu%d = %+v, want base 2400000 min 800000 max 3600000", id, got) + } + } + if got := byCPU[4].Base; got != 1800000 { + t.Errorf("cpu4 base = %d, want 1800000", got) + } + if _, ok := byCPU[8]; ok { + t.Error("cpu8 got a frequency it was not given") + } + // nothing says anything about the governor, so it stays unknown + if got := byCPU[0].EPP; got != EPPUnknown { + t.Errorf("EPP = %s, want unknown", got) + } + + for _, bad := range []string{`not json`, `[{"cpus": "nope"}]`} { + if _, err := parseFreqOverrides(bad); err == nil { + t.Errorf("parseFreqOverrides(%s) succeeded", bad) + } + } +} + +func TestWithEnvOverrides(t *testing.T) { + t.Run("applied", func(t *testing.T) { + t.Setenv(envAtomCPUs, "2-3") + t.Setenv(envCoreCPUs, "0-1") + t.Setenv(envCPUFreq, `[{"cpus": "0-3", "base": 1000000}]`) + + m, err := Discover(WithFS(syntheticFS(4, true)), WithEnvOverrides()) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + if got, want := m.CoreKindCPUs(PerformanceCore).String(), "0-1"; got != want { + t.Errorf("P-cores = %s, want %s", got, want) + } + if got, want := m.CoreKindCPUs(EfficientCore).String(), "2-3"; got != want { + t.Errorf("E-cores = %s, want %s", got, want) + } + if got := m.CPU(0).Freq().Base; got != 1000000 { + t.Errorf("cpu0 base frequency = %d, want the override 1000000", got) + } + + checkMachineInvariants(t, m) + }) + + t.Run("bad-value-names-the-variable", func(t *testing.T) { + t.Setenv(envCaches, "not json") + + _, err := Discover(WithFS(syntheticFS(2, true)), WithEnvOverrides()) + if err == nil { + t.Fatal("a malformed override was accepted") + } + + var oe *overrideError + if !errors.As(err, &oe) { + t.Fatalf("error %v is not an overrideError", err) + } + if oe.name != envCaches { + t.Errorf("the error names %s, want %s", oe.name, envCaches) + } + if errors.Unwrap(oe) == nil { + t.Error("the error does not wrap the parse failure") + } + }) + + t.Run("unset-changes-nothing", func(t *testing.T) { + // t.Setenv with an empty value is still "set", so clear them explicitly + for _, name := range []string{envCoreCPUs, envAtomCPUs, envCaches, envCPUFreq} { + t.Setenv(name, "") + } + + withOverrides, err := Discover(WithFS(syntheticFS(4, true)), WithEnvOverrides()) + if err != nil { + t.Fatalf("Discover: %v", err) + } + plain, err := Discover(WithFS(syntheticFS(4, true))) + if err != nil { + t.Fatalf("Discover: %v", err) + } + + if !withOverrides.CoreKindCPUs(PerformanceCore). + Equals(plain.CoreKindCPUs(PerformanceCore)) { + t.Error("empty overrides changed the core kinds") + } + }) +} + +func TestParseSizeAndSizeString(t *testing.T) { + for _, tc := range []struct { + in string + want int64 + fail bool + }{ + {in: "", want: 0}, + {in: "0", want: 0}, + {in: "32K", want: 32 << 10}, + {in: "32k", want: 32 << 10}, + {in: "2M", want: 2 << 20}, + {in: "128M", want: 128 << 20}, + {in: "1G", want: 1 << 30}, + {in: "1024", want: 1024}, + {in: "big", fail: true}, + {in: "M", fail: true}, + } { + got, err := parseSize(tc.in) + switch { + case tc.fail && err == nil: + t.Errorf("parseSize(%q) = %d, want an error", tc.in, got) + case !tc.fail && err != nil: + t.Errorf("parseSize(%q): %v", tc.in, err) + case !tc.fail && got != tc.want: + t.Errorf("parseSize(%q) = %d, want %d", tc.in, got, tc.want) + } + } + + // the two round-trip for the sizes the kernel actually writes + for _, s := range []string{"32K", "2M", "128M", "1G"} { + size, err := parseSize(s) + if err != nil { + t.Fatalf("parseSize(%q): %v", s, err) + } + if got := sizeString(size); got != s { + t.Errorf("sizeString(parseSize(%q)) = %q", s, got) + } + } + if got := sizeString(0); got != "0" { + t.Errorf("sizeString(0) = %q, want %q", got, "0") + } + if got := sizeString(1500); got != "1500" { + t.Errorf("sizeString(1500) = %q, want %q", got, "1500") + } +} diff --git a/pkg/lib/hardware/read.go b/pkg/lib/hardware/read.go new file mode 100644 index 000000000..aa5565946 --- /dev/null +++ b/pkg/lib/hardware/read.go @@ -0,0 +1,256 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "fmt" + "io/fs" + "os" + "path" + "path/filepath" + "slices" + "strconv" + "strings" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// All reading goes through these. They take paths relative to the host root and +// slash-separated, as [fs.FS] requires: build them with [path.Join], never +// filepath.Join, or an absolute path will slip in and fs.ValidPath will reject +// it at runtime. + +// readFile returns the contents of a file with any trailing newlines removed. +// sysfs attributes are newline-terminated and nothing here wants the newline. +func readFile(fsys fs.FS, name string) (string, error) { + blob, err := fs.ReadFile(fsys, name) + if err != nil { + return "", err + } + return strings.Trim(string(blob), "\n"), nil +} + +// readInt reads a file and parses its contents as a signed integer. A leading +// 0x or 0b is honoured, as strconv does with a base of 0. +func readInt(fsys fs.FS, name string) (int, error) { + str, err := readFile(fsys, name) + if err != nil { + return 0, err + } + i, err := strconv.ParseInt(str, 0, strconv.IntSize) + if err != nil { + return 0, fmt.Errorf("%s: invalid integer %q: %w", name, str, err) + } + return int(i), nil +} + +// readUint64 reads a file and parses its contents as an unsigned integer. +func readUint64(fsys fs.FS, name string) (uint64, error) { + str, err := readFile(fsys, name) + if err != nil { + return 0, err + } + u, err := strconv.ParseUint(str, 0, 64) + if err != nil { + return 0, fmt.Errorf("%s: invalid unsigned integer %q: %w", name, str, err) + } + return u, nil +} + +// readCPUs reads a file and parses its contents as a kernel CPU list, the +// "0-3,8,12-15" form sysfs uses for every set of CPUs. The result is sealed, so +// it is safe to hand out and to share. +func readCPUs(fsys fs.FS, name string) (*libcpu.CpuMask, error) { + str, err := readFile(fsys, name) + if err != nil { + return nil, err + } + cpus, err := libcpu.ParseCpuMask(str) + if err != nil { + return nil, fmt.Errorf("%s: invalid CPU list %q: %w", name, str, err) + } + cpus.Seal() + return cpus, nil +} + +// readInts reads a file and parses its contents as a list of integers separated +// by sep. The NUMA distance vector is the only thing which needs it, and it is +// space separated rather than a CPU list. +func readInts(fsys fs.FS, name, sep string) ([]int, error) { + str, err := readFile(fsys, name) + if err != nil { + return nil, err + } + + var out []int + for _, field := range strings.Split(str, sep) { + if field == "" { + continue + } + i, err := strconv.Atoi(field) + if err != nil { + return nil, fmt.Errorf("%s: invalid integer %q: %w", name, field, err) + } + out = append(out, i) + } + + return out, nil +} + +// glob returns the names matching pattern, in the order fs.Glob gives them, +// which is lexical: cpu10 comes before cpu2. Anything which needs numeric order +// has to sort for itself. +func glob(fsys fs.FS, pattern string) ([]string, error) { + return fs.Glob(fsys, pattern) +} + +// globIDs returns the names matching pattern together with the trailing number +// of each, sorted by that number. It is how the cpuN, nodeN and indexN +// directories are enumerated. +func globIDs(fsys fs.FS, pattern string) ([]string, []ID, error) { + names, err := glob(fsys, pattern) + if err != nil { + return nil, nil, err + } + + type entry struct { + name string + id ID + } + + entries := make([]entry, 0, len(names)) + for _, name := range names { + id, ok := trailingID(name) + if !ok { + continue + } + entries = append(entries, entry{name: name, id: id}) + } + + slices.SortFunc(entries, func(a, b entry) int { return a.id - b.id }) + + sorted := make([]string, len(entries)) + ids := make([]ID, len(entries)) + for i, e := range entries { + sorted[i], ids[i] = e.name, e.id + } + + return sorted, ids, nil +} + +// trailingID returns the number a name ends in, as "cpu12" ends in 12. +func trailingID(name string) (ID, bool) { + base := path.Base(name) + + end := len(base) + for end > 0 && base[end-1] >= '0' && base[end-1] <= '9' { + end-- + } + if end == len(base) { + return 0, false + } + + id, err := strconv.Atoi(base[end:]) + if err != nil { + return 0, false + } + + return id, true +} + +// exists reports whether a path is there at all. +func exists(fsys fs.FS, name string) bool { + _, err := fs.Stat(fsys, name) + return err == nil +} + +// +// Writing +// + +// WriterFS is an [fs.FS] which also supports writing. Discovery never needs it; +// it is here for callers which read a topology through an fs.FS and then want to +// write to the same tree. [HostFS] below is one. +type WriterFS interface { + fs.FS + + // WriteFile writes data to an existing file in a single write. It neither + // creates nor truncates the file, which is what writing a sysfs attribute + // requires. + WriteFile(name string, data []byte) error +} + +// writeFile writes to a file through fsys, which has to be a [WriterFS] for +// that to be possible at all. +func writeFile(fsys fs.FS, name string, data []byte) error { + w, ok := fsys.(WriterFS) + if !ok { + return fmt.Errorf("cannot write %s: %T is read-only", name, fsys) + } + return w.WriteFile(name, data) +} + +// +// The default filesystem +// + +// hostFS is the real filesystem below a root, with writing. +type hostFS struct { + fs.FS + root string +} + +// HostFS returns a [WriterFS] for the real filesystem below root. HostFS("") +// and HostFS("/") both mean the whole filesystem. +func HostFS(root string) WriterFS { + if root == "" { + root = "/" + } + return &hostFS{FS: os.DirFS(root), root: root} +} + +// WriteFile writes data to an existing file in a single write. It opens the file +// write-only and neither creates nor truncates it, which is what writing a sysfs +// attribute requires: os.WriteFile's O_CREATE|O_TRUNC is wrong for a kernel +// attribute. +func (h *hostFS) WriteFile(name string, data []byte) (err error) { + if !fs.ValidPath(name) { + return &fs.PathError{Op: "write", Path: name, Err: fs.ErrInvalid} + } + + f, err := os.OpenFile(h.osPath(name), os.O_WRONLY, 0) + if err != nil { + return err + } + defer func() { + if cerr := f.Close(); cerr != nil && err == nil { + err = cerr + } + }() + + _, err = f.Write(data) + + return err +} + +// osPath turns an fs.FS name into a path in the operating system's own form. +func (h *hostFS) osPath(name string) string { + return filepath.Join(h.root, filepath.FromSlash(name)) +} + +// String names the root, so that an error mentioning the filesystem says which. +func (h *hostFS) String() string { + return "host filesystem at " + h.root +} diff --git a/pkg/lib/hardware/read_test.go b/pkg/lib/hardware/read_test.go new file mode 100644 index 000000000..552c03793 --- /dev/null +++ b/pkg/lib/hardware/read_test.go @@ -0,0 +1,494 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "errors" + "io/fs" + "os" + "path/filepath" + "slices" + "testing" + "testing/fstest" +) + +// file is shorthand for one entry in a synthetic filesystem. +func file(contents string) *fstest.MapFile { + return &fstest.MapFile{Data: []byte(contents)} +} + +func TestReadFile(t *testing.T) { + fsys := fstest.MapFS{ + "plain": file("value"), + "newline": file("value\n"), + "many-newlines": file("value\n\n\n"), + "empty": file(""), + "only-newline": file("\n"), + "inner-whitespace": file(" spaced out \n"), + "multi-line": file("first\nsecond\n"), + } + + for _, tc := range []struct { + name string + want string + }{ + {"plain", "value"}, + {"newline", "value"}, + {"many-newlines", "value"}, + {"empty", ""}, + {"only-newline", ""}, + // only newlines are trimmed, not other whitespace: a sysfs attribute + // which contains spaces means them + {"inner-whitespace", " spaced out "}, + {"multi-line", "first\nsecond"}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := readFile(fsys, tc.name) + if err != nil { + t.Fatalf("readFile(%q): %v", tc.name, err) + } + if got != tc.want { + t.Errorf("readFile(%q) = %q, want %q", tc.name, got, tc.want) + } + }) + } + + t.Run("missing", func(t *testing.T) { + if _, err := readFile(fsys, "nope"); !errors.Is(err, fs.ErrNotExist) { + t.Errorf("readFile of a missing file: %v, want fs.ErrNotExist", err) + } + }) +} + +func TestReadInt(t *testing.T) { + fsys := fstest.MapFS{ + "zero": file("0\n"), + "positive": file("42\n"), + "negative": file("-1\n"), + "hex": file("0x10\n"), + "padded": file("007\n"), + "empty": file("\n"), + "words": file("not a number\n"), + "float": file("1.5\n"), + } + + for _, tc := range []struct { + name string + want int + fail bool + }{ + {name: "zero", want: 0}, + {name: "positive", want: 42}, + // a die_id or cluster_id the kernel does not know reads as -1 + {name: "negative", want: -1}, + {name: "hex", want: 16}, + {name: "padded", want: 7}, + {name: "empty", fail: true}, + {name: "words", fail: true}, + {name: "float", fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := readInt(fsys, tc.name) + switch { + case tc.fail && err == nil: + t.Errorf("readInt(%q) = %d, want an error", tc.name, got) + case !tc.fail && err != nil: + t.Errorf("readInt(%q): %v", tc.name, err) + case !tc.fail && got != tc.want: + t.Errorf("readInt(%q) = %d, want %d", tc.name, got, tc.want) + } + }) + } +} + +func TestReadUint64(t *testing.T) { + fsys := fstest.MapFS{ + "zero": file("0\n"), + "freq": file("2400000\n"), + "big": file("18446744073709551615\n"), + "negative": file("-1\n"), + "overflow": file("18446744073709551616\n"), + } + + for _, tc := range []struct { + name string + want uint64 + fail bool + }{ + {name: "zero", want: 0}, + {name: "freq", want: 2400000}, + {name: "big", want: 1<<64 - 1}, + {name: "negative", fail: true}, + {name: "overflow", fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := readUint64(fsys, tc.name) + switch { + case tc.fail && err == nil: + t.Errorf("readUint64(%q) = %d, want an error", tc.name, got) + case !tc.fail && err != nil: + t.Errorf("readUint64(%q): %v", tc.name, err) + case !tc.fail && got != tc.want: + t.Errorf("readUint64(%q) = %d, want %d", tc.name, got, tc.want) + } + }) + } +} + +func TestReadCPUs(t *testing.T) { + fsys := fstest.MapFS{ + "single": file("0\n"), + "range": file("0-3\n"), + "list": file("0,2,4\n"), + "mixed": file("0-3,8,12-15\n"), + "high": file("960-1023\n"), + "unordered": file("3,1,0,2\n"), + "empty": file("\n"), + "words": file("nope\n"), + "bad-range": file("5-3\n"), + "open-range": file("0-\n"), + } + + for _, tc := range []struct { + name string + want []int + fail bool + }{ + {name: "single", want: []int{0}}, + {name: "range", want: []int{0, 1, 2, 3}}, + {name: "list", want: []int{0, 2, 4}}, + {name: "mixed", want: []int{0, 1, 2, 3, 8, 12, 13, 14, 15}}, + {name: "high", want: cpuRange(960, 1023)}, + {name: "unordered", want: []int{0, 1, 2, 3}}, + // an empty attribute is an empty set, not an error: isolated is empty on + // most machines + {name: "empty", want: []int{}}, + {name: "words", fail: true}, + {name: "bad-range", fail: true}, + {name: "open-range", fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := readCPUs(fsys, tc.name) + if tc.fail { + if err == nil { + t.Errorf("readCPUs(%q) = %s, want an error", tc.name, got) + } + return + } + if err != nil { + t.Fatalf("readCPUs(%q): %v", tc.name, err) + } + if !slices.Equal(got.List(), tc.want) { + t.Errorf("readCPUs(%q) = %v, want %v", tc.name, got.List(), tc.want) + } + // every set discovery hands out has to be sealed + assertSealed(t, got) + }) + } +} + +func TestReadInts(t *testing.T) { + fsys := fstest.MapFS{ + "distance": file("10 21\n"), + "one": file("10\n"), + "four": file("10 21 31 41\n"), + "extra-gaps": file("10 21\n"), + "words": file("10 x\n"), + } + + for _, tc := range []struct { + name string + want []int + fail bool + }{ + {name: "distance", want: []int{10, 21}}, + {name: "one", want: []int{10}}, + {name: "four", want: []int{10, 21, 31, 41}}, + // empty fields are skipped, so repeated separators are harmless + {name: "extra-gaps", want: []int{10, 21}}, + {name: "words", fail: true}, + } { + t.Run(tc.name, func(t *testing.T) { + got, err := readInts(fsys, tc.name, " ") + if tc.fail { + if err == nil { + t.Errorf("readInts(%q) = %v, want an error", tc.name, got) + } + return + } + if err != nil { + t.Fatalf("readInts(%q): %v", tc.name, err) + } + if !slices.Equal(got, tc.want) { + t.Errorf("readInts(%q) = %v, want %v", tc.name, got, tc.want) + } + }) + } +} + +func TestGlobIDs(t *testing.T) { + fsys := fstest.MapFS{ + "sys/devices/system/cpu/cpu0/x": file(""), + "sys/devices/system/cpu/cpu1/x": file(""), + "sys/devices/system/cpu/cpu2/x": file(""), + "sys/devices/system/cpu/cpu10/x": file(""), + "sys/devices/system/cpu/cpu11/x": file(""), + "sys/devices/system/cpu/cpu100/x": file(""), + "sys/devices/system/cpu/cpufreq/x": file(""), + "sys/devices/system/cpu/online": file(""), + "sys/devices/system/cpu/cpuidle/x": file(""), + "sys/devices/system/cpu/microcode/x": file(""), + } + + names, ids, err := globIDs(fsys, "sys/devices/system/cpu/cpu[0-9]*") + if err != nil { + t.Fatalf("globIDs: %v", err) + } + + // numeric order, not the lexical order fs.Glob returns: cpu2 before cpu10 + want := []ID{0, 1, 2, 10, 11, 100} + if !slices.Equal(ids, want) { + t.Errorf("ids = %v, want %v", ids, want) + } + if len(names) != len(ids) { + t.Fatalf("got %d names for %d ids", len(names), len(ids)) + } + for i, name := range names { + if got, _ := trailingID(name); got != ids[i] { + t.Errorf("names[%d] = %q does not match ids[%d] = %d", i, name, i, ids[i]) + } + } +} + +func TestTrailingID(t *testing.T) { + for _, tc := range []struct { + name string + want ID + ok bool + }{ + {"cpu0", 0, true}, + {"cpu12", 12, true}, + {"node1", 1, true}, + {"index3", 3, true}, + {"sys/devices/system/cpu/cpu7", 7, true}, + {"cpu", 0, false}, + {"cpufreq", 0, false}, + {"", 0, false}, + {"sys/devices/system/cpu/online", 0, false}, + } { + t.Run(tc.name, func(t *testing.T) { + got, ok := trailingID(tc.name) + if ok != tc.ok { + t.Fatalf("trailingID(%q) ok = %v, want %v", tc.name, ok, tc.ok) + } + if ok && got != tc.want { + t.Errorf("trailingID(%q) = %d, want %d", tc.name, got, tc.want) + } + }) + } +} + +func TestExists(t *testing.T) { + fsys := fstest.MapFS{"a/b": file("")} + + for _, tc := range []struct { + name string + want bool + }{ + {"a/b", true}, + {"a", true}, + {"a/c", false}, + {"b", false}, + } { + if got := exists(fsys, tc.name); got != tc.want { + t.Errorf("exists(%q) = %v, want %v", tc.name, got, tc.want) + } + } +} + +// Machine.FS hands back the filesystem the machine was discovered from, so that +// a caller reading or writing more of the same tree cannot end up in a different +// one. Discovering from an injected fs.FS has to yield that fs.FS, not a +// filesystem rooted somewhere the machine knows nothing about. +func TestMachineFS(t *testing.T) { + // a pointer, so that identity can be compared at all: fstest.MapFS is a map + fsys := &wrappedFS{syntheticFS(1, true)} + + m, err := Discover(WithFS(fsys)) + if err != nil { + t.Fatalf("Discover: %v", err) + } + if got := m.FS(); got != fs.FS(fsys) { + t.Errorf("FS() = %#v, want the injected one", got) + } + + // the default is the real filesystem, which can be written through + m, err = Discover(WithRoot(t.TempDir())) + if err == nil { + if _, ok := m.FS().(WriterFS); !ok { + t.Error("the default filesystem is not a WriterFS") + } + } +} + +// wrappedFS is an fs.FS which can be compared for identity. +type wrappedFS struct { + fs.FS +} + +// A read-only fs.FS is enough to discover with, so writing through one has to +// fail with something a caller can act on rather than panicking. +func TestWriteFileNeedsAWriterFS(t *testing.T) { + err := writeFile(fstest.MapFS{"a": file("")}, "a", []byte("1")) + if err == nil { + t.Fatal("writing through a read-only fs.FS succeeded") + } + if got := err.Error(); !contains(got, "read-only") { + t.Errorf("error %q does not say the filesystem is read-only", got) + } +} + +func TestHostFS(t *testing.T) { + root := t.TempDir() + + if err := os.MkdirAll(filepath.Join(root, "sys", "devices"), 0o755); err != nil { + t.Fatal(err) + } + target := filepath.Join(root, "sys", "devices", "attr") + if err := os.WriteFile(target, []byte("original\n"), 0o644); err != nil { + t.Fatal(err) + } + + h := HostFS(root) + + t.Run("read", func(t *testing.T) { + got, err := readFile(h, "sys/devices/attr") + if err != nil { + t.Fatalf("readFile: %v", err) + } + if got != "original" { + t.Errorf("readFile = %q, want %q", got, "original") + } + }) + + t.Run("glob", func(t *testing.T) { + names, err := glob(h, "sys/devices/*") + if err != nil { + t.Fatalf("glob: %v", err) + } + if !slices.Equal(names, []string{"sys/devices/attr"}) { + t.Errorf("glob = %v", names) + } + }) + + t.Run("write", func(t *testing.T) { + if err := writeFile(h, "sys/devices/attr", []byte("newvalue1")); err != nil { + t.Fatalf("writeFile: %v", err) + } + blob, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + // exactly the bytes given, with no newline appended: what a caller + // writes is what the kernel is handed + if got := string(blob); got != "newvalue1" { + t.Errorf("file holds %q, want %q", got, "newvalue1") + } + }) + + // The file is opened write-only and is neither created nor truncated, which + // is what a sysfs attribute wants: each write is a whole command and the + // file has no length worth preserving. On a regular file that shows up as a + // short write leaving the tail of the previous contents behind. pkg/sysfs + // does the same, so this is deliberate, but it is worth pinning: switching + // to os.WriteFile would add O_CREATE|O_TRUNC and change it. + t.Run("write-does-not-truncate", func(t *testing.T) { + if err := os.WriteFile(target, []byte("original\n"), 0o644); err != nil { + t.Fatal(err) + } + if err := writeFile(h, "sys/devices/attr", []byte("changed")); err != nil { + t.Fatalf("writeFile: %v", err) + } + blob, err := os.ReadFile(target) + if err != nil { + t.Fatal(err) + } + if got, want := string(blob), "changedl\n"; got != want { + t.Errorf("file holds %q, want %q: the write should not truncate", got, want) + } + }) + + t.Run("write-missing", func(t *testing.T) { + // writing must not create: a sysfs attribute which is not there is a + // kernel which does not support it, not a file to make + err := writeFile(h, "sys/devices/absent", []byte("1")) + if !errors.Is(err, fs.ErrNotExist) { + t.Errorf("writing a missing file: %v, want fs.ErrNotExist", err) + } + if exists(h, "sys/devices/absent") { + t.Error("writing a missing file created it") + } + }) + + t.Run("invalid-path", func(t *testing.T) { + // fs.FS names are unrooted; an absolute path is a programming error and + // has to be rejected rather than silently escaping the root + for _, name := range []string{"/sys/devices/attr", "../escape", "./here"} { + if err := writeFile(h, name, []byte("1")); err == nil { + t.Errorf("writeFile(%q) succeeded, want an error", name) + } + } + }) + + t.Run("empty-root-is-slash", func(t *testing.T) { + if got, want := HostFS("").(*hostFS).root, "/"; got != want { + t.Errorf("HostFS(\"\").root = %q, want %q", got, want) + } + }) +} + +// +// helpers +// + +func cpuRange(lo, hi int) []int { + out := make([]int, 0, hi-lo+1) + for i := lo; i <= hi; i++ { + out = append(out, i) + } + return out +} + +func contains(haystack, needle string) bool { + return len(haystack) >= len(needle) && (func() bool { + for i := 0; i+len(needle) <= len(haystack); i++ { + if haystack[i:i+len(needle)] == needle { + return true + } + } + return false + })() +} + +// assertSealed checks that a set discovery produced cannot be modified. +func assertSealed(t *testing.T, cpus interface{ Set(...int) }) { + t.Helper() + defer func() { + if recover() == nil { + t.Error("set is not sealed") + } + }() + cpus.Set(0) +} diff --git a/pkg/lib/hardware/topology.go b/pkg/lib/hardware/topology.go new file mode 100644 index 000000000..81bfa26c6 --- /dev/null +++ b/pkg/lib/hardware/topology.go @@ -0,0 +1,459 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + "fmt" + "slices" + "strings" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// TopologyIndex is a [Machine]'s CPU topology flattened into one lookup +// table: which CPUs are in each package, each die, each cluster, each NUMA +// node, each cache, each core. +// +// It answers the same questions as searching the zones, and it holds the same +// [libcpu.CpuMask] values rather than copies of them, so it costs little beyond +// the maps themselves. What it adds is a lookup by coordinate: a caller which +// keeps reaching for "the CPUs of package 1, die 0" in a comparison function +// wants one map read, not a search through Zones. Code which reads the topology +// once at startup should use [Machine] and its zones; code which consults it per +// allocation should take a TopologyIndex and keep it. +// +// A TopologyIndex is derived once and never changes, so it is safe for +// concurrent use. Every set it returns is sealed, and a coordinate the machine +// does not have yields an empty set rather than an error. +type TopologyIndex struct { + m *Machine + + pkg map[ID]*libcpu.CpuMask + die map[DieID]*libcpu.CpuMask + cluster map[ClusterID]*libcpu.CpuMask + node map[ID]*libcpu.CpuMask + cache map[CacheID]*libcpu.CpuMask + core map[CoreID]*libcpu.CpuMask + threads map[ID]*libcpu.CpuMask + + pkgs []ID + dies []DieID + clusters []ClusterID + nodes []ID + caches []CacheID + cores []CoreID +} + +// buildIndex flattens a machine's zones into the maps a TopologyIndex answers +// from. The masks are the zones' own, not copies. +func (m *Machine) buildIndex() *TopologyIndex { + t := &TopologyIndex{ + m: m, + pkg: map[ID]*libcpu.CpuMask{}, + die: map[DieID]*libcpu.CpuMask{}, + cluster: map[ClusterID]*libcpu.CpuMask{}, + node: map[ID]*libcpu.CpuMask{}, + cache: map[CacheID]*libcpu.CpuMask{}, + core: map[CoreID]*libcpu.CpuMask{}, + threads: map[ID]*libcpu.CpuMask{}, + } + + for _, z := range m.zones[LevelPackage] { + t.pkg[z.id] = z.cpus + t.pkgs = append(t.pkgs, z.id) + } + for _, z := range m.zones[LevelDie] { + id := z.DieID() + t.die[id] = z.cpus + t.dies = append(t.dies, id) + } + for _, z := range m.zones[LevelCluster] { + id := z.ClusterID() + t.cluster[id] = z.cpus + t.clusters = append(t.clusters, id) + } + for _, id := range m.nodeIDs { + t.node[id] = m.nodes[id].cpus + t.nodes = append(t.nodes, id) + } + for key, cache := range m.caches { + t.cache[key] = cache.cpus + t.caches = append(t.caches, key) + } + for _, z := range m.zones[LevelCore] { + id := CoreID{Package: z.pkg, Core: z.id} + t.core[id] = z.cpus + t.cores = append(t.cores, id) + } + for _, id := range m.cpuIDs { + t.threads[id] = m.cpus[id].Threads() + } + + slices.Sort(t.pkgs) + slices.Sort(t.nodes) + slices.SortFunc(t.dies, compareDieIDs) + slices.SortFunc(t.clusters, compareClusterIDs) + slices.SortFunc(t.caches, compareCacheIDs) + slices.SortFunc(t.cores, compareCoreIDs) + + return t +} + +func compareDieIDs(a, b DieID) int { + if a.Package != b.Package { + return a.Package - b.Package + } + return a.Die - b.Die +} + +func compareClusterIDs(a, b ClusterID) int { + if d := compareDieIDs(DieID{a.Package, a.Die}, DieID{b.Package, b.Die}); d != 0 { + return d + } + return a.Cluster - b.Cluster +} + +func compareCacheIDs(a, b CacheID) int { + if a.Level != b.Level { + return a.Level - b.Level + } + if a.Kind != b.Kind { + return int(a.Kind) - int(b.Kind) + } + return a.ID - b.ID +} + +func compareCoreIDs(a, b CoreID) int { + if a.Package != b.Package { + return a.Package - b.Package + } + return a.Core - b.Core +} + +// TopologyIndex returns this machine's topology as a flat lookup table. The +// result is computed once and shared, so calling this repeatedly is cheap. +func (m *Machine) TopologyIndex() *TopologyIndex { + return m.index +} + +// +// Coordinates +// +// These name a place in the topology and are usable as map keys, so that a +// caller which groups CPUs by die or by cache can key its own maps the same way +// this one does. +// + +// DieID identifies a die within a package. +type DieID struct { + Package ID + Die ID +} + +// ClusterID identifies a cluster within a die. +type ClusterID struct { + Package ID + Die ID + Cluster ID +} + +// CacheID identifies a cache. The kernel numbers caches within a level and a +// kind, so all three are needed: a machine's L1 data and L1 instruction caches +// both start at id 0. +type CacheID struct { + Level int + Kind CacheKind + ID ID +} + +// CoreID identifies a physical core within a package, since the kernel numbers +// cores per package rather than per machine. +type CoreID struct { + Package ID + Core ID +} + +// String returns "package#

/die#". +func (d DieID) String() string { + return fmt.Sprintf("package#%d/die#%d", d.Package, d.Die) +} + +// String returns "package#

/die#/cluster#". +func (c ClusterID) String() string { + return fmt.Sprintf("package#%d/die#%d/cluster#%d", c.Package, c.Die, c.Cluster) +} + +// String returns "L#". +func (c CacheID) String() string { + return fmt.Sprintf("L%d%s#%d", c.Level, c.Kind.suffix(), c.ID) +} + +// String returns "package#

/core#". +func (c CoreID) String() string { + return fmt.Sprintf("package#%d/core#%d", c.Package, c.Core) +} + +// +// CPUs by coordinate +// + +// PackageCPUs returns the CPUs of the given package. +func (t *TopologyIndex) PackageCPUs(pkg ID) *libcpu.CpuMask { + if cpus, ok := t.pkg[pkg]; ok { + return cpus + } + return emptyCPUs +} + +// DieCPUs returns the CPUs of the given die. +func (t *TopologyIndex) DieCPUs(die DieID) *libcpu.CpuMask { + if cpus, ok := t.die[die]; ok { + return cpus + } + return emptyCPUs +} + +// ClusterCPUs returns the CPUs of the given cluster. +func (t *TopologyIndex) ClusterCPUs(cluster ClusterID) *libcpu.CpuMask { + if cpus, ok := t.cluster[cluster]; ok { + return cpus + } + return emptyCPUs +} + +// MemoryNodeCPUs returns the CPUs local to the given NUMA node, which is empty +// for a node holding only memory. +func (t *TopologyIndex) MemoryNodeCPUs(node ID) *libcpu.CpuMask { + if cpus, ok := t.node[node]; ok { + return cpus + } + return emptyCPUs +} + +// CacheCPUs returns the CPUs sharing the given cache. +func (t *TopologyIndex) CacheCPUs(cache CacheID) *libcpu.CpuMask { + if cpus, ok := t.cache[cache]; ok { + return cpus + } + return emptyCPUs +} + +// CoreCPUs returns the CPUs of the given core, i.e. its hardware threads. +func (t *TopologyIndex) CoreCPUs(core CoreID) *libcpu.CpuMask { + if cpus, ok := t.core[core]; ok { + return cpus + } + return emptyCPUs +} + +// ThreadsOf returns the CPUs sharing a core with the given CPU, including it. +// It is the lookup by CPU id that [TopologyIndex.CoreCPUs] is by core id, and +// what a caller iterating CPUs rather than cores wants. +func (t *TopologyIndex) ThreadsOf(cpu ID) *libcpu.CpuMask { + if cpus, ok := t.threads[cpu]; ok { + return cpus + } + return emptyCPUs +} + +// CoreKindCPUs returns the CPUs of the given core kind. +func (t *TopologyIndex) CoreKindCPUs(kind CoreKind) *libcpu.CpuMask { + return t.m.CoreKindCPUs(kind) +} + +// +// Coordinates present in the machine +// +// These enumerate what there is to look up, so that a caller can iterate a +// dimension without knowing the machine. All are ordered, and stable across +// calls. +// + +// PackageIDs returns the ids of all packages. +func (t *TopologyIndex) PackageIDs() []ID { + return t.pkgs +} + +// DieIDs returns every die of the machine, or the dies of the given packages +// if any are named. +func (t *TopologyIndex) DieIDs(pkgs ...ID) []DieID { + if len(pkgs) == 0 { + return t.dies + } + var out []DieID + for _, die := range t.dies { + if slices.Contains(pkgs, die.Package) { + out = append(out, die) + } + } + return out +} + +// ClusterIDs returns every cluster of the machine, or the clusters of the +// given dies if any are named. It is empty on a machine whose clustering says +// nothing; see [LevelCluster]. +func (t *TopologyIndex) ClusterIDs(dies ...DieID) []ClusterID { + if len(dies) == 0 { + return t.clusters + } + var out []ClusterID + for _, cl := range t.clusters { + if slices.Contains(dies, DieID{Package: cl.Package, Die: cl.Die}) { + out = append(out, cl) + } + } + return out +} + +// MemoryNodeIDs returns the ids of all NUMA nodes. +func (t *TopologyIndex) MemoryNodeIDs() []ID { + return t.nodes +} + +// CacheIDs returns every cache of the machine, or the caches at the given +// levels if any are named. +func (t *TopologyIndex) CacheIDs(levels ...int) []CacheID { + if len(levels) == 0 { + return t.caches + } + var out []CacheID + for _, id := range t.caches { + if slices.Contains(levels, id.Level) { + out = append(out, id) + } + } + return out +} + +// CacheLevels returns the cache levels the machine has, lowest first. +func (t *TopologyIndex) CacheLevels() []int { + return t.m.CacheLevels() +} + +// CoreIDs returns every core of the machine, or the cores of the given +// packages if any are named. +func (t *TopologyIndex) CoreIDs(pkgs ...ID) []CoreID { + if len(pkgs) == 0 { + return t.cores + } + var out []CoreID + for _, core := range t.cores { + if slices.Contains(pkgs, core.Package) { + out = append(out, core) + } + } + return out +} + +// CoreKinds returns the core kinds the machine has. +func (t *TopologyIndex) CoreKinds() []CoreKind { + return t.m.CoreKinds() +} + +// +// Whole-machine sets +// +// The same sets [Machine] reports, repeated here so that a caller holding +// only a TopologyIndex does not have to keep the Machine as well. +// + +// AllCPUs returns every CPU the machine has, online or not. +func (t *TopologyIndex) AllCPUs() *libcpu.CpuMask { + return t.m.PresentCPUs() +} + +// OnlineCPUs returns the CPUs which are online. +func (t *TopologyIndex) OnlineCPUs() *libcpu.CpuMask { + return t.m.OnlineCPUs() +} + +// OfflineCPUs returns the CPUs which are present but not online. +func (t *TopologyIndex) OfflineCPUs() *libcpu.CpuMask { + return t.m.OfflineCPUs() +} + +// IsolatedCPUs returns the CPUs the kernel was told to isolate. +func (t *TopologyIndex) IsolatedCPUs() *libcpu.CpuMask { + return t.m.IsolatedCPUs() +} + +// +// Reverse lookup +// + +// CoordinatesOf returns where the given CPU sits in the topology. The ids of +// coordinates the machine does not have, or which are not known for an offline +// CPU, are -1. +func (t *TopologyIndex) CoordinatesOf(cpu ID) Coordinates { + c := t.m.CPU(cpu) + if !c.Valid() { + return Coordinates{ + CPU: unknownID, Package: unknownID, Die: unknownID, + Cluster: unknownID, MemoryNode: unknownID, Core: unknownID, + } + } + return c.coordinates() +} + +// Coordinates is where one CPU sits in the topology: everything a caller would +// otherwise ask a [CPU] handle for one field at a time. +type Coordinates struct { + // CPU is the id of the CPU itself. + CPU ID + // Package is the package it is in. + Package ID + // Die is the die it is in. + Die ID + // Cluster is the cluster it is in, or -1 if the machine has no meaningful + // clustering. + Cluster ID + // MemoryNode is the NUMA node it is local to. + MemoryNode ID + // Core is the core it is a thread of, numbered within Package. + Core ID + // Kind is whether it is a performance or an efficiency core. + Kind CoreKind +} + +// String returns the coordinates in the form +// "cpu#3 package#0/die#0/cluster#1/node#0/core#3 (P-core)". +func (c Coordinates) String() string { + return fmt.Sprintf( + "cpu#%d package#%d/die#%d/cluster#%d/node#%d/core#%d (%s)", + c.CPU, c.Package, c.Die, c.Cluster, c.MemoryNode, c.Core, c.Kind) +} + +// String returns a multi-line description of the topology, for logs. +func (t *TopologyIndex) String() string { + var b strings.Builder + + fmt.Fprintf(&b, "%d CPUs (%s), %d online, %d isolated\n", + t.AllCPUs().Size(), t.AllCPUs(), t.OnlineCPUs().Size(), + t.IsolatedCPUs().Size()) + for _, pkg := range t.PackageIDs() { + fmt.Fprintf(&b, " package#%d: %s\n", pkg, t.PackageCPUs(pkg)) + for _, die := range t.DieIDs(pkg) { + fmt.Fprintf(&b, " %s: %s\n", die, t.DieCPUs(die)) + } + } + for _, node := range t.MemoryNodeIDs() { + fmt.Fprintf(&b, " node#%d: %s\n", node, t.MemoryNodeCPUs(node)) + } + for _, cache := range t.CacheIDs() { + fmt.Fprintf(&b, " %s: %s\n", cache, t.CacheCPUs(cache)) + } + + return b.String() +} diff --git a/pkg/lib/hardware/zone.go b/pkg/lib/hardware/zone.go new file mode 100644 index 000000000..185dff29d --- /dev/null +++ b/pkg/lib/hardware/zone.go @@ -0,0 +1,204 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package hardware + +import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" +) + +// Level names a kind of hardware which CPUs can share. +// +// A machine has zones at only some of these levels, and which ones is a property +// of the hardware. Ask [Machine.Levels] rather than assuming. +// +// The constants are a tag, not an order. Which level contains which is a +// property of the machine, not of this list: on one machine a cluster is a few +// cores within a NUMA node, on another a single cluster covers a whole die. Ask +// with [ZonesWithin] or [ZoneOf] when it matters. +type Level int + +const ( + // LevelPackage is a physical package, i.e. a socket. + LevelPackage Level = iota + // LevelDie is a die within a package. + LevelDie + // LevelCluster is a group of CPUs the kernel reports as a cluster, from + // topology/cluster_id and topology/cluster_cpus_list. What a cluster shares + // is up to the hardware, and on some machines the grouping is real and + // worth allocating along. On others it says nothing: every cluster is one + // core, or a single cluster covers everything. There is no cluster level in + // those cases. See also [LogicalClusters], for machines which report one + // cluster per core only because of how their threads are counted. + LevelCluster + // LevelNUMANode is a NUMA node: CPUs with the same locality to a piece of + // memory. See [MemoryNode] for the memory itself. + LevelNUMANode + // LevelL3Cache is a group of CPUs sharing a level 3 cache. + LevelL3Cache + // LevelL2Cache is a group of CPUs sharing a level 2 cache. + LevelL2Cache + // LevelCore is a physical core, i.e. a group of hardware threads. + LevelCore + // LevelThread is a single CPU as the kernel counts them. + LevelThread + + // numLevels is how many levels there are, for sizing an array by Level. + numLevels = int(LevelThread) + 1 +) + +// String returns the name of the level. +func (l Level) String() string { + if l < 0 || int(l) >= numLevels { + return "unknown level" + } + return levelNames[l] +} + +// levelNames name the levels, indexed by [Level]. They double as the prefix +// [Zone.Name] is built from. +var levelNames = [...]string{ + LevelPackage: "package", + LevelDie: "die", + LevelCluster: "cluster", + LevelNUMANode: "node", + LevelL3Cache: "L3", + LevelL2Cache: "L2", + LevelCore: "core", + LevelThread: "thread", +} + +// allLevels are the levels which may have zones, in the order +// [Machine.Levels] reports them. It is a reporting order and nothing more. +var allLevels = []Level{ + LevelPackage, LevelDie, LevelCluster, LevelNUMANode, + LevelL3Cache, LevelL2Cache, LevelCore, LevelThread, +} + +// Zone is a set of CPUs which share a piece of hardware at one [Level]: a +// package, a die, a cluster, a NUMA node, a cache, a core, a thread. +// +// Zones do not form a tree. The hardware does not describe one: whether a +// cluster falls inside a NUMA node or spans several is a property of the +// machine, and which levels are worth nesting is a decision each caller makes +// differently. [Machine.Zones] lists the zones of a level, [ZonesWithin] and +// [ZonesOverlapping] answer containment questions, and a caller which wants a +// hierarchy builds the one it wants out of those. +// +// A Zone is a handle into the [Machine] which produced it: never nil, and safe +// to use even when it refers to nothing, in which case Valid() is false. +type Zone struct { + m *Machine + valid bool + + level Level + id ID + name string + cpus *libcpu.CpuMask + + // where it sits, as the kernel numbers things. unknownID for a coordinate + // which does not apply or which the machine does not report. + pkg ID + die ID + cluster ID + node ID + + cache *Cache // set for the cache levels +} + +// invalidZone is what a lookup for a zone the machine does not have returns. Its +// methods answer with zero values. +var invalidZone = &Zone{ + id: unknownID, pkg: unknownID, die: unknownID, + cluster: unknownID, node: unknownID, +} + +// Valid reports whether the zone refers to real hardware. +func (z *Zone) Valid() bool { + return z.valid +} + +// Level returns the level of hardware this zone represents. +func (z *Zone) Level() Level { + return z.level +} + +// ID returns the id of the hardware this zone represents, as the kernel +// numbers it. Ids are unique within a level. +func (z *Zone) ID() ID { + return z.id +} + +// Name returns a stable name for the zone, spelling out where it sits, for +// instance "package#1/die#0/node#2". Names are unique within a [Machine] +// and are meant for logs and for keying by position; use [Zone.ID] to identify +// the hardware itself. +func (z *Zone) Name() string { + return z.name +} + +// CPUs returns the CPUs in this zone. The set is sealed. +func (z *Zone) CPUs() *libcpu.CpuMask { + if z.cpus == nil { + return emptyCPUs + } + return z.cpus +} + +// MemoryNode returns the NUMA node this zone's CPUs are local to, or an invalid +// node if they span more than one. +func (z *Zone) MemoryNode() *MemoryNode { + if z.m == nil || z.node == unknownID { + return invalidMemoryNode + } + return z.m.MemoryNode(z.node) +} + +// Cache returns the cache this zone represents, or an invalid cache if the zone +// is not a cache level. +func (z *Zone) Cache() *Cache { + if z.cache == nil { + return invalidCache + } + return z.cache +} + +// DieID returns the coordinates of the die this zone is in, or is, with both ids +// -1 if it is not within a single die. Unlike [Zone.ID] this says which package +// the die belongs to, which matters because the kernel numbers dies per package +// rather than per machine. +func (z *Zone) DieID() DieID { + if z.pkg == unknownID || z.die == unknownID { + return DieID{Package: unknownID, Die: unknownID} + } + return DieID{Package: z.pkg, Die: z.die} +} + +// ClusterID returns the coordinates of the cluster this zone is in, or is, with +// all ids -1 if it is not within a single cluster or the machine reports no +// meaningful clustering. +func (z *Zone) ClusterID() ClusterID { + if z.pkg == unknownID || z.die == unknownID || z.cluster == unknownID { + return ClusterID{Package: unknownID, Die: unknownID, Cluster: unknownID} + } + return ClusterID{Package: z.pkg, Die: z.die, Cluster: z.cluster} +} + +// String returns [Zone.Name] and the zone's CPUs. +func (z *Zone) String() string { + if !z.valid { + return "" + } + return z.name + " (" + z.CPUs().String() + ")" +} From e9db4f7d6508ef91be99cb466c30177c9b722571 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 9 Sep 2026 18:49:52 +0300 Subject: [PATCH 13/39] lib/hardware/system: drop-in for pkg/sysfs. Reimplement pkg/sysfs on top of pkg/lib/hardware, with the same names and the same signatures, so that a consumer moves over by changing one import line. Most already import pkg/sysfs aliased to "system", which is why the package is called that: for those files even the alias stays. This is a migration step and a proof. pkg/sysfs stays in the tree beside it, so equivalence_test.go can run every method of both against the same recorded topology and compare the answers. It compares everything, including the methods no caller uses, since those are the ones a reimplementation gets wrong unnoticed. test-setup.sh unpacks the six recorded machines the other packages already keep, rather than adding another copy of them here. internal/dropin holds one body of code compiled against both packages, so the compiler enforces that the surfaces match, and a test enforces that the two files are the same code. One pkg/sysfs quirk is copied deliberately: a NUMA node with no CPUs of its own reports package 0, which is the zero value of a field pkg/sysfs never assigns rather than a real package. hardware.MemoryNode says -1 instead. SST lives here rather than in hardware, and moves on to whoever still wants it. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/lib/hardware/system/doc.go | 88 +++ pkg/lib/hardware/system/equivalence_test.go | 665 ++++++++++++++++++ .../hardware/system/internal/dropin/doc.go | 24 + .../system/internal/dropin/dropin_test.go | 67 ++ .../system/internal/dropin/viasysfs/api.go | 179 +++++ .../system/internal/dropin/viasystem/api.go | 179 +++++ pkg/lib/hardware/system/objects.go | 570 +++++++++++++++ pkg/lib/hardware/system/shim_test.go | 367 ++++++++++ pkg/lib/hardware/system/system.go | 665 ++++++++++++++++++ pkg/lib/hardware/system/test-cleanup.sh | 4 + pkg/lib/hardware/system/test-setup.sh | 44 ++ pkg/lib/hardware/system/types.go | 294 ++++++++ 12 files changed, 3146 insertions(+) create mode 100644 pkg/lib/hardware/system/doc.go create mode 100644 pkg/lib/hardware/system/equivalence_test.go create mode 100644 pkg/lib/hardware/system/internal/dropin/doc.go create mode 100644 pkg/lib/hardware/system/internal/dropin/dropin_test.go create mode 100644 pkg/lib/hardware/system/internal/dropin/viasysfs/api.go create mode 100644 pkg/lib/hardware/system/internal/dropin/viasystem/api.go create mode 100644 pkg/lib/hardware/system/objects.go create mode 100644 pkg/lib/hardware/system/shim_test.go create mode 100644 pkg/lib/hardware/system/system.go create mode 100755 pkg/lib/hardware/system/test-cleanup.sh create mode 100755 pkg/lib/hardware/system/test-setup.sh create mode 100644 pkg/lib/hardware/system/types.go diff --git a/pkg/lib/hardware/system/doc.go b/pkg/lib/hardware/system/doc.go new file mode 100644 index 000000000..fc2df84b8 --- /dev/null +++ b/pkg/lib/hardware/system/doc.go @@ -0,0 +1,88 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package system is pkg/sysfs reimplemented on top of +// [github.com/containers/nri-plugins/pkg/lib/hardware]. +// +// Everything pkg/sysfs exports is exported here with the same name and the same +// signature, so a consumer moves over by changing one import line. Most already +// import pkg/sysfs aliased to "system", which is why this package is called +// that: for those files even the alias stays as it was. +// +// # Why it exists +// +// It is a migration step, and a proof. pkg/sysfs stays in the tree beside it, so +// both implementations are in one build and equivalence_test.go can run every +// method of both against the same recorded sysfs trees and compare the answers. +// That is a much stronger statement than a rewritten pkg/sysfs could make, where +// the only reference left would be in git history. +// +// It also splits the work into reviewable pieces. This package plus the hardware +// package underneath it change no behaviour and no caller, so they can land on +// their own. Moving consumers off pkg/sysfs, and deleting it, comes after. +// +// Nothing new should be built on this package. New code should use +// [github.com/containers/nri-plugins/pkg/lib/hardware] directly; this is +// here to be deleted. +// +// # Where it is faithful, and where it cannot be +// +// The intent is bug-for-bug compatibility, including the parts of pkg/sysfs +// which are odd. Some of it is worth calling out. +// +// Cache is an exported struct with unexported fields, and at least one caller +// uses a *Cache as a map key, relying on pkg/sysfs handing out one pointer per +// cache. This package interns its *Cache values the same way, so pointer +// identity keeps working. +// +// MemoryInfo reads the machine on every call in pkg/sysfs. It does the same +// here, rather than answering from hardware's cached capacity, because callers +// use it to read current usage. +// +// A NUMA node with no CPUs of its own reports package 0 and die 0, which is what +// pkg/sysfs reports -- not because such a node is in package 0 but because that +// is the zero value of a field it never assigns. hardware.MemoryNode says -1, +// which is honest; this says 0, which is compatible. It is the last thing here +// which copies a pkg/sysfs quirk rather than an intent. +// +// Sst, SstInfo and SstClos expose Intel Speed Select through goresctrl types. +// The hardware package deliberately has nothing to do with SST, so the +// discovery for those three lives here. It is the last thing to move, and it +// moves to whoever ends up wanting it -- today only pkg/cpuallocator does. +// +// SetCpusOnline, SetCPUFrequencyLimits and CPU.SetFrequencyLimits write to +// sysfs. The hardware package only reads, so these are implemented here over +// [hardware.WriterFS], obtained from [hardware.Machine.FS] so that a write goes +// to the tree the topology was read from. They have no callers left in the tree; +// they are here because the interface has them. +// +// Discover re-runs discovery on an existing System and mutates it in place. +// Nothing calls it with anything the constructors did not already ask for. +// +// The DiscoveryFlag values are accepted and ignored, exactly as pkg/sysfs +// ignores them: DiscoverSystem discards its arguments before passing them on, +// and the flags inside Discover all take the same path anyway. +// +// SetSysRoot keeps its package-global behaviour, so that the one caller in +// pkg/resmgr does not have to change. The global is read when a System is +// constructed and not consulted afterwards. +// +// # What it costs +// +// Every set crosses a representation boundary: hardware works in +// [libcpu.CpuMask], these interfaces in cpuset.CPUSet and idset.IDSet, so each +// call which returns a set converts one. That is fine for a layer whose purpose +// is to be removed, and it is a reason not to leave it in place longer than +// necessary. +package system diff --git a/pkg/lib/hardware/system/equivalence_test.go b/pkg/lib/hardware/system/equivalence_test.go new file mode 100644 index 000000000..1a8505d98 --- /dev/null +++ b/pkg/lib/hardware/system/equivalence_test.go @@ -0,0 +1,665 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// This is the reason pkg/sysfs stays in the tree while this package exists: both +// implementations are in one build, so every method of both can be run against +// the same recorded topology and the answers compared. Nothing else states +// equivalence as strongly. +// +// It compares everything, including the methods no caller uses, because the ones +// nobody calls are exactly the ones a reimplementation gets wrong unnoticed. +// +// Once every consumer has moved off pkg/sysfs, this file and pkg/sysfs go +// together. + +package system_test + +import ( + "os" + "path/filepath" + "slices" + "strconv" + "testing" + + "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// trees are the recorded sysfs topologies to compare over, as test-setup.sh +// unpacks them. They come from pkg/sysfs, pkg/cpuallocator and the +// topology-aware policy, which each keep some for their own tests. +var trees = []string{ + "sample1", + "sample2", + "2-socket-4-node-40-core", + "4-socket-server-nosnc", + "desktop", + "server", +} + +// pair is one topology discovered through both implementations. +type pair struct { + name string + old sysfs.System + new system.System +} + +// discoverBoth reads one recorded tree through pkg/sysfs and through this +// package. Both take the sysfs mount point, so both get the same argument. +func discoverBoth(t *testing.T, tree string) pair { + t.Helper() + + root, err := filepath.Abs(filepath.Join("testdata", tree, "sys")) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(root); err != nil { + t.Skipf("recorded tree %s is not unpacked: run ./test-setup.sh", tree) + } + + old, err := sysfs.DiscoverSystemAt(root) + if err != nil { + t.Fatalf("pkg/sysfs failed to discover %s: %v", tree, err) + } + + new, err := system.DiscoverSystemAt(root) + if err != nil { + t.Fatalf("the drop-in failed to discover %s: %v", tree, err) + } + + return pair{name: tree, old: old, new: new} +} + +func TestEquivalentSystem(t *testing.T) { + for _, tree := range trees { + t.Run(tree, func(t *testing.T) { + p := discoverBoth(t, tree) + + // whole-machine sets + eqCPUSet(t, "CPUSet", p.old.CPUSet(), p.new.CPUSet()) + eqCPUSet(t, "PossibleCPUs", p.old.PossibleCPUs(), p.new.PossibleCPUs()) + eqCPUSet(t, "PresentCPUs", p.old.PresentCPUs(), p.new.PresentCPUs()) + eqCPUSet(t, "OnlineCPUs", p.old.OnlineCPUs(), p.new.OnlineCPUs()) + eqCPUSet(t, "OfflineCPUs", p.old.OfflineCPUs(), p.new.OfflineCPUs()) + eqCPUSet(t, "IsolatedCPUs", p.old.IsolatedCPUs(), p.new.IsolatedCPUs()) + eqCPUSet(t, "Offlined", p.old.Offlined(), p.new.Offlined()) + eqCPUSet(t, "Isolated", p.old.Isolated(), p.new.Isolated()) + + // counts + eqInt(t, "PackageCount", p.old.PackageCount(), p.new.PackageCount()) + eqInt(t, "SocketCount", p.old.SocketCount(), p.new.SocketCount()) + eqInt(t, "CPUCount", p.old.CPUCount(), p.new.CPUCount()) + eqInt(t, "NUMANodeCount", p.old.NUMANodeCount(), p.new.NUMANodeCount()) + eqInt(t, "MinThreadCount", p.old.MinThreadCount(), p.new.MinThreadCount()) + eqInt(t, "MaxThreadCount", p.old.MaxThreadCount(), p.new.MaxThreadCount()) + + // id lists + eqIDs(t, "PackageIDs", p.old.PackageIDs(), p.new.PackageIDs()) + eqIDs(t, "NodeIDs", p.old.NodeIDs(), p.new.NodeIDs()) + eqIDs(t, "CPUIDs", p.old.CPUIDs(), p.new.CPUIDs()) + + eqCoreKinds(t, p) + eqPackages(t, p) + eqNodes(t, p) + eqCPUs(t, p) + eqDerivedSets(t, p) + eqNodeFilters(t, p) + eqNodeHints(t, p) + }) + } +} + +func eqCoreKinds(t *testing.T, p pair) { + t.Helper() + + oldKinds, newKinds := p.old.CoreKinds(), p.new.CoreKinds() + if len(oldKinds) != len(newKinds) { + t.Errorf("CoreKinds: %d vs %d kinds", len(oldKinds), len(newKinds)) + } + + // compare by kind rather than by position: pkg/sysfs iterates a map + for _, kind := range []int{int(sysfs.PerformanceCore), int(sysfs.EfficientCore)} { + eqCPUSet(t, "CoreKindCPUs("+sysfs.CoreKind(kind).String()+")", + p.old.CoreKindCPUs(sysfs.CoreKind(kind)), + p.new.CoreKindCPUs(system.CoreKind(kind))) + + inOld := slices.Contains(oldKinds, sysfs.CoreKind(kind)) + inNew := slices.Contains(newKinds, system.CoreKind(kind)) + if inOld != inNew { + t.Errorf("CoreKinds: %s present=%v vs %v", + sysfs.CoreKind(kind), inOld, inNew) + } + } +} + +func eqPackages(t *testing.T, p pair) { + t.Helper() + + for _, id := range p.old.PackageIDs() { + var ( + op = p.old.Package(id) + np = p.new.Package(id) + at = "package#" + itoa(id) + ) + if np == nil { + t.Errorf("%s: the drop-in has no such package", at) + continue + } + + eqInt(t, at+" ID", op.ID(), np.ID()) + eqCPUSet(t, at+" CPUSet", op.CPUSet(), np.CPUSet()) + eqIDs(t, at+" DieIDs", op.DieIDs(), np.DieIDs()) + eqIDs(t, at+" NodeIDs", op.NodeIDs(), np.NodeIDs()) + eqIDs(t, at+" L3CacheIDs", op.L3CacheIDs(), np.L3CacheIDs()) + + for _, l3 := range op.L3CacheIDs() { + eqCPUSet(t, at+" L3CacheCPUSet("+itoa(l3)+")", + op.L3CacheCPUSet(l3), np.L3CacheCPUSet(l3)) + } + + for _, die := range op.DieIDs() { + d := at + "/die#" + itoa(die) + eqCPUSet(t, d+" DieCPUSet", op.DieCPUSet(die), np.DieCPUSet(die)) + eqIDs(t, d+" DieNodeIDs", op.DieNodeIDs(die), np.DieNodeIDs(die)) + eqIDs(t, d+" DieClusterIDs", + op.DieClusterIDs(die), np.DieClusterIDs(die)) + eqIDs(t, d+" LogicalDieClusterIDs", + op.LogicalDieClusterIDs(die), np.LogicalDieClusterIDs(die)) + + for _, cl := range op.DieClusterIDs(die) { + eqCPUSet(t, d+" DieClusterCPUSet("+itoa(cl)+")", + op.DieClusterCPUSet(die, cl), np.DieClusterCPUSet(die, cl)) + } + for _, cl := range op.LogicalDieClusterIDs(die) { + eqCPUSet(t, d+" LogicalDieClusterCPUSet("+itoa(cl)+")", + op.LogicalDieClusterCPUSet(die, cl), + np.LogicalDieClusterCPUSet(die, cl)) + } + } + + // a die which is not there answers the same way in both + eqCPUSet(t, at+" DieCPUSet(absent)", + op.DieCPUSet(1<<20), np.DieCPUSet(1<<20)) + eqIDs(t, at+" DieNodeIDs(absent)", + op.DieNodeIDs(1<<20), np.DieNodeIDs(1<<20)) + eqIDs(t, at+" DieClusterIDs(absent)", + op.DieClusterIDs(1<<20), np.DieClusterIDs(1<<20)) + eqCPUSet(t, at+" L3CacheCPUSet(absent)", + op.L3CacheCPUSet(1<<20), np.L3CacheCPUSet(1<<20)) + } +} + +func eqNodes(t *testing.T, p pair) { + t.Helper() + + for _, id := range p.old.NodeIDs() { + var ( + on = p.old.Node(id) + nn = p.new.Node(id) + at = "node#" + itoa(id) + ) + if nn == nil { + t.Errorf("%s: the drop-in has no such node", at) + continue + } + + eqInt(t, at+" ID", on.ID(), nn.ID()) + eqInt(t, at+" PackageID", on.PackageID(), nn.PackageID()) + eqInt(t, at+" DieID", on.DieID(), nn.DieID()) + eqCPUSet(t, at+" CPUSet", on.CPUSet(), nn.CPUSet()) + eqInts(t, at+" Distance", on.Distance(), nn.Distance()) + eqBool(t, at+" HasNormalMemory", + on.HasNormalMemory(), nn.HasNormalMemory()) + eqStr(t, at+" GetMemoryType", + on.GetMemoryType().String(), nn.GetMemoryType().String()) + + for _, to := range p.old.NodeIDs() { + eqInt(t, at+" DistanceFrom("+itoa(to)+")", + on.DistanceFrom(to), nn.DistanceFrom(to)) + } + eqInt(t, at+" DistanceFrom(absent)", + on.DistanceFrom(1<<20), nn.DistanceFrom(1<<20)) + + oNodes, oDist := on.ClosestNodes() + nNodes, nDist := nn.ClosestNodes() + eqInts(t, at+" ClosestNodes distances", oDist, nDist) + eqIDSets(t, at+" ClosestNodes", oNodes, nNodes) + + // MemoryInfo reads the machine, so free and used can move between the two + // calls. Only the total is stable enough to compare. + oi, oerr := on.MemoryInfo() + ni, nerr := nn.MemoryInfo() + switch { + case (oerr == nil) != (nerr == nil): + t.Errorf("%s MemoryInfo: errors differ: %v vs %v", at, oerr, nerr) + case oerr == nil && oi.MemTotal != ni.MemTotal: + t.Errorf("%s MemoryInfo MemTotal: %d vs %d", at, oi.MemTotal, ni.MemTotal) + } + } +} + +func eqCPUs(t *testing.T, p pair) { + t.Helper() + + for _, id := range p.old.CPUIDs() { + var ( + oc = p.old.CPU(id) + nc = p.new.CPU(id) + at = "cpu#" + itoa(id) + ) + if nc == nil { + t.Errorf("%s: the drop-in has no such CPU", at) + continue + } + + eqInt(t, at+" ID", oc.ID(), nc.ID()) + eqInt(t, at+" PackageID", oc.PackageID(), nc.PackageID()) + eqInt(t, at+" DieID", oc.DieID(), nc.DieID()) + eqInt(t, at+" ClusterID", oc.ClusterID(), nc.ClusterID()) + eqInt(t, at+" NodeID", oc.NodeID(), nc.NodeID()) + eqInt(t, at+" CoreID", oc.CoreID(), nc.CoreID()) + eqCPUSet(t, at+" ThreadCPUSet", oc.ThreadCPUSet(), nc.ThreadCPUSet()) + eqBool(t, at+" Online", oc.Online(), nc.Online()) + eqBool(t, at+" Isolated", oc.Isolated(), nc.Isolated()) + eqStr(t, at+" CoreKind", oc.CoreKind().String(), nc.CoreKind().String()) + eqStr(t, at+" EPP", oc.EPP().String(), nc.EPP().String()) + eqInt(t, at+" SstClos", oc.SstClos(), nc.SstClos()) + eqInt(t, at+" CacheCount", oc.CacheCount(), nc.CacheCount()) + + if got, want := nc.BaseFrequency(), oc.BaseFrequency(); got != want { + t.Errorf("%s BaseFrequency: %d vs %d", at, want, got) + } + of, nf := oc.FrequencyRange(), nc.FrequencyRange() + if of.Base != nf.Base || of.Min != nf.Min || of.Max != nf.Max { + t.Errorf("%s FrequencyRange: %+v vs %+v", at, of, nf) + } + + eqCaches(t, at+" GetCaches", oc.GetCaches(), nc.GetCaches()) + eqCaches(t, at+" GetLastLevelCaches", + oc.GetLastLevelCaches(), nc.GetLastLevelCaches()) + eqCPUSet(t, at+" GetLastLevelCacheCPUSet", + oc.GetLastLevelCacheCPUSet(), nc.GetLastLevelCacheCPUSet()) + + for level := 0; level <= 4; level++ { + eqCaches(t, at+" GetCachesByLevel("+itoa(level)+")", + oc.GetCachesByLevel(level), nc.GetCachesByLevel(level)) + eqCPUSet(t, at+" GetNthLevelCacheCPUSet("+itoa(level)+")", + oc.GetNthLevelCacheCPUSet(level), nc.GetNthLevelCacheCPUSet(level)) + } + for idx := -1; idx <= oc.CacheCount(); idx++ { + eqCache(t, at+" GetCacheByIndex("+itoa(idx)+")", + oc.GetCacheByIndex(idx), nc.GetCacheByIndex(idx)) + } + } +} + +// eqCacheIdentity checks that both implementations hand out one *Cache per cache, +// which the balloons policy relies on by keying a map with it. +func TestEquivalentCacheIdentity(t *testing.T) { + for _, tree := range trees { + t.Run(tree, func(t *testing.T) { + p := discoverBoth(t, tree) + + oldSeen := map[*sysfs.Cache]cpuset.CPUSet{} + newSeen := map[*system.Cache]cpuset.CPUSet{} + + for _, id := range p.old.CPUIDs() { + for _, c := range p.old.CPU(id).GetCaches() { + oldSeen[c] = c.SharedCPUSet() + } + for _, c := range p.new.CPU(id).GetCaches() { + newSeen[c] = c.SharedCPUSet() + } + } + + if len(oldSeen) != len(newSeen) { + t.Errorf("distinct caches by pointer: %d vs %d", + len(oldSeen), len(newSeen)) + } + + // the same cache asked for twice is the same pointer + for _, id := range p.old.CPUIDs() { + a := p.new.CPU(id).GetCaches() + b := p.new.CPU(id).GetCaches() + for i := range a { + if a[i] != b[i] { + t.Fatalf("cpu#%d cache %d is a different pointer each time", + id, i) + } + } + } + }) + } +} + +func eqDerivedSets(t *testing.T, p pair) { + t.Helper() + + for _, tc := range sampleSets(p) { + eqCPUSet(t, "AllThreadsForCPUs("+tc.String()+")", + p.old.AllThreadsForCPUs(tc), p.new.AllThreadsForCPUs(tc)) + eqCPUSet(t, "SingleThreadForCPUs("+tc.String()+")", + p.old.SingleThreadForCPUs(tc), p.new.SingleThreadForCPUs(tc)) + + for level := 0; level <= 4; level++ { + eqCPUSet(t, + "AllCPUsSharingNthLevelCacheWithCPUs("+itoa(level)+", "+tc.String()+")", + p.old.AllCPUsSharingNthLevelCacheWithCPUs(level, tc), + p.new.AllCPUsSharingNthLevelCacheWithCPUs(level, tc)) + } + + // IDSetForCPUs, with each of the ids a caller might ask for + eqIDSet(t, "IDSetForCPUs(package)", + p.old.IDSetForCPUs(tc, func(c sysfs.CPU) idset.ID { return c.PackageID() }), + p.new.IDSetForCPUs(tc, func(c system.CPU) idset.ID { return c.PackageID() })) + eqIDSet(t, "IDSetForCPUs(node)", + p.old.IDSetForCPUs(tc, func(c sysfs.CPU) idset.ID { return c.NodeID() }), + p.new.IDSetForCPUs(tc, func(c system.CPU) idset.ID { return c.NodeID() })) + eqIDSet(t, "IDSetForCPUs(die)", + p.old.IDSetForCPUs(tc, func(c sysfs.CPU) idset.ID { return c.DieID() }), + p.new.IDSetForCPUs(tc, func(c system.CPU) idset.ID { return c.DieID() })) + } +} + +func eqNodeFilters(t *testing.T, p pair) { + t.Helper() + + filters := []struct { + name string + old sysfs.NodeFilter + new system.NodeFilter + }{ + {"NodeOfDRAMType", sysfs.NodeOfDRAMType, system.NodeOfDRAMType}, + {"NodeOfPMEMType", sysfs.NodeOfPMEMType, system.NodeOfPMEMType}, + {"NodeOfHBMType", sysfs.NodeOfHBMType, system.NodeOfHBMType}, + {"NodeHasMemory", sysfs.NodeHasMemory, system.NodeHasMemory}, + {"NodeHasNoMemory", sysfs.NodeHasNoMemory, system.NodeHasNoMemory}, + {"NodeHasLocalCPUs", sysfs.NodeHasLocalCPUs, system.NodeHasLocalCPUs}, + {"NodeHasNoLocalCPUs", sysfs.NodeHasNoLocalCPUs, system.NodeHasNoLocalCPUs}, + } + + ids := p.old.NodeIDs() + + for _, f := range filters { + eqIDSet(t, "FilterNodes("+f.name+")", + p.old.FilterNodes(ids, f.old), p.new.FilterNodes(ids, f.new)) + + for _, id := range ids { + eqBool(t, "FilterNode("+itoa(id)+", "+f.name+")", + p.old.FilterNode(id, f.old), p.new.FilterNode(id, f.new)) + + oNodes, oDist := p.old.ClosestNodes(id, f.old) + nNodes, nDist := p.new.ClosestNodes(id, f.new) + eqInts(t, "ClosestNodes("+itoa(id)+", "+f.name+") distances", oDist, nDist) + eqIDSets(t, "ClosestNodes("+itoa(id)+", "+f.name+")", oNodes, nNodes) + } + } + + // no filters at all, and a combination, as pools.go uses them + eqIDSet(t, "FilterNodes()", p.old.FilterNodes(ids), p.new.FilterNodes(ids)) + eqIDSet(t, "FilterNodes(PMEM+HasMemory+NoLocalCPUs)", + p.old.FilterNodes(ids, sysfs.NodeOfPMEMType, sysfs.NodeHasMemory, + sysfs.NodeHasNoLocalCPUs), + p.new.FilterNodes(ids, system.NodeOfPMEMType, system.NodeHasMemory, + system.NodeHasNoLocalCPUs)) + + // a node which is not there + eqBool(t, "FilterNode(absent)", + p.old.FilterNode(1<<20), p.new.FilterNode(1<<20)) + oNodes, oDist := p.old.ClosestNodes(1 << 20) + nNodes, nDist := p.new.ClosestNodes(1 << 20) + eqInts(t, "ClosestNodes(absent) distances", oDist, nDist) + eqIDSets(t, "ClosestNodes(absent)", oNodes, nNodes) +} + +func eqNodeHints(t *testing.T, p pair) { + t.Helper() + + hints := []string{"", "0", "0-1", "1", itoa(1 << 20), "0," + itoa(1<<20), "bad"} + for _, hint := range hints { + eqStr(t, "NodeHintToCPUs("+hint+")", + p.old.NodeHintToCPUs(hint), p.new.NodeHintToCPUs(hint)) + } +} + +// TestEquivalentAbsentHardware checks that both implementations answer, rather +// than panicking, when asked about hardware the machine does not have. +// +// pkg/sysfs used to return a nil pointer inside a non-nil interface here, so its +// callers' nil checks were dead and the call after them panicked. That is fixed, +// so the drop-in has nothing to copy: both return an untyped nil and both +// tolerate an absent CPU in a set. +func TestEquivalentAbsentHardware(t *testing.T) { + for _, tree := range trees { + t.Run(tree, func(t *testing.T) { + p := discoverBoth(t, tree) + absent := cpuset.New(1<<20, 1<<20+1) + + // the lookups return a nil which compares equal to nil + if p.old.CPU(1<<20) != nil || p.new.CPU(1<<20) != nil { + t.Error("CPU(absent) is not nil in one of them") + } + if p.old.Node(1<<20) != nil || p.new.Node(1<<20) != nil { + t.Error("Node(absent) is not nil in one of them") + } + if p.old.Package(1<<20) != nil || p.new.Package(1<<20) != nil { + t.Error("Package(absent) is not nil in one of them") + } + + // the set helpers tolerate a CPU which is not there + eqCPUSet(t, "SingleThreadForCPUs(absent)", + p.old.SingleThreadForCPUs(absent), + p.new.SingleThreadForCPUs(absent)) + eqCPUSet(t, "AllThreadsForCPUs(absent)", + p.old.AllThreadsForCPUs(absent), p.new.AllThreadsForCPUs(absent)) + eqCPUSet(t, "AllCPUsSharingNthLevelCacheWithCPUs(2, absent)", + p.old.AllCPUsSharingNthLevelCacheWithCPUs(2, absent), + p.new.AllCPUsSharingNthLevelCacheWithCPUs(2, absent)) + eqIDSet(t, "IDSetForCPUs(absent)", + p.old.IDSetForCPUs(absent, func(c sysfs.CPU) idset.ID { return c.ID() }), + p.new.IDSetForCPUs(absent, func(c system.CPU) idset.ID { return c.ID() })) + + // and so does a set mixing absent CPUs with real ones + mixed := p.old.OnlineCPUs().Union(absent) + eqCPUSet(t, "SingleThreadForCPUs(mixed)", + p.old.SingleThreadForCPUs(mixed), p.new.SingleThreadForCPUs(mixed)) + + // an unknown node has no distance rather than a panic + eqInt(t, "NodeDistance(absent, 0)", + p.old.NodeDistance(1<<20, 0), p.new.NodeDistance(1<<20, 0)) + if got := p.new.NodeDistance(1<<20, 0); got != -1 { + t.Errorf("NodeDistance(absent, 0) = %d, want -1", got) + } + }) + } +} + +// TestEquivalentSetSysRoot checks that the package-global sys root behaves as it +// does in pkg/sysfs, including the cases SetSysRoot special-cases. +func TestEquivalentSetSysRoot(t *testing.T) { + // Both keep this in a package global, so discovering through it and through + // DiscoverSystemAt has to agree. + root, err := filepath.Abs(filepath.Join("testdata", trees[0])) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(root, "sys")); err != nil { + t.Skipf("recorded tree %s is not unpacked: run ./test-setup.sh", trees[0]) + } + + sysfs.SetSysRoot(root) + system.SetSysRoot(root) + t.Cleanup(func() { + sysfs.SetSysRoot("") + system.SetSysRoot("") + }) + + old, err := sysfs.DiscoverSystem() + if err != nil { + t.Fatalf("pkg/sysfs: %v", err) + } + new, err := system.DiscoverSystem() + if err != nil { + t.Fatalf("the drop-in: %v", err) + } + + eqCPUSet(t, "CPUSet via SetSysRoot", old.CPUSet(), new.CPUSet()) + eqIDs(t, "NodeIDs via SetSysRoot", old.NodeIDs(), new.NodeIDs()) +} + +// TestDiscoverSystemAtRejectsNonSysfs checks the one place the drop-in is +// stricter than pkg/sysfs, deliberately: it needs the host root above the mount +// point, so it has to recognise the mount point to find it. +func TestDiscoverSystemAtRejectsNonSysfs(t *testing.T) { + if _, err := system.DiscoverSystemAt("testdata/sample1"); err == nil { + t.Error("a path which is not a sysfs mount point was accepted") + } +} + +// +// comparison helpers +// +// Each says which method disagreed and how, since a bare "not equal" in a matrix +// this size is not enough to act on. +// + +func eqCPUSet(t *testing.T, what string, old, new cpuset.CPUSet) { + t.Helper() + if !old.Equals(new) { + t.Errorf("%s: pkg/sysfs %s, drop-in %s", what, old, new) + } +} + +func eqInt(t *testing.T, what string, old, new int) { + t.Helper() + if old != new { + t.Errorf("%s: pkg/sysfs %d, drop-in %d", what, old, new) + } +} + +func eqBool(t *testing.T, what string, old, new bool) { + t.Helper() + if old != new { + t.Errorf("%s: pkg/sysfs %v, drop-in %v", what, old, new) + } +} + +func eqStr(t *testing.T, what string, old, new string) { + t.Helper() + if old != new { + t.Errorf("%s: pkg/sysfs %q, drop-in %q", what, old, new) + } +} + +func eqIDs(t *testing.T, what string, old, new []idset.ID) { + t.Helper() + // both are documented as sorted, so compare them as they come + if !slices.Equal(old, new) { + t.Errorf("%s: pkg/sysfs %v, drop-in %v", what, old, new) + } +} + +func eqInts(t *testing.T, what string, old, new []int) { + t.Helper() + if !slices.Equal(old, new) { + t.Errorf("%s: pkg/sysfs %v, drop-in %v", what, old, new) + } +} + +func eqIDSet(t *testing.T, what string, old, new idset.IDSet) { + t.Helper() + if !slices.Equal(old.SortedMembers(), new.SortedMembers()) { + t.Errorf("%s: pkg/sysfs %v, drop-in %v", + what, old.SortedMembers(), new.SortedMembers()) + } +} + +func eqIDSets(t *testing.T, what string, old, new []idset.IDSet) { + t.Helper() + if len(old) != len(new) { + t.Errorf("%s: pkg/sysfs %d groups, drop-in %d", what, len(old), len(new)) + return + } + for i := range old { + eqIDSet(t, what+" group "+itoa(i), old[i], new[i]) + } +} + +func eqCache(t *testing.T, what string, old *sysfs.Cache, new *system.Cache) { + t.Helper() + + if (old == nil) != (new == nil) { + t.Errorf("%s: pkg/sysfs nil=%v, drop-in nil=%v", what, old == nil, new == nil) + return + } + if old == nil { + return + } + + eqInt(t, what+" ID", old.ID(), new.ID()) + eqInt(t, what+" Level", old.Level(), new.Level()) + eqStr(t, what+" Type", old.Type().String(), new.Type().String()) + if old.Size() != new.Size() { + t.Errorf("%s Size: pkg/sysfs %d, drop-in %d", what, old.Size(), new.Size()) + } + eqCPUSet(t, what+" SharedCPUSet", old.SharedCPUSet(), new.SharedCPUSet()) +} + +func eqCaches(t *testing.T, what string, old []*sysfs.Cache, new []*system.Cache) { + t.Helper() + + if len(old) != len(new) { + t.Errorf("%s: pkg/sysfs %d caches, drop-in %d", what, len(old), len(new)) + return + } + for i := range old { + eqCache(t, what+"["+itoa(i)+"]", old[i], new[i]) + } +} + +// sampleSets returns the CPU sets to exercise the set-to-set methods with: +// nothing, one CPU, one core, one node, one package, everything, and a set which +// deliberately cuts across cores and caches. +func sampleSets(p pair) []cpuset.CPUSet { + sets := []cpuset.CPUSet{cpuset.New(), p.old.CPUSet(), p.old.OnlineCPUs()} + + ids := p.old.OnlineCPUs().List() + if len(ids) == 0 { + return sets + } + + sets = append(sets, cpuset.New(ids[0])) + sets = append(sets, p.old.CPU(ids[0]).ThreadCPUSet()) + + if nodes := p.old.NodeIDs(); len(nodes) > 0 { + sets = append(sets, p.old.Node(nodes[0]).CPUSet()) + } + if pkgs := p.old.PackageIDs(); len(pkgs) > 0 { + sets = append(sets, p.old.Package(pkgs[0]).CPUSet()) + } + + ragged := []int{} + for i := 0; i < len(ids); i += 2 { + ragged = append(ragged, ids[i]) + } + sets = append(sets, cpuset.New(ragged...)) + + return sets +} + +func itoa(i int) string { + return strconv.Itoa(i) +} diff --git a/pkg/lib/hardware/system/internal/dropin/doc.go b/pkg/lib/hardware/system/internal/dropin/doc.go new file mode 100644 index 000000000..48969808a --- /dev/null +++ b/pkg/lib/hardware/system/internal/dropin/doc.go @@ -0,0 +1,24 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package dropin checks that pkg/lib/hardware/system is a drop-in +// replacement for pkg/sysfs, by compiling one body of code against both. +// +// viasysfs/api.go and viasystem/api.go reference every exported pkg/sysfs +// symbol. They differ only in the import line and the package clause. Building +// them both proves the two packages present the same surface; the test below +// proves the two files really are the same code, so that the check cannot be +// quietly weakened by editing one of them. + +package dropin diff --git a/pkg/lib/hardware/system/internal/dropin/dropin_test.go b/pkg/lib/hardware/system/internal/dropin/dropin_test.go new file mode 100644 index 000000000..d829d72a8 --- /dev/null +++ b/pkg/lib/hardware/system/internal/dropin/dropin_test.go @@ -0,0 +1,67 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package dropin + +import ( + "os" + "strings" + "testing" +) + +// TestSourcesAreIdentical checks that the two api.go files differ only in their +// package clause and their import of the package under test. +func TestSourcesAreIdentical(t *testing.T) { + viaSysfs := mustRead(t, "viasysfs/api.go") + viaSystem := mustRead(t, "viasystem/api.go") + + if len(viaSysfs) != len(viaSystem) { + t.Fatalf("the two files have %d and %d lines; they must differ only in "+ + "the package clause, the package doc and the import", + len(viaSysfs), len(viaSystem)) + } + + for i := range viaSysfs { + a, b := viaSysfs[i], viaSystem[i] + if a == b || allowedDifference(a, b) { + continue + } + t.Errorf("line %d differs beyond the import:\n viasysfs: %s\n viasystem: %s", + i+1, a, b) + } +} + +// allowedDifference reports whether two corresponding lines are allowed to +// differ: the package clause, the package doc naming the other file, and the +// import of the package under test. +func allowedDifference(a, b string) bool { + switch { + case strings.HasPrefix(a, "package ") && strings.HasPrefix(b, "package "): + return true + case strings.HasPrefix(strings.TrimSpace(a), "//"): + return strings.HasPrefix(strings.TrimSpace(b), "//") + case strings.Contains(a, "nri-plugins/pkg/sysfs"): + return strings.Contains(b, "nri-plugins/pkg/lib/hardware/system") + } + return false +} + +func mustRead(t *testing.T, path string) []string { + t.Helper() + blob, err := os.ReadFile(path) + if err != nil { + t.Fatalf("failed to read %s: %v", path, err) + } + return strings.Split(strings.TrimRight(string(blob), "\n"), "\n") +} diff --git a/pkg/lib/hardware/system/internal/dropin/viasysfs/api.go b/pkg/lib/hardware/system/internal/dropin/viasysfs/api.go new file mode 100644 index 000000000..4537c8784 --- /dev/null +++ b/pkg/lib/hardware/system/internal/dropin/viasysfs/api.go @@ -0,0 +1,179 @@ +// Package viasysfs references every exported pkg/sysfs symbol, against +// pkg/sysfs. +// +// It exists so that the compiler enforces the drop-in property. This file and +// its sibling in ../viasystem/api.go differ only in which package they +// import; TestSourcesAreIdentical in the parent directory checks that, and +// building both checks that the two packages really are interchangeable. +// +// Nothing imports this. It goes when pkg/sysfs goes. +package viasysfs + +import ( + sysfs "github.com/containers/nri-plugins/pkg/sysfs" + + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "github.com/intel/goresctrl/pkg/sst" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// interfaces, in the shapes consumers actually use them +var ( + _ sysfs.System = nil + _ sysfs.CPU = nil + _ sysfs.CPUPackage = nil + _ sysfs.Node = nil + _ *sysfs.Cache = nil + _ sysfs.NodeFilter = nil + _ sysfs.PickEntryFn = nil +) + +// enums and structs +var ( + _ sysfs.DiscoveryFlag = sysfs.DiscoverCPUTopology | sysfs.DiscoverMemTopology | + sysfs.DiscoverCache | sysfs.DiscoverSst | sysfs.DiscoverNone | + sysfs.DiscoverAll | sysfs.DiscoverDefault + _ sysfs.MemoryType = sysfs.MemoryTypeDRAM + _ sysfs.MemoryType = sysfs.MemoryTypePMEM + _ sysfs.MemoryType = sysfs.MemoryTypeHBM + _ sysfs.CacheType = sysfs.DataCache + _ sysfs.CacheType = sysfs.InstructionCache + _ sysfs.CacheType = sysfs.UnifiedCache + _ int = sysfs.NumCacheTypes + _ sysfs.EPP = sysfs.EPPPerformance + _ sysfs.EPP = sysfs.EPPBalancePerformance + _ sysfs.EPP = sysfs.EPPBalancePower + _ sysfs.EPP = sysfs.EPPPower + _ sysfs.EPP = sysfs.EPPUnknown + _ sysfs.CoreKind = sysfs.PerformanceCore + _ sysfs.CoreKind = sysfs.EfficientCore + _ sysfs.CPUFreq = sysfs.CPUFreq{Base: 1, Min: 2, Max: 3} + _ sysfs.MemInfo = sysfs.MemInfo{MemTotal: 1, MemFree: 2, MemUsed: 3} +) + +// predefined filters, used both bare and in a []NodeFilter as pools.go does +var _ = []sysfs.NodeFilter{ + sysfs.NodeOfDRAMType, sysfs.NodeOfPMEMType, sysfs.NodeOfHBMType, + sysfs.NodeHasMemory, sysfs.NodeHasNoMemory, + sysfs.NodeHasLocalCPUs, sysfs.NodeHasNoLocalCPUs, +} + +// stringers and parsers +var ( + _ = sysfs.MemoryTypeDRAM.String() + _ = sysfs.DataCache.String() + _ = sysfs.EPPPerformance.String() + _ = sysfs.PerformanceCore.String() + _ = sysfs.EPPFromString("performance") +) + +// functions +func useFuncs(sys sysfs.System, c sysfs.CPU, cset cpuset.CPUSet, ids idset.IDSet) { //nolint:unused // compiled, not called + sysfs.SetSysRoot("/host") + _, _ = sysfs.DiscoverSystem() + _, _ = sysfs.DiscoverSystem(sysfs.DiscoverCPUTopology | sysfs.DiscoverCache) + _, _ = sysfs.DiscoverSystemAt("/sys") + _, _ = sysfs.DiscoverSystemAt("/sys", sysfs.DiscoverAll) + _ = sysfs.IDSetFromCPUSet(cset) + _ = sysfs.CPUSetFromIDSet(ids) + _ = sysfs.GetMemoryCapacity() + _ = sysfs.ParseFileEntries("/proc/meminfo", map[string]any{}, nil) + _ = sysfs.NodeOfType(sysfs.MemoryTypeDRAM) + _ = sysfs.NodeFilterAnd(sysfs.NodeHasMemory, sysfs.NodeHasLocalCPUs) + _ = sysfs.NodeFilterOr(sysfs.NodeHasMemory) + _ = sysfs.NodeFilterNot(sysfs.NodeHasMemory) + + // every System method, called the way consumers call them + _ = sys.Discover(sysfs.DiscoverAll) + _, _ = sys.SetCpusOnline(true, ids) + _ = sys.SetCPUFrequencyLimits(1, 2, ids) + _ = sys.PackageIDs() + _ = sys.NodeIDs() + _ = sys.FilterNodes(sys.NodeIDs(), sysfs.NodeHasMemory) + _ = sys.FilterNode(0) + _, _ = sys.ClosestNodes(0, sysfs.NodeOfDRAMType, sysfs.NodeHasLocalCPUs) + _ = sys.CPUIDs() + _, _, _, _ = sys.PackageCount(), sys.SocketCount(), sys.CPUCount(), sys.NUMANodeCount() + _, _ = sys.MinThreadCount(), sys.MaxThreadCount() + _ = sys.CPUSet() + _ = sys.NodeDistance(0, 1) + _, _, _ = sys.PossibleCPUs(), sys.PresentCPUs(), sys.OnlineCPUs() + _, _ = sys.IsolatedCPUs(), sys.OfflineCPUs() + _ = sys.CoreKindCPUs(sysfs.PerformanceCore) + _ = sys.CoreKinds() + _ = sys.IDSetForCPUs(cset, func(c sysfs.CPU) idset.ID { return c.PackageID() }) + _ = sys.AllThreadsForCPUs(cset) + _ = sys.SingleThreadForCPUs(cset) + _ = sys.AllCPUsSharingNthLevelCacheWithCPUs(2, cset) + _, _ = sys.Offlined(), sys.Isolated() + _ = sys.NodeHintToCPUs("0-1") + var _ *sst.Platform = sys.Sst() //nolint:staticcheck // the type is the check + + usePackage(sys.Package(0)) + useNode(sys.Node(0)) + useCPU(sys.CPU(0)) + useCPU(c) +} + +func usePackage(p sysfs.CPUPackage) { //nolint:unused // compiled, not called + _ = p.ID() + _ = p.CPUSet() + _ = p.DieIDs() + _ = p.NodeIDs() + _ = p.DieNodeIDs(0) + _ = p.DieCPUSet(0) + _ = p.DieClusterIDs(0) + _ = p.DieClusterCPUSet(0, 0) + _ = p.LogicalDieClusterIDs(0) + _ = p.LogicalDieClusterCPUSet(0, 0) + _ = p.L3CacheIDs() + _ = p.L3CacheCPUSet(0) + var _ *sst.PackageStatus = p.SstInfo() //nolint:staticcheck // the type is the check +} + +func useNode(n sysfs.Node) { //nolint:unused // compiled, not called + _ = n.ID() + _ = n.PackageID() + _ = n.DieID() + _ = n.CPUSet() + _ = n.Distance() + _ = n.DistanceFrom(0) + _, _ = n.ClosestNodes() + _, _ = n.MemoryInfo() + _ = n.GetMemoryType() + _ = n.HasNormalMemory() +} + +func useCPU(c sysfs.CPU) { //nolint:unused // compiled, not called + _ = c.ID() + _ = c.PackageID() + _ = c.DieID() + _ = c.ClusterID() + _ = c.NodeID() + _ = c.CoreID() + _ = c.ThreadCPUSet() + _ = c.BaseFrequency() + _ = c.FrequencyRange() + _ = c.EPP() + _ = c.Online() + _ = c.Isolated() + _ = c.SetFrequencyLimits(1, 2) + _ = c.SstClos() + _ = c.CacheCount() + _ = c.GetCaches() + _ = c.GetCachesByLevel(2) + _ = c.GetCacheByIndex(0) + _ = c.GetNthLevelCacheCPUSet(2) + _ = c.GetLastLevelCaches() + _ = c.GetLastLevelCacheCPUSet() + _ = c.CoreKind() + + // balloons keys a map by *Cache, so this must compile and be comparable + seen := map[*sysfs.Cache]struct{}{} + for _, cc := range c.GetCaches() { + seen[cc] = struct{}{} + _, _, _, _ = cc.ID(), cc.Level(), cc.Type(), cc.Size() + _ = cc.SharedCPUSet() + } + _ = seen +} diff --git a/pkg/lib/hardware/system/internal/dropin/viasystem/api.go b/pkg/lib/hardware/system/internal/dropin/viasystem/api.go new file mode 100644 index 000000000..0c17a1df6 --- /dev/null +++ b/pkg/lib/hardware/system/internal/dropin/viasystem/api.go @@ -0,0 +1,179 @@ +// Package viasystem references every exported pkg/sysfs symbol, against +// the drop-in. +// +// It exists so that the compiler enforces the drop-in property. This file and +// its sibling in ../viasysfs/api.go differ only in which package they +// import; TestSourcesAreIdentical in the parent directory checks that, and +// building both checks that the two packages really are interchangeable. +// +// Nothing imports this. It goes when pkg/sysfs goes. +package viasystem + +import ( + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "github.com/intel/goresctrl/pkg/sst" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// interfaces, in the shapes consumers actually use them +var ( + _ sysfs.System = nil + _ sysfs.CPU = nil + _ sysfs.CPUPackage = nil + _ sysfs.Node = nil + _ *sysfs.Cache = nil + _ sysfs.NodeFilter = nil + _ sysfs.PickEntryFn = nil +) + +// enums and structs +var ( + _ sysfs.DiscoveryFlag = sysfs.DiscoverCPUTopology | sysfs.DiscoverMemTopology | + sysfs.DiscoverCache | sysfs.DiscoverSst | sysfs.DiscoverNone | + sysfs.DiscoverAll | sysfs.DiscoverDefault + _ sysfs.MemoryType = sysfs.MemoryTypeDRAM + _ sysfs.MemoryType = sysfs.MemoryTypePMEM + _ sysfs.MemoryType = sysfs.MemoryTypeHBM + _ sysfs.CacheType = sysfs.DataCache + _ sysfs.CacheType = sysfs.InstructionCache + _ sysfs.CacheType = sysfs.UnifiedCache + _ int = sysfs.NumCacheTypes + _ sysfs.EPP = sysfs.EPPPerformance + _ sysfs.EPP = sysfs.EPPBalancePerformance + _ sysfs.EPP = sysfs.EPPBalancePower + _ sysfs.EPP = sysfs.EPPPower + _ sysfs.EPP = sysfs.EPPUnknown + _ sysfs.CoreKind = sysfs.PerformanceCore + _ sysfs.CoreKind = sysfs.EfficientCore + _ sysfs.CPUFreq = sysfs.CPUFreq{Base: 1, Min: 2, Max: 3} + _ sysfs.MemInfo = sysfs.MemInfo{MemTotal: 1, MemFree: 2, MemUsed: 3} +) + +// predefined filters, used both bare and in a []NodeFilter as pools.go does +var _ = []sysfs.NodeFilter{ + sysfs.NodeOfDRAMType, sysfs.NodeOfPMEMType, sysfs.NodeOfHBMType, + sysfs.NodeHasMemory, sysfs.NodeHasNoMemory, + sysfs.NodeHasLocalCPUs, sysfs.NodeHasNoLocalCPUs, +} + +// stringers and parsers +var ( + _ = sysfs.MemoryTypeDRAM.String() + _ = sysfs.DataCache.String() + _ = sysfs.EPPPerformance.String() + _ = sysfs.PerformanceCore.String() + _ = sysfs.EPPFromString("performance") +) + +// functions +func useFuncs(sys sysfs.System, c sysfs.CPU, cset cpuset.CPUSet, ids idset.IDSet) { //nolint:unused // compiled, not called + sysfs.SetSysRoot("/host") + _, _ = sysfs.DiscoverSystem() + _, _ = sysfs.DiscoverSystem(sysfs.DiscoverCPUTopology | sysfs.DiscoverCache) + _, _ = sysfs.DiscoverSystemAt("/sys") + _, _ = sysfs.DiscoverSystemAt("/sys", sysfs.DiscoverAll) + _ = sysfs.IDSetFromCPUSet(cset) + _ = sysfs.CPUSetFromIDSet(ids) + _ = sysfs.GetMemoryCapacity() + _ = sysfs.ParseFileEntries("/proc/meminfo", map[string]any{}, nil) + _ = sysfs.NodeOfType(sysfs.MemoryTypeDRAM) + _ = sysfs.NodeFilterAnd(sysfs.NodeHasMemory, sysfs.NodeHasLocalCPUs) + _ = sysfs.NodeFilterOr(sysfs.NodeHasMemory) + _ = sysfs.NodeFilterNot(sysfs.NodeHasMemory) + + // every System method, called the way consumers call them + _ = sys.Discover(sysfs.DiscoverAll) + _, _ = sys.SetCpusOnline(true, ids) + _ = sys.SetCPUFrequencyLimits(1, 2, ids) + _ = sys.PackageIDs() + _ = sys.NodeIDs() + _ = sys.FilterNodes(sys.NodeIDs(), sysfs.NodeHasMemory) + _ = sys.FilterNode(0) + _, _ = sys.ClosestNodes(0, sysfs.NodeOfDRAMType, sysfs.NodeHasLocalCPUs) + _ = sys.CPUIDs() + _, _, _, _ = sys.PackageCount(), sys.SocketCount(), sys.CPUCount(), sys.NUMANodeCount() + _, _ = sys.MinThreadCount(), sys.MaxThreadCount() + _ = sys.CPUSet() + _ = sys.NodeDistance(0, 1) + _, _, _ = sys.PossibleCPUs(), sys.PresentCPUs(), sys.OnlineCPUs() + _, _ = sys.IsolatedCPUs(), sys.OfflineCPUs() + _ = sys.CoreKindCPUs(sysfs.PerformanceCore) + _ = sys.CoreKinds() + _ = sys.IDSetForCPUs(cset, func(c sysfs.CPU) idset.ID { return c.PackageID() }) + _ = sys.AllThreadsForCPUs(cset) + _ = sys.SingleThreadForCPUs(cset) + _ = sys.AllCPUsSharingNthLevelCacheWithCPUs(2, cset) + _, _ = sys.Offlined(), sys.Isolated() + _ = sys.NodeHintToCPUs("0-1") + var _ *sst.Platform = sys.Sst() //nolint:staticcheck // the type is the check + + usePackage(sys.Package(0)) + useNode(sys.Node(0)) + useCPU(sys.CPU(0)) + useCPU(c) +} + +func usePackage(p sysfs.CPUPackage) { //nolint:unused // compiled, not called + _ = p.ID() + _ = p.CPUSet() + _ = p.DieIDs() + _ = p.NodeIDs() + _ = p.DieNodeIDs(0) + _ = p.DieCPUSet(0) + _ = p.DieClusterIDs(0) + _ = p.DieClusterCPUSet(0, 0) + _ = p.LogicalDieClusterIDs(0) + _ = p.LogicalDieClusterCPUSet(0, 0) + _ = p.L3CacheIDs() + _ = p.L3CacheCPUSet(0) + var _ *sst.PackageStatus = p.SstInfo() //nolint:staticcheck // the type is the check +} + +func useNode(n sysfs.Node) { //nolint:unused // compiled, not called + _ = n.ID() + _ = n.PackageID() + _ = n.DieID() + _ = n.CPUSet() + _ = n.Distance() + _ = n.DistanceFrom(0) + _, _ = n.ClosestNodes() + _, _ = n.MemoryInfo() + _ = n.GetMemoryType() + _ = n.HasNormalMemory() +} + +func useCPU(c sysfs.CPU) { //nolint:unused // compiled, not called + _ = c.ID() + _ = c.PackageID() + _ = c.DieID() + _ = c.ClusterID() + _ = c.NodeID() + _ = c.CoreID() + _ = c.ThreadCPUSet() + _ = c.BaseFrequency() + _ = c.FrequencyRange() + _ = c.EPP() + _ = c.Online() + _ = c.Isolated() + _ = c.SetFrequencyLimits(1, 2) + _ = c.SstClos() + _ = c.CacheCount() + _ = c.GetCaches() + _ = c.GetCachesByLevel(2) + _ = c.GetCacheByIndex(0) + _ = c.GetNthLevelCacheCPUSet(2) + _ = c.GetLastLevelCaches() + _ = c.GetLastLevelCacheCPUSet() + _ = c.CoreKind() + + // balloons keys a map by *Cache, so this must compile and be comparable + seen := map[*sysfs.Cache]struct{}{} + for _, cc := range c.GetCaches() { + seen[cc] = struct{}{} + _, _, _, _ = cc.ID(), cc.Level(), cc.Type(), cc.Size() + _ = cc.SharedCPUSet() + } + _ = seen +} diff --git a/pkg/lib/hardware/system/objects.go b/pkg/lib/hardware/system/objects.go new file mode 100644 index 000000000..9f798b7bd --- /dev/null +++ b/pkg/lib/hardware/system/objects.go @@ -0,0 +1,570 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package system + +import ( + "fmt" + "path" + "slices" + "strconv" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "github.com/intel/goresctrl/pkg/sst" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// toCPUSet converts a hardware CPU set to the cpuset.CPUSet these interfaces +// speak. Every accessor which returns a set pays for one of these; it is the +// price of the layer and a reason to remove it once nothing needs it. +func toCPUSet(cpus interface{ List() []int }) cpuset.CPUSet { + return cpuset.New(cpus.List()...) +} + +// +// CPU +// + +// cpu implements [CPU] over a hardware.CPU. +type cpu struct { + sys *system + hw *hardware.CPU +} + +func (c *cpu) ID() idset.ID { + return c.hw.ID() +} + +func (c *cpu) PackageID() idset.ID { + return c.hw.PackageID() +} + +func (c *cpu) DieID() idset.ID { + return c.hw.DieID() +} + +func (c *cpu) ClusterID() idset.ID { + return c.hw.ClusterID() +} + +func (c *cpu) NodeID() idset.ID { + return c.hw.NodeID() +} + +func (c *cpu) CoreID() idset.ID { + return c.hw.CoreID() +} + +func (c *cpu) ThreadCPUSet() cpuset.CPUSet { + return toCPUSet(c.hw.Threads()) +} + +func (c *cpu) BaseFrequency() uint64 { + return c.hw.Freq().Base +} + +func (c *cpu) FrequencyRange() CPUFreq { + freq := c.hw.Freq() + return CPUFreq{Base: freq.Base, Min: freq.Min, Max: freq.Max} +} + +func (c *cpu) EPP() EPP { + return EPP(c.hw.Freq().EPP) +} + +func (c *cpu) Online() bool { + return c.hw.Online() +} + +func (c *cpu) Isolated() bool { + return c.hw.Isolated() +} + +// SetFrequencyLimits writes the scaling limits for this CPU, clamping them to +// the range cpufreq reports, as pkg/sysfs does. A CPU with no cpufreq support +// reports no minimum and is left alone. +func (c *cpu) SetFrequencyLimits(min, max uint64) error { + freq := c.hw.Freq() + if freq.Min == 0 { + return nil + } + + min /= 1000 + max /= 1000 + if min < freq.Min && min != 0 { + min = freq.Min + } + if min > freq.Max { + min = freq.Max + } + if max < freq.Min && max != 0 { + max = freq.Min + } + if max > freq.Max { + max = freq.Max + } + + dir := path.Join(sysCPUDir, "cpu"+strconv.Itoa(c.hw.ID()), "cpufreq") + if err := c.sys.write(path.Join(dir, "scaling_min_freq"), min); err != nil { + return err + } + + return c.sys.write(path.Join(dir, "scaling_max_freq"), max) +} + +// SstClos returns the SST-CP CLOS this CPU is in, or -1 when SST prioritization +// is not in effect. +func (c *cpu) SstClos() int { + if clos, ok := c.sys.sstClos[c.hw.ID()]; ok { + return clos + } + return -1 +} + +func (c *cpu) CacheCount() int { + return len(c.hw.Caches()) +} + +func (c *cpu) GetCaches() []*Cache { + return c.sys.wrapCaches(c.hw.Caches()) +} + +// GetCachesByLevel returns this CPU's caches of one level. As in pkg/sysfs it +// relies on the caches being ordered by level and stops at the first higher one. +func (c *cpu) GetCachesByLevel(level int) []*Cache { + var caches []*Cache + + for _, cache := range c.hw.Caches() { + if cache.Level() == level { + caches = append(caches, c.sys.wrapCache(cache)) + } else if cache.Level() > level { + break + } + } + + return caches +} + +func (c *cpu) GetCacheByIndex(idx int) *Cache { + caches := c.hw.Caches() + if 0 <= idx && idx < len(caches) { + return c.sys.wrapCache(caches[idx]) + } + return nil +} + +// GetNthLevelCacheCPUSet returns the CPUs sharing any of this CPU's caches at +// one level, or its thread siblings when it has no caches at all. +// +// Note the union: a level with a separate data and instruction cache contributes +// both. hardware.CPU.Cache returns only one, which is why this does not use it. +func (c *cpu) GetNthLevelCacheCPUSet(n int) cpuset.CPUSet { + caches := c.hw.Caches() + if len(caches) == 0 { + return c.ThreadCPUSet() + } + + cpus := cpuset.New() + for _, cache := range caches { + if cache.Level() == n { + cpus = cpus.Union(toCPUSet(cache.CPUs())) + } else if cache.Level() > n { + break + } + } + + return cpus +} + +// GetLastLevelCaches returns this CPU's caches of its highest level. The order +// is the reverse of GetCaches, as in pkg/sysfs, which walks backwards and +// appends. +func (c *cpu) GetLastLevelCaches() []*Cache { + hw := c.hw.Caches() + if len(hw) < 1 { + return nil + } + + var ( + caches []*Cache + lastIndex = len(hw) - 1 + lastLevel = hw[lastIndex].Level() + ) + + for idx := lastIndex; idx >= 0; idx-- { + caches = append(caches, c.sys.wrapCache(hw[idx])) + if hw[idx].Level() != lastLevel { + break + } + } + + return caches +} + +// GetLastLevelCacheCPUSet returns the CPUs sharing this CPU's highest level +// caches, or its thread siblings when it has none. +func (c *cpu) GetLastLevelCacheCPUSet() cpuset.CPUSet { + hw := c.hw.Caches() + if len(hw) < 1 { + return c.ThreadCPUSet() + } + + var ( + lastIndex = len(hw) - 1 + lastLevel = hw[lastIndex].Level() + cpus = cpuset.New() + ) + + for idx := lastIndex; idx >= 0; idx-- { + cpus = cpus.Union(toCPUSet(hw[idx].CPUs())) + if hw[idx].Level() != lastLevel { + break + } + } + + return cpus +} + +func (c *cpu) CoreKind() CoreKind { + return CoreKind(c.hw.Kind()) +} + +var _ CPU = (*cpu)(nil) + +// +// Node +// + +// node implements [Node] over a hardware.MemoryNode. +type node struct { + sys *system + hw *hardware.MemoryNode +} + +func (n *node) ID() idset.ID { + return n.hw.ID() +} + +// PackageID returns the package this node belongs to. +// +// A node with no CPUs of its own belongs to no package, and pkg/sysfs reports 0 +// for it -- not because it is in package 0 but because that is the zero value of +// a field it never assigns. hardware.MemoryNode.PackageID says -1, which is +// honest; this says 0, which is compatible. +func (n *node) PackageID() idset.ID { + if id := n.hw.PackageID(); id >= 0 { + return id + } + return 0 +} + +// DieID returns the die this node belongs to, with the same caveat as +// [node.PackageID]. +func (n *node) DieID() idset.ID { + if id := n.hw.DieID(); id >= 0 { + return id + } + return 0 +} + +func (n *node) CPUSet() cpuset.CPUSet { + return toCPUSet(n.hw.CPUs()) +} + +func (n *node) Distance() []int { + return n.hw.Distances() +} + +func (n *node) DistanceFrom(id idset.ID) int { + return n.hw.Distance(id) +} + +// ClosestNodes returns the other nodes grouped by distance, nearest first. It is +// the unfiltered form; System.ClosestNodes is the one which takes filters. +func (n *node) ClosestNodes() ([]idset.IDSet, []int) { + groups := hardware.ClosestMemoryNodes(n.sys.hw, n.hw.ID(), nil) + + nodes := make([]idset.IDSet, 0, len(groups)) + distances := make([]int, 0, len(groups)) + for _, g := range groups { + nodes = append(nodes, idset.NewIDSet(g.Nodes...)) + distances = append(distances, g.Distance) + } + + return nodes, distances +} + +// MemoryInfo reads this node's memory usage now, as pkg/sysfs does, rather than +// answering total from the capacity discovery read once. +func (n *node) MemoryInfo() (*MemInfo, error) { + info, err := n.hw.Usage() + if err != nil { + return nil, err + } + + return &MemInfo{ + MemTotal: uint64(info.Total), + MemFree: uint64(info.Free), + MemUsed: uint64(info.Used), + }, nil +} + +// GetMemoryType returns the kind of memory this node holds. +// +// A node hardware could not classify is reported as DRAM. pkg/sysfs never +// returns anything else because it refuses to finish discovery at all on such a +// machine; see the note in doc.go. +func (n *node) GetMemoryType() MemoryType { + switch n.hw.Kind() { + case hardware.MemoryKindPMEM: + return MemoryTypePMEM + case hardware.MemoryKindHBM: + return MemoryTypeHBM + } + return MemoryTypeDRAM +} + +func (n *node) HasNormalMemory() bool { + return n.hw.HasNormalMemory() +} + +var _ Node = (*node)(nil) + +// +// CPUPackage +// + +// cpuPackage implements [CPUPackage] over the zones of one package. +type cpuPackage struct { + sys *system + hw *hardware.Zone +} + +func (p *cpuPackage) ID() idset.ID { + return p.hw.ID() +} + +func (p *cpuPackage) CPUSet() cpuset.CPUSet { + return toCPUSet(p.hw.CPUs()) +} + +func (p *cpuPackage) DieIDs() []idset.ID { + var ids []idset.ID + for _, die := range p.dies() { + ids = append(ids, die.ID()) + } + return ids +} + +// NodeIDs returns the NUMA nodes whose CPUs are in this package. +func (p *cpuPackage) NodeIDs() []idset.ID { + return sortedIDs(hardware.MemoryNodesFor(p.sys.hw, p.hw.CPUs())) +} + +func (p *cpuPackage) DieNodeIDs(id idset.ID) []idset.ID { + die := p.die(id) + if die == nil { + return []idset.ID{} + } + return sortedIDs(hardware.MemoryNodesFor(p.sys.hw, die.CPUs())) +} + +func (p *cpuPackage) DieCPUSet(id idset.ID) cpuset.CPUSet { + die := p.die(id) + if die == nil { + return cpuset.New() + } + return toCPUSet(die.CPUs()) +} + +func (p *cpuPackage) DieClusterIDs(die idset.ID) []idset.ID { + var ids []idset.ID + for _, cluster := range p.clusters(die) { + ids = append(ids, cluster.ID()) + } + slices.Sort(ids) + return ids +} + +func (p *cpuPackage) DieClusterCPUSet(die, cluster idset.ID) cpuset.CPUSet { + for _, z := range p.clusters(die) { + if z.ID() == cluster { + return toCPUSet(z.CPUs()) + } + } + return cpuset.New() +} + +// LogicalDieClusterIDs returns the cluster ids of a die with the clusters which +// hold nothing but one core's threads merged into a single one, which is what +// pkg/sysfs reports. +func (p *cpuPackage) LogicalDieClusterIDs(die idset.ID) []idset.ID { + var ids []idset.ID + for _, cpus := range p.logicalClusters(die) { + ids = append(ids, p.sys.hw.CPU(cpus.List()[0]).ClusterID()) + } + slices.Sort(ids) + return ids +} + +func (p *cpuPackage) LogicalDieClusterCPUSet(die, cluster idset.ID) cpuset.CPUSet { + for _, cpus := range p.logicalClusters(die) { + if p.sys.hw.CPU(cpus.List()[0]).ClusterID() == cluster { + return toCPUSet(cpus) + } + } + return cpuset.New() +} + +// logicalClusters returns the clusters of a die of this package, merging the +// single-core ones as pkg/sysfs does. +func (p *cpuPackage) logicalClusters(die idset.ID) []*libcpu.CpuMask { + return hardware.LogicalClusters(p.sys.hw, p.hw.ID(), die, + hardware.MergeSingleCoreClusters) +} + +// L3CacheIDs returns the ids of the level 3 caches this package's CPUs use. +func (p *cpuPackage) L3CacheIDs() []idset.ID { + var ids []idset.ID + for _, z := range p.l3Caches() { + ids = append(ids, z.ID()) + } + slices.Sort(ids) + return ids +} + +func (p *cpuPackage) L3CacheCPUSet(id idset.ID) cpuset.CPUSet { + for _, z := range p.l3Caches() { + if z.ID() == id { + return toCPUSet(z.CPUs()) + } + } + return cpuset.New() +} + +// SstInfo returns the SST status of this package, or nil when SST is not in use. +func (p *cpuPackage) SstInfo() *sst.PackageStatus { + return p.sys.sstPkg[p.hw.ID()] +} + +// dies returns the die zones of this package, ordered by id. +func (p *cpuPackage) dies() []*hardware.Zone { + return hardware.ZonesWithin(p.sys.hw, hardware.LevelDie, p.hw.CPUs()) +} + +// die returns one die zone of this package, or nil. +func (p *cpuPackage) die(id idset.ID) *hardware.Zone { + for _, z := range p.dies() { + if z.ID() == id { + return z + } + } + return nil +} + +// clusters returns the cluster zones of one die of this package. +func (p *cpuPackage) clusters(die idset.ID) []*hardware.Zone { + z := p.die(die) + if z == nil { + return nil + } + return hardware.ZonesWithin(p.sys.hw, hardware.LevelCluster, z.CPUs()) +} + +// l3Caches returns the level 3 cache zones this package's CPUs use. Overlapping +// rather than within: a cache shared across packages still belongs to both, and +// pkg/sysfs reports its whole CPU set for each. +func (p *cpuPackage) l3Caches() []*hardware.Zone { + return hardware.ZonesOverlapping(p.sys.hw, hardware.LevelL3Cache, p.hw.CPUs()) +} + +var _ CPUPackage = (*cpuPackage)(nil) + +// +// Cache +// + +// Cache has details about a CPU cache. +// +// This mirrors pkg/sysfs: an exported struct with unexported fields, whose +// methods tolerate a nil receiver. One *Cache is handed out per cache, so that +// callers which use it as a map key keep working. +type Cache struct { + hw *hardware.Cache +} + +// ID returns the id of the cache, or 0 for a nil receiver. +func (c *Cache) ID() int { + if c == nil { + return 0 + } + return c.hw.ID() +} + +// Level returns the level of the cache, or 0 for a nil receiver. +func (c *Cache) Level() int { + if c == nil { + return 0 + } + return c.hw.Level() +} + +// Type returns the type of the cache, or 0 for a nil receiver. +func (c *Cache) Type() CacheType { + if c == nil { + return 0 + } + return CacheType(c.hw.Kind()) +} + +// Size returns the size of the cache in bytes, or 0 for a nil receiver. +func (c *Cache) Size() uint64 { + if c == nil { + return 0 + } + return uint64(c.hw.Size()) +} + +// SharedCPUSet returns the CPUs sharing the cache, or an empty set for a nil +// receiver. +func (c *Cache) SharedCPUSet() cpuset.CPUSet { + if c == nil { + return cpuset.New() + } + return toCPUSet(c.hw.CPUs()) +} + +// String describes the cache, for logs. +func (c *Cache) String() string { + if c == nil { + return "" + } + return fmt.Sprintf("L%d#%d", c.Level(), c.ID()) +} + +// +// helpers +// + +// sortedIDs returns ids in increasing order, never nil. +func sortedIDs(ids []idset.ID) []idset.ID { + out := slices.Clone(ids) + if out == nil { + out = []idset.ID{} + } + slices.Sort(out) + return out +} diff --git a/pkg/lib/hardware/system/shim_test.go b/pkg/lib/hardware/system/shim_test.go new file mode 100644 index 000000000..b616920de --- /dev/null +++ b/pkg/lib/hardware/system/shim_test.go @@ -0,0 +1,367 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The parts of the drop-in the differential test cannot reach: the write paths, +// which nothing in the tree calls and which need a writable sysfs; the pieces +// which do not depend on a discovered machine; and FromMachine. + +package system_test + +import ( + "os" + "os/exec" + "path/filepath" + "strings" + "testing" + + "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// TestEquivalentEPPParsing checks the EPP names round-trip the same way in both. +func TestEquivalentEPPParsing(t *testing.T) { + for _, name := range []string{ + "performance", "balance_performance", "balance_power", "power", + "", "nonsense", + } { + old := sysfs.EPPFromString(name) + new := system.EPPFromString(name) + if int(old) != int(new) { + t.Errorf("EPPFromString(%q): pkg/sysfs %d, drop-in %d", name, old, new) + } + if old.String() != new.String() { + t.Errorf("EPPFromString(%q).String(): pkg/sysfs %q, drop-in %q", + name, old.String(), new.String()) + } + } + + // every value's name round-trips + for e := 0; e <= int(system.EPPUnknown); e++ { + name := system.EPP(e).String() + if name == "" { + continue + } + if got := system.EPPFromString(name); int(got) != e { + t.Errorf("EPP(%d).String() = %q parses back as %d", e, name, got) + } + } +} + +// TestEquivalentFilterCombinators checks the And/Or/Not combinators, which the +// differential test does not reach because no caller uses them. +func TestEquivalentFilterCombinators(t *testing.T) { + p := discoverBoth(t, trees[0]) + ids := p.old.NodeIDs() + + cases := []struct { + name string + old sysfs.NodeFilter + new system.NodeFilter + }{ + { + name: "And(HasMemory, HasLocalCPUs)", + old: sysfs.NodeFilterAnd(sysfs.NodeHasMemory, sysfs.NodeHasLocalCPUs), + new: system.NodeFilterAnd(system.NodeHasMemory, system.NodeHasLocalCPUs), + }, + { + name: "Or(PMEM, HBM)", + old: sysfs.NodeFilterOr(sysfs.NodeOfPMEMType, sysfs.NodeOfHBMType), + new: system.NodeFilterOr(system.NodeOfPMEMType, system.NodeOfHBMType), + }, + { + name: "Not(HasLocalCPUs)", + old: sysfs.NodeFilterNot(sysfs.NodeHasLocalCPUs), + new: system.NodeFilterNot(system.NodeHasLocalCPUs), + }, + { + name: "And()", + old: sysfs.NodeFilterAnd(), + new: system.NodeFilterAnd(), + }, + { + name: "Or()", + old: sysfs.NodeFilterOr(), + new: system.NodeFilterOr(), + }, + } + + for _, tc := range cases { + eqIDSet(t, "FilterNodes("+tc.name+")", + p.old.FilterNodes(ids, tc.old), p.new.FilterNodes(ids, tc.new)) + } + + // NodeOfType, for each type + for _, ty := range []int{ + int(system.MemoryTypeDRAM), int(system.MemoryTypePMEM), int(system.MemoryTypeHBM), + } { + eqIDSet(t, "FilterNodes(NodeOfType)", + p.old.FilterNodes(ids, sysfs.NodeOfType(sysfs.MemoryType(ty))), + p.new.FilterNodes(ids, system.NodeOfType(system.MemoryType(ty)))) + } +} + +// TestEquivalentUtilities checks the helpers which only live in pkg/sysfs by +// accident and which the drop-in repeats so that an import swap is complete. +func TestEquivalentUtilities(t *testing.T) { + if got, want := system.GetMemoryCapacity(), sysfs.GetMemoryCapacity(); got != want { + t.Errorf("GetMemoryCapacity: pkg/sysfs %d, drop-in %d", want, got) + } + + cpus := mustParse(t, "0-3,8") + eqIDSet(t, "IDSetFromCPUSet", + sysfs.IDSetFromCPUSet(cpus), system.IDSetFromCPUSet(cpus)) + eqCPUSet(t, "CPUSetFromIDSet", + sysfs.CPUSetFromIDSet(idset.NewIDSet(0, 1, 2, 3, 8)), + system.CPUSetFromIDSet(idset.NewIDSet(0, 1, 2, 3, 8))) + + // ParseFileEntries, over /proc/meminfo as the cgroup code uses it + pick := func(line string) (string, string, error) { + fields := strings.Fields(line) + if len(fields) < 2 { + return "", "", nil + } + return fields[0], fields[1], nil + } + + var oldTotal, newTotal uint64 + oerr := sysfs.ParseFileEntries("/proc/meminfo", + map[string]any{"MemTotal:": &oldTotal}, pick) + nerr := system.ParseFileEntries("/proc/meminfo", + map[string]any{"MemTotal:": &newTotal}, pick) + + if (oerr == nil) != (nerr == nil) { + t.Errorf("ParseFileEntries: errors differ: %v vs %v", oerr, nerr) + } + if oerr == nil && oldTotal != newTotal { + t.Errorf("ParseFileEntries MemTotal: %d vs %d", oldTotal, newTotal) + } + + // a file which is not there fails in both + oerr = sysfs.ParseFileEntries("/nonexistent", map[string]any{}, pick) + nerr = system.ParseFileEntries("/nonexistent", map[string]any{}, pick) + if (oerr == nil) != (nerr == nil) { + t.Errorf("ParseFileEntries of a missing file: %v vs %v", oerr, nerr) + } +} + +// TestFromMachine checks the seam a caller which already has a Machine uses. +func TestFromMachine(t *testing.T) { + root, err := filepath.Abs(filepath.Join("testdata", trees[0])) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(root, "sys")); err != nil { + t.Skipf("recorded tree %s is not unpacked: run ./test-setup.sh", trees[0]) + } + + m, err := hardware.Discover(hardware.WithRoot(root)) + if err != nil { + t.Fatalf("hardware.Discover: %v", err) + } + + wrapped := system.FromMachine(m) + direct, err := system.DiscoverSystemAt(filepath.Join(root, "sys")) + if err != nil { + t.Fatalf("DiscoverSystemAt: %v", err) + } + + eqCPUSet(t, "FromMachine CPUSet", direct.CPUSet(), wrapped.CPUSet()) + eqIDs(t, "FromMachine NodeIDs", direct.NodeIDs(), wrapped.NodeIDs()) + eqIDs(t, "FromMachine PackageIDs", direct.PackageIDs(), wrapped.PackageIDs()) + + // Discover on an existing System is a no-op which reports success, as + // nothing calls it with anything the constructor did not already read + if err := wrapped.Discover(system.DiscoverAll); err != nil { + t.Errorf("Discover on an existing System: %v", err) + } + + // SST is probed for a wrapped Machine too, so that this seam yields a System + // which is complete in the same way the discovering constructors do. Compare + // the two rather than asserting absence, so this holds on a machine with SST + // as well as on one without. + if (wrapped.Sst() == nil) != (direct.Sst() == nil) { + t.Errorf("SST platform presence differs: FromMachine %v, discovered %v", + wrapped.Sst() != nil, direct.Sst() != nil) + } + cpu0 := m.CPUIDs()[0] + if got, want := wrapped.CPU(cpu0).SstClos(), direct.CPU(cpu0).SstClos(); got != want { + t.Errorf("SstClos = %d, want %d", got, want) + } + for _, id := range wrapped.PackageIDs() { + w, d := wrapped.Package(id).SstInfo(), direct.Package(id).SstInfo() + if (w == nil) != (d == nil) { + t.Errorf("package#%d SST info presence differs: FromMachine %v, "+ + "discovered %v", id, w != nil, d != nil) + } + } +} + +// TestWritePaths exercises SetCpusOnline, SetCPUFrequencyLimits and +// CPU.SetFrequencyLimits, which nothing in the tree calls and which the +// differential test cannot run against a read-only recorded tree. +// +// It copies a recorded tree somewhere writable, adds the attributes the writes +// touch, and checks what lands in them. +func TestWritePaths(t *testing.T) { + src, err := filepath.Abs(filepath.Join("testdata", trees[0])) + if err != nil { + t.Fatal(err) + } + if _, err := os.Stat(filepath.Join(src, "sys")); err != nil { + t.Skipf("recorded tree %s is not unpacked: run ./test-setup.sh", trees[0]) + } + + root := t.TempDir() + if out, err := runCp(src+"/sys", root); err != nil { + t.Skipf("cannot copy the recorded tree: %v: %s", err, out) + } + + sys, err := system.DiscoverSystemAt(filepath.Join(root, "sys")) + if err != nil { + t.Fatalf("DiscoverSystemAt: %v", err) + } + + cpus := sys.CPUIDs() + if len(cpus) < 2 { + t.Skip("need at least two CPUs") + } + + // the attributes the writes touch, which a recorded tree does not have + cpuDir := filepath.Join(root, "sys", "devices", "system", "cpu", + "cpu"+itoa(cpus[1])) + freqDir := filepath.Join(cpuDir, "cpufreq") + if err := os.MkdirAll(freqDir, 0o755); err != nil { + t.Fatal(err) + } + for name, contents := range map[string]string{ + filepath.Join(cpuDir, "online"): "1\n", + filepath.Join(freqDir, "scaling_min_freq"): "0000000\n", + filepath.Join(freqDir, "scaling_max_freq"): "0000000\n", + } { + if err := os.WriteFile(name, []byte(contents), 0o644); err != nil { + t.Fatal(err) + } + } + + t.Run("SetCpusOnline", func(t *testing.T) { + changed, err := sys.SetCpusOnline(false, idset.NewIDSet(cpus[1])) + if err != nil { + t.Fatalf("SetCpusOnline: %v", err) + } + if !changed.Has(cpus[1]) { + t.Errorf("cpu%d is not among the changed CPUs %v", + cpus[1], changed.SortedMembers()) + } + if got := readFile(t, filepath.Join(cpuDir, "online")); got != "0\n" { + t.Errorf("online holds %q, want %q", got, "0\n") + } + + // cpu0 is never taken offline, as pkg/sysfs also refuses + changed, err = sys.SetCpusOnline(false, idset.NewIDSet(0)) + if err != nil { + t.Fatalf("SetCpusOnline(cpu0): %v", err) + } + if changed.Has(0) { + t.Error("cpu0 was taken offline") + } + }) + + t.Run("SetFrequencyLimits", func(t *testing.T) { + c := sys.CPU(cpus[1]) + if c.FrequencyRange().Min == 0 { + t.Skip("the recorded tree has no cpufreq range to clamp against") + } + + if err := c.SetFrequencyLimits(1_000_000, 9_000_000_000); err != nil { + t.Fatalf("SetFrequencyLimits: %v", err) + } + + // the values are clamped to the range cpufreq reports + freq := c.FrequencyRange() + if got, want := readFile(t, filepath.Join(freqDir, "scaling_max_freq")), + itoa(int(freq.Max))+"\n"; got != want { + t.Errorf("scaling_max_freq holds %q, want %q (clamped)", got, want) + } + }) + + t.Run("SetCPUFrequencyLimits", func(t *testing.T) { + // the whole-machine form, which walks every CPU; the ones without the + // attributes fail, so pass only the one which has them + err := sys.SetCPUFrequencyLimits(1_000_000, 2_000_000, + idset.NewIDSet(cpus[1])) + if err != nil { + t.Fatalf("SetCPUFrequencyLimits: %v", err) + } + }) + + // A System from FromMachine writes to the machine's own tree. It has no root + // of its own to write to, so if it ever built one from a remembered path + // instead of asking the machine, a write here would land in the real /sys. + t.Run("FromMachine writes to the machine's tree", func(t *testing.T) { + m, err := hardware.Discover(hardware.WithRoot(root)) + if err != nil { + t.Fatalf("hardware.Discover: %v", err) + } + + // The machine reads who is online from the cpu-level "online" list, which + // the earlier subtest did not touch, so it still sees cpu1 as online and + // taking it offline is a change it will act on. + online := filepath.Join(cpuDir, "online") + if err := os.WriteFile(online, []byte("1\n"), 0o644); err != nil { + t.Fatal(err) + } + + changed, err := system.FromMachine(m).SetCpusOnline(false, + idset.NewIDSet(cpus[1])) + if err != nil { + t.Fatalf("SetCpusOnline: %v", err) + } + if !changed.Has(cpus[1]) { + t.Errorf("cpu%d is not among the changed CPUs %v", + cpus[1], changed.SortedMembers()) + } + if got := readFile(t, online); got != "0\n" { + t.Errorf("online under the machine's root holds %q, want %q", got, "0\n") + } + }) +} + +func readFile(t *testing.T, name string) string { + t.Helper() + blob, err := os.ReadFile(name) + if err != nil { + t.Fatal(err) + } + return string(blob) +} + +// runCp copies a directory tree, since Go has no library call for it and the +// write tests need a writable copy of a recorded one. +func runCp(from, to string) (string, error) { + out, err := exec.Command("cp", "-a", from, to).CombinedOutput() + return string(out), err +} + +// mustParse parses a cpuset or fails the test. +func mustParse(t *testing.T, s string) cpuset.CPUSet { + t.Helper() + cpus, err := cpuset.Parse(s) + if err != nil { + t.Fatalf("cpuset.Parse(%q): %v", s, err) + } + return cpus +} diff --git a/pkg/lib/hardware/system/system.go b/pkg/lib/hardware/system/system.go new file mode 100644 index 000000000..786ab0964 --- /dev/null +++ b/pkg/lib/hardware/system/system.go @@ -0,0 +1,665 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package system + +import ( + "fmt" + "path" + "path/filepath" + "strconv" + + "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + "github.com/intel/goresctrl/pkg/sst" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// System devices +// +// Declared exactly as pkg/sysfs declares it. Do not tidy it: the point is that +// it matches, so that a consumer can switch to this package without touching +// anything but the import. +type System interface { + Discover(flags DiscoveryFlag) error + SetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error) + SetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error + PackageIDs() []idset.ID + NodeIDs() []idset.ID + FilterNodes(ids []idset.ID, filters ...NodeFilter) idset.IDSet + FilterNode(id idset.ID, filters ...NodeFilter) bool + ClosestNodes(id idset.ID, filters ...NodeFilter) ([]idset.IDSet, []int) + CPUIDs() []idset.ID + PackageCount() int + SocketCount() int + CPUCount() int + NUMANodeCount() int + MinThreadCount() int + MaxThreadCount() int + CPUSet() cpuset.CPUSet + Package(id idset.ID) CPUPackage + Node(id idset.ID) Node + NodeDistance(from, to idset.ID) int + CPU(id idset.ID) CPU + PossibleCPUs() cpuset.CPUSet + PresentCPUs() cpuset.CPUSet + OnlineCPUs() cpuset.CPUSet + IsolatedCPUs() cpuset.CPUSet + OfflineCPUs() cpuset.CPUSet + CoreKindCPUs(CoreKind) cpuset.CPUSet + CoreKinds() []CoreKind + IDSetForCPUs(cpuset.CPUSet, func(CPU) idset.ID) idset.IDSet + AllThreadsForCPUs(cpuset.CPUSet) cpuset.CPUSet + SingleThreadForCPUs(cpuset.CPUSet) cpuset.CPUSet + AllCPUsSharingNthLevelCacheWithCPUs(int, cpuset.CPUSet) cpuset.CPUSet + + Offlined() cpuset.CPUSet + Isolated() cpuset.CPUSet + + NodeHintToCPUs(string) string + + Sst() *sst.Platform +} + +// CPUPackage is a physical package (a collection of CPUs). +type CPUPackage interface { + ID() idset.ID + CPUSet() cpuset.CPUSet + DieIDs() []idset.ID + NodeIDs() []idset.ID + DieNodeIDs(idset.ID) []idset.ID + DieCPUSet(idset.ID) cpuset.CPUSet + DieClusterIDs(idset.ID) []idset.ID + DieClusterCPUSet(idset.ID, idset.ID) cpuset.CPUSet + LogicalDieClusterIDs(idset.ID) []idset.ID + LogicalDieClusterCPUSet(idset.ID, idset.ID) cpuset.CPUSet + L3CacheIDs() []idset.ID + L3CacheCPUSet(idset.ID) cpuset.CPUSet + SstInfo() *sst.PackageStatus +} + +// Node represents a NUMA node. +type Node interface { + ID() idset.ID + PackageID() idset.ID + DieID() idset.ID + CPUSet() cpuset.CPUSet + Distance() []int + DistanceFrom(id idset.ID) int + ClosestNodes() ([]idset.IDSet, []int) + MemoryInfo() (*MemInfo, error) + GetMemoryType() MemoryType + HasNormalMemory() bool +} + +// CPU is a CPU core. +type CPU interface { + ID() idset.ID + PackageID() idset.ID + DieID() idset.ID + ClusterID() idset.ID + NodeID() idset.ID + CoreID() idset.ID + ThreadCPUSet() cpuset.CPUSet + BaseFrequency() uint64 + FrequencyRange() CPUFreq + EPP() EPP + Online() bool + Isolated() bool + SetFrequencyLimits(min, max uint64) error + SstClos() int + CacheCount() int + GetCaches() []*Cache + GetCachesByLevel(int) []*Cache + GetCacheByIndex(int) *Cache + GetNthLevelCacheCPUSet(n int) cpuset.CPUSet + GetLastLevelCaches() []*Cache + GetLastLevelCacheCPUSet() cpuset.CPUSet + CoreKind() CoreKind +} + +// +// Construction +// + +// sysRoot is the parent directory of the host's /sys, as [SetSysRoot] last set +// it. Package-global, as in pkg/sysfs, and read when a System is built. +var sysRoot string + +// SetSysRoot sets the sys root directory. +func SetSysRoot(root string) { + if root == "" { + sysRoot = "" + return + } + + root = filepath.Clean(root) + if root != "" && !filepath.IsAbs(root) { + abs, err := filepath.Abs(root) + if err != nil { + panic(fmt.Errorf("failed to resolve %q to absolute path: %v", root, err)) + } + root = abs + } + if root == "/" { + root = "" + } + + sysRoot = root +} + +// DiscoverSystem performs discovery of the running systems details. +// +// The flags are accepted and ignored, as they are in pkg/sysfs, whose +// DiscoverSystem drops them before passing them on. +func DiscoverSystem(args ...DiscoveryFlag) (System, error) { + return discover(filepath.Join("/", sysRoot)) +} + +// DiscoverSystemAt performs discovery of the running systems details from sysfs +// mounted at path. +// +// path names the sysfs mount point, so it ends in "/sys", whereas the hardware +// package takes the host root above it. The two are translated here. +func DiscoverSystemAt(path string, args ...DiscoveryFlag) (System, error) { + root, base := filepath.Split(filepath.Clean(path)) + if base != "sys" { + return nil, fmt.Errorf("%q does not look like a sysfs mount point", path) + } + if root == "" { + root = "." + } + + return discover(filepath.Clean(root)) +} + +// FromMachine wraps an already discovered [hardware.Machine] in the pkg/sysfs +// interface. It is the seam a caller which has moved on to hardware uses to keep +// feeding a caller which has not, so it has to yield a System which is complete +// in every way the ones below do: SST is probed here too, and the write paths go +// to the same tree the machine was read from. +func FromMachine(m *hardware.Machine) System { + sys := newSystem(m) + sys.discoverSst() + return sys +} + +// discover reads a machine below root and wraps it. +func discover(root string) (System, error) { + m, err := hardware.Discover(hardware.WithRoot(root), hardware.WithEnvOverrides()) + if err != nil { + return nil, err + } + + return FromMachine(m), nil +} + +// system implements [System] over a [hardware.Machine]. +type system struct { + hw *hardware.Machine + x *hardware.TopologyIndex + + cpus map[idset.ID]*cpu + nodes map[idset.ID]*node + pkgs map[idset.ID]*cpuPackage + caches map[*hardware.Cache]*Cache + + // SST, which the hardware package deliberately knows nothing about + sst *sst.Platform + sstClos map[idset.ID]int + sstPkg map[idset.ID]*sst.PackageStatus +} + +// newSystem wraps a machine, interning one wrapper per piece of hardware so that +// callers which compare or key by them keep working. +func newSystem(m *hardware.Machine) *system { + sys := &system{ + hw: m, + x: m.TopologyIndex(), + cpus: map[idset.ID]*cpu{}, + nodes: map[idset.ID]*node{}, + pkgs: map[idset.ID]*cpuPackage{}, + caches: map[*hardware.Cache]*Cache{}, + sstClos: map[idset.ID]int{}, + sstPkg: map[idset.ID]*sst.PackageStatus{}, + } + + for _, id := range m.CPUIDs() { + sys.cpus[id] = &cpu{sys: sys, hw: m.CPU(id)} + } + for _, n := range m.MemoryNodes() { + sys.nodes[n.ID()] = &node{sys: sys, hw: n} + } + for _, z := range m.Zones(hardware.LevelPackage) { + sys.pkgs[z.ID()] = &cpuPackage{sys: sys, hw: z} + } + + return sys +} + +// wrapCache returns the one wrapper for a cache, so that a caller which keys a +// map by *Cache -- as the balloons policy does -- sees the same pointer for the +// same cache. +func (s *system) wrapCache(hw *hardware.Cache) *Cache { + if c, ok := s.caches[hw]; ok { + return c + } + c := &Cache{hw: hw} + s.caches[hw] = c + return c +} + +// wrapCaches wraps a list of caches. +func (s *system) wrapCaches(hw []*hardware.Cache) []*Cache { + out := make([]*Cache, 0, len(hw)) + for _, c := range hw { + out = append(out, s.wrapCache(c)) + } + return out +} + +// write writes a number to a sysfs attribute of the machine. +// +// Through the machine's own filesystem, not a fresh one built from a root we +// remembered: that is the only way a write lands in the tree the topology was +// read from, whether that is the host root, a recorded tree or an injected fs.FS. +func (s *system) write(name string, value uint64) error { + fsys, ok := s.hw.FS().(hardware.WriterFS) + if !ok { + return fmt.Errorf("cannot write %s: the filesystem is read-only", name) + } + // pkg/sysfs appends a newline; the kernel does not care but keep it identical + return fsys.WriteFile(name, []byte(strconv.FormatUint(value, 10)+"\n")) +} + +// +// SST +// +// The hardware package has nothing to do with Intel Speed Select, so the three +// interface methods which expose it are served from here. This is a port of +// pkg/sysfs.discoverSst. It is the last thing which should move out of this +// package, and it moves to whoever still wants it -- today only pkg/cpuallocator. +// + +// discoverSst probes Speed Select and records what it says. Failure is not +// fatal: SST is simply reported as absent, as pkg/sysfs does. +func (s *system) discoverSst() { + if !sst.SstSupported() { + return + } + + platform, err := sst.Init() + if err != nil || platform == nil { + return + } + s.sst = platform + + for id := range s.pkgs { + pkg, ok := platform.Package(id) + if !ok { + continue + } + status, err := pkg.GetStatus() + if err != nil { + continue + } + + for _, punit := range status.Punits { + if !punit.CP.Supported || !punit.CP.Enabled { + continue + } + for _, cpu := range punit.CPUs.SortedMembers() { + clos, err := platform.GetCPUClosID(cpu) + if err != nil { + continue + } + s.sstClos[cpu] = clos + } + } + + s.sstPkg[id] = status + } +} + +func (s *system) Discover(flags DiscoveryFlag) error { + // Everything the flags could ask for was discovered already. pkg/sysfs + // re-reads and mutates in place; nothing calls it with anything new. + return nil +} + +func (s *system) SetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error) { + if cpus == nil { + cpus = idset.NewIDSet(s.CPUIDs()...) + } + + desired := map[bool]uint64{false: 0, true: 1}[online] + changed := idset.NewIDSet() + + for _, id := range cpus.SortedMembers() { + if id <= 0 { + // cpu0 cannot be taken offline, and pkg/sysfs skips it + continue + } + c, ok := s.cpus[id] + if !ok { + continue + } + if c.hw.Online() == online { + continue + } + + name := path.Join(sysCPUDir, "cpu"+strconv.Itoa(id), "online") + if err := s.write(name, desired); err != nil { + return nil, err + } + changed.Add(id) + } + + // The machine is immutable, so what it says about who is online is now out + // of date. Nothing in the tree calls this; a caller which starts to would + // have to rediscover. + return changed, nil +} + +func (s *system) SetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error { + if cpus == nil { + cpus = idset.NewIDSet(s.CPUIDs()...) + } + + for _, id := range cpus.SortedMembers() { + c, ok := s.cpus[id] + if !ok { + continue + } + if err := c.SetFrequencyLimits(min, max); err != nil { + return err + } + } + + return nil +} + +func (s *system) PackageIDs() []idset.ID { + ids := make([]idset.ID, 0, len(s.pkgs)) + for _, z := range s.hw.Zones(hardware.LevelPackage) { + ids = append(ids, z.ID()) + } + return ids +} + +func (s *system) NodeIDs() []idset.ID { + return s.hw.MemoryNodeIDs() +} + +func (s *system) FilterNodes(ids []idset.ID, filters ...NodeFilter) idset.IDSet { + out := idset.NewIDSet() + for _, id := range ids { + if s.FilterNode(id, filters...) { + out.Add(id) + } + } + return out +} + +func (s *system) FilterNode(id idset.ID, filters ...NodeFilter) bool { + n, ok := s.nodes[id] + if !ok { + return false + } + for _, filter := range filters { + if !filter(n) { + return false + } + } + return true +} + +func (s *system) ClosestNodes(id idset.ID, filters ...NodeFilter) ([]idset.IDSet, []int) { + if _, ok := s.nodes[id]; !ok { + return nil, nil + } + + match := func(n *hardware.MemoryNode) bool { + return s.FilterNode(n.ID(), filters...) + } + + groups := hardware.ClosestMemoryNodes(s.hw, id, match) + if len(groups) == 0 { + return nil, nil + } + + nodes := make([]idset.IDSet, 0, len(groups)) + distances := make([]int, 0, len(groups)) + for _, g := range groups { + nodes = append(nodes, idset.NewIDSet(g.Nodes...)) + distances = append(distances, g.Distance) + } + + return nodes, distances +} + +func (s *system) CPUIDs() []idset.ID { + return s.hw.CPUIDs() +} + +func (s *system) PackageCount() int { + return len(s.pkgs) +} + +func (s *system) SocketCount() int { + return len(s.pkgs) +} + +func (s *system) CPUCount() int { + return len(s.cpus) +} + +func (s *system) NUMANodeCount() int { + return max(len(s.nodes), 1) +} + +func (s *system) MinThreadCount() int { + min, _ := s.threadCounts() + return min +} + +func (s *system) MaxThreadCount() int { + _, max := s.threadCounts() + return max +} + +func (s *system) CPUSet() cpuset.CPUSet { + return cpuset.New(s.hw.CPUIDs()...) +} + +// Package returns the package with the given id, or nil if there is none. +func (s *system) Package(id idset.ID) CPUPackage { + if pkg, ok := s.pkgs[id]; ok { + return pkg + } + return nil +} + +// Node returns the NUMA node with the given id, or nil if there is none. +func (s *system) Node(id idset.ID) Node { + if n, ok := s.nodes[id]; ok { + return n + } + return nil +} + +// NodeDistance returns the distance between two NUMA nodes, or -1 if either is +// unknown. +func (s *system) NodeDistance(from, to idset.ID) int { + n, ok := s.nodes[from] + if !ok { + return -1 + } + return n.DistanceFrom(to) +} + +// CPU returns the CPU with the given id, or nil if there is none. +func (s *system) CPU(id idset.ID) CPU { + if c, ok := s.cpus[id]; ok { + return c + } + return nil +} + +func (s *system) PossibleCPUs() cpuset.CPUSet { + return toCPUSet(s.hw.PossibleCPUs()) +} + +func (s *system) PresentCPUs() cpuset.CPUSet { + return toCPUSet(s.hw.PresentCPUs()) +} + +func (s *system) OnlineCPUs() cpuset.CPUSet { + return toCPUSet(s.hw.OnlineCPUs()) +} + +func (s *system) IsolatedCPUs() cpuset.CPUSet { + return toCPUSet(s.hw.IsolatedCPUs()) +} + +func (s *system) OfflineCPUs() cpuset.CPUSet { + return toCPUSet(s.hw.OfflineCPUs()) +} + +func (s *system) CoreKindCPUs(kind CoreKind) cpuset.CPUSet { + return toCPUSet(s.hw.CoreKindCPUs(hardware.CoreKind(kind))) +} + +func (s *system) CoreKinds() []CoreKind { + kinds := s.hw.CoreKinds() + out := make([]CoreKind, 0, len(kinds)) + for _, kind := range kinds { + out = append(out, CoreKind(kind)) + } + return out +} + +func (s *system) IDSetForCPUs(cpus cpuset.CPUSet, idForCPU func(CPU) idset.ID) idset.IDSet { + ids := idset.NewIDSet() + for _, id := range cpus.UnsortedList() { + if c, ok := s.cpus[id]; ok { + ids.Add(idForCPU(c)) + } + } + return ids +} + +func (s *system) AllThreadsForCPUs(cpus cpuset.CPUSet) cpuset.CPUSet { + all := cpuset.New() + for _, id := range cpus.UnsortedList() { + if c, ok := s.cpus[id]; ok { + all = all.Union(c.ThreadCPUSet()) + } + } + return all +} + +func (s *system) SingleThreadForCPUs(cpus cpuset.CPUSet) cpuset.CPUSet { + var ( + result = make([]int, 0, cpus.Size()) + handled = make(map[int]struct{}, cpus.Size()) + ) + + for _, id := range cpus.List() { + if _, ok := handled[id]; ok { + continue + } + handled[id] = struct{}{} + result = append(result, id) + c, ok := s.cpus[id] + if !ok { + continue + } + for _, sibling := range c.ThreadCPUSet().UnsortedList() { + handled[sibling] = struct{}{} + } + } + + return cpuset.New(result...) +} + +func (s *system) AllCPUsSharingNthLevelCacheWithCPUs(n int, cpus cpuset.CPUSet) cpuset.CPUSet { + all := cpuset.New() + for _, id := range cpus.UnsortedList() { + if all.Contains(id) { + continue + } + if c, ok := s.cpus[id]; ok { + all = all.Union(c.GetNthLevelCacheCPUSet(n)) + } + } + return all +} + +func (s *system) Offlined() cpuset.CPUSet { + return s.OfflineCPUs() +} + +func (s *system) Isolated() cpuset.CPUSet { + return s.IsolatedCPUs() +} + +func (s *system) NodeHintToCPUs(nodes string) string { + mset, err := cpuset.Parse(nodes) + if err != nil { + return "" + } + + cset := cpuset.New() + for _, id := range mset.List() { + if n, ok := s.nodes[id]; ok { + cset = cset.Union(n.CPUSet()) + } + } + + return cset.Intersection(s.OnlineCPUs()).String() +} + +func (s *system) Sst() *sst.Platform { + return s.sst +} + +// threadCounts returns the smallest and largest number of threads per core the +// machine has. As in pkg/sysfs a CPU with no thread siblings, i.e. an offline +// one, is not counted. +func (s *system) threadCounts() (int, int) { + var min, max int + + for _, id := range s.hw.CPUIDs() { + n := s.hw.CPU(id).Threads().Size() + if n == 0 { + continue + } + if min == 0 || n < min { + min = n + } + if max == 0 || n > max { + max = n + } + } + + return min, max +} + +// sysCPUDir is where the per-CPU attributes the write paths touch live, relative +// to the host root. +const sysCPUDir = "sys/devices/system/cpu" + +// system must satisfy System. +var _ System = (*system)(nil) diff --git a/pkg/lib/hardware/system/test-cleanup.sh b/pkg/lib/hardware/system/test-cleanup.sh new file mode 100755 index 000000000..bfdbead58 --- /dev/null +++ b/pkg/lib/hardware/system/test-cleanup.sh @@ -0,0 +1,4 @@ +#!/bin/bash + +cd "$(dirname "$0")" +rm -rf testdata diff --git a/pkg/lib/hardware/system/test-setup.sh b/pkg/lib/hardware/system/test-setup.sh new file mode 100755 index 000000000..717ba1975 --- /dev/null +++ b/pkg/lib/hardware/system/test-setup.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# Unpack the recorded sysfs trees the equivalence tests compare over. They are +# the trees pkg/sysfs, pkg/cpuallocator and the topology-aware policy already +# keep for their own tests; this only unpacks them somewhere shared, rather than +# adding another copy of the same machines to the repository. + +set -e + +cd "$(dirname "$0")" + +REPO=../../../.. +OUT=testdata + +mkdir -p $OUT + +# pkg/sysfs keeps two trees, one machine each, as .tar.xz holding /sys. +for tree in sample1 sample2; do + [ -d "$OUT/$tree" ] && continue + tar -C $OUT -xJf $REPO/pkg/sysfs/test-data-$tree.tar.xz +done + +# The other two archives are .tar.bz2 holding sysfs//sys/... for several +# machines each. Lift each machine up to $OUT/, so that every tree looks +# the same to the tests. +unpack_bz2() { + local archive="$1" tmp + + tmp=$(mktemp -d) + # shellcheck disable=SC2064 + trap "rm -rf '$tmp'" RETURN + + tar -C "$tmp" -xjf "$archive" + for dir in "$tmp"/sysfs/*/; do + local name + name=$(basename "$dir") + [ -d "$OUT/$name" ] && continue + mkdir -p "$OUT/$name" + cp -a "$dir/sys" "$OUT/$name/sys" + done +} + +unpack_bz2 $REPO/pkg/cpuallocator/testdata/sysfs.tar.bz2 +unpack_bz2 $REPO/cmd/plugins/topology-aware/policy/testdata/sysfs.tar.bz2 diff --git a/pkg/lib/hardware/system/types.go b/pkg/lib/hardware/system/types.go new file mode 100644 index 000000000..e5680c4c4 --- /dev/null +++ b/pkg/lib/hardware/system/types.go @@ -0,0 +1,294 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package system + +import ( + "fmt" + + "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// The enumerations below repeat pkg/sysfs, values included. Anything which +// persists one of these numbers, or compares it against a literal, keeps +// working. + +// DiscoveryFlag controls what hardware details to discover. +// +// The values are accepted and ignored, as they are in pkg/sysfs. +type DiscoveryFlag uint + +const ( + // DiscoverCPUTopology requests discovering CPU topology details. + DiscoverCPUTopology DiscoveryFlag = 1 << iota + // DiscoverMemTopology requests discovering memory topology details. + DiscoverMemTopology + // DiscoverCache requests discovering CPU cache details. + DiscoverCache + // DiscoverSst requests discovering details of Intel Speed Select Technology + DiscoverSst + // DiscoverNone is the zero value for discovery flags. + DiscoverNone DiscoveryFlag = 0 + // DiscoverAll requests full supported discovery. + DiscoverAll DiscoveryFlag = 0xffffffff + // DiscoverDefault is the default set of discovery flags. + DiscoverDefault DiscoveryFlag = DiscoverAll +) + +// MemoryType is an enum for the Node memory +type MemoryType int + +const ( + // MemoryTypeDRAM means that the node has regular DRAM-type memory + MemoryTypeDRAM MemoryType = iota + // MemoryTypePMEM means that the node has persistent memory + MemoryTypePMEM + // MemoryTypeHBM means that the node has high bandwidth memory + MemoryTypeHBM +) + +// String returns the name of the memory type, or a %!(BAD-MemoryType:n) marker +// for an unknown one, as pkg/sysfs does. +func (t MemoryType) String() string { + switch t { + case MemoryTypeDRAM: + return "DRAM" + case MemoryTypePMEM: + return "PMEM" + case MemoryTypeHBM: + return "HBM" + } + return fmt.Sprintf("%%(BAD-MemoryType:%d)", t) +} + +// CacheType specifies a cache type. +type CacheType int + +const ( + // DataCache is a data only cache + DataCache CacheType = iota + // InstructionCache is an instruction only cache. + InstructionCache + // UnifiedCache is a unified data and instruction cache. + UnifiedCache + numCacheTypes + // NumCacheTypes is the number of cache types. + NumCacheTypes = int(numCacheTypes) +) + +// String returns "Data", "Instruction", "Unified", or "" for an unknown type. +func (t CacheType) String() string { + switch t { + case DataCache: + return "Data" + case InstructionCache: + return "Instruction" + case UnifiedCache: + return "Unified" + } + return "" +} + +// EPP represents the value of a CPU energy performance profile +type EPP int + +const ( + EPPPerformance EPP = iota + EPPBalancePerformance + EPPBalancePower + EPPPower + EPPUnknown +) + +// String returns EPP value as string +func (e EPP) String() string { + if int(e) < len(eppStrings) { + return eppStrings[e] + } + return "" +} + +// EPPFromString converts string to EPP value +func EPPFromString(s string) EPP { + if v, ok := eppValues[s]; ok { + return v + } + return EPPUnknown +} + +// CoreKind represents high-level classification of CPU cores, currently P- and +// E-cores +type CoreKind int + +const ( + PerformanceCore CoreKind = iota + EfficientCore +) + +// String returns "P-core" or "E-core". +func (k CoreKind) String() string { + switch k { + case PerformanceCore: + return "P-core" + case EfficientCore: + return "E-core" + } + return "" +} + +// CPUFreq is a CPU frequency scaling range +type CPUFreq struct { + Base uint64 // base frequency + Min uint64 // minimum frequency (kHz) + Max uint64 // maximum frequency (kHz) +} + +// MemInfo contains data read from a NUMA node meminfo file. +type MemInfo struct { + MemTotal uint64 + MemFree uint64 + MemUsed uint64 +} + +// +// Node filters +// + +// NodeFilter is a function for filtering nodes. A node passes a filter if the +// filter returns true for the node. +type NodeFilter func(n Node) bool + +// NodeOfType filters nodes with the given memory type. +func NodeOfType(t MemoryType) NodeFilter { + return func(n Node) bool { + return n.GetMemoryType() == t + } +} + +var ( + // NodeOfDRAMType filters nodes with DRAM memory. + NodeOfDRAMType = NodeOfType(MemoryTypeDRAM) + // NodeOfPMEMType filters nodes with PMEM memory. + NodeOfPMEMType = NodeOfType(MemoryTypePMEM) + // NodeOfHBMType filters nodes with HBM memory. + NodeOfHBMType = NodeOfType(MemoryTypeHBM) + // NodeHasMemory filters nodes with some attached memory. + // + // As in pkg/sysfs a node whose meminfo cannot be read counts as having + // memory, so that an unreadable node is not silently excluded from + // everything. + NodeHasMemory = func(n Node) bool { + mi, _ := n.MemoryInfo() + return mi == nil || mi.MemTotal > 0 + } + // NodeHasNoMemory filters nodes with no any attached memory. + NodeHasNoMemory = func(n Node) bool { + mi, _ := n.MemoryInfo() + return mi != nil && mi.MemTotal == 0 + } + // NodeHasLocalCPUs filters nodes which has close CPUs. + NodeHasLocalCPUs = func(n Node) bool { + return !n.CPUSet().IsEmpty() + } + // NodeHasNoLocalCPUs filters nodes which don't have close CPUs. + NodeHasNoLocalCPUs = func(n Node) bool { + return n.CPUSet().IsEmpty() + } +) + +// eppStrings are the kernel's names for the energy performance preferences, +// indexed by EPP. Built this way to catch a change in the enum. +var eppStrings = func() [EPPUnknown]string { + var e [EPPUnknown]string + e[EPPPerformance] = "performance" + e[EPPBalancePerformance] = "balance_performance" + e[EPPBalancePower] = "balance_power" + e[EPPPower] = "power" + return e +}() + +// eppValues is eppStrings the other way round. +var eppValues = func() map[string]EPP { + m := make(map[string]EPP, len(eppStrings)) + for i, v := range eppStrings { + m[v] = EPP(i) + } + return m +}() + +// NodeFilterAnd returns a filter that is the logical AND of the given filters. +func NodeFilterAnd(filters ...NodeFilter) NodeFilter { + return func(n Node) bool { + for _, f := range filters { + if !f(n) { + return false + } + } + return true + } +} + +// NodeFilterOr returns a filter that is the logical OR of the given filters. +func NodeFilterOr(filters ...NodeFilter) NodeFilter { + return func(n Node) bool { + for _, f := range filters { + if f(n) { + return true + } + } + return false + } +} + +// NodeFilterNot returns a filter that is the logical NOT of the given filter. +func NodeFilterNot(f NodeFilter) NodeFilter { + return func(n Node) bool { + return !f(n) + } +} + +// +// Utilities +// +// These have nothing to do with topology, they just live in pkg/sysfs. They are +// repeated here so that a consumer's import swap is complete, and they should +// end up somewhere under pkg/utils rather than following the topology into +// hardware. +// + +// PickEntryFn picks a given input line apart into an entry of key and value. +type PickEntryFn func(string) (string, string, error) + +// ParseFileEntries parses a sysfs files for the given entries. +func ParseFileEntries(path string, values map[string]any, pickFn PickEntryFn) error { + return sysfs.ParseFileEntries(path, values, sysfs.PickEntryFn(pickFn)) +} + +// IDSetFromCPUSet returns an id set corresponding to a cpuset.CPUSet. +func IDSetFromCPUSet(cset cpuset.CPUSet) idset.IDSet { + return idset.NewIDSetFromIntSlice(cset.List()...) +} + +// CPUSetFromIDSet returns a cpuset.CPUSet corresponding to an id set. +func CPUSetFromIDSet(s idset.IDSet) cpuset.CPUSet { + return cpuset.New(s.Members()...) +} + +// GetMemoryCapacity parses memory capacity from /proc/meminfo (mimicking +// cAdvisor). +func GetMemoryCapacity() int64 { + return sysfs.GetMemoryCapacity() +} From 4fced887bb9c8b1c6fe4332266165dd54554435a Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 9 Sep 2026 19:25:18 +0300 Subject: [PATCH 14/39] resmgr,plugins: import the hardware topology drop-in. Swap pkg/sysfs for pkg/lib/hardware/system everywhere but pkg/sysfs's own tests and the three places whose job is to compare against it. Nothing else changes: the drop-in presents the same names and signatures, so files which imported pkg/sysfs aliased to "system" keep even the alias, and the rest keep the name under an explicit one. This extra intermediate step lets us do an extra round of verification for the new implementation by running end-to-end tests and checking that everything passes. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- cmd/plugins/balloons/policy/cputree.go | 2 +- cmd/plugins/memory-policy/main.go | 2 +- cmd/plugins/topology-aware/policy/coldstart_test.go | 2 +- cmd/plugins/topology-aware/policy/hint.go | 2 +- cmd/plugins/topology-aware/policy/libmem_test.go | 2 +- cmd/plugins/topology-aware/policy/metrics_test.go | 2 +- cmd/plugins/topology-aware/policy/mocks_test.go | 2 +- cmd/plugins/topology-aware/policy/node.go | 2 +- cmd/plugins/topology-aware/policy/pools.go | 2 +- cmd/plugins/topology-aware/policy/pools_test.go | 2 +- cmd/plugins/topology-aware/policy/resources.go | 2 +- cmd/plugins/topology-aware/policy/topology-aware-policy.go | 2 +- pkg/cgroups/cgroupstats.go | 2 +- pkg/cpuallocator/allocator.go | 2 +- pkg/cpuallocator/cpuallocator_test.go | 2 +- pkg/kubernetes/resources.go | 2 +- pkg/resmgr/cpuclass/cpuclass.go | 2 +- pkg/resmgr/cpuclass/handler_commit_test.go | 2 +- pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go | 2 +- pkg/resmgr/cpuclass/internal/cpufreq/platform.go | 2 +- pkg/resmgr/cpuclass/internal/pct/pct.go | 2 +- pkg/resmgr/cpuclass/internal/pct/pct_test.go | 2 +- pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go | 2 +- pkg/resmgr/lib/memory/allocator.go | 2 +- pkg/resmgr/lib/memory/allocator_test.go | 2 +- pkg/resmgr/lib/memory/types.go | 2 +- pkg/resmgr/lib/memory/types_test.go | 2 +- pkg/resmgr/policy/metrics.go | 2 +- pkg/resmgr/policy/policy.go | 2 +- pkg/resmgr/resource-manager.go | 2 +- pkg/utils/topology/hints.go | 2 +- 31 files changed, 31 insertions(+), 31 deletions(-) diff --git a/cmd/plugins/balloons/policy/cputree.go b/cmd/plugins/balloons/policy/cputree.go index b3ed6dcb3..78c28f949 100644 --- a/cmd/plugins/balloons/policy/cputree.go +++ b/cmd/plugins/balloons/policy/cputree.go @@ -21,7 +21,7 @@ import ( "sort" "strings" - system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/cmd/plugins/memory-policy/main.go b/cmd/plugins/memory-policy/main.go index 0b2faf72a..4f137363d 100644 --- a/cmd/plugins/memory-policy/main.go +++ b/cmd/plugins/memory-policy/main.go @@ -30,8 +30,8 @@ import ( "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/stub" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) diff --git a/cmd/plugins/topology-aware/policy/coldstart_test.go b/cmd/plugins/topology-aware/policy/coldstart_test.go index 0c9f6d1d0..6187a5748 100644 --- a/cmd/plugins/topology-aware/policy/coldstart_test.go +++ b/cmd/plugins/topology-aware/policy/coldstart_test.go @@ -20,11 +20,11 @@ import ( "testing" "time" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - system "github.com/containers/nri-plugins/pkg/sysfs" idset "github.com/intel/goresctrl/pkg/utils" ) diff --git a/cmd/plugins/topology-aware/policy/hint.go b/cmd/plugins/topology-aware/policy/hint.go index 519a45c58..b189c0ad8 100644 --- a/cmd/plugins/topology-aware/policy/hint.go +++ b/cmd/plugins/topology-aware/policy/hint.go @@ -18,7 +18,7 @@ import ( "strconv" "strings" - system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" diff --git a/cmd/plugins/topology-aware/policy/libmem_test.go b/cmd/plugins/topology-aware/policy/libmem_test.go index d0032e22c..57e4f6d94 100644 --- a/cmd/plugins/topology-aware/policy/libmem_test.go +++ b/cmd/plugins/topology-aware/policy/libmem_test.go @@ -21,8 +21,8 @@ import ( "testing" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/testutils" ) diff --git a/cmd/plugins/topology-aware/policy/metrics_test.go b/cmd/plugins/topology-aware/policy/metrics_test.go index d620d4cd0..734b11929 100644 --- a/cmd/plugins/topology-aware/policy/metrics_test.go +++ b/cmd/plugins/topology-aware/policy/metrics_test.go @@ -25,9 +25,9 @@ import ( "go.opentelemetry.io/otel/sdk/metric/metricdata" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/metrics" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/testutils" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 91f26197d..fdaea070e 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -22,9 +22,9 @@ import ( "github.com/containers/nri-plugins/pkg/agent/podresapi" resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1" "github.com/containers/nri-plugins/pkg/cpuallocator" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cache" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" "github.com/intel/goresctrl/pkg/sst" diff --git a/cmd/plugins/topology-aware/policy/node.go b/cmd/plugins/topology-aware/policy/node.go index 4d71d109b..5c662e518 100644 --- a/cmd/plugins/topology-aware/policy/node.go +++ b/cmd/plugins/topology-aware/policy/node.go @@ -18,7 +18,7 @@ import ( "fmt" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" - system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index 00567da83..95fcd40cb 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -25,10 +25,10 @@ import ( "k8s.io/apimachinery/pkg/types" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/dra" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - system "github.com/containers/nri-plugins/pkg/sysfs" idset "github.com/intel/goresctrl/pkg/utils" "tags.cncf.io/container-device-interface/pkg/parser" ) diff --git a/cmd/plugins/topology-aware/policy/pools_test.go b/cmd/plugins/topology-aware/policy/pools_test.go index 46d191540..5d8088e68 100644 --- a/cmd/plugins/topology-aware/policy/pools_test.go +++ b/cmd/plugins/topology-aware/policy/pools_test.go @@ -28,7 +28,7 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/dra" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - system "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/testutils" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 4be1b32cc..f2813d24e 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/containers/nri-plugins/pkg/agent/podresapi" - "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index cf1489eb1..d4904d0a6 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -32,8 +32,8 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/events" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - system "github.com/containers/nri-plugins/pkg/sysfs" ) const ( diff --git a/pkg/cgroups/cgroupstats.go b/pkg/cgroups/cgroupstats.go index 631588272..884517d78 100644 --- a/pkg/cgroups/cgroupstats.go +++ b/pkg/cgroups/cgroupstats.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" ) // BlkioDeviceBytes contains a single operations line of blkio.throttle.io_service_bytes_recursive file diff --git a/pkg/cpuallocator/allocator.go b/pkg/cpuallocator/allocator.go index db513a042..ae430fdf7 100644 --- a/pkg/cpuallocator/allocator.go +++ b/pkg/cpuallocator/allocator.go @@ -21,8 +21,8 @@ import ( "github.com/containers/nri-plugins/pkg/utils/cpuset" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils" "github.com/intel/goresctrl/pkg/sst" idset "github.com/intel/goresctrl/pkg/utils" diff --git a/pkg/cpuallocator/cpuallocator_test.go b/pkg/cpuallocator/cpuallocator_test.go index e9bddc2dd..396f94bce 100644 --- a/pkg/cpuallocator/cpuallocator_test.go +++ b/pkg/cpuallocator/cpuallocator_test.go @@ -22,7 +22,7 @@ import ( "github.com/containers/nri-plugins/pkg/testutils" "github.com/containers/nri-plugins/pkg/utils/cpuset" - "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" ) diff --git a/pkg/kubernetes/resources.go b/pkg/kubernetes/resources.go index d0eec9c7b..868eaecd7 100644 --- a/pkg/kubernetes/resources.go +++ b/pkg/kubernetes/resources.go @@ -20,7 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" - "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" ) const ( diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index f5dad786f..578213917 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -30,13 +30,13 @@ import ( "sort" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpufreq" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpuidle" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/uncorefreq" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/pkg/resmgr/cpuclass/handler_commit_test.go b/pkg/resmgr/cpuclass/handler_commit_test.go index b2b4d37d7..392cf47d4 100644 --- a/pkg/resmgr/cpuclass/handler_commit_test.go +++ b/pkg/resmgr/cpuclass/handler_commit_test.go @@ -20,11 +20,11 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpufreq" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpuidle" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/uncorefreq" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go b/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go index c58eef3df..ade8c35d3 100644 --- a/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go +++ b/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go @@ -24,9 +24,9 @@ import ( "slices" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/pkg/resmgr/cpuclass/internal/cpufreq/platform.go b/pkg/resmgr/cpuclass/internal/cpufreq/platform.go index 18ce2e177..91f3ef207 100644 --- a/pkg/resmgr/cpuclass/internal/cpufreq/platform.go +++ b/pkg/resmgr/cpuclass/internal/cpufreq/platform.go @@ -17,7 +17,7 @@ package cpufreq import ( "fmt" - "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" ) // platformTurboInfo holds platform-level turbo frequency capabilities diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index 68283e293..aaf6f71da 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -21,9 +21,9 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index d1ea60b7c..175e29180 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -23,8 +23,8 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go b/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go index 33c589571..316edcc3d 100644 --- a/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go +++ b/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go @@ -24,9 +24,9 @@ import ( "github.com/intel/goresctrl/pkg/utils" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" - "github.com/containers/nri-plugins/pkg/sysfs" ) var log = logger.NewLogger("cpuclass") diff --git a/pkg/resmgr/lib/memory/allocator.go b/pkg/resmgr/lib/memory/allocator.go index 4939c4469..5d1267f24 100644 --- a/pkg/resmgr/lib/memory/allocator.go +++ b/pkg/resmgr/lib/memory/allocator.go @@ -21,7 +21,7 @@ import ( "slices" "strings" - "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) diff --git a/pkg/resmgr/lib/memory/allocator_test.go b/pkg/resmgr/lib/memory/allocator_test.go index f5c7f88dd..915182bd3 100644 --- a/pkg/resmgr/lib/memory/allocator_test.go +++ b/pkg/resmgr/lib/memory/allocator_test.go @@ -20,8 +20,8 @@ import ( "github.com/stretchr/testify/require" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" . "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/pkg/resmgr/lib/memory/types.go b/pkg/resmgr/lib/memory/types.go index a6a9e6ae3..5e6658c4a 100644 --- a/pkg/resmgr/lib/memory/types.go +++ b/pkg/resmgr/lib/memory/types.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" ) // Type represents known types of memory. diff --git a/pkg/resmgr/lib/memory/types_test.go b/pkg/resmgr/lib/memory/types_test.go index 05dbf76e8..2065eb97e 100644 --- a/pkg/resmgr/lib/memory/types_test.go +++ b/pkg/resmgr/lib/memory/types_test.go @@ -15,8 +15,8 @@ package libmem_test import ( + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" . "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/sysfs" "testing" diff --git a/pkg/resmgr/policy/metrics.go b/pkg/resmgr/policy/metrics.go index 1b6ca7f25..f4fbec974 100644 --- a/pkg/resmgr/policy/metrics.go +++ b/pkg/resmgr/policy/metrics.go @@ -23,9 +23,9 @@ import ( "go.opentelemetry.io/otel/metric" v1 "k8s.io/api/core/v1" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/metrics" "github.com/containers/nri-plugins/pkg/resmgr/cache" - system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index ff6f4d952..c7ebd2050 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -26,8 +26,8 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/events" "github.com/prometheus/client_golang/prometheus" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" - system "github.com/containers/nri-plugins/pkg/sysfs" // nrt "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" ) diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index e556d933d..3d64936e8 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -23,12 +23,12 @@ import ( "github.com/containers/nri-plugins/pkg/agent" "github.com/containers/nri-plugins/pkg/healthz" "github.com/containers/nri-plugins/pkg/instrumentation" + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/pidfile" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/control" "github.com/containers/nri-plugins/pkg/resmgr/policy" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/topology" "k8s.io/client-go/kubernetes" "sigs.k8s.io/yaml" diff --git a/pkg/utils/topology/hints.go b/pkg/utils/topology/hints.go index 171f0e3d2..df8621919 100644 --- a/pkg/utils/topology/hints.go +++ b/pkg/utils/topology/hints.go @@ -15,8 +15,8 @@ package topology import ( + sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) From 332d9d9d6ad9c609c52e08fd156b899f4f396e8d Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 9 Sep 2026 22:54:04 +0300 Subject: [PATCH 15/39] resmgr: discover the hardware topology at startup. The topology was discovered in policy.NewPolicy, as a sysfs.System. Discover it in NewResourceManager instead, as a hardware.Machine, and hand it down through policy.Options to the backends. For now we keep both sysfs.System and hardware.Machine in the policy backend options, but the former essentially come from the latter through the compatibility wrapper. Once we have converted everything to hardware.Machine, sysfs.System can be removed from the options. Note that the env overrides have to be asked for explicitly here, as the drop-in asked for them on the callers' behalf. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/resmgr/policy/policy.go | 32 ++++++++++++++++++++------------ pkg/resmgr/resource-manager.go | 33 ++++++++++++++++++++++++--------- 2 files changed, 44 insertions(+), 21 deletions(-) diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index c7ebd2050..f36421c92 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -26,6 +26,7 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/events" "github.com/prometheus/client_golang/prometheus" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" // nrt "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" @@ -55,6 +56,8 @@ type ConstraintSet map[Domain]Constraint // Options describes policy options type Options struct { + // Machine is the CPU and memory topology, discovered by the resource manager. + Machine *hardware.Machine // SendEvent is the function for delivering events back to the resource manager. SendEvent SendEventFn // KubeClientFn returns the shared kubernetes client, or a nil interface @@ -69,7 +72,10 @@ type Options struct { // BackendOptions describes the options for a policy backend instance type BackendOptions struct { - // System provides system/HW/topology information + // Machine provides system/HW/topology information. + Machine *hardware.Machine + // System is Machine behind the pkg/sysfs interface, for backends which have + // not been moved over to Machine yet. It goes with the last of them. System system.System // System state/cache Cache cache.Cache @@ -247,11 +253,12 @@ type ZoneAttribute struct { // Policy instance/state. type policy struct { - options Options // policy options - cache cache.Cache // system state cache - active Backend // our active backend - system system.System // system/HW/topology info - scollect *SystemCollector // system metrics collector + options Options // policy options + cache cache.Cache // system state cache + active Backend // our active backend + machine *hardware.Machine // CPU and memory topology + system system.System // the same, behind the pkg/sysfs interface + scollect *SystemCollector // system metrics collector } // Out logger instance. @@ -261,18 +268,18 @@ var log logger.Logger = logger.NewLogger("policy") func NewPolicy(backend Backend, cache cache.Cache, o *Options) (Policy, error) { log.Infof("creating '%s' policy...", backend.Name()) + if o.Machine == nil { + return nil, policyError("no machine topology given") + } + p := &policy{ cache: cache, options: *o, active: backend, + machine: o.Machine, + system: system.FromMachine(o.Machine), } - sys, err := system.DiscoverSystem() - if err != nil { - return nil, policyError("failed to discover system topology: %v", err) - } - p.system = sys - return p, nil } @@ -289,6 +296,7 @@ func (p *policy) Start(cfg any) error { if err := p.active.Setup(&BackendOptions{ Cache: p.cache, + Machine: p.machine, System: p.system, SendEvent: p.options.SendEvent, Config: cfg, diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index 3d64936e8..8f18bab4f 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -23,6 +23,7 @@ import ( "github.com/containers/nri-plugins/pkg/agent" "github.com/containers/nri-plugins/pkg/healthz" "github.com/containers/nri-plugins/pkg/instrumentation" + "github.com/containers/nri-plugins/pkg/lib/hardware" sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/pidfile" @@ -54,14 +55,15 @@ type resmgr struct { sync.RWMutex agent *agent.Agent cfg cfgapi.ResmgrConfig - cache cache.Cache // cached state - policy policy.Policy // resource manager policy - control control.Control // policy controllers/enforcement - events chan any // channel for delivering events - stop chan any // channel for signalling shutdown to goroutines - nri *nriPlugin // NRI plugins, if we're running as such - rdt *rdtControl // control for RDT allocation and monitoring - blkio *blkioControl // control for block I/O prioritization and throttling + cache cache.Cache // cached state + machine *hardware.Machine // CPU and memory topology, discovered once + policy policy.Policy // resource manager policy + control control.Control // policy controllers/enforcement + events chan any // channel for delivering events + stop chan any // channel for signalling shutdown to goroutines + nri *nriPlugin // NRI plugins, if we're running as such + rdt *rdtControl // control for RDT allocation and monitoring + blkio *blkioControl // control for block I/O prioritization and throttling running bool } @@ -83,8 +85,20 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag irq.SetProcRoot(opt.HostRoot) } + // The topology is discovered once here and handed down. Anything which still + // wants the pkg/sysfs interface wraps this with sysfs.FromMachine, so there is + // one discovery and one view of the hardware however it is reached. + machine, err := hardware.Discover( + hardware.WithRoot(opt.HostRoot), + hardware.WithEnvOverrides(), + ) + if err != nil { + return nil, resmgrError("failed to discover hardware topology: %v", err) + } + m := &resmgr{ - agent: agt, + agent: agt, + machine: machine, } if err := m.setupCache(); err != nil { @@ -278,6 +292,7 @@ func (m *resmgr) setupPolicy(backend policy.Backend) error { } p, err := policy.NewPolicy(backend, m.cache, &policy.Options{ + Machine: m.machine, SendEvent: m.SendEvent, KubeClientFn: m.kubeClientFn, NodeName: m.agent.NodeName(), From ca9f0415bccaa1cc981afe89d6c1a3a7e51fe8a6 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Wed, 9 Sep 2026 22:58:23 +0300 Subject: [PATCH 16/39] balloons: read the topology from hardware.Machine. The policy takes its topology from the Machine the resource manager discovered, rather than from the pkg/sysfs interface wrapped around it. cpuallocator, cpuclass and libmem still want that interface, so they keep being handed the System from the backend options; the policy itself no longer reads it. toCpuSet and toCpuMask convert at the seam. The policy is written in cpuset.CPUSet and stays that way for now. Once we have removed all the remaining dependencies on sysfs.System, we can update the internals here to use libcpu.CpuMask, which should scale much better with the number of CPUs present. Building the L2 cache level no longer iterates a map, so a node's cache children come out in cache id order instead of a different order on every run. The set of nodes is unchanged; on all six recorded machines the tree matches what the previous code built. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 27 ++++---- cmd/plugins/balloons/policy/cputree.go | 63 +++++++++++-------- 2 files changed, 51 insertions(+), 39 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 37f0a01e0..170c463a6 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -31,6 +31,7 @@ import ( "github.com/containers/nri-plugins/pkg/cpuallocator" "github.com/containers/nri-plugins/pkg/irq" "github.com/containers/nri-plugins/pkg/kubernetes" + "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" @@ -80,6 +81,7 @@ const ( type balloons struct { options *policy.BackendOptions // configuration common to all policies bpoptions *BalloonsOptions // balloons-specific configuration + machine *hardware.Machine // CPU and memory topology cch cache.Cache // nri-resource-policy cache allowed cpuset.CPUSet // bounding set of CPUs we're allowed to use reserved cpuset.CPUSet // system-/kube-reserved CPUs @@ -217,11 +219,12 @@ func (p *balloons) Setup(policyOptions *policy.BackendOptions) error { bpoptions = bpoptions.DeepCopy() p.options = policyOptions + p.machine = policyOptions.Machine p.cch = policyOptions.Cache p.cpuAllocator = cpuallocator.NewCPUAllocator(policyOptions.System) log.Infof("setting up %s policy...", PolicyName) - p.cpuTree = NewCpuTreeFromSystem(policyOptions.System) + p.cpuTree = NewCpuTreeFromMachine(policyOptions.Machine) log.Debugf("CPU topology: %s", p.cpuTree) // Handle policy-specific options @@ -1063,11 +1066,11 @@ func (p *balloons) updateLoadedVirtDev(allocatorOptions *cpuTreeAllocatorOptions switch virtDev.level { case CPUTopologyLevelCore: // add all CPUs from same cores of virtual device CPUs - allocatorOptions.virtDevCpusets[virtDevName] = []cpuset.CPUSet{prevCpus.Union(p.cpuTree.system().AllThreadsForCPUs(vdCpus))} + allocatorOptions.virtDevCpusets[virtDevName] = []cpuset.CPUSet{prevCpus.Union(toCpuSet(hardware.AllThreads(p.machine, toCpuMask(vdCpus))))} case CPUTopologyLevelL2Cache: // add all CPUs from the same L2 cache of virtual // device CPUs - allocatorOptions.virtDevCpusets[virtDevName] = []cpuset.CPUSet{prevCpus.Union(p.cpuTree.system().AllCPUsSharingNthLevelCacheWithCPUs(2, vdCpus))} + allocatorOptions.virtDevCpusets[virtDevName] = []cpuset.CPUSet{prevCpus.Union(toCpuSet(hardware.CPUsSharingCache(p.machine, 2, toCpuMask(vdCpus))))} default: log.Errorf("internal error: not implemented load level %q used in virtual device %q", virtDev.level, virtDevName) } @@ -1223,7 +1226,7 @@ func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool, c cache.Contain preferFarFromDevices: append([]string(nil), blnDef.PreferFarFromDevices...), virtDevCpusets: map[string][]cpuset.CPUSet{ virtDevReservedCpus: {p.reserved}, - virtDevIsolatedCpus: {p.options.System.Isolated()}, + virtDevIsolatedCpus: {toCpuSet(p.machine.IsolatedCPUs())}, virtDevECores: {p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityLow]}, virtDevPCores: {p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityHigh]}, }, @@ -1897,16 +1900,16 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { if err != nil { return balloonsError("failed to parse available CPU cpuset '%s': %w", amount, err) } - availableCpus = p.options.System.CPUSet().Difference(cset) + availableCpus = toCpuSet(p.machine.PresentCPUs()).Difference(cset) case cfgapi.AmountQuantity: return balloonsError("can't handle CPU resources given as resource.Quantity (%v)", amount) case cfgapi.AmountAbsent: // Available CPUs not specified, default to system CPUs. - availableCpus = p.options.System.CPUSet() + availableCpus = toCpuSet(p.machine.PresentCPUs()) } // Allocation of only online CPUs is allowed. - p.allowed = availableCpus.Intersection(p.options.System.OnlineCPUs()) + p.allowed = availableCpus.Intersection(toCpuSet(p.machine.OnlineCPUs())) setOmittedDefaults(bpoptions) @@ -2293,12 +2296,12 @@ func (p *balloons) containerDeviceCpus(c cache.Container, resourceName string) ( // unknown id Node() returns a nil *node wrapped in a non-nil // Node interface. FilterNode without filters is an existence // check. - if !p.options.System.FilterNode(numaID) { + if !p.machine.MemoryNode(numaID).Valid() { log.Errorf("unknown NUMA node %d in the topology of device %q of container %s", numaID, resourceName, c.PrettyName()) continue } - cpus = cpus.Union(p.options.System.Node(numaID).CPUSet()) + cpus = cpus.Union(toCpuSet(p.machine.MemoryNode(numaID).CPUs())) } if cpus.IsEmpty() { return emptyCpuSet, false @@ -2397,7 +2400,7 @@ func (p *balloons) fillFarFromDevices(blnDefs []*BalloonDef) { // beginning of the list will be more effectively avoided than // devices later in the list. avoidDevs := []string{} - if p.options.System.Isolated().Size() != 0 { + if p.machine.IsolatedCPUs().Size() != 0 { avoidDevs = append(avoidDevs, virtDevIsolatedCpus) } for _, blnDef := range blnDefs { @@ -2580,7 +2583,7 @@ func (p *balloons) updatePinning(blns ...*Balloon) { if c, ok := p.cch.LookupContainer(cID); ok { if runWithoutHyperthreads(c, bln) { if cpusNoHt.Size() == 0 { - cpusNoHt = p.cpuTree.system().SingleThreadForCPUs(pinnableCpus) + cpusNoHt = toCpuSet(hardware.SingleThreadPerCore(p.machine, toCpuMask(pinnableCpus))) } allowedCpus = cpusNoHt } else { @@ -2617,7 +2620,7 @@ func (p *balloons) shareIdleCpus(addCpus, removeCpus cpuset.CPUSet) []*Balloon { } } } - addCpus = addCpus.Difference(p.options.System.Isolated()) + addCpus = addCpus.Difference(toCpuSet(p.machine.IsolatedCPUs())) if addCpus.Size() > 0 { for blnIdx, bln := range p.balloons { topoLevel := bln.Def.ShareIdleCpusInSame diff --git a/cmd/plugins/balloons/policy/cputree.go b/cmd/plugins/balloons/policy/cputree.go index 78c28f949..ef9271d2f 100644 --- a/cmd/plugins/balloons/policy/cputree.go +++ b/cmd/plugins/balloons/policy/cputree.go @@ -21,7 +21,8 @@ import ( "sort" "strings" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) @@ -33,7 +34,6 @@ type cpuTreeNode struct { parent *cpuTreeNode children []*cpuTreeNode cpus cpuset.CPUSet // union of CPUs of child nodes - sys system.System } // cpuTreeNodeAttributes contains various attributes of a CPU tree @@ -78,6 +78,18 @@ type cpuTreeAllocatorOptions struct { var emptyCpuSet = cpuset.New() +// toCpuSet and toCpuMask convert between the set the hardware package speaks and +// the one this policy is written in. They are the seam left by moving the policy +// onto hardware without rewriting its allocation logic; they disappear if the +// policy ever switches to libcpu sets throughout. +func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { + return cpuset.New(cpus.List()...) +} + +func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { + return libcpu.NewCpuMask(cpus.List()...) +} + // String returns string representation of a CPU tree node. func (t *cpuTreeNode) String() string { if len(t.children) == 0 { @@ -101,13 +113,6 @@ func (t *cpuTreeNode) PrettyPrint() string { return strings.Join(lines, "\n") } -func (t *cpuTreeNode) system() system.System { - if t.sys != nil || t.parent == nil { - return t.sys - } - return t.parent.system() -} - // String returns cpuTreeNodeAttributes as a string. func (tna cpuTreeNodeAttributes) String() string { return fmt.Sprintf("%s{%d,%v,%d,%d}", tna.t.name, tna.depth, @@ -261,51 +266,55 @@ func (t *cpuTreeNode) CpuLocations(cpus cpuset.CPUSet) [][]string { return names } -// NewCpuTreeFromSystem returns the root node of the topology tree -// constructed from the given system. -func NewCpuTreeFromSystem(sys system.System) *cpuTreeNode { +// NewCpuTreeFromMachine returns the root node of the topology tree constructed +// from the given machine. +func NewCpuTreeFromMachine(m *hardware.Machine) *cpuTreeNode { // TODO: split deep nested loops into functions + x := m.TopologyIndex() sysTree := NewCpuTree("system") - sysTree.sys = sys sysTree.level = CPUTopologyLevelSystem - for _, packageID := range sys.PackageIDs() { + for _, packageID := range x.PackageIDs() { packageTree := NewCpuTree(fmt.Sprintf("p%d", packageID)) packageTree.level = CPUTopologyLevelPackage - cpuPackage := sys.Package(packageID) sysTree.AddChild(packageTree) - for _, dieID := range cpuPackage.DieIDs() { - dieTree := NewCpuTree(fmt.Sprintf("%sd%d", packageTree.name, dieID)) + for _, dieID := range x.DieIDs(packageID) { + dieTree := NewCpuTree(fmt.Sprintf("%sd%d", packageTree.name, dieID.Die)) dieTree.level = CPUTopologyLevelDie packageTree.AddChild(dieTree) - for _, nodeID := range cpuPackage.DieNodeIDs(dieID) { + dieCpus := x.DieCPUs(dieID) + for _, nodeID := range slices.Sorted( + slices.Values(hardware.MemoryNodesFor(m, dieCpus))) { nodeTree := NewCpuTree(fmt.Sprintf("%sn%d", dieTree.name, nodeID)) nodeTree.level = CPUTopologyLevelNuma dieTree.AddChild(nodeTree) - node := sys.Node(nodeID) // Find all level 2 caches (l2c) shared by CPUs of this node. - l2cs := map[*system.Cache]struct{}{} - for _, cpuID := range node.CPUSet().List() { - for _, cache := range sys.CPU(cpuID).GetCachesByLevel(2) { - l2cs[cache] = struct{}{} + l2cs := []*hardware.Cache{} + for _, cpuID := range m.MemoryNode(nodeID).CPUs().List() { + for _, cache := range m.CPU(cpuID).Caches() { + if cache.Level() == 2 && !slices.Contains(l2cs, cache) { + l2cs = append(l2cs, cache) + } } } + slices.SortFunc(l2cs, func(a, b *hardware.Cache) int { + return a.ID() - b.ID() + }) - for cache := range l2cs { + for _, cache := range l2cs { l2cTree := NewCpuTree(fmt.Sprintf("%s$%d", nodeTree.name, cache.ID())) l2cTree.level = CPUTopologyLevelL2Cache nodeTree.AddChild(l2cTree) threadsSeen := map[int]struct{}{} - for _, cpuID := range cache.SharedCPUSet().List() { + for _, cpuID := range cache.CPUs().List() { if _, alreadySeen := threadsSeen[cpuID]; alreadySeen { continue } - cpu := sys.CPU(cpuID) coreTree := NewCpuTree(fmt.Sprintf("%scpu%d", nodeTree.name, cpuID)) coreTree.level = CPUTopologyLevelCore l2cTree.AddChild(coreTree) - for _, threadID := range cpu.ThreadCPUSet().List() { + for _, threadID := range m.CPU(cpuID).Threads().List() { threadsSeen[threadID] = struct{}{} threadTree := NewCpuTree(fmt.Sprintf("%st%d", coreTree.name, threadID)) threadTree.level = CPUTopologyLevelThread From 5efd7935ff1b71399e13041f8ed9a0950e7bf71c Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 00:20:15 +0300 Subject: [PATCH 17/39] libmem: take memory nodes from a hardware.Machine. WithSystemNodes read the NUMA nodes through the pkg/sysfs interface. WithMachineNodes reads them from an already discovered hardware.Machine instead, so the package no longer depends on that interface at all. A node hardware could not classify maps to TypeDRAM rather than panicking as TypeForSysfs did on an unknown type. There is no type here for "do not know", and ordinary memory is what such a node arrived as before. Capacities are read during discovery rather than being re-read here. They cannot have changed, and a node whose meminfo cannot be read fails discovery, so the error this used to return now comes earlier. The public set types are unchanged: this is about where the topology comes from, not how sets are represented. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 2 +- .../topology-aware/policy/coldstart_test.go | 5 +- .../topology-aware/policy/libmem_test.go | 14 ++++- .../topology-aware/policy/metrics_test.go | 11 +++- .../topology-aware/policy/pools_test.go | 28 +++++++--- .../policy/topology-aware-policy.go | 2 +- pkg/resmgr/lib/memory/allocator.go | 36 +++++++------ pkg/resmgr/lib/memory/allocator_test.go | 14 ++--- pkg/resmgr/lib/memory/types.go | 51 +++++++++++-------- pkg/resmgr/lib/memory/types_test.go | 19 ++++--- 10 files changed, 114 insertions(+), 68 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 170c463a6..3fd26a692 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -1963,7 +1963,7 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { // Create new memory allocator to clear any allocations with previous configuration. // Other allocators are stateless in that respect. - malloc, err := libmem.NewAllocator(libmem.WithSystemNodes(p.options.System)) + malloc, err := libmem.NewAllocator(libmem.WithMachineNodes(p.machine)) if err != nil { return balloonsError("failed to create memory allocator: %w", err) } diff --git a/cmd/plugins/topology-aware/policy/coldstart_test.go b/cmd/plugins/topology-aware/policy/coldstart_test.go index 6187a5748..e0d9a8578 100644 --- a/cmd/plugins/topology-aware/policy/coldstart_test.go +++ b/cmd/plugins/topology-aware/policy/coldstart_test.go @@ -105,7 +105,10 @@ func TestColdStart(t *testing.T) { } policy.allocations.policy = policy policy.options.SendEvent = sendEvent - ma, err := libmem.NewAllocator(libmem.WithSystemNodes(policy.sys)) + // No nodes: the allocator takes them from a hardware.Machine now, + // and the mocked system above cannot stand in for one. Moot while + // the test is skipped, which is for the same reason. + ma, err := libmem.NewAllocator() if err != nil { panic(err) } diff --git a/cmd/plugins/topology-aware/policy/libmem_test.go b/cmd/plugins/topology-aware/policy/libmem_test.go index 57e4f6d94..710c500cc 100644 --- a/cmd/plugins/topology-aware/policy/libmem_test.go +++ b/cmd/plugins/topology-aware/policy/libmem_test.go @@ -21,6 +21,7 @@ import ( "testing" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/lib/hardware/system" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/testutils" @@ -49,10 +50,19 @@ func setupTestPolicy(t *testing.T) (*policy, string) { t.Fatalf("failed to discover system: %v", err) } + machine, err := hardware.Discover(hardware.WithRoot(path.Dir(sysPath))) + if err != nil { + if rerr := os.RemoveAll(dir); rerr != nil { + t.Logf("failed to remove temp dir %q: %v", dir, rerr) + } + t.Fatalf("failed to discover machine: %v", err) + } + p := New().(*policy) if err := p.Setup(&policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, + Cache: &mockCache{}, + System: sys, + Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{cfgapi.CPU: "750m"}, }, diff --git a/cmd/plugins/topology-aware/policy/metrics_test.go b/cmd/plugins/topology-aware/policy/metrics_test.go index 734b11929..f518384a5 100644 --- a/cmd/plugins/topology-aware/policy/metrics_test.go +++ b/cmd/plugins/topology-aware/policy/metrics_test.go @@ -25,6 +25,7 @@ import ( "go.opentelemetry.io/otel/sdk/metric/metricdata" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/metrics" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" @@ -96,10 +97,16 @@ func newServerPolicyWithMetrics(t *testing.T) (*policy, *TopologyAwareMetrics, * if err != nil { t.Fatalf("failed to discover system: %v", err) } + machine, err := hardware.Discover( + hardware.WithRoot(path.Join(dir, "sysfs", "server"))) + if err != nil { + t.Fatalf("failed to discover machine: %v", err) + } opts := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, + Cache: &mockCache{}, + System: sys, + Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ cfgapi.CPU: "750m", diff --git a/cmd/plugins/topology-aware/policy/pools_test.go b/cmd/plugins/topology-aware/policy/pools_test.go index 5d8088e68..2f02e8aff 100644 --- a/cmd/plugins/topology-aware/policy/pools_test.go +++ b/cmd/plugins/topology-aware/policy/pools_test.go @@ -28,6 +28,7 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/dra" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/testutils" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -138,10 +139,15 @@ func TestPoolCreation(t *testing.T) { if err != nil { panic(err) } + machine, err := hardware.Discover(hardware.WithRoot(path.Dir(tc.path))) + if err != nil { + panic(err) + } policyOptions := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, + Cache: &mockCache{}, + System: sys, + Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ cfgapi.CPU: "750m", @@ -263,10 +269,15 @@ func TestWorkloadPlacement(t *testing.T) { if err != nil { panic(err) } + machine, err := hardware.Discover(hardware.WithRoot(path.Dir(tc.path))) + if err != nil { + panic(err) + } policyOptions := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, + Cache: &mockCache{}, + System: sys, + Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ cfgapi.CPU: "750m", @@ -523,10 +534,15 @@ func TestAffinities(t *testing.T) { if err != nil { panic(err) } + machine, err := hardware.Discover(hardware.WithRoot(path.Dir(tc.path))) + if err != nil { + panic(err) + } policyOptions := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, + Cache: &mockCache{}, + System: sys, + Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ cfgapi.CPU: "750m", diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index d4904d0a6..a5b3ab363 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -165,7 +165,7 @@ func (p *policy) Setup(opts *policyapi.BackendOptions) error { p.sys = opts.System p.options = opts p.cpuAllocator = cpuallocator.NewCPUAllocator(opts.System) - p.memAllocator, err = libmem.NewAllocator(libmem.WithSystemNodes(opts.System)) + p.memAllocator, err = libmem.NewAllocator(libmem.WithMachineNodes(opts.Machine)) if err != nil { return policyError("failed to initialize %s policy: %w", err) } diff --git a/pkg/resmgr/lib/memory/allocator.go b/pkg/resmgr/lib/memory/allocator.go index 5d1267f24..0e5f66962 100644 --- a/pkg/resmgr/lib/memory/allocator.go +++ b/pkg/resmgr/lib/memory/allocator.go @@ -21,7 +21,7 @@ import ( "slices" "strings" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -70,25 +70,29 @@ const ( // AllocatorOption is an opaque option for an Allocator. type AllocatorOption func(*Allocator) error -// WithSystemNodes is an option to request an allocator to perform -// automatic NUMA node discovery using the given sysfs instance. -func WithSystemNodes(sys sysfs.System) AllocatorOption { +// WithMachineNodes is an option to request an allocator to take its NUMA nodes +// from an already discovered machine. +// +// The node capacities are the ones read during that discovery, rather than being +// re-read here. They cannot have changed, and a node whose meminfo could not be +// read at all fails discovery, so there is nothing left to go wrong by the time +// a Machine exists. +func WithMachineNodes(m *hardware.Machine) AllocatorOption { return func(a *Allocator) error { - nodes := []*Node{} + if m == nil { + return fmt.Errorf("no machine to take memory nodes from") + } - for _, id := range sys.NodeIDs() { - sysNode := sys.Node(id) - info, err := sysNode.MemoryInfo() - if err != nil { - return fmt.Errorf("failed to discover system node #%d: %w", id, err) - } + nodes := []*Node{} + for _, node := range m.MemoryNodes() { var ( - memType = TypeForSysfs(sysNode.GetMemoryType()) - capacity = int64(info.MemTotal) - isNormal = sysNode.HasNormalMemory() - closeCPUs = sysNode.CPUSet() - distance = sysNode.Distance() + id = node.ID() + memType = TypeForKind(node.Kind()) + capacity = node.Capacity() + isNormal = node.HasNormalMemory() + closeCPUs = cpuset.New(node.CPUs().List()...) + distance = node.Distances() ) n, err := NewNode(id, memType, capacity, isNormal, closeCPUs, distance) diff --git a/pkg/resmgr/lib/memory/allocator_test.go b/pkg/resmgr/lib/memory/allocator_test.go index 915182bd3..89742e7d7 100644 --- a/pkg/resmgr/lib/memory/allocator_test.go +++ b/pkg/resmgr/lib/memory/allocator_test.go @@ -20,24 +20,24 @@ import ( "github.com/stretchr/testify/require" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" . "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) -func TestNewAllocatorWithSystemNodes(t *testing.T) { +func TestNewAllocatorWithMachineNodes(t *testing.T) { var ( sysRoot = "./testdata/sample2" - sys sysfs.System + m *hardware.Machine err error a *Allocator ) - sys, err = sysfs.DiscoverSystemAt(sysRoot + "/sys") - require.Nil(t, err, "sysfs discovery error for "+sysRoot) - require.NotNil(t, sys, "sysfs discovery for "+sysRoot) + m, err = hardware.Discover(hardware.WithRoot(sysRoot)) + require.Nil(t, err, "hardware discovery error for "+sysRoot) + require.NotNil(t, m, "discovered machine for "+sysRoot) - a, err = NewAllocator(WithSystemNodes(sys)) + a, err = NewAllocator(WithMachineNodes(m)) require.Nil(t, err, "allocator creation error") require.NotNil(t, a, "created allocator") } diff --git a/pkg/resmgr/lib/memory/types.go b/pkg/resmgr/lib/memory/types.go index 5e6658c4a..36adf99a3 100644 --- a/pkg/resmgr/lib/memory/types.go +++ b/pkg/resmgr/lib/memory/types.go @@ -19,7 +19,7 @@ import ( "fmt" "strings" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" ) // Type represents known types of memory. @@ -32,15 +32,21 @@ const ( ) var ( - sysToType = map[sysfs.MemoryType]Type{ - sysfs.MemoryTypeDRAM: TypeDRAM, - sysfs.MemoryTypePMEM: TypePMEM, - sysfs.MemoryTypeHBM: TypeHBM, + kindToType = map[hardware.MemoryKind]Type{ + hardware.MemoryKindDRAM: TypeDRAM, + hardware.MemoryKindPMEM: TypePMEM, + hardware.MemoryKindHBM: TypeHBM, + // A node the hardware package could not classify counts as DRAM. There + // is no type here for "do not know", and ordinary memory is the reading + // which keeps the node's capacity available to allocations. Spelled out + // rather than left to TypeDRAM being the zero value, so that it reads as + // a decision. + hardware.MemoryKindUnknown: TypeDRAM, } - typeToSys = map[Type]sysfs.MemoryType{ - TypeDRAM: sysfs.MemoryTypeDRAM, - TypePMEM: sysfs.MemoryTypePMEM, - TypeHBM: sysfs.MemoryTypeHBM, + typeToKind = map[Type]hardware.MemoryKind{ + TypeDRAM: hardware.MemoryKindDRAM, + TypePMEM: hardware.MemoryKindPMEM, + TypeHBM: hardware.MemoryKindHBM, } typeToString = map[Type]string{ TypeDRAM: "DRAM", @@ -54,19 +60,19 @@ var ( } ) -// TypeForSysfs returns the memory type for the given sysfs memory type. -func TypeForSysfs(sysType sysfs.MemoryType) Type { - if t, ok := sysToType[sysType]; ok { +// TypeForKind returns the memory type for the given hardware memory kind. +func TypeForKind(kind hardware.MemoryKind) Type { + if t, ok := kindToType[kind]; ok { return t } - panic(fmt.Errorf("unknown sysfs memory type %v", sysType)) + panic(fmt.Errorf("unknown hardware memory kind %v", kind)) } -// Sysfs returns the sysfs memory type for the given memory type. -func (t Type) Sysfs() sysfs.MemoryType { - if sysType, ok := typeToSys[t]; ok { - return sysType +// Kind returns the hardware memory kind for the given memory type. +func (t Type) Kind() hardware.MemoryKind { + if kind, ok := typeToKind[t]; ok { + return kind } panic(fmt.Errorf("unknown libmem memory type %d", t)) @@ -99,7 +105,7 @@ func (t Type) Mask() TypeMask { // IsValid returns true if the memory type is valid/known. func (t Type) IsValid() bool { - _, ok := typeToSys[t] + _, ok := typeToKind[t] return ok } @@ -162,11 +168,12 @@ func NewTypeMask(types ...Type) TypeMask { return m & TypeMaskAll } -// NewTypeMaskForSysfs returns a TypeMask containing th given sysfs memory types. -func NewTypeMaskForSysfs(sysTypes ...sysfs.MemoryType) TypeMask { +// NewTypeMaskForKinds returns a TypeMask containing the given hardware memory +// kinds. +func NewTypeMaskForKinds(kinds ...hardware.MemoryKind) TypeMask { m := TypeMask(0) - for _, st := range sysTypes { - m |= (1 << TypeForSysfs(st)) + for _, kind := range kinds { + m |= (1 << TypeForKind(kind)) } return m & TypeMaskAll } diff --git a/pkg/resmgr/lib/memory/types_test.go b/pkg/resmgr/lib/memory/types_test.go index 2065eb97e..4cca61c32 100644 --- a/pkg/resmgr/lib/memory/types_test.go +++ b/pkg/resmgr/lib/memory/types_test.go @@ -15,7 +15,7 @@ package libmem_test import ( - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" . "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" "testing" @@ -26,33 +26,32 @@ import ( func TestTypes(t *testing.T) { type testCase struct { name string - sysType sysfs.MemoryType + kind hardware.MemoryKind memType Type } for _, tc := range []*testCase{ { name: "DRAM", - sysType: sysfs.MemoryTypeDRAM, + kind: hardware.MemoryKindDRAM, memType: TypeDRAM, }, { name: "PMEM", - sysType: sysfs.MemoryTypePMEM, + kind: hardware.MemoryKindPMEM, memType: TypePMEM, }, { name: "HBM", - sysType: sysfs.MemoryTypeHBM, + kind: hardware.MemoryKindHBM, memType: TypeHBM, }, } { - t.Run(tc.name+" TypeForSysfs", func(t *testing.T) { - memType := TypeForSysfs(tc.sysType) - require.Equal(t, tc.memType, memType) + t.Run(tc.name+" TypeForKind", func(t *testing.T) { + require.Equal(t, tc.memType, TypeForKind(tc.kind)) }) - t.Run(tc.name+" Sysfs", func(t *testing.T) { - require.Equal(t, tc.sysType, tc.memType.Sysfs()) + t.Run(tc.name+" Kind", func(t *testing.T) { + require.Equal(t, tc.kind, tc.memType.Kind()) }) t.Run(tc.name+" MustParseType", func(t *testing.T) { require.Equal(t, tc.memType, MustParseType(tc.name)) From 066939876854818ac20b24150623c9c266728ecd Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 00:28:47 +0300 Subject: [PATCH 18/39] cpuallocator: take the topology from a hardware.Machine. NewCPUAllocator took a pkg/sysfs System. It takes an already discovered hardware.Machine now, and the topology cache is built from that and its TopologyIndex, so the package no longer depends on that interface. SST moves here with it. Topology discovery has nothing to do with SST -- it is a property of the running platform rather than of its shape -- and this is the only package which ever asked, so the probing is now done here, once per machine rather than once per package. Clusters come from hardware.LogicalClusters/MergeSingleCoreClusters call, which is equivalent to what the old sysfs interface reported. The public set types are unchanged. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 2 +- .../policy/topology-aware-policy.go | 2 +- pkg/cpuallocator/allocator.go | 260 ++++++++++-------- pkg/cpuallocator/cpuallocator_test.go | 111 ++++---- pkg/cpuallocator/sst.go | 115 ++++++++ pkg/cpuallocator/topology.go | 54 ++++ 6 files changed, 367 insertions(+), 177 deletions(-) create mode 100644 pkg/cpuallocator/sst.go create mode 100644 pkg/cpuallocator/topology.go diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 3fd26a692..ea9dd5526 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -221,7 +221,7 @@ func (p *balloons) Setup(policyOptions *policy.BackendOptions) error { p.options = policyOptions p.machine = policyOptions.Machine p.cch = policyOptions.Cache - p.cpuAllocator = cpuallocator.NewCPUAllocator(policyOptions.System) + p.cpuAllocator = cpuallocator.NewCPUAllocator(policyOptions.Machine) log.Infof("setting up %s policy...", PolicyName) p.cpuTree = NewCpuTreeFromMachine(policyOptions.Machine) diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index a5b3ab363..c89b617cd 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -164,7 +164,7 @@ func (p *policy) Setup(opts *policyapi.BackendOptions) error { p.cache = opts.Cache p.sys = opts.System p.options = opts - p.cpuAllocator = cpuallocator.NewCPUAllocator(opts.System) + p.cpuAllocator = cpuallocator.NewCPUAllocator(opts.Machine) p.memAllocator, err = libmem.NewAllocator(libmem.WithMachineNodes(opts.Machine)) if err != nil { return policyError("failed to initialize %s policy: %w", err) diff --git a/pkg/cpuallocator/allocator.go b/pkg/cpuallocator/allocator.go index ae430fdf7..4ffc2c456 100644 --- a/pkg/cpuallocator/allocator.go +++ b/pkg/cpuallocator/allocator.go @@ -21,7 +21,7 @@ import ( "github.com/containers/nri-plugins/pkg/utils/cpuset" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/utils" "github.com/intel/goresctrl/pkg/sst" @@ -49,14 +49,14 @@ const ( // allocatorHelper encapsulates state for allocating CPUs. type allocatorHelper struct { - logger.Logger // allocatorHelper logger instance - sys sysfs.System // sysfs CPU and topology information - topology topologyCache // cached topology information - flags AllocFlag // allocation preferences - from cpuset.CPUSet // set of CPUs to allocate from - prefer CPUPriority // CPU priority to prefer - cnt int // number of CPUs to allocate - result cpuset.CPUSet // set of CPUs allocated + logger.Logger // allocatorHelper logger instance + machine *hardware.Machine // CPU and memory topology + topology topologyCache // cached topology information + flags AllocFlag // allocation preferences + from cpuset.CPUSet // set of CPUs to allocate from + prefer CPUPriority // CPU priority to prefer + cnt int // number of CPUs to allocate + result cpuset.CPUSet // set of CPUs allocated } // CPUAllocator is an interface for a generic CPU allocator @@ -101,8 +101,8 @@ func WithAllocFlags(flags AllocFlag) Option { type cpuAllocator struct { logger.Logger - sys sysfs.System // wrapped sysfs.System instance - topologyCache topologyCache // topology lookups + machine *hardware.Machine // CPU and memory topology + topologyCache topologyCache // topology lookups } // topologyCache caches topology lookups @@ -110,7 +110,7 @@ type topologyCache struct { pkg map[idset.ID]cpuset.CPUSet node map[idset.ID]cpuset.CPUSet core map[idset.ID]cpuset.CPUSet - kind map[sysfs.CoreKind]cpuset.CPUSet + kind map[hardware.CoreKind]cpuset.CPUSet cpuPriorities cpuPriorities // CPU priority mapping clusters []*cpuCluster // CPU clusters @@ -124,7 +124,7 @@ type cpuCluster struct { die idset.ID cluster idset.ID cpus cpuset.CPUSet - kind sysfs.CoreKind + kind hardware.CoreKind } type cacheGroup struct { @@ -133,7 +133,7 @@ type cacheGroup struct { die idset.ID node idset.ID cpus cpuset.CPUSet - kind sysfs.CoreKind + kind hardware.CoreKind } // IDFilter helps filtering Ids. @@ -146,11 +146,11 @@ type IDSorter func(int, int) bool var log = logger.NewLogger(logSource) // NewCPUAllocator return a new cpuAllocator instance -func NewCPUAllocator(sys sysfs.System) CPUAllocator { +func NewCPUAllocator(m *hardware.Machine) CPUAllocator { ca := cpuAllocator{ Logger: log, - sys: sys, - topologyCache: newTopologyCache(sys), + machine: m, + topologyCache: newTopologyCache(m), } return &ca @@ -172,10 +172,10 @@ func pickIds(idSlice []idset.ID, f IDFilter) []idset.ID { } // newAllocatorHelper creates a new CPU allocatorHelper. -func newAllocatorHelper(sys sysfs.System, topo topologyCache) *allocatorHelper { +func newAllocatorHelper(m *hardware.Machine, topo topologyCache) *allocatorHelper { a := &allocatorHelper{ Logger: log, - sys: sys, + machine: m, topology: topo, flags: AllocDefault, } @@ -187,10 +187,10 @@ func newAllocatorHelper(sys sysfs.System, topo topologyCache) *allocatorHelper { func (a *allocatorHelper) takeIdlePackages() { a.Debugf("* takeIdlePackages()...") - offline := a.sys.Offlined() + offline := toCpuSet(a.machine.OfflineCPUs()) // pick idle packages - pkgs := pickIds(a.sys.PackageIDs(), + pkgs := pickIds(a.machine.TopologyIndex().PackageIDs(), func(id idset.ID) bool { // Consider a package idle if all online preferred CPUs are idle. // In particular, on hybrid core architectures exclude @@ -241,16 +241,16 @@ var ( // Allocate full idle CPU clusters. func (a *allocatorHelper) takeIdleClusters() { var ( - offline = a.sys.OfflineCPUs() + offline = toCpuSet(a.machine.OfflineCPUs()) pickIdle = func(c *cpuCluster) (bool, cpuset.CPUSet) { if len(a.topology.kind) > 1 { // we only take E-clusters for low-prio requests - if a.prefer != PriorityLow && c.kind == sysfs.EfficientCore { + if a.prefer != PriorityLow && c.kind == hardware.EfficientCore { a.Debugf(" - omit %s, CPU preference is %s", c, a.prefer) return false, emptyCPUSet } // we only take P-clusters for other than low-prio requests - if a.prefer == PriorityLow && c.kind == sysfs.PerformanceCore { + if a.prefer == PriorityLow && c.kind == hardware.PerformanceCore { a.Debugf(" - omit %s, CPU preference is %s", c, a.prefer) return false, emptyCPUSet } @@ -467,16 +467,16 @@ func (a *allocatorHelper) takeCacheGroups() { // o fragment fewest groups possible (take from small to large, preserve large groups) var ( - offline = a.sys.OfflineCPUs() + offline = toCpuSet(a.machine.OfflineCPUs()) pickGroups = func(g *cacheGroup) (pickVerdict, cpuset.CPUSet) { if len(a.topology.kind) > 1 { // only take E-groups for low-prio requests, or if we have none other - if a.prefer != PriorityLow && g.kind == sysfs.EfficientCore { + if a.prefer != PriorityLow && g.kind == hardware.EfficientCore { log.Debugf(" - ignore %s (CPU preference is %s)", g, a.prefer) return pickIgnore, emptyCPUSet } // only take P-groups for other than low-prio requests, or if we have none other - if a.prefer == PriorityLow && g.kind == sysfs.PerformanceCore { + if a.prefer == PriorityLow && g.kind == hardware.PerformanceCore { log.Debugf(" - ignore %s (CPU preference is %s)", g, a.prefer) return pickIgnore, emptyCPUSet } @@ -760,7 +760,7 @@ func (a *allocatorHelper) takeCacheGroups() { } // partially allocate the rest from this group - ta := newAllocatorHelper(a.sys, a.topology) + ta := newAllocatorHelper(a.machine, a.topology) ta.prefer = a.prefer ta.flags = AllocIdleCores ta.from = cset @@ -916,7 +916,7 @@ func (a *allocatorHelper) takeCacheGroups() { g := sorter.usable[grpCnt-1] cset := sorter.cpus[g] - ta := newAllocatorHelper(a.sys, a.topology) + ta := newAllocatorHelper(a.machine, a.topology) ta.prefer = a.prefer ta.flags = AllocIdleCores ta.from = cset @@ -945,10 +945,10 @@ func (a *allocatorHelper) takeCacheGroups() { func (a *allocatorHelper) takeIdleCores() { a.Debugf("* takeIdleCores()...") - offline := a.sys.Offlined() + offline := toCpuSet(a.machine.OfflineCPUs()) // pick (first id for all) idle cores - cores := pickIds(a.sys.CPUIDs(), + cores := pickIds(a.machine.CPUIDs(), func(id idset.ID) bool { cset := a.topology.core[id].Difference(offline) if cset.IsEmpty() { @@ -987,10 +987,10 @@ func (a *allocatorHelper) takeIdleCores() { // Allocate idle CPU hyperthreads. func (a *allocatorHelper) takeIdleThreads() { - offline := a.sys.Offlined() + offline := toCpuSet(a.machine.OfflineCPUs()) // pick all threads with free capacity - cores := pickIds(a.sys.CPUIDs(), + cores := pickIds(a.machine.CPUIDs(), func(id idset.ID) bool { return a.from.Difference(offline).Contains(int(id)) }) @@ -1011,8 +1011,8 @@ func (a *allocatorHelper) takeIdleThreads() { func(i, j int) bool { iCore := cores[i] jCore := cores[j] - iPkg := a.sys.CPU(iCore).PackageID() - jPkg := a.sys.CPU(jCore).PackageID() + iPkg := a.machine.CPU(iCore).PackageID() + jPkg := a.machine.CPU(jCore).PackageID() iCoreSet := a.topology.core[iCore] jCoreSet := a.topology.core[jCore] @@ -1087,7 +1087,7 @@ func (a *allocatorHelper) takeAny() { // Perform CPU allocation. func (a *allocatorHelper) allocate() cpuset.CPUSet { - if a.sys != nil { + if a.machine != nil { if (a.flags & AllocIdlePackages) != 0 { a.takeIdlePackages() } @@ -1360,7 +1360,7 @@ func (ca *cpuAllocator) allocateCpus(from *cpuset.CPUSet, cnt int, options ...Op case from.Size() == cnt: result, err, *from = from.Clone(), nil, cpuset.New() default: - a := newAllocatorHelper(ca.sys, ca.topologyCache) + a := newAllocatorHelper(ca.machine, ca.topologyCache) for _, o := range options { if err := o(a); err != nil { return cpuset.New(), err @@ -1404,51 +1404,59 @@ func (ca *cpuAllocator) GetCPUPriorities() map[CPUPriority]cpuset.CPUSet { return prios } -func newTopologyCache(sys sysfs.System) topologyCache { +func newTopologyCache(m *hardware.Machine) topologyCache { c := topologyCache{ pkg: make(map[idset.ID]cpuset.CPUSet), node: make(map[idset.ID]cpuset.CPUSet), core: make(map[idset.ID]cpuset.CPUSet), } - if sys != nil { - for _, id := range sys.PackageIDs() { - c.pkg[id] = sys.Package(id).CPUSet() + if m != nil { + x := m.TopologyIndex() + for _, id := range x.PackageIDs() { + c.pkg[id] = toCpuSet(x.PackageCPUs(id)) } - for _, id := range sys.NodeIDs() { - c.node[id] = sys.Node(id).CPUSet() + for _, id := range m.MemoryNodeIDs() { + c.node[id] = toCpuSet(m.MemoryNode(id).CPUs()) } - for _, id := range sys.CPUIDs() { - c.core[id] = sys.CPU(id).ThreadCPUSet() + for _, id := range m.CPUIDs() { + c.core[id] = toCpuSet(m.CPU(id).Threads()) } } - c.discoverCPUClusters(sys) - c.discoverCacheGroups(sys) - c.discoverCPUPriorities(sys) + c.discoverCPUClusters(m) + c.discoverCacheGroups(m) + c.discoverCPUPriorities(m) return c } -func (c *topologyCache) discoverCPUPriorities(sys sysfs.System) { - if sys == nil { +func (c *topologyCache) discoverCPUPriorities(m *hardware.Machine) { + if m == nil { return } var prio cpuPriorities + // Probe Speed Select once for the whole machine rather than per package. + pkgIDs := make([]idset.ID, 0, len(c.pkg)) + for id := range c.pkg { + pkgIDs = append(pkgIDs, id) + } + ss := discoverSpeedSelect(pkgIDs) + // Discover on per-package basis for id := range c.pkg { - cpuPriorities, sstActive := c.discoverSstCPUPriority(sys, id) + cpuPriorities, sstActive := c.discoverSstCPUPriority(m, ss, id) if !sstActive { - cpuPriorities = c.discoverCpufreqPriority(sys, id) + cpuPriorities = c.discoverCpufreqPriority(m, id) } - ecores := c.kind[sysfs.EfficientCore] - ocores := sys.OnlineCPUs().Difference(ecores) + ecores := c.kind[hardware.EfficientCore] + ocores := toCpuSet(m.OnlineCPUs()).Difference(ecores) for p, cpus := range cpuPriorities { source := map[bool]string{true: "sst", false: "cpufreq"}[sstActive] - cset := sysfs.CPUSetFromIDSet(idset.NewIDSet(cpus...)) + cset := cpuset.New(cpus...) if p != int(PriorityLow) && ocores.Size() > 0 { cset = cset.Difference(ecores) @@ -1461,13 +1469,12 @@ func (c *topologyCache) discoverCPUPriorities(sys sysfs.System) { c.cpuPriorities = prio } -func (c *topologyCache) discoverSstCPUPriority(sys sysfs.System, pkgID idset.ID) ([NumCPUPriorities][]idset.ID, bool) { +func (c *topologyCache) discoverSstCPUPriority(m *hardware.Machine, ss *speedSelect, pkgID idset.ID) ([NumCPUPriorities][]idset.ID, bool) { var ret [NumCPUPriorities][]idset.ID active := false - pkg := sys.Package(pkgID) - pkgStatus := pkg.SstInfo() + pkgStatus := ss.PackageStatus(pkgID) prios := make(map[idset.ID]CPUPriority, c.pkg[pkgID].Size()) if pkgStatus == nil { @@ -1493,7 +1500,7 @@ func (c *topologyCache) discoverSstCPUPriority(sys sysfs.System, pkgID idset.ID) id := idset.ID(i) p := PriorityLow // First two CLOSes are prioritized by SST - if sys.CPU(id).SstClos() < 2 { + if ss.Clos(id) < 2 { p = PriorityHigh } prios[id] = p @@ -1505,7 +1512,7 @@ func (c *topologyCache) discoverSstCPUPriority(sys sysfs.System, pkgID idset.ID) for _, i := range cpuIDs { id := idset.ID(i) - clos := sys.CPU(id).SstClos() + clos := ss.Clos(id) p := closPrio[clos] if p != PriorityNormal { active = true @@ -1519,7 +1526,7 @@ func (c *topologyCache) discoverSstCPUPriority(sys sysfs.System, pkgID idset.ID) log.Debugf("package #%d punit #%d: using SST-BF based CPU prioritization", pkgID, punitID) currentPerfLevel := punit.PP.CurrentLevel if plInfos == nil { - sstPkg, ok := sys.Sst().Package(pkgID) + sstPkg, ok := ss.Package(pkgID) if !ok { log.Debugf("package #%d punit #%d: unable to get package for SST-PP info, skipping", pkgID, punitID) continue @@ -1599,20 +1606,20 @@ func (c *topologyCache) sstClosPriority(pkgID idset.ID, punitID idset.ID, punit return closPriority } -func (c *topologyCache) discoverCpufreqPriority(sys sysfs.System, pkgID idset.ID) [NumCPUPriorities][]idset.ID { +func (c *topologyCache) discoverCpufreqPriority(m *hardware.Machine, pkgID idset.ID) [NumCPUPriorities][]idset.ID { var prios [NumCPUPriorities][]idset.ID // Group cpus by base frequency, core kind and energy performance profile freqs := map[uint64][]idset.ID{} - epps := map[sysfs.EPP][]idset.ID{} + epps := map[hardware.EPP][]idset.ID{} cpuIDs := c.pkg[pkgID].List() for _, num := range cpuIDs { id := idset.ID(num) - cpu := sys.CPU(id) - bf := cpu.BaseFrequency() + cpu := m.CPU(id) + bf := cpu.Freq().Base freqs[bf] = append(freqs[bf], id) - epp := cpu.EPP() + epp := cpu.Freq().EPP epps[epp] = append(epps[epp], id) } @@ -1627,7 +1634,7 @@ func (c *topologyCache) discoverCpufreqPriority(sys sysfs.System, pkgID idset.ID eppList := []int{} for e := range epps { - if e != sysfs.EPPUnknown { + if e != hardware.EPPUnknown { eppList = append(eppList, int(e)) } } @@ -1636,11 +1643,11 @@ func (c *topologyCache) discoverCpufreqPriority(sys sysfs.System, pkgID idset.ID // Finally, determine priority of each CPU for _, num := range cpuIDs { id := idset.ID(num) - cpu := sys.CPU(id) + cpu := m.CPU(id) p := PriorityNormal if len(freqList) > 1 { - bf := cpu.BaseFrequency() + bf := cpu.Freq().Base // All cpus NOT in the lowest base frequency bin are considered high prio if bf > freqList[0] { @@ -1653,11 +1660,11 @@ func (c *topologyCache) discoverCpufreqPriority(sys sysfs.System, pkgID idset.ID // All E-cores are unconditionally considered low prio. // All cpus NOT in the lowest performance epp are considered high prio. // NOTE: higher EPP value denotes lower performance preference - if cpu.CoreKind() == sysfs.EfficientCore { + if cpu.Kind() == hardware.EfficientCore { p = PriorityLow } else { if len(eppList) > 1 { - epp := cpu.EPP() + epp := cpu.Freq().EPP if int(epp) < eppList[len(eppList)-1] { p = PriorityHigh } else { @@ -1672,25 +1679,31 @@ func (c *topologyCache) discoverCpufreqPriority(sys sysfs.System, pkgID idset.ID return prios } -func (c *topologyCache) discoverCPUClusters(sys sysfs.System) { - if sys == nil { +func (c *topologyCache) discoverCPUClusters(m *hardware.Machine) { + if m == nil { return } - for _, id := range sys.PackageIDs() { - pkg := sys.Package(id) + x := m.TopologyIndex() + for _, id := range x.PackageIDs() { clusters := []*cpuCluster{} - for _, die := range pkg.DieIDs() { - for _, cl := range pkg.LogicalDieClusterIDs(die) { - if cpus := pkg.LogicalDieClusterCPUSet(die, cl); cpus.Size() > 0 { - clusters = append(clusters, &cpuCluster{ - pkg: id, - die: die, - cluster: cl, - cpus: cpus, - kind: sys.CPU(cpus.List()[0]).CoreKind(), - }) + for _, die := range x.DieIDs(id) { + // Merged, not omitted: a machine which reports one cluster per core + // still gets those CPUs grouped, which is what this allocator has + // always seen and what its CPU picks depend on. + for _, cpus := range hardware.LogicalClusters(m, die.Package, die.Die, + hardware.MergeSingleCoreClusters) { + if cpus.Size() == 0 { + continue } + first := m.CPU(cpus.List()[0]) + clusters = append(clusters, &cpuCluster{ + pkg: id, + die: die.Die, + cluster: first.ClusterID(), + cpus: toCpuSet(cpus), + kind: first.Kind(), + }) } } if len(clusters) > 1 { @@ -1703,32 +1716,39 @@ func (c *topologyCache) discoverCPUClusters(sys sysfs.System) { } } - c.kind = map[sysfs.CoreKind]cpuset.CPUSet{} - for _, kind := range sys.CoreKinds() { - c.kind[kind] = sys.CoreKindCPUs(kind) + c.kind = map[hardware.CoreKind]cpuset.CPUSet{} + for _, kind := range m.CoreKinds() { + c.kind[kind] = toCpuSet(m.CoreKindCPUs(kind)) } } -func (c *topologyCache) pickCacheLevelForGrouping(sys sysfs.System) int { - if sys == nil { +func (c *topologyCache) pickCacheLevelForGrouping(m *hardware.Machine) int { + if m == nil { return -1 } - online := sys.OnlineCPUs() + x := m.TopologyIndex() + online := toCpuSet(m.OnlineCPUs()) for _, id := range online.List() { - cpu := sys.CPU(id) - pkg := sys.Package(cpu.PackageID()) - for n := cpu.CacheCount() - 1; n > 0; n-- { - cpus := cpu.GetNthLevelCacheCPUSet(n) + cpu := m.CPU(id) + var ( + pkgCPUs = toCpuSet(x.PackageCPUs(cpu.PackageID())) + dieCPUs = toCpuSet(x.DieCPUs(hardware.DieID{ + Package: cpu.PackageID(), + Die: cpu.DieID(), + })) + ) + for n := len(cpu.Caches()) - 1; n > 0; n-- { + cpus := cacheCPUsAtLevel(m, id, n) switch { case cpus.Size() == 0 || cpus.Size() == 1: continue - case cpus.Equals(cpu.ThreadCPUSet().Intersection(online)): + case cpus.Equals(toCpuSet(cpu.Threads()).Intersection(online)): continue - case cpus.Equals(pkg.DieCPUSet(cpu.DieID()).Intersection(online)): + case cpus.Equals(dieCPUs.Intersection(online)): continue - case cpus.Equals(pkg.CPUSet().Intersection(online)): + case cpus.Equals(pkgCPUs.Intersection(online)): continue } @@ -1739,12 +1759,12 @@ func (c *topologyCache) pickCacheLevelForGrouping(sys sysfs.System) int { return -1 } -func (c *topologyCache) discoverCacheGroups(sys sysfs.System) { - if sys == nil { +func (c *topologyCache) discoverCacheGroups(m *hardware.Machine) { + if m == nil { return } - n := c.pickCacheLevelForGrouping(sys) + n := c.pickCacheLevelForGrouping(m) if n < 0 { log.Infof("no cache level provides useful CPU grouping") return @@ -1752,28 +1772,33 @@ func (c *topologyCache) discoverCacheGroups(sys sysfs.System) { log.Infof("picked cache level %d for CPU grouping", n) - online := sys.OnlineCPUs() - for _, id := range sys.PackageIDs() { - pkg := sys.Package(id) + x := m.TopologyIndex() + online := toCpuSet(m.OnlineCPUs()) + for _, id := range x.PackageIDs() { + pkgCPUs := toCpuSet(x.PackageCPUs(id)) groups := []*cacheGroup{} assigned := idset.NewIDSet() - for _, cpuID := range pkg.CPUSet().Intersection(online).List() { + for _, cpuID := range pkgCPUs.Intersection(online).List() { if assigned.Has(cpuID) { continue } - cpu := sys.CPU(cpuID) - cpus := cpu.GetNthLevelCacheCPUSet(n).Intersection(online) + cpu := m.CPU(cpuID) + cpus := cacheCPUsAtLevel(m, cpuID, n).Intersection(online) + dieCPUs := toCpuSet(x.DieCPUs(hardware.DieID{ + Package: cpu.PackageID(), + Die: cpu.DieID(), + })) switch { case cpus.Size() == 0 || cpus.Size() == 1: continue - case cpus.Equals(cpu.ThreadCPUSet().Intersection(online)): + case cpus.Equals(toCpuSet(cpu.Threads()).Intersection(online)): continue - case cpus.Equals(pkg.DieCPUSet(cpu.DieID()).Intersection(online)): + case cpus.Equals(dieCPUs.Intersection(online)): continue - case cpus.Equals(pkg.CPUSet().Intersection(online)): + case cpus.Equals(pkgCPUs.Intersection(online)): continue } @@ -1782,7 +1807,7 @@ func (c *topologyCache) discoverCacheGroups(sys sysfs.System) { die: cpu.DieID(), node: cpu.NodeID(), cpus: cpus.Clone(), - kind: cpu.CoreKind(), // TODO(klihub): maybe verify all CPUs are the same kind + kind: cpu.Kind(), // TODO(klihub): maybe verify all CPUs are the same kind }) assigned.Add(cpus.UnsortedList()...) } @@ -1815,7 +1840,7 @@ func (c *topologyCache) discoverCacheGroups(sys sysfs.System) { g.id = idx for _, cpuID := range g.cpus.UnsortedList() { - cpu := sys.CPU(cpuID) + cpu := m.CPU(cpuID) if cpu.PackageID() != g.pkg { log.Warnf("CPU #%d in cache group #%d has package #%d != #%d", cpuID, g.id, cpu.PackageID(), g.pkg) @@ -1836,7 +1861,7 @@ func (c *topologyCache) discoverCacheGroups(sys sysfs.System) { } } - cpu := sys.CPU(g.cpus.List()[0]) + cpu := m.CPU(g.cpus.List()[0]) log.Debugf("cache group #%d: pkg #%d/die #%d/node #%d %s cpus %s", g.id, cpu.PackageID(), cpu.DieID(), cpu.NodeID(), g.kind, g.cpus) } @@ -1850,7 +1875,7 @@ func (c *topologyCache) discoverCacheGroups(sys sysfs.System) { for idx, g := range c.cacheGroups { g.id = idx - cpu := sys.CPU(g.cpus.List()[0]) + cpu := m.CPU(g.cpus.List()[0]) log.Debugf("cache group #%d: pkg #%d/die #%d/node #%d %s cpus %s", g.id, cpu.PackageID(), cpu.DieID(), cpu.NodeID(), g.kind, g.cpus) } @@ -1959,12 +1984,11 @@ func (c *cacheGroup) PackageID() int { return c.pkg } -func (c *cacheGroup) DieID(sys sysfs.System) int { - cpu := sys.CPU(c.cpus.List()[0]) - return cpu.DieID() +func (c *cacheGroup) DieID(m *hardware.Machine) int { + return m.CPU(c.cpus.List()[0]).DieID() } -func (c *cacheGroup) SmallestCoreID(sys sysfs.System) int { +func (c *cacheGroup) SmallestCoreID(m *hardware.Machine) int { return c.cpus.List()[0] } diff --git a/pkg/cpuallocator/cpuallocator_test.go b/pkg/cpuallocator/cpuallocator_test.go index 396f94bce..bb7d7f625 100644 --- a/pkg/cpuallocator/cpuallocator_test.go +++ b/pkg/cpuallocator/cpuallocator_test.go @@ -22,7 +22,7 @@ import ( "github.com/containers/nri-plugins/pkg/testutils" "github.com/containers/nri-plugins/pkg/utils/cpuset" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" ) @@ -40,13 +40,12 @@ func TestAllocatorHelper(t *testing.T) { } // Discover mock system from the testdata - sys, err := sysfs.DiscoverSystemAt( - path.Join(tmpdir, "sysfs", "2-socket-4-node-40-core", "sys"), - sysfs.DiscoverCPUTopology, sysfs.DiscoverMemTopology) + m, err := hardware.Discover( + hardware.WithRoot(path.Join(tmpdir, "sysfs", "2-socket-4-node-40-core"))) if err != nil { t.Fatalf("failed to discover mock system: %v", err) } - topoCache := newTopologyCache(sys) + topoCache := newTopologyCache(m) // Fake cpu priorities: 5 cores from pkg #0 as high prio // Package CPUs: #0: [0-19,40-59], #1: [20-39,60-79] @@ -89,7 +88,7 @@ func TestAllocatorHelper(t *testing.T) { // Run tests for _, tc := range tcs { t.Run(tc.description, func(t *testing.T) { - a := newAllocatorHelper(sys, topoCache) + a := newAllocatorHelper(m, topoCache) a.from = tc.from a.prefer = tc.prefer a.cnt = tc.cnt @@ -118,13 +117,12 @@ func TestClusteredAllocation(t *testing.T) { } // Discover mock system from the testdata - sys, err := sysfs.DiscoverSystemAt( - path.Join(tmpdir, "sysfs", "2-socket-4-node-40-core", "sys"), - sysfs.DiscoverCPUTopology, sysfs.DiscoverMemTopology) + m, err := hardware.Discover( + hardware.WithRoot(path.Join(tmpdir, "sysfs", "2-socket-4-node-40-core"))) if err != nil { t.Fatalf("failed to discover mock system: %v", err) } - topoCache := newTopologyCache(sys) + topoCache := newTopologyCache(m) // Fake cpu priorities: 5 cores from pkg #0 as high prio // Package CPUs: #0: [0-19,40-59], #1: [20-39,60-79] @@ -306,7 +304,7 @@ func TestClusteredAllocation(t *testing.T) { // Run tests for _, tc := range tcs { t.Run(tc.description, func(t *testing.T) { - a := newAllocatorHelper(sys, topoCache) + a := newAllocatorHelper(m, topoCache) a.from = tc.from a.cnt = tc.cnt result := a.allocate() @@ -334,9 +332,8 @@ func TestClusteredCoreKindAllocation(t *testing.T) { } // Discover mock system from the testdata - sys, err := sysfs.DiscoverSystemAt( - path.Join(tmpdir, "sysfs", "2-socket-4-node-40-core", "sys"), - sysfs.DiscoverCPUTopology, sysfs.DiscoverMemTopology) + m, err := hardware.Discover( + hardware.WithRoot(path.Join(tmpdir, "sysfs", "2-socket-4-node-40-core"))) if err != nil { t.Fatalf("failed to discover mock system: %v", err) } @@ -347,70 +344,70 @@ func TestClusteredCoreKindAllocation(t *testing.T) { die: 0, cluster: 0, cpus: cpuset.MustParse("0-3"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 1, cpus: cpuset.MustParse("4-7"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 2, cpus: cpuset.MustParse("8-11"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 3, cpus: cpuset.MustParse("12-15"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 4, cpus: cpuset.MustParse("16-19"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, { pkg: 0, die: 0, cluster: 5, cpus: cpuset.MustParse("40-43"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 6, cpus: cpuset.MustParse("44-47"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 7, cpus: cpuset.MustParse("48-51"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 8, cpus: cpuset.MustParse("52-55"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 9, cpus: cpuset.MustParse("56-59"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, { @@ -418,70 +415,70 @@ func TestClusteredCoreKindAllocation(t *testing.T) { die: 0, cluster: 0, cpus: cpuset.MustParse("20,22,24,26"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 1, cpus: cpuset.MustParse("21,23,25,27"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 2, cpus: cpuset.MustParse("28-31"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 3, cpus: cpuset.MustParse("32-35"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 4, cpus: cpuset.MustParse("36-39"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, { pkg: 1, die: 0, cluster: 5, cpus: cpuset.MustParse("60-63"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 6, cpus: cpuset.MustParse("64-67"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 7, cpus: cpuset.MustParse("68-71"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 8, cpus: cpuset.MustParse("72-75"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 9, cpus: cpuset.MustParse("76-79"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, } @@ -491,70 +488,70 @@ func TestClusteredCoreKindAllocation(t *testing.T) { die: 0, cluster: 0, cpus: cpuset.MustParse("0-3"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 1, cpus: cpuset.MustParse("4-7"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 2, cpus: cpuset.MustParse("8-11"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 3, cpus: cpuset.MustParse("12-15"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 4, cpus: cpuset.MustParse("16-19"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, { pkg: 0, die: 0, cluster: 5, cpus: cpuset.MustParse("40-43"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 6, cpus: cpuset.MustParse("44-47"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 7, cpus: cpuset.MustParse("48-51"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 8, cpus: cpuset.MustParse("52-55"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 9, cpus: cpuset.MustParse("56-59"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, { @@ -562,77 +559,77 @@ func TestClusteredCoreKindAllocation(t *testing.T) { die: 0, cluster: 0, cpus: cpuset.MustParse("20,22,24,26"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 1, cpus: cpuset.MustParse("21,23,25,27"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 2, cpus: cpuset.MustParse("28-31"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 3, cpus: cpuset.MustParse("32-35"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 4, cpus: cpuset.MustParse("36-37"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, { pkg: 1, die: 0, cluster: 5, cpus: cpuset.MustParse("38-39"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 6, cpus: cpuset.MustParse("60-63"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 7, cpus: cpuset.MustParse("64-67"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 8, cpus: cpuset.MustParse("68-71"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 9, cpus: cpuset.MustParse("72-75"), - kind: sysfs.PerformanceCore, + kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 10, cpus: cpuset.MustParse("76-79"), - kind: sysfs.EfficientCore, + kind: hardware.EfficientCore, }, } @@ -733,9 +730,9 @@ func TestClusteredCoreKindAllocation(t *testing.T) { // Run tests for _, tc := range tcs { t.Run(tc.description, func(t *testing.T) { - topoCache := newTopologyCache(sys) + topoCache := newTopologyCache(m) topoCache.clusters = tc.clusters - a := newAllocatorHelper(sys, topoCache) + a := newAllocatorHelper(m, topoCache) a.from = tc.from a.prefer = tc.prefer a.cnt = tc.cnt diff --git a/pkg/cpuallocator/sst.go b/pkg/cpuallocator/sst.go new file mode 100644 index 000000000..6b6cd0b03 --- /dev/null +++ b/pkg/cpuallocator/sst.go @@ -0,0 +1,115 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cpuallocator + +import ( + "github.com/intel/goresctrl/pkg/sst" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// speedSelect is what Intel Speed Select Technology says about the machine. +// +// This lives here because this package is the only one which asks. Topology +// discovery deliberately knows nothing about SST -- it is a feature of the +// running platform rather than a property of its shape -- so the probing sits +// next to the CPU prioritization which is the only thing that wants it. +// +// A nil *speedSelect means SST told us nothing, either because the platform does +// not support it or because probing failed. Every method tolerates that, so a +// caller does not have to check first. +type speedSelect struct { + platform *sst.Platform + status map[idset.ID]*sst.PackageStatus + clos map[idset.ID]int +} + +// discoverSpeedSelect probes SST for the given packages. It returns nil when +// there is nothing to report; failing to probe is not an error, it just means +// prioritization has to be decided some other way. +func discoverSpeedSelect(pkgIDs []idset.ID) *speedSelect { + if !sst.SstSupported() { + return nil + } + + platform, err := sst.Init() + if err != nil || platform == nil { + log.Debugf("SST is supported but could not be initialized: %v", err) + return nil + } + + s := &speedSelect{ + platform: platform, + status: map[idset.ID]*sst.PackageStatus{}, + clos: map[idset.ID]int{}, + } + + for _, id := range pkgIDs { + pkg, ok := platform.Package(id) + if !ok { + continue + } + status, err := pkg.GetStatus() + if err != nil { + log.Debugf("package #%d: no SST status: %v", id, err) + continue + } + + for _, punit := range status.Punits { + if !punit.CP.Supported || !punit.CP.Enabled { + continue + } + for _, cpu := range punit.CPUs.SortedMembers() { + clos, err := platform.GetCPUClosID(cpu) + if err != nil { + continue + } + s.clos[cpu] = clos + } + } + + s.status[id] = status + } + + return s +} + +// PackageStatus returns what SST says about a package, or nil if it says nothing. +func (s *speedSelect) PackageStatus(pkgID idset.ID) *sst.PackageStatus { + if s == nil { + return nil + } + return s.status[pkgID] +} + +// Clos returns the SST-CP class of service a CPU is in, or -1 when SST +// prioritization is not in effect for it. +func (s *speedSelect) Clos(cpu idset.ID) int { + if s == nil { + return -1 + } + if clos, ok := s.clos[cpu]; ok { + return clos + } + return -1 +} + +// Package returns the SST view of a package, for the perf level information +// which only it can answer. +func (s *speedSelect) Package(pkgID idset.ID) (*sst.Package, bool) { + if s == nil || s.platform == nil { + return nil, false + } + return s.platform.Package(pkgID) +} diff --git a/pkg/cpuallocator/topology.go b/pkg/cpuallocator/topology.go new file mode 100644 index 000000000..eb9cfa7da --- /dev/null +++ b/pkg/cpuallocator/topology.go @@ -0,0 +1,54 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package cpuallocator + +import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// toCpuSet converts a set the hardware package returns to the one this package +// keeps its topology in. It goes away if this package ever switches to libcpu +// sets throughout. +func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { + return cpuset.New(cpus.List()...) +} + +// cacheCPUsAtLevel returns the CPUs sharing any of a CPU's caches at one level, +// or its thread siblings when it has no caches at all. +// +// Note the union: a level with a separate data and instruction cache contributes +// both, which is why this does not just take hardware.CPU.Cache. +func cacheCPUsAtLevel(m *hardware.Machine, id idset.ID, level int) cpuset.CPUSet { + c := m.CPU(id) + + caches := c.Caches() + if len(caches) == 0 { + return toCpuSet(c.Threads()) + } + + cpus := cpuset.New() + for _, cache := range caches { + if cache.Level() == level { + cpus = cpus.Union(toCpuSet(cache.CPUs())) + } else if cache.Level() > level { + break + } + } + + return cpus +} From 1b8adf0dc0c7f2174ba12961fe0b610edc5b7e3f Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 10:10:37 +0300 Subject: [PATCH 19/39] utils/parse: move file entry parsing out of sysfs. ParseFileEntries has nothing to do with CPU or memory topology. It parses a file of key and value lines, and lived in pkg/sysfs only because that is where it was written. Move ParseFileEntries to a pkg/utils/parse of its own, beside pkg/utils/cpuset and pkg/utils/topology, so that any further parsing helpers have somewhere to land without crowding the names in pkg/utils. It loses the prefix its package now carries and is parse.FileEntries. Its tests move with it. Fix an error with wrong formatting verbs vs. arguments. Also, remove the unused ParseEnabled in pkg/utils/parse.go instead of moving it here. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/cgroups/cgroupstats.go | 4 +- pkg/lib/hardware/system/types.go | 12 +- pkg/sysfs/parsers.go | 151 +-------------- pkg/utils/parse.go | 32 ---- pkg/utils/parse/fileentries.go | 177 ++++++++++++++++++ .../parse/fileentries_test.go} | 23 ++- 6 files changed, 207 insertions(+), 192 deletions(-) delete mode 100644 pkg/utils/parse.go create mode 100644 pkg/utils/parse/fileentries.go rename pkg/{sysfs/parsers_test.go => utils/parse/fileentries_test.go} (93%) diff --git a/pkg/cgroups/cgroupstats.go b/pkg/cgroups/cgroupstats.go index 884517d78..4ea1a0cd3 100644 --- a/pkg/cgroups/cgroupstats.go +++ b/pkg/cgroups/cgroupstats.go @@ -22,7 +22,7 @@ import ( "strconv" "strings" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/utils/parse" ) // BlkioDeviceBytes contains a single operations line of blkio.throttle.io_service_bytes_recursive file @@ -476,7 +476,7 @@ func GetGlobalNumaStats() (map[int64]GlobalNumaStats, error) { nodeStat := GlobalNumaStats{} numastat := path.Join(dir, "numastat") - err = sysfs.ParseFileEntries(numastat, + err = parse.FileEntries(numastat, map[string]any{ "numa_hit": &nodeStat.NumaHit, "numa_miss": &nodeStat.NumaMiss, diff --git a/pkg/lib/hardware/system/types.go b/pkg/lib/hardware/system/types.go index e5680c4c4..47b174fae 100644 --- a/pkg/lib/hardware/system/types.go +++ b/pkg/lib/hardware/system/types.go @@ -19,6 +19,7 @@ import ( "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/utils/cpuset" + "github.com/containers/nri-plugins/pkg/utils/parse" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -263,18 +264,17 @@ func NodeFilterNot(f NodeFilter) NodeFilter { // // Utilities // -// These have nothing to do with topology, they just live in pkg/sysfs. They are -// repeated here so that a consumer's import swap is complete, and they should -// end up somewhere under pkg/utils rather than following the topology into -// hardware. +// These have nothing to do with topology. They are repeated here only so that a +// consumer's import swap is complete; a consumer which wants them and not the +// topology should take them from where they live instead. // // PickEntryFn picks a given input line apart into an entry of key and value. -type PickEntryFn func(string) (string, string, error) +type PickEntryFn = parse.PickEntryFn // ParseFileEntries parses a sysfs files for the given entries. func ParseFileEntries(path string, values map[string]any, pickFn PickEntryFn) error { - return sysfs.ParseFileEntries(path, values, sysfs.PickEntryFn(pickFn)) + return parse.FileEntries(path, values, pickFn) } // IDSetFromCPUSet returns an id set corresponding to a cpuset.CPUSet. diff --git a/pkg/sysfs/parsers.go b/pkg/sysfs/parsers.go index dbc4815e0..cb65b40d2 100644 --- a/pkg/sysfs/parsers.go +++ b/pkg/sysfs/parsers.go @@ -1,4 +1,4 @@ -// Copyright 2019 Intel Corporation. All Rights Reserved. +// Copyright 2020 Intel Corporation. All Rights Reserved. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -15,154 +15,17 @@ package sysfs import ( - "os" - "strconv" - "strings" + "github.com/containers/nri-plugins/pkg/utils/parse" ) -// unit multipliers -const ( - k = (int64(1) << 10) - M = (int64(1) << 20) - G = (int64(1) << 30) - T = (int64(1) << 40) -) - -// unit name to multiplier mapping -var units = map[string]int64{ - "k": k, "kB": k, - "M": M, "MB": M, - "G": G, "GB": G, - "T": T, "TB": T, -} +// ParseFileEntries and the key/value file parsing behind it have nothing to do +// with topology; they live in pkg/utils now. These are kept so that this +// package's interface is unchanged for as long as it is still here. // PickEntryFn picks a given input line apart into an entry of key and value. -type PickEntryFn func(string) (string, string, error) - -// splitNumericAndUnit splits a string into a numeric and a unit part. -func splitNumericAndUnit(path string, value string) (string, int64, error) { - fields := strings.Fields(value) - - switch len(fields) { - case 1: - return fields[0], 1, nil - case 2: - num := fields[0] - unit, ok := units[fields[1]] - if !ok { - return "", -1, sysfsError(path, "failed to parse '%s', invalid unit '%s'", - value, num, unit) - } - return num, unit, nil - } - - return "", -1, sysfsError(path, "invalid numeric value %s", value) -} - -// PparseNumberic parses a numeric string into integer of the right size. -func parseNumeric(path, value string, ptr any) error { - var numstr string - var num, unit int64 - var f float64 - var err error - - if numstr, unit, err = splitNumericAndUnit(path, value); err != nil { - return err - } - - switch ptr := ptr.(type) { - case *int: - num, err = strconv.ParseInt(numstr, 0, strconv.IntSize) - *ptr = int(num * unit) - case *int8: - num, err = strconv.ParseInt(numstr, 0, 8) - *ptr = int8(num * unit) - case *int16: - num, err = strconv.ParseInt(numstr, 0, 16) - *ptr = int16(num * unit) - case *int32: - num, err = strconv.ParseInt(numstr, 0, 32) - *ptr = int32(num * unit) - case *int64: - num, err = strconv.ParseInt(numstr, 0, 64) - *ptr = int64(num * unit) - case *uint: - num, err = strconv.ParseInt(numstr, 0, strconv.IntSize) - *ptr = uint(num * unit) - case *uint8: - num, err = strconv.ParseInt(numstr, 0, 8) - *ptr = uint8(num * unit) - case *uint16: - num, err = strconv.ParseInt(numstr, 0, 16) - *ptr = uint16(num * unit) - case *uint32: - num, err = strconv.ParseInt(numstr, 0, 32) - *ptr = uint32(num * unit) - case *uint64: - num, err = strconv.ParseInt(numstr, 0, 64) - *ptr = uint64(num * unit) - case *float32: - f, err = strconv.ParseFloat(numstr, 32) - *ptr = float32(f) * float32(unit) - case *float64: - f, err = strconv.ParseFloat(numstr, 64) - *ptr = f * float64(unit) - - default: - err = sysfsError(path, "can't parse numeric value '%s' into type %T", value, ptr) - } - - return err -} +type PickEntryFn = parse.PickEntryFn // ParseFileEntries parses a sysfs files for the given entries. func ParseFileEntries(path string, values map[string]any, pickFn PickEntryFn) error { - var err error - - data, err := os.ReadFile(path) - if err != nil { - return sysfsError(path, "failed to read file: %v", err) - } - - left := len(values) - for line := range strings.SplitSeq(string(data), "\n") { - key, value, err := pickFn(line) - if err != nil { - return err - } - - ptr, ok := values[key] - if !ok { - continue - } - - switch ptr := ptr.(type) { - case *int, *int8, *int32, *int16, *int64, *uint, *uint8, *uint16, *uint32, *uint64: - if err = parseNumeric(path, value, ptr); err != nil { - return err - } - case *float32, *float64: - if err = parseNumeric(path, value, ptr); err != nil { - return err - } - case *string: - *ptr = value - case *bool: - *ptr, err = strconv.ParseBool(value) - if err != nil { - return sysfsError(path, "failed to parse line %s, value '%s' for boolean key '%s'", - line, value, key) - } - default: - return sysfsError(path, "don't know how to parse key '%s' of type %T", key, ptr) - - } - - left-- - if left == 0 { - break - } - } - - return nil + return parse.FileEntries(path, values, pickFn) } diff --git a/pkg/utils/parse.go b/pkg/utils/parse.go deleted file mode 100644 index f303a7b89..000000000 --- a/pkg/utils/parse.go +++ /dev/null @@ -1,32 +0,0 @@ -// Copyright 2020 Intel Corporation. All Rights Reserved. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package utils - -import ( - "fmt" - "strings" -) - -// ParseEnabled returns whether the given string represents an 'enabled' state. -func ParseEnabled(value string) (bool, error) { - switch strings.ToLower(value) { - case "true", "on", "enable", "enabled", "1": - return true, nil - case "false", "off", "disable", "disabled", "0": - return false, nil - default: - return false, fmt.Errorf("invalid enabled string %q", value) - } -} diff --git a/pkg/utils/parse/fileentries.go b/pkg/utils/parse/fileentries.go new file mode 100644 index 000000000..4da39254a --- /dev/null +++ b/pkg/utils/parse/fileentries.go @@ -0,0 +1,177 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package parse + +import ( + "fmt" + "os" + "strconv" + "strings" +) + +// unit multipliers +const ( + unitK = (int64(1) << 10) + unitM = (int64(1) << 20) + unitG = (int64(1) << 30) + unitT = (int64(1) << 40) +) + +// unit name to multiplier mapping +var units = map[string]int64{ + "k": unitK, "kB": unitK, + "M": unitM, "MB": unitM, + "G": unitG, "GB": unitG, + "T": unitT, "TB": unitT, +} + +// PickEntryFn picks a given input line apart into an entry of key and value. +type PickEntryFn func(string) (string, string, error) + +// fileError prefixes an error with the file which produced it. +func fileError(path, format string, args ...any) error { + return fmt.Errorf(path+": "+format, args...) +} + +// splitNumericAndUnit splits a string into a numeric and a unit part. +func splitNumericAndUnit(path string, value string) (string, int64, error) { + fields := strings.Fields(value) + + switch len(fields) { + case 1: + return fields[0], 1, nil + case 2: + num := fields[0] + unit, ok := units[fields[1]] + if !ok { + return "", -1, fileError(path, "failed to parse '%s', invalid unit '%s'", + value, fields[1]) + } + return num, unit, nil + } + + return "", -1, fileError(path, "invalid numeric value %s", value) +} + +// parseNumeric parses a numeric string into an integer of the right size. +func parseNumeric(path, value string, ptr any) error { + var numstr string + var num, unit int64 + var f float64 + var err error + + if numstr, unit, err = splitNumericAndUnit(path, value); err != nil { + return err + } + + switch ptr := ptr.(type) { + case *int: + num, err = strconv.ParseInt(numstr, 0, strconv.IntSize) + *ptr = int(num * unit) + case *int8: + num, err = strconv.ParseInt(numstr, 0, 8) + *ptr = int8(num * unit) + case *int16: + num, err = strconv.ParseInt(numstr, 0, 16) + *ptr = int16(num * unit) + case *int32: + num, err = strconv.ParseInt(numstr, 0, 32) + *ptr = int32(num * unit) + case *int64: + num, err = strconv.ParseInt(numstr, 0, 64) + *ptr = int64(num * unit) + case *uint: + num, err = strconv.ParseInt(numstr, 0, strconv.IntSize) + *ptr = uint(num * unit) + case *uint8: + num, err = strconv.ParseInt(numstr, 0, 8) + *ptr = uint8(num * unit) + case *uint16: + num, err = strconv.ParseInt(numstr, 0, 16) + *ptr = uint16(num * unit) + case *uint32: + num, err = strconv.ParseInt(numstr, 0, 32) + *ptr = uint32(num * unit) + case *uint64: + num, err = strconv.ParseInt(numstr, 0, 64) + *ptr = uint64(num * unit) + case *float32: + f, err = strconv.ParseFloat(numstr, 32) + *ptr = float32(f) * float32(unit) + case *float64: + f, err = strconv.ParseFloat(numstr, 64) + *ptr = f * float64(unit) + + default: + err = fileError(path, "can't parse numeric value '%s' into type %T", value, ptr) + } + + return err +} + +// FileEntries parses the given entries out of a file of key and value lines, as +// the files under /sys and /proc are. pickFn splits one line into its key and +// value; values maps a key to a pointer to store its value in, which says how +// the value is parsed. Parsing stops once every key has been seen. +func FileEntries(path string, values map[string]any, pickFn PickEntryFn) error { + var err error + + data, err := os.ReadFile(path) + if err != nil { + return fileError(path, "failed to read file: %v", err) + } + + left := len(values) + for line := range strings.SplitSeq(string(data), "\n") { + key, value, err := pickFn(line) + if err != nil { + return err + } + + ptr, ok := values[key] + if !ok { + continue + } + + switch ptr := ptr.(type) { + case *int, *int8, *int32, *int16, *int64, *uint, *uint8, *uint16, *uint32, *uint64: + if err = parseNumeric(path, value, ptr); err != nil { + return err + } + case *float32, *float64: + if err = parseNumeric(path, value, ptr); err != nil { + return err + } + case *string: + *ptr = value + case *bool: + *ptr, err = strconv.ParseBool(value) + if err != nil { + return fileError(path, "failed to parse line %s, value '%s' for boolean key '%s'", + line, value, key) + } + default: + return fileError(path, "don't know how to parse key '%s' of type %T", key, ptr) + + } + + left-- + if left == 0 { + break + } + } + + return nil +} diff --git a/pkg/sysfs/parsers_test.go b/pkg/utils/parse/fileentries_test.go similarity index 93% rename from pkg/sysfs/parsers_test.go rename to pkg/utils/parse/fileentries_test.go index b54a2bbb4..0e2369a5e 100644 --- a/pkg/sysfs/parsers_test.go +++ b/pkg/utils/parse/fileentries_test.go @@ -12,14 +12,14 @@ // See the License for the specific language governing permissions and // limitations under the License. -package sysfs_test +package parse_test import ( "os" "strings" "testing" - "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils/parse" ) // colonPickFn splits lines of the form "key: value" into key and value. @@ -49,7 +49,7 @@ func writeTempFile(t *testing.T, content string) string { func TestParseFileEntries_Int(t *testing.T) { path := writeTempFile(t, "count: 42\n") var count int - if err := sysfs.ParseFileEntries(path, map[string]any{"count": &count}, colonPickFn); err != nil { + if err := parse.FileEntries(path, map[string]any{"count": &count}, colonPickFn); err != nil { t.Fatalf("unexpected error: %v", err) } if count != 42 { @@ -60,7 +60,7 @@ func TestParseFileEntries_Int(t *testing.T) { func TestParseFileEntries_IntWithUnit(t *testing.T) { path := writeTempFile(t, "size: 4 kB\n") var size int64 - if err := sysfs.ParseFileEntries(path, map[string]any{"size": &size}, colonPickFn); err != nil { + if err := parse.FileEntries(path, map[string]any{"size": &size}, colonPickFn); err != nil { t.Fatalf("unexpected error: %v", err) } if want := int64(4 * 1024); size != want { @@ -72,7 +72,7 @@ func TestParseFileEntries_StringAndBool(t *testing.T) { path := writeTempFile(t, "name: hello\nenabled: true\n") var name string var enabled bool - err := sysfs.ParseFileEntries(path, map[string]any{ + err := parse.FileEntries(path, map[string]any{ "name": &name, "enabled": &enabled, }, colonPickFn) @@ -89,7 +89,7 @@ func TestParseFileEntries_StringAndBool(t *testing.T) { func TestParseFileEntries_MissingFile(t *testing.T) { var count int - err := sysfs.ParseFileEntries("/nonexistent/path/file", map[string]any{"count": &count}, colonPickFn) + err := parse.FileEntries("/nonexistent/path/file", map[string]any{"count": &count}, colonPickFn) if err == nil { t.Errorf("expected an error for missing file, got nil") } @@ -99,7 +99,7 @@ func TestParseFileEntries_MissingFile(t *testing.T) { func parseVal(t *testing.T, fieldContent string, dest any) error { t.Helper() path := writeTempFile(t, "v: "+fieldContent+"\n") - return sysfs.ParseFileEntries(path, map[string]any{"v": dest}, colonPickFn) + return parse.FileEntries(path, map[string]any{"v": dest}, colonPickFn) } // TestParseNumericUnits_Positive tests every recognized unit string against every @@ -338,8 +338,15 @@ func TestParseNumericUnits_Negative(t *testing.T) { for _, unit := range []string{"KB", "kb", "Kb", "mb", "gb", "tb", "xyz", "MiB", "GiB"} { t.Run(unit, func(t *testing.T) { var v int64 - if err := parseVal(t, "1 "+unit, &v); err == nil { + err := parseVal(t, "1 "+unit, &v) + if err == nil { t.Errorf("expected error for unit %q, got nil", unit) + return + } + // and it has to name the unit it did not recognize, rather than + // the number in front of it + if !strings.Contains(err.Error(), "'"+unit+"'") { + t.Errorf("error %q does not name the unit %q", err, unit) } }) } From 768df5f92f9847b25e40354a7cd59fcb31dfb77f Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 10:11:48 +0300 Subject: [PATCH 20/39] kubernetes,utils: sort out the memory capacity accessors. GetMemoryCapacity reads MemTotal out of /proc/meminfo and has nothing to do with CPU or memory topology either. Move it from pkg/sysfs to pkg/utils. Not to pkg/utils/parse with the file entry parsing: it reads one fixed file rather than parsing what it is handed, so a parse.MemoryCapacity would promise the wrong thing. pkg/kubernetes takes it from there now, which is all it wanted from the topology packages. pkg/sysfs and the drop-in keep it, forwarding, as they do the parsing. With this the drop-in no longer imports pkg/sysfs for anything, which is what has to be true before pkg/sysfs can go. pkg/kubernetes has accessors of its own for the capacity the OOM adjustment estimates are calculated against, and neither needs to be exported. SetMemoryCapacity said why it was: so that the estimator tests could vary the capacity, which they could only do from outside because they were in kubernetes_test rather than in the package. Move the test into the package, as cpuset_test.go already is, and the setter can be unexported. Its GetMemoryCapacity goes altogether, never having been called by anything. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/kubernetes/resources.go | 15 +++----- pkg/kubernetes/resources_test.go | 6 ++-- pkg/lib/hardware/system/types.go | 10 +++--- pkg/sysfs/utils.go | 37 +++----------------- pkg/utils/meminfo.go | 59 ++++++++++++++++++++++++++++++++ 5 files changed, 76 insertions(+), 51 deletions(-) create mode 100644 pkg/utils/meminfo.go diff --git a/pkg/kubernetes/resources.go b/pkg/kubernetes/resources.go index 868eaecd7..c9a4e0738 100644 --- a/pkg/kubernetes/resources.go +++ b/pkg/kubernetes/resources.go @@ -20,7 +20,7 @@ import ( corev1 "k8s.io/api/core/v1" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/utils" ) const ( @@ -196,10 +196,9 @@ func CalculateOomAdjToMemReqEstimates() map[int64]int64 { return adjToReq } -// Set memory capacity for OOM adjustment to memory request estimation. -// Exported to allow testing the estimator code with different memory -// capacities. -func SetMemoryCapacity(capacity int64) { +// setMemoryCapacity sets the memory capacity the OOM adjustment to memory +// request estimates are calculated against, and recalculates them. +func setMemoryCapacity(capacity int64) { if capacity == 0 { panic(fmt.Errorf("failed to set memory capacity, invalid capacity 0")) } @@ -208,10 +207,6 @@ func SetMemoryCapacity(capacity int64) { oomAdjToMemReqEstimates = CalculateOomAdjToMemReqEstimates() } -func GetMemoryCapacity() int64 { - return memCapacity -} - func init() { - SetMemoryCapacity(sysfs.GetMemoryCapacity()) + setMemoryCapacity(utils.GetMemoryCapacity()) } diff --git a/pkg/kubernetes/resources_test.go b/pkg/kubernetes/resources_test.go index 7b715e4a5..87f633fbd 100644 --- a/pkg/kubernetes/resources_test.go +++ b/pkg/kubernetes/resources_test.go @@ -12,14 +12,12 @@ // See the License for the specific language governing permissions and // limitations under the License. -package kubernetes_test +package kubernetes import ( "testing" "github.com/stretchr/testify/require" - - . "github.com/containers/nri-plugins/pkg/kubernetes" ) func TestCalculateOomAdjToMemReqEstimates(t *testing.T) { @@ -30,7 +28,7 @@ func TestCalculateOomAdjToMemReqEstimates(t *testing.T) { ) for capacity := int64(4 * G); capacity <= (1024+512)*G; capacity += 4 * G { - SetMemoryCapacity(capacity) + setMemoryCapacity(capacity) for adj := int64(MinBurstableOOMScoreAdj); adj <= MaxBurstableOOMScoreAdj; adj++ { req := OomAdjToMemReq(adj, capacity) require.NotNil(t, req, diff --git a/pkg/lib/hardware/system/types.go b/pkg/lib/hardware/system/types.go index 47b174fae..9e22ee0da 100644 --- a/pkg/lib/hardware/system/types.go +++ b/pkg/lib/hardware/system/types.go @@ -17,7 +17,7 @@ package system import ( "fmt" - "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/utils" "github.com/containers/nri-plugins/pkg/utils/cpuset" "github.com/containers/nri-plugins/pkg/utils/parse" idset "github.com/intel/goresctrl/pkg/utils" @@ -264,9 +264,9 @@ func NodeFilterNot(f NodeFilter) NodeFilter { // // Utilities // -// These have nothing to do with topology. They are repeated here only so that a -// consumer's import swap is complete; a consumer which wants them and not the -// topology should take them from where they live instead. +// These have nothing to do with topology. They live in pkg/utils, and are +// repeated here only so that a consumer's import swap is complete; a consumer +// which wants them and not the topology should take them from there directly. // // PickEntryFn picks a given input line apart into an entry of key and value. @@ -290,5 +290,5 @@ func CPUSetFromIDSet(s idset.IDSet) cpuset.CPUSet { // GetMemoryCapacity parses memory capacity from /proc/meminfo (mimicking // cAdvisor). func GetMemoryCapacity() int64 { - return sysfs.GetMemoryCapacity() + return utils.GetMemoryCapacity() } diff --git a/pkg/sysfs/utils.go b/pkg/sysfs/utils.go index 3fb6a23b6..28ed3a0cd 100644 --- a/pkg/sysfs/utils.go +++ b/pkg/sysfs/utils.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "github.com/containers/nri-plugins/pkg/utils" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -261,37 +262,9 @@ func CPUSetFromIDSet(s idset.IDSet) cpuset.CPUSet { return cpuset.New(s.Members()...) } -// GetMemoryCapacity parses memory capacity from /proc/meminfo (mimicking cAdvisor). +// GetMemoryCapacity parses memory capacity from /proc/meminfo (mimicking +// cAdvisor). It lives in pkg/utils now; this is kept so that this package's +// interface is unchanged for as long as it is still here. func GetMemoryCapacity() int64 { - var ( - data []byte - err error - capa int64 - ) - - if data, err = os.ReadFile("/proc/meminfo"); err != nil { - return -1 - } - - for line := range strings.SplitSeq(string(data), "\n") { - keyval := strings.Split(line, ":") - if len(keyval) != 2 || keyval[0] != "MemTotal" { - continue - } - - valunit := strings.Split(strings.TrimSpace(keyval[1]), " ") - if len(valunit) != 2 || valunit[1] != "kB" { - return -1 - } - - capa, err = strconv.ParseInt(valunit[0], 10, 64) - if err != nil { - return -1 - } - - capa *= 1024 - break - } - - return capa + return utils.GetMemoryCapacity() } diff --git a/pkg/utils/meminfo.go b/pkg/utils/meminfo.go new file mode 100644 index 000000000..4965a86d7 --- /dev/null +++ b/pkg/utils/meminfo.go @@ -0,0 +1,59 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package utils + +import ( + "os" + "strconv" + "strings" +) + +// GetMemoryCapacity parses memory capacity from /proc/meminfo (mimicking +// cAdvisor). It returns -1 if the amount cannot be determined. +// +// Note that this reads the real /proc/meminfo, not one below any host root. +func GetMemoryCapacity() int64 { + var ( + data []byte + err error + capa int64 + ) + + if data, err = os.ReadFile("/proc/meminfo"); err != nil { + return -1 + } + + for line := range strings.SplitSeq(string(data), "\n") { + keyval := strings.Split(line, ":") + if len(keyval) != 2 || keyval[0] != "MemTotal" { + continue + } + + valunit := strings.Split(strings.TrimSpace(keyval[1]), " ") + if len(valunit) != 2 || valunit[1] != "kB" { + return -1 + } + + capa, err = strconv.ParseInt(valunit[0], 10, 64) + if err != nil { + return -1 + } + + capa *= 1024 + break + } + + return capa +} From e0c3cf9166383e7365def782cf98c80c05524aa9 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 11:17:43 +0300 Subject: [PATCH 21/39] memory-policy: take the topology from a hardware.Machine. The plugin discovered a pkg/sysfs System of its own. It discovers a hardware.Machine instead and reads the topology from that, so it no longer depends on that interface. It never set a sysfs root, so the machine is discovered below "/" as before. This also fixes which nodes "cpu-packages" selects. It asks each memory node which package it is in, and a node with no CPUs of its own has no package to answer with: pkg/sysfs said 0 for those, not because they are in package 0 but because that is the zero value of a field it never assigned. On a machine with HBM, CXL or PMEM that meant every such node counted as package 0, so a container on package 0 got all of them, including the ones attached to another package, and a container on any other package got none. Such a node now belongs to the package of the nearest node which does have CPUs, which is what the kernel's distances say about where the memory is and the only thing there is to go on. On the topology the n6-hbm-cxl end-to-end suite describes, the two HBM and two CXL nodes are each unambiguously nearer one of the two packages, and that is what they are reported as now. Nothing exercised "cpu-packages": it appears in the sample and helm configurations as an example class and in no test, and the two lines of comment above it were its only specification. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- cmd/plugins/memory-policy/main.go | 34 ++++--- cmd/plugins/memory-policy/topology.go | 67 +++++++++++++ cmd/plugins/memory-policy/topology_test.go | 109 +++++++++++++++++++++ 3 files changed, 194 insertions(+), 16 deletions(-) create mode 100644 cmd/plugins/memory-policy/topology.go create mode 100644 cmd/plugins/memory-policy/topology_test.go diff --git a/cmd/plugins/memory-policy/main.go b/cmd/plugins/memory-policy/main.go index 4f137363d..3fe60e4a6 100644 --- a/cmd/plugins/memory-policy/main.go +++ b/cmd/plugins/memory-policy/main.go @@ -30,7 +30,7 @@ import ( "github.com/containerd/nri/pkg/api" "github.com/containerd/nri/pkg/stub" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" @@ -69,8 +69,8 @@ const ( ) var ( - sys system.System - log *logrus.Logger + machine *hardware.Machine + log *logrus.Logger verbose bool veryVerbose bool @@ -328,14 +328,14 @@ func (policySpec *MemoryPolicySpec) ToLinuxMemoryPolicy(ctr *api.Container) (*Li } // Resolve nodes based on the policy specification. - ctrCpuset := sys.OnlineCPUs() + ctrCpuset := cpuset.New(machine.OnlineCPUs().List()...) if ctrCpus := ctr.GetLinux().GetResources().GetCpu().GetCpus(); ctrCpus != "" { ctrCpuset, err = cpuset.Parse(ctrCpus) if err != nil { return nil, fmt.Errorf("failed to parse allowed CPUs %q: %v", ctrCpus, err) } } - allowedMemsMask := libmem.NewNodeMask(sys.NodeIDs()...) + allowedMemsMask := libmem.NewNodeMask(machine.MemoryNodeIDs()...) ctrMems := ctr.GetLinux().GetResources().GetCpu().GetMems() if ctrMems != "" { if parsedMask, err := libmem.ParseNodeMask(ctrMems); err == nil { @@ -349,7 +349,7 @@ func (policySpec *MemoryPolicySpec) ToLinuxMemoryPolicy(ctr *api.Container) (*Li switch { // "all" includes all nodes into the mask. case policySpec.Nodes == "all": - nodeMask = libmem.NewNodeMask(sys.NodeIDs()...) + nodeMask = libmem.NewNodeMask(machine.MemoryNodeIDs()...) log.Tracef("- nodes %q (all)", nodeMask.MemsetString()) // "allowed-mems" includes only allowed memory nodes into the mask. @@ -360,21 +360,23 @@ func (policySpec *MemoryPolicySpec) ToLinuxMemoryPolicy(ctr *api.Container) (*Li // "cpu-packages" includes all nodes that are in the same package // as the CPUs in the container's cpuset. case policySpec.Nodes == "cpu-packages": - pkgs := sys.IDSetForCPUs(ctrCpuset, func(cpu system.CPU) idset.ID { + pkgs := idsForCPUs(ctrCpuset, func(cpu *hardware.CPU) idset.ID { return cpu.PackageID() }) nodeMask = libmem.NewNodeMask() - for _, nodeId := range sys.NodeIDs() { - nodePkgId := sys.Node(nodeId).PackageID() - if pkgs.Has(nodePkgId) { - nodeMask = nodeMask.Set(nodeId) + for _, nodeId := range machine.MemoryNodeIDs() { + for _, pkgId := range packagesOfNode(nodeId) { + if pkgs.Has(pkgId) { + nodeMask = nodeMask.Set(nodeId) + break + } } } log.Tracef("- nodes: %q (cpu-packages %q)", nodeMask.MemsetString(), pkgs) // "cpu-nodes" includes all nodes in the cpuset of the container. case policySpec.Nodes == "cpu-nodes": - nodeIds := sys.IDSetForCPUs(ctrCpuset, func(cpu system.CPU) idset.ID { + nodeIds := idsForCPUs(ctrCpuset, func(cpu *hardware.CPU) idset.ID { return cpu.NodeID() }) nodeMask = libmem.NewNodeMask(nodeIds.Members()...) @@ -389,12 +391,12 @@ func (policySpec *MemoryPolicySpec) ToLinuxMemoryPolicy(ctr *api.Container) (*Li return nil, fmt.Errorf("failed to parse max-dist %q: %v", maxDist, err) } nodeMask = libmem.NewNodeMask() - fromNodes := sys.IDSetForCPUs(ctrCpuset, func(cpu system.CPU) idset.ID { + fromNodes := idsForCPUs(ctrCpuset, func(cpu *hardware.CPU) idset.ID { return cpu.NodeID() }) for _, fromNode := range fromNodes.Members() { - for _, toNode := range sys.NodeIDs() { - if sys.NodeDistance(fromNode, toNode) <= maxDistInt { + for _, toNode := range machine.MemoryNodeIDs() { + if machine.MemoryNode(fromNode).Distance(toNode) <= maxDistInt { nodeMask = nodeMask.Set(toNode) } } @@ -571,7 +573,7 @@ func main() { } } - sys, err = system.DiscoverSystem(system.DiscoverCPUTopology) + machine, err = hardware.Discover(hardware.WithEnvOverrides()) if err != nil { log.Fatalf("failed to discover CPU topology: %v", err) } diff --git a/cmd/plugins/memory-policy/topology.go b/cmd/plugins/memory-policy/topology.go new file mode 100644 index 000000000..d99df731b --- /dev/null +++ b/cmd/plugins/memory-policy/topology.go @@ -0,0 +1,67 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// idsForCPUs returns the set of ids idOf gives for the given CPUs. CPUs the +// machine does not have are skipped. +func idsForCPUs(cpus cpuset.CPUSet, idOf func(*hardware.CPU) idset.ID) idset.IDSet { + ids := idset.NewIDSet() + for _, id := range cpus.List() { + if cpu := machine.CPU(id); cpu.Valid() { + ids.Add(idOf(cpu)) + } + } + return ids +} + +// packagesOfNode returns the CPU packages a memory node belongs to. +// +// A node with CPUs of its own belongs to the package those CPUs are in. A node +// with none -- HBM, CXL and PMEM nodes are commonly reported this way -- belongs +// to the package of the nearest node which does have CPUs, of which there can be +// more than one if two are equally near. That is what the kernel's distances say +// about where such memory is, and there is nothing else to go on: such a node has +// no package of its own to read. +func packagesOfNode(nodeId idset.ID) []idset.ID { + node := machine.MemoryNode(nodeId) + if !node.Valid() { + return nil + } + + if pkg := node.PackageID(); pkg >= 0 { + return []idset.ID{pkg} + } + + groups := hardware.ClosestMemoryNodes(machine, nodeId, + func(n *hardware.MemoryNode) bool { return n.CPUs().Size() > 0 }) + if len(groups) == 0 { + return nil + } + + pkgs := idset.NewIDSet() + for _, id := range groups[0].Nodes { + if pkg := machine.MemoryNode(id).PackageID(); pkg >= 0 { + pkgs.Add(pkg) + } + } + + return pkgs.SortedMembers() +} diff --git a/cmd/plugins/memory-policy/topology_test.go b/cmd/plugins/memory-policy/topology_test.go new file mode 100644 index 000000000..ade37eef5 --- /dev/null +++ b/cmd/plugins/memory-policy/topology_test.go @@ -0,0 +1,109 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "slices" + "testing" + "testing/fstest" + + "github.com/containers/nri-plugins/pkg/lib/hardware" + idset "github.com/intel/goresctrl/pkg/utils" +) + +func file(s string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(s)} } + +// hbmCxlFS is the shape of the n6-hbm-cxl e2e topology, reduced: two packages +// with CPUs and DRAM, and two memory nodes with no CPUs of their own, one near +// each package. +// +// node0 DRAM, cpus 0-1, package 0 +// node1 DRAM, cpus 2-3, package 1 +// node2 no cpus, 15 from node0, 30 from node1 +// node3 no cpus, 15 from node1, 30 from node0 +func hbmCxlFS() fstest.MapFS { + fsys := fstest.MapFS{ + "proc/meminfo": file("MemTotal: 8388608 kB\n"), + "sys/devices/system/cpu/online": file("0-3\n"), + "sys/devices/system/cpu/present": file("0-3\n"), + "sys/devices/system/cpu/possible": file("0-3\n"), + } + for cpu, pkg := range map[int]int{0: 0, 1: 0, 2: 1, 3: 1} { + dir := "sys/devices/system/cpu/cpu" + itoa(cpu) + "/topology" + fsys[dir+"/physical_package_id"] = file(itoa(pkg) + "\n") + fsys[dir+"/core_id"] = file(itoa(cpu) + "\n") + fsys[dir+"/core_cpus_list"] = file(itoa(cpu) + "\n") + } + for node, spec := range map[int]struct { + cpus string + distance string + }{ + 0: {"0-1", "10 20 15 30"}, + 1: {"2-3", "20 10 30 15"}, + 2: {"", "15 30 10 35"}, + 3: {"", "30 15 35 10"}, + } { + dir := "sys/devices/system/node/node" + itoa(node) + fsys[dir+"/cpulist"] = file(spec.cpus + "\n") + fsys[dir+"/distance"] = file(spec.distance + "\n") + fsys[dir+"/meminfo"] = file("Node " + itoa(node) + " MemTotal: 2097152 kB\n") + } + return fsys +} + +func itoa(n int) string { + if n == 0 { + return "0" + } + var b []byte + for n > 0 { + b = append([]byte{byte('0' + n%10)}, b...) + n /= 10 + } + return string(b) +} + +// A memory node with no CPUs of its own belongs to the package of the nearest +// node which has them. Reading the node's own package id instead yields -1, and +// "cpu-packages" then matches no such node at all -- which on a machine with HBM +// or CXL means the memory those policies exist to steer towards is never chosen. +func TestPackagesOfNode(t *testing.T) { + m, err := hardware.Discover(hardware.WithFS(hbmCxlFS())) + if err != nil { + t.Fatalf("Discover: %v", err) + } + saved := machine + machine = m + defer func() { machine = saved }() + + for _, tc := range []struct { + node idset.ID + want []idset.ID + }{ + {0, []idset.ID{0}}, + {1, []idset.ID{1}}, + {2, []idset.ID{0}}, // no CPUs, nearest is node0 + {3, []idset.ID{1}}, // no CPUs, nearest is node1 + } { + got := packagesOfNode(tc.node) + if !slices.Equal(got, tc.want) { + t.Errorf("packagesOfNode(%d) = %v, want %v", tc.node, got, tc.want) + } + } + + if got := packagesOfNode(1 << 20); got != nil { + t.Errorf("packagesOfNode(absent) = %v, want nil", got) + } +} From 66f3cb657367dabb780c6e498361f5353fd00725 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 11:33:32 +0300 Subject: [PATCH 22/39] cpuclass: take the topology from a hardware.Machine. The handler and its cpufreq, pct and uncorefreq internals read the topology through the pkg/sysfs interface. They take an already discovered hardware.Machine now, so none of them depends on that interface. Two of them get smaller rather than just different. pct declared a four-method subset of the interface for its Allocator to depend on, of which it called two, and those only to find one online CPU's frequency range. Its Sys is those two methods now, which a Machine satisfies as it is. uncorefreq's DiesForCpus walked every die of a CPU's package looking for the die which contained the CPU. A hardware.CPU answers with its own die, so it asks. The tests keep their shape. pct's fake was a Sys, a CPUPackage and a CPU built by embedding the interfaces and overriding a few methods; it is two methods returning nothing now, which is all the tests ever exercised through it. The handler's uncore tests described a cpu -> (pkg, die) layout to a fake System; they write the same layout out as sysfs and discover a machine from it, since a Machine cannot be faked. Those tests exercise real discovery now, and a new case covers DiesForCpus across several packages and dies, which none of them did. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 2 +- .../policy/topology-aware-policy.go | 2 +- pkg/resmgr/cpuclass/cpuclass.go | 18 +- pkg/resmgr/cpuclass/handler_commit_test.go | 195 +++++++++--------- .../cpuclass/internal/cpufreq/cpufreq.go | 18 +- .../cpuclass/internal/cpufreq/platform.go | 20 +- pkg/resmgr/cpuclass/internal/pct/pct.go | 23 ++- pkg/resmgr/cpuclass/internal/pct/pct_test.go | 125 +++-------- .../internal/uncorefreq/uncorefreq.go | 39 ++-- 9 files changed, 175 insertions(+), 267 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index ea9dd5526..6f46d78e2 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -1920,7 +1920,7 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { // Construct the CPU class handler that fronts both cpufreq and // PCT/SST internals. if p.cpuClasses == nil { - h, err := cpuclass.New(p.options.System) + h, err := cpuclass.New(p.machine) if err != nil { return balloonsError("failed to create CPU class handler: %w", err) } diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index c89b617cd..7846e7542 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -810,7 +810,7 @@ func (p *policy) initialize() error { opt.UnlimitedBurstable = p.findExistingTopologyLevel(opt.UnlimitedBurstable) if len(opt.CPUClasses) > 0 { - cc, err := cpuclass.New(p.options.System) + cc, err := cpuclass.New(p.options.Machine) if err != nil { return policyError("failed to create CPU class handler: %w", err) } diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index 578213917..a0f2d2c91 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -17,7 +17,7 @@ // Intel Priority Core Turbo state implied by a list of user-facing // CPU class definitions. // -// Policies talk to a single *Handler, constructed with New(sys). +// Policies talk to a single *Handler, constructed with New(machine). // Configure(spec) installs (or replaces) the class set; UseClass // pins given CPUs to a named class; Commit() flushes deferred // per-CPU sysfs writes; Hints() returns placement preferences a @@ -30,7 +30,7 @@ import ( "sort" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpufreq" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpuidle" @@ -79,7 +79,7 @@ type ConfigSpec struct { // construction and configuration of the per-technology allocators // (cpufreq, pct) and writers (cpufreq, cpuidle, uncorefreq). type Handler struct { - sys sysfs.System + machine *hardware.Machine allowed cpuset.CPUSet cpufreq *cpufreq.Allocator @@ -110,9 +110,9 @@ type Handler struct { // New constructs a Handler with both internal allocators (cpufreq // and pct) ready in a "no configuration applied" state. Configure // must be called before the handler is usable. -func New(sys sysfs.System) (*Handler, error) { +func New(m *hardware.Machine) (*Handler, error) { h := &Handler{ - sys: sys, + machine: m, defs: map[string]types.ClassDef{}, cpuClass: map[int]string{}, dirtyCPUs: map[int]bool{}, @@ -120,11 +120,11 @@ func New(sys sysfs.System) (*Handler, error) { idleWriter: cpuidle.NewWriter(cpuidle.Hooks{}), uncoreWriter: uncorefreq.NewWriter(uncorefreq.Hooks{}), } - freq, err := cpufreq.New(sys, h) + freq, err := cpufreq.New(m, h) if err != nil { return nil, fmt.Errorf("cpuclass: failed to create cpufreq allocator: %w", err) } - pctA, err := pct.NewAllocator(sys) + pctA, err := pct.NewAllocator(m) if err != nil { return nil, fmt.Errorf("cpuclass: failed to create pct allocator: %w", err) } @@ -328,8 +328,8 @@ func (h *Handler) Commit() error { firstErr = err } } - dirtyDies := uncorefreq.DiesForCpus(h.sys, h.dirtyCPUs) - if err := h.uncoreWriter.Enforce(h.sys, h.defs, h.cpuClass, dirtyDies); err != nil && firstErr == nil { + dirtyDies := uncorefreq.DiesForCpus(h.machine, h.dirtyCPUs) + if err := h.uncoreWriter.Enforce(h.machine, h.defs, h.cpuClass, dirtyDies); err != nil && firstErr == nil { firstErr = err } h.dirtyCPUs = map[int]bool{} diff --git a/pkg/resmgr/cpuclass/handler_commit_test.go b/pkg/resmgr/cpuclass/handler_commit_test.go index 392cf47d4..51632a4ff 100644 --- a/pkg/resmgr/cpuclass/handler_commit_test.go +++ b/pkg/resmgr/cpuclass/handler_commit_test.go @@ -15,12 +15,13 @@ package cpuclass import ( + "fmt" + "sort" "sync" "testing" + "testing/fstest" - idset "github.com/intel/goresctrl/pkg/utils" - - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpufreq" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpuidle" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" @@ -28,105 +29,52 @@ import ( "github.com/containers/nri-plugins/pkg/utils/cpuset" ) -// dieFakePackage extends the package fake with die support so the -// uncore writer can enumerate (pkg, die) tuples. -type dieFakePackage struct { - sysfs.CPUPackage - id idset.ID - cpus cpuset.CPUSet - dies []idset.ID - dieCpus map[idset.ID]cpuset.CPUSet -} - -func (p *dieFakePackage) ID() idset.ID { return p.id } -func (p *dieFakePackage) CPUSet() cpuset.CPUSet { return p.cpus } -func (p *dieFakePackage) DieIDs() []idset.ID { return p.dies } -func (p *dieFakePackage) DieCPUSet(d idset.ID) cpuset.CPUSet { return p.dieCpus[d] } - -// dieFakeCPU augments the cpu fake with package id. -type dieFakeCPU struct { - sysfs.CPU - id idset.ID - pkg idset.ID -} - -func (c *dieFakeCPU) ID() idset.ID { return c.id } -func (c *dieFakeCPU) PackageID() idset.ID { return c.pkg } - -// dieFakeSys is the minimum sysfs.System surface used by the -// uncore writer (Package, CPU, PackageIDs, DieIDs, DieCPUSet). -// Unimplemented methods panic via the embedded nil interface. -type dieFakeSys struct { - sysfs.System - packages map[idset.ID]*dieFakePackage - cpuPkg map[int]idset.ID +// dieFakeCpu specifies the (pkg, die) location of a single CPU when building a +// machine for these tests. +type dieFakeCpu struct { + pkg int + die int } -func (s *dieFakeSys) PackageIDs() []idset.ID { - ids := make([]idset.ID, 0, len(s.packages)) - for id := range s.packages { - ids = append(ids, id) - } - return ids -} +// newDieMachine returns a machine with the given cpu -> (pkg, die) layout. +// +// Handler takes a *hardware.Machine, which is a concrete type and cannot be +// faked, so the layout is written out as the kernel would present it in sysfs and +// read back through real discovery. One NUMA node holds everything: the uncore +// writer cares about packages and dies only. +func newDieMachine(t *testing.T, cpus map[int]dieFakeCpu) *hardware.Machine { + t.Helper() -func (s *dieFakeSys) Package(id idset.ID) sysfs.CPUPackage { - if p, ok := s.packages[id]; ok { - return p + file := func(s string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(s)} } + ids := make([]int, 0, len(cpus)) + for cpu := range cpus { + ids = append(ids, cpu) } - return nil -} + sort.Ints(ids) -func (s *dieFakeSys) CPU(id idset.ID) sysfs.CPU { - pkg, ok := s.cpuPkg[int(id)] - if !ok { - return nil + all := cpuset.New(ids...).String() + fsys := fstest.MapFS{ + "proc/meminfo": file("MemTotal: 1048576 kB\n"), + "sys/devices/system/cpu/online": file(all + "\n"), + "sys/devices/system/cpu/present": file(all + "\n"), + "sys/devices/system/cpu/possible": file(all + "\n"), + "sys/devices/system/node/node0/cpulist": file(all + "\n"), + "sys/devices/system/node/node0/distance": file("10\n"), + "sys/devices/system/node/node0/meminfo": file("Node 0 MemTotal: 1048576 kB\n"), } - return &dieFakeCPU{id: id, pkg: pkg} -} - -// dieFakeCpu specifies the (pkg, die) location of a single CPU when -// building a dieFakeSys. -type dieFakeCpu struct { - pkg int - die int -} - -// newDieFakeSys builds a dieFakeSys from a map cpu -> (pkg, die). -func newDieFakeSys(cpus map[int]dieFakeCpu) *dieFakeSys { - pkgs := map[idset.ID]*dieFakePackage{} - cpuPkg := map[int]idset.ID{} - type pkgDieKey struct{ pkg, die int } - dieCpus := map[pkgDieKey]cpuset.CPUSet{} - pkgCpus := map[int]cpuset.CPUSet{} - pkgDies := map[int]map[int]bool{} for cpu, loc := range cpus { - cpuPkg[cpu] = idset.ID(loc.pkg) - pkgCpus[loc.pkg] = pkgCpus[loc.pkg].Union(cpuset.New(cpu)) - k := pkgDieKey(loc) - dieCpus[k] = dieCpus[k].Union(cpuset.New(cpu)) - if pkgDies[loc.pkg] == nil { - pkgDies[loc.pkg] = map[int]bool{} - } - pkgDies[loc.pkg][loc.die] = true + dir := fmt.Sprintf("sys/devices/system/cpu/cpu%d/topology", cpu) + fsys[dir+"/physical_package_id"] = file(fmt.Sprintf("%d\n", loc.pkg)) + fsys[dir+"/die_id"] = file(fmt.Sprintf("%d\n", loc.die)) + fsys[dir+"/core_id"] = file(fmt.Sprintf("%d\n", cpu)) + fsys[dir+"/core_cpus_list"] = file(fmt.Sprintf("%d\n", cpu)) } - for pkg, dies := range pkgDies { - dList := make([]idset.ID, 0, len(dies)) - for d := range dies { - dList = append(dList, idset.ID(d)) - } - dc := map[idset.ID]cpuset.CPUSet{} - for d := range dies { - dc[idset.ID(d)] = dieCpus[pkgDieKey{pkg, d}] - } - pkgs[idset.ID(pkg)] = &dieFakePackage{ - id: idset.ID(pkg), - cpus: pkgCpus[pkg], - dies: dList, - dieCpus: dc, - } + + m, err := hardware.Discover(hardware.WithFS(fsys)) + if err != nil { + t.Fatalf("failed to discover the test machine: %v", err) } - return &dieFakeSys{packages: pkgs, cpuPkg: cpuPkg} + return m } // recordingWriters captures the per-CPU and per-die writes issued by @@ -202,7 +150,7 @@ func (r *recordingWriters) installOn(h *Handler) { } // newBareHandler returns a Handler with empty state, no sysfs -// topology (callers may set h.sys), and the recording writers +// topology (callers may set h.machine), and the recording writers // installed. The cpuidle writer is left in a state where Enforce // will return early because no class has DisabledCstates. func newBareHandler() (*Handler, *recordingWriters) { @@ -279,12 +227,12 @@ func TestAssignToEmptyClassDoesNotWriteCpufreq(t *testing.T) { // TestUncoreSkipBothZero verifies that a die with effective min=0 // and max=0 produces no uncore writes. func TestUncoreSkipBothZero(t *testing.T) { - sys := newDieFakeSys(map[int]dieFakeCpu{ + m := newDieMachine(t, map[int]dieFakeCpu{ 0: {pkg: 0, die: 0}, 1: {pkg: 0, die: 0}, }) h, r := newBareHandler() - h.sys = sys + h.machine = m h.SetClassDef("idle@d0", types.ClassDef{MinFreq: 800_000}) h.AssignCPUs("idle@d0", []int{0, 1}) if err := h.Commit(); err != nil { @@ -298,12 +246,12 @@ func TestUncoreSkipBothZero(t *testing.T) { // TestUncoreMaxWinsAcrossClasses verifies the per-die max-wins // reduction when multiple classes are active on the same die. func TestUncoreMaxWinsAcrossClasses(t *testing.T) { - sys := newDieFakeSys(map[int]dieFakeCpu{ + m := newDieMachine(t, map[int]dieFakeCpu{ 0: {pkg: 0, die: 0}, 1: {pkg: 0, die: 0}, }) h, r := newBareHandler() - h.sys = sys + h.machine = m h.SetClassDef("lo@d0", types.ClassDef{UncoreMinFreq: 800_000, UncoreMaxFreq: 1_500_000}) h.SetClassDef("hi@d0", types.ClassDef{UncoreMinFreq: 1_200_000, UncoreMaxFreq: 2_400_000}) h.AssignCPUs("lo@d0", []int{0}) @@ -324,12 +272,12 @@ func TestUncoreMaxWinsAcrossClasses(t *testing.T) { // winner class from a die triggers a fresh write with the loser's // (lower) values. func TestUncoreRecomputesOnAssignmentChange(t *testing.T) { - sys := newDieFakeSys(map[int]dieFakeCpu{ + m := newDieMachine(t, map[int]dieFakeCpu{ 0: {pkg: 0, die: 0}, 1: {pkg: 0, die: 0}, }) h, r := newBareHandler() - h.sys = sys + h.machine = m h.SetClassDef("lo@d0", types.ClassDef{UncoreMaxFreq: 1_500_000}) h.SetClassDef("hi@d0", types.ClassDef{UncoreMaxFreq: 2_400_000}) h.AssignCPUs("lo@d0", []int{0}) @@ -345,3 +293,50 @@ func TestUncoreRecomputesOnAssignmentChange(t *testing.T) { t.Errorf("uncore max after hi removed = %d, want 1_500_000", got) } } + +// TestDiesForCpus covers the (pkg, die) lookup across more than one of each, +// which the tests above do not: they all describe a single die. A CPU knows its +// own die, so this is also what says newDieMachine lays the topology out the way +// its callers describe it. +func TestDiesForCpus(t *testing.T) { + m := newDieMachine(t, map[int]dieFakeCpu{ + 0: {pkg: 0, die: 0}, + 1: {pkg: 0, die: 0}, + 2: {pkg: 0, die: 1}, + 3: {pkg: 1, die: 0}, + 4: {pkg: 1, die: 1}, + }) + + for _, tc := range []struct { + name string + cpus []int + want []uncorefreq.DieKey + }{ + {"one die", []int{0, 1}, []uncorefreq.DieKey{{Pkg: 0, Die: 0}}}, + {"two dies of one package", []int{1, 2}, + []uncorefreq.DieKey{{Pkg: 0, Die: 0}, {Pkg: 0, Die: 1}}}, + {"across packages", []int{0, 3}, + []uncorefreq.DieKey{{Pkg: 0, Die: 0}, {Pkg: 1, Die: 0}}}, + {"every die", []int{0, 1, 2, 3, 4}, []uncorefreq.DieKey{ + {Pkg: 0, Die: 0}, {Pkg: 0, Die: 1}, {Pkg: 1, Die: 0}, {Pkg: 1, Die: 1}, + }}, + {"a CPU the machine does not have", []int{1 << 20}, nil}, + } { + t.Run(tc.name, func(t *testing.T) { + cpus := map[int]bool{} + for _, cpu := range tc.cpus { + cpus[cpu] = true + } + + got := uncorefreq.DiesForCpus(m, cpus) + if len(got) != len(tc.want) { + t.Fatalf("DiesForCpus(%v) = %v, want %v", tc.cpus, got, tc.want) + } + for _, key := range tc.want { + if !got[key] { + t.Errorf("DiesForCpus(%v) = %v, missing %v", tc.cpus, got, key) + } + } + }) + } +} diff --git a/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go b/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go index ade8c35d3..a3849208c 100644 --- a/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go +++ b/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go @@ -24,7 +24,7 @@ import ( "slices" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -43,7 +43,7 @@ type Sink interface { // Allocator owns the per-turbo-domain class state for cpufreq. type Allocator struct { - sys sysfs.System + machine *hardware.Machine sink Sink classes []*policyapi.CPUClass classByName map[string]*policyapi.CPUClass @@ -77,15 +77,15 @@ const ( // New returns an Allocator that publishes class definitions and // per-CPU assignments to sink. The constructor does not push any // class definitions; the caller follows up with Configure(). -func New(sys sysfs.System, sink Sink) (*Allocator, error) { - if sys == nil { - return nil, fmt.Errorf("cpufreq: missing required argument sys") +func New(m *hardware.Machine, sink Sink) (*Allocator, error) { + if m == nil { + return nil, fmt.Errorf("cpufreq: missing required argument machine") } if sink == nil { return nil, fmt.Errorf("cpufreq: missing required argument sink") } a := &Allocator{ - sys: sys, + machine: m, sink: sink, activeCpus: map[domainID]map[string]cpuset.CPUSet{}, winnerPrio: map[domainID]int{}, @@ -207,12 +207,12 @@ func (a *Allocator) buildCpuDomains() { if mode == "" { mode = turboDomainPackage } - for _, cpuID := range a.sys.CPUIDs() { + for _, cpuID := range a.machine.CPUIDs() { if a.allowed.Size() > 0 && !a.allowed.Contains(int(cpuID)) { continue } - c := a.sys.CPU(cpuID) - if c == nil { + c := a.machine.CPU(cpuID) + if !c.Valid() { continue } var d domainID diff --git a/pkg/resmgr/cpuclass/internal/cpufreq/platform.go b/pkg/resmgr/cpuclass/internal/cpufreq/platform.go index 91f3ef207..3d0f8f0be 100644 --- a/pkg/resmgr/cpuclass/internal/cpufreq/platform.go +++ b/pkg/resmgr/cpuclass/internal/cpufreq/platform.go @@ -17,7 +17,7 @@ package cpufreq import ( "fmt" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" ) // platformTurboInfo holds platform-level turbo frequency capabilities @@ -28,10 +28,10 @@ type platformTurboInfo struct { minFreqKHz uint } -// discoverPlatformInfo populates a.turboInfo from sysfs. Failure is +// discoverPlatformInfo populates a.turboInfo from the machine. Failure is // non-fatal: symbolic frequencies then resolve to 0. func (a *Allocator) discoverPlatformInfo() { - info, err := discoverTurboInfo(a.sys) + info, err := discoverTurboInfo(a.machine) if err != nil { log.Warnf("cpufreq: cannot discover platform turbo info: %v", err) return @@ -39,20 +39,20 @@ func (a *Allocator) discoverPlatformInfo() { a.turboInfo = info } -// discoverTurboInfo reads platform turbo capabilities from sysfs. It +// discoverTurboInfo reads platform turbo capabilities from the machine. It // uses the first online CPU's frequency range as representative. -func discoverTurboInfo(sys sysfs.System) (*platformTurboInfo, error) { - cpuIDs := sys.CPUIDs() +func discoverTurboInfo(m *hardware.Machine) (*platformTurboInfo, error) { + cpuIDs := m.CPUIDs() if len(cpuIDs) == 0 { return nil, fmt.Errorf("no CPUs found in system topology") } for _, id := range cpuIDs { - cpu := sys.CPU(id) - if cpu == nil || !cpu.Online() { + cpu := m.CPU(id) + if !cpu.Valid() || !cpu.Online() { continue } - freq := cpu.FrequencyRange() - baseFreq := cpu.BaseFrequency() + freq := cpu.Freq() + baseFreq := freq.Base if freq.Min == 0 && freq.Max == 0 { log.Warnf("cannot detect cpu%d frequency range, skipping platform turbo info", id) continue diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index aaf6f71da..ed796bbf7 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -21,7 +21,7 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -54,14 +54,13 @@ type pctClassPlan struct { MaxFreq uint // kHz, 0 = leave alone } -// Sys is the subset of sysfs.System that Allocator depends -// on. Defined here so tests can substitute a fake without -// implementing the full sysfs.System surface. +// Sys is the part of the machine topology Allocator depends on, which is the +// frequency range of one online CPU and nothing else. A [hardware.Machine] +// satisfies it; it is an interface so that tests can supply a machine with no +// CPUs without having to synthesize one. type Sys interface { - PackageIDs() []idset.ID - Package(id idset.ID) sysfs.CPUPackage - CPU(id idset.ID) sysfs.CPU CPUIDs() []idset.ID + CPU(id idset.ID) *hardware.CPU } // Allocator manages Intel Priority Core Turbo CLOS associations @@ -1196,7 +1195,7 @@ type turboInfo struct { minFreqKHz uint } -// discoverTurboInfo reads platform turbo capabilities from sysfs via +// discoverTurboInfo reads platform turbo capabilities from the machine via // the first online CPU. Returns nil if no online CPU exposes valid // frequency data. func discoverTurboInfo(sys Sys) (*turboInfo, error) { @@ -1205,12 +1204,14 @@ func discoverTurboInfo(sys Sys) (*turboInfo, error) { return nil, fmt.Errorf("no CPUs found in system topology") } for _, id := range cpuIDs { + // A Machine never hands out a nil CPU, but Sys is an interface and a + // test implementation may. cpu := sys.CPU(id) - if cpu == nil || !cpu.Online() { + if cpu == nil || !cpu.Valid() || !cpu.Online() { continue } - freq := cpu.FrequencyRange() - baseFreq := cpu.BaseFrequency() + freq := cpu.Freq() + baseFreq := freq.Base if freq.Min == 0 && freq.Max == 0 { continue } diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index 175e29180..1a88c8594 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -23,84 +23,23 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) var errFakeSstNoClos = errors.New("fakeSst: no CLOS for CPU") -// --- minimal sysfs.System / CPUPackage / CPU fakes ------------------ +// --- minimal Sys fake ---------------------------------------------- -// fakePackage implements sysfs.CPUPackage via an embedded nil -// interface. Methods not overridden here panic if called, which is -// the desired guardrail in unit tests. -type fakePackage struct { - sysfs.CPUPackage - id idset.ID - cpus cpuset.CPUSet -} - -func (p *fakePackage) ID() idset.ID { return p.id } -func (p *fakePackage) CPUSet() cpuset.CPUSet { return p.cpus } - -// fakeCPU implements sysfs.CPU likewise. -type fakeCPU struct { - sysfs.CPU - id idset.ID - pkg idset.ID -} - -func (c *fakeCPU) ID() idset.ID { return c.id } -func (c *fakeCPU) PackageID() idset.ID { return c.pkg } - -// fakeSys is a minimal Sys implementation built from package -// CPU maps. -type fakeSys struct { - packageCpus map[idset.ID]cpuset.CPUSet // pkgID -> cpus - cpuPkg map[int]idset.ID // cpu -> pkgID -} - -func (s *fakeSys) PackageIDs() []idset.ID { - ids := make([]idset.ID, 0, len(s.packageCpus)) - for id := range s.packageCpus { - ids = append(ids, id) - } - return ids -} +// fakeSys reports no CPUs. Allocator asks the machine for one online CPU's +// frequency range and nothing else, so the tests below -- which are about CLOS +// planning and association -- have no topology to provide. Turbo info then stays +// nil, which is the same as on a platform whose CPUs expose no frequency data. +type fakeSys struct{} -func (s *fakeSys) Package(id idset.ID) sysfs.CPUPackage { - cpus, ok := s.packageCpus[id] - if !ok { - return nil - } - return &fakePackage{id: id, cpus: cpus} -} - -func (s *fakeSys) CPU(id idset.ID) sysfs.CPU { - pkg, ok := s.cpuPkg[int(id)] - if !ok { - return nil - } - return &fakeCPU{id: id, pkg: pkg} -} - -func (s *fakeSys) CPUIDs() []idset.ID { return nil } - -// newTwoPackageFakeSys returns a fakeSys with two packages of 4 CPUs -// each: pkg0=0..3, pkg1=4..7. -func newTwoPackageFakeSys() *fakeSys { - return &fakeSys{ - packageCpus: map[idset.ID]cpuset.CPUSet{ - 0: cpuset.MustParse("0-3"), - 1: cpuset.MustParse("4-7"), - }, - cpuPkg: map[int]idset.ID{ - 0: 0, 1: 0, 2: 0, 3: 0, - 4: 1, 5: 1, 6: 1, 7: 1, - }, - } -} +func (*fakeSys) CPUIDs() []idset.ID { return nil } +func (*fakeSys) CPU(idset.ID) *hardware.CPU { return nil } // --- minimal sst fake ------------------------------------------------ @@ -275,7 +214,7 @@ func pctTestWirePunits(a *Allocator) { // TestPctHintsNoClassNoOp covers the "no plan and not managed-with-HP" // branch where hints() must return an empty types.AllocationHints. func TestPctHintsNoClassNoOp(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{supported: true} // disabled allocator: hints must short-circuit to empty. @@ -303,7 +242,7 @@ func TestPctHintsNoClassNoOp(t *testing.T) { // branch in assoc-only mode: hints prefer free CPUs already // associated to the class's CLOS, enabling bin packing. func TestPctHintsAssocOnlyPreferClosCpus(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, // cpus 1, 2 and 3 already on CLOS 1, others on default CLOS 0. @@ -345,7 +284,7 @@ func TestPctHintsAssocOnlyPreferClosCpus(t *testing.T) { // branch: hints contain (a) free CPUs already on the HP CLOS for bin // packing and (b) the HP-reserve preference (largest-room package). func TestPctHintsHighPriorityReserveAndClosCpus(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, // cpus 0 and 1 already on CLOS 0 (HP), cpu 0 in use. @@ -409,7 +348,7 @@ func TestPctHintsHighPriorityReserveAndClosCpus(t *testing.T) { // hosting HP-class CPUs, so non-HP classes do not steal HP turbo // budget. THIS BRANCH IS NOT COVERED IN test19 e2e. func TestPctHintsManagedNonHpAvoidsHpInUse(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, cpuClos: map[int]int{}, @@ -464,7 +403,7 @@ func TestPctHintsManagedNonHpAvoidsHpInUse(t *testing.T) { // Allowed (via the handler-level intersectHints + pct-internal // allowed intersections). func TestPctHintsAllowedBoundsResults(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, cpuClos: map[int]int{1: 0, 4: 0}, // HP cpus on both packages @@ -511,24 +450,6 @@ func TestPctHintsAllowedBoundsResults(t *testing.T) { // --- Tier A/B/C reservation tests ---------------------------------- -// newTwoPunitFakeSys returns a fakeSys whose package layout matches -// the standard two-punit-per-package fixture below: pkg0 = 0..7 -// (punit-0 = 0..3, punit-1 = 4..7), pkg1 = 8..15 (punit-2 = 8..11, -// punit-3 = 12..15). The synthesis function does not know about -// punits, only packages. -func newTwoPunitFakeSys() *fakeSys { - return &fakeSys{ - packageCpus: map[idset.ID]cpuset.CPUSet{ - 0: cpuset.MustParse("0-7"), - 1: cpuset.MustParse("8-15"), - }, - cpuPkg: map[int]idset.ID{ - 0: 0, 1: 0, 2: 0, 3: 0, 4: 0, 5: 0, 6: 0, 7: 0, - 8: 1, 9: 1, 10: 1, 11: 1, 12: 1, 13: 1, 14: 1, 15: 1, - }, - } -} - // makeTwoPunitsPerPkg returns four punits laid out as in // newTwoPunitFakeSys, with the given MaxHpCpus per punit. func makeTwoPunitsPerPkg(hp0, hp1, hp2, hp3 int) []pctPunit { @@ -544,7 +465,7 @@ func makeTwoPunitsPerPkg(hp0, hp1, hp2, hp3 int) []pctPunit { // HP work, punit-1 in the same package has full HP room. A request // for 1 HP CPU must steer to punit-1 (Tier A), not to pkg1. func TestPctHints_HpRoomTierAPunitWins(t *testing.T) { - sys := newTwoPunitFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, punits: makeTwoPunitsPerPkg(2, 2, 2, 2), @@ -594,7 +515,7 @@ func TestPctHints_HpRoomTierAPunitWins(t *testing.T) { // enough for the request. Pkg1 has only 1 HP slot in total. The // Tier-B aggregate must steer to pkg0 (free CPUs of both punits). func TestPctHints_HpRoomTierBSamePackage(t *testing.T) { - sys := newTwoPunitFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, punits: makeTwoPunitsPerPkg(2, 2, 1, 0), @@ -642,7 +563,7 @@ func TestPctHints_HpRoomTierBSamePackage(t *testing.T) { // allocator must return no HP-reserve hint so the caller falls back // to topology-only placement on the same socket. func TestPctHints_HpRoomTierCNoCrossPackage(t *testing.T) { - sys := newTwoPunitFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, // pkg0 has 2 HP CPUs total, pkg1 has 2 HP CPUs total. @@ -676,7 +597,7 @@ func TestPctHints_HpRoomTierCNoCrossPackage(t *testing.T) { // entire package. This is a regression guard for the punit-keyed // rewrite of hpInUseCpus. func TestPctHints_HpInUseIsPunitGranular(t *testing.T) { - sys := newTwoPunitFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, punits: makeTwoPunitsPerPkg(2, 2, 2, 2), @@ -937,7 +858,7 @@ func newAssocOnlyPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans m // -- not zero. (Pre-fix the result was 0 because closCpus(HP CLOS) // was empty.) func TestFreeClassCapacity_AssocOnlyHpFromFallbackCLOS(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, // All CPUs are on CLOS 3 (the LP/fallback CLOS). The HP @@ -984,7 +905,7 @@ func TestFreeClassCapacity_AssocOnlyHpFromFallbackCLOS(t *testing.T) { // GuaranteedHpCpus is non-zero. Prevents over-publishing HP // capacity on nodes that cannot actually deliver top turbo. func TestFreeClassCapacity_AssocOnlyHpTFDisabledPunitExcluded(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, punits: []pctPunit{ @@ -1012,7 +933,7 @@ func TestFreeClassCapacity_AssocOnlyHpTFDisabledPunitExcluded(t *testing.T) { // where no class was classified HP (e.g. no CLOS has a programmed // MaxFreq) falls through to the non-HP formula |Allowed \ held|. func TestFreeClassCapacity_AssocOnlyNoHpClassification(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, punits: []pctPunit{ @@ -1037,7 +958,7 @@ func TestFreeClassCapacity_AssocOnlyNoHpClassification(t *testing.T) { // (PrepareManagedMode enables SST-TF) and the result is the // guaranteed-top-turbo sum, capped by per-punit free CPUs. func TestFreeClassCapacity_ManagedHpRespectsEligibility(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{ supported: true, punits: []pctPunit{ @@ -1075,7 +996,7 @@ func TestFreeClassCapacity_ManagedHpRespectsEligibility(t *testing.T) { // TestFreeClassCapacity_UnknownClassReturnsZero: unknown class // (no PCT plan) yields 0 regardless of mode. func TestFreeClassCapacity_UnknownClassReturnsZero(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := &fakeSys{} sst := &fakeSst{supported: true} a := newManagedPctForTest(t, []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, map[string]*pctClassPlan{"hp": {ClosID: 0}}, diff --git a/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go b/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go index 316edcc3d..8772f8fbe 100644 --- a/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go +++ b/pkg/resmgr/cpuclass/internal/uncorefreq/uncorefreq.go @@ -24,7 +24,7 @@ import ( "github.com/intel/goresctrl/pkg/utils" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" ) @@ -120,13 +120,13 @@ func UnavailableError(className string) error { // // Returns the first error encountered. Skips silently when the // uncore driver is unavailable. -func (w *Writer) Enforce(sys sysfs.System, defs map[string]types.ClassDef, cpuClass map[int]string, dirtyDies map[DieKey]bool) error { +func (w *Writer) Enforce(m *hardware.Machine, defs map[string]types.ClassDef, cpuClass map[int]string, dirtyDies map[DieKey]bool) error { if !w.available || len(dirtyDies) == 0 { return nil } var firstErr error for key := range dirtyDies { - min, max, minCls, maxCls := effectiveUncoreFreqs(sys, key, defs, cpuClass) + min, max, minCls, maxCls := effectiveUncoreFreqs(m, key, defs, cpuClass) if min == 0 && max == 0 { log.Debugf("uncore: pkg/die %d/%d: no limits in effect", key.Pkg, key.Die) continue @@ -165,12 +165,11 @@ func (w *Writer) Enforce(sys sysfs.System, defs map[string]types.ClassDef, cpuCl // effectiveUncoreFreqs computes the effective uncore min and max for // a single die. Returns 0,0 when no class with uncore limits is // active on the die. -func effectiveUncoreFreqs(sys sysfs.System, key DieKey, defs map[string]types.ClassDef, cpuClass map[int]string) (minFreq, maxFreq uint, minCls, maxCls string) { - pkg := sys.Package(utils.ID(key.Pkg)) - if pkg == nil { - return 0, 0, "", "" - } - dieCPUs := pkg.DieCPUSet(utils.ID(key.Die)) +func effectiveUncoreFreqs(m *hardware.Machine, key DieKey, defs map[string]types.ClassDef, cpuClass map[int]string) (minFreq, maxFreq uint, minCls, maxCls string) { + dieCPUs := m.TopologyIndex().DieCPUs(hardware.DieID{ + Package: key.Pkg, + Die: key.Die, + }) seen := map[string]bool{} for _, cpu := range dieCPUs.UnsortedList() { name, ok := cpuClass[cpu] @@ -199,27 +198,19 @@ func effectiveUncoreFreqs(sys sysfs.System, key DieKey, defs map[string]types.Cl // DiesForCpus returns the set of (pkg, die) keys that contain at // least one cpu from cpus. -func DiesForCpus(sys sysfs.System, cpus map[int]bool) map[DieKey]bool { +func DiesForCpus(m *hardware.Machine, cpus map[int]bool) map[DieKey]bool { out := map[DieKey]bool{} - if sys == nil { + if m == nil { return out } + // A CPU knows which die it is on, so there is no need to look for the die + // which contains it. for cpu := range cpus { - c := sys.CPU(utils.ID(cpu)) - if c == nil { - continue - } - pkgID := int(c.PackageID()) - pkg := sys.Package(utils.ID(pkgID)) - if pkg == nil { + c := m.CPU(cpu) + if !c.Valid() { continue } - for _, die := range pkg.DieIDs() { - if pkg.DieCPUSet(die).Contains(cpu) { - out[DieKey{Pkg: pkgID, Die: int(die)}] = true - break - } - } + out[DieKey{Pkg: c.PackageID(), Die: c.DieID()}] = true } return out } From 3883585d60c4fe88ea951ec0299942f6016684ea Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 14:43:21 +0300 Subject: [PATCH 23/39] topology-aware: build test topologies from real machines. The tests faked the pkg/sysfs interface: a mockSystem with all 35 of its methods, plus a mockCPUPackage, a mockCPU and a mockSystemNode, each built by embedding the interface and overriding the few methods a test needed. That works only for an interface, and the topology this policy is being moved onto is concrete types. Describe the machine instead and read it back through discovery. synthMachine writes a topology out as sysfs and discovers it, so a test says what nodes and CPUs it wants and gets the real thing. A node with memory and no CPUs of its own comes back as PMEM or HBM depending on its size, which is how the hardware package classifies one, so the memory kinds a test asks for are expressed as sizes rather than asserted into a fake. The mocks were worth less than their 328 lines suggest. The hint tests passed an entirely empty mockSystem, twice, to check that an absent socket and an absent NUMA node yield nothing; a machine with one CPU says the same. The coldstart test is the only other user and has been skipped for a while, for the very reason this addresses: it could not mock enough of the system. Its fixture is kept, as nodes to synthesize, and it stays skipped here. mockContainer, mockPod, mockCache and mockCPUAllocator stay. They have nothing to do with topology and five test files use them. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../topology-aware/policy/coldstart_test.go | 14 +- .../topology-aware/policy/hint_test.go | 5 +- .../topology-aware/policy/machine_test.go | 123 +++++++ .../topology-aware/policy/mocks_test.go | 331 ------------------ 4 files changed, 133 insertions(+), 340 deletions(-) create mode 100644 cmd/plugins/topology-aware/policy/machine_test.go diff --git a/cmd/plugins/topology-aware/policy/coldstart_test.go b/cmd/plugins/topology-aware/policy/coldstart_test.go index e0d9a8578..ff3930726 100644 --- a/cmd/plugins/topology-aware/policy/coldstart_test.go +++ b/cmd/plugins/topology-aware/policy/coldstart_test.go @@ -52,7 +52,7 @@ func TestColdStart(t *testing.T) { tcases := []struct { name string - numaNodes []system.Node + numaNodes []synthNode req Request affinities map[int]int32 container cache.Container @@ -64,9 +64,11 @@ func TestColdStart(t *testing.T) { }{ { name: "three node cold start", - numaNodes: []system.Node{ - &mockSystemNode{id: 0, memFree: 10000, memTotal: 10000, memType: system.MemoryTypeDRAM, distance: []int{1, 5}}, - &mockSystemNode{id: 1, memFree: 50000, memTotal: 50000, memType: system.MemoryTypePMEM, distance: []int{5, 1}}, + // node0 has CPUs, so it is ordinary memory. node1 has none and is + // larger, which is how a persistent memory node presents itself. + numaNodes: []synthNode{ + {cpus: "0-1", memKB: 10000, distance: []int{10, 50}}, + {cpus: "", memKB: 50000, distance: []int{50, 10}}, }, container: &mockContainer{ name: "demo-coldstart-container", @@ -90,9 +92,7 @@ func TestColdStart(t *testing.T) { t.Skipf("Coldstart tests are disabled (can't mock enough of the system, lacks CPUs)") policy := &policy{ - sys: &mockSystem{ - nodes: tc.numaNodes, - }, + sys: system.FromMachine(synthMachine(t, tc.numaNodes)), cache: &mockCache{ returnValue1ForLookupContainer: tc.container, returnValue2ForLookupContainer: true, diff --git a/cmd/plugins/topology-aware/policy/hint_test.go b/cmd/plugins/topology-aware/policy/hint_test.go index d111613fc..849ef68c4 100644 --- a/cmd/plugins/topology-aware/policy/hint_test.go +++ b/cmd/plugins/topology-aware/policy/hint_test.go @@ -17,6 +17,7 @@ package topologyaware import ( "testing" + "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" @@ -163,7 +164,7 @@ func TestHintCpus(t *testing.T) { supply: &supply{ node: &node{ policy: &policy{ - sys: &mockSystem{}, + sys: system.FromMachine(oneCpuMachine(t)), }, }, }, @@ -183,7 +184,7 @@ func TestHintCpus(t *testing.T) { supply: &supply{ node: &node{ policy: &policy{ - sys: &mockSystem{}, + sys: system.FromMachine(oneCpuMachine(t)), }, }, }, diff --git a/cmd/plugins/topology-aware/policy/machine_test.go b/cmd/plugins/topology-aware/policy/machine_test.go new file mode 100644 index 000000000..5b8cd7881 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/machine_test.go @@ -0,0 +1,123 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "fmt" + "strings" + "testing" + "testing/fstest" + + "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/utils/cpuset" +) + +// synthNode describes one NUMA node of a machine to build for a test. +// +// A node with CPUs is ordinary memory. A node with memory and no CPUs is +// something special, and which special sort is inferred from its size against the +// DRAM nodes: larger is persistent memory, smaller is high-bandwidth memory. That +// is how the hardware package classifies a real machine, so describing the sizes +// is how a test asks for a PMEM or an HBM node. +type synthNode struct { + cpus string // cpulist, empty for a node with no CPUs of its own + memKB int // MemTotal in kB + distance []int // distance to every node, this one included +} + +// synthMachine builds a machine from the given nodes by writing the topology out +// as sysfs and discovering it. +// +// The policy reads topology from a *hardware.Machine, which is a concrete type +// and cannot be faked, so a test describes the machine it wants and reads it back +// through real discovery. All CPUs are single-threaded cores; the node a CPU is +// in also gives its package, one package per node with CPUs. +func synthMachine(t *testing.T, nodes []synthNode) *hardware.Machine { + t.Helper() + + file := func(s string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(s)} } + fsys := fstest.MapFS{} + + var ( + online = cpuset.New() + normal = cpuset.New() + pkg = 0 + ) + for id, node := range nodes { + dir := fmt.Sprintf("sys/devices/system/node/node%d", id) + + cpus := cpuset.New() + if node.cpus != "" { + var err error + if cpus, err = cpuset.Parse(node.cpus); err != nil { + t.Fatalf("node%d: bad cpulist %q: %v", id, node.cpus, err) + } + } + online = online.Union(cpus) + + fsys[dir+"/cpulist"] = file(node.cpus + "\n") + fsys[dir+"/meminfo"] = file( + fmt.Sprintf("Node %d MemTotal: %d kB\n", id, node.memKB)) + + if node.memKB > 0 { + normal = normal.Union(cpuset.New(id)) + } + + dist := make([]string, 0, len(node.distance)) + for _, d := range node.distance { + dist = append(dist, fmt.Sprintf("%d", d)) + } + fsys[dir+"/distance"] = file(strings.Join(dist, " ") + "\n") + + if cpus.IsEmpty() { + continue + } + for _, cpu := range cpus.List() { + topo := fmt.Sprintf("sys/devices/system/cpu/cpu%d/topology", cpu) + fsys[topo+"/physical_package_id"] = file(fmt.Sprintf("%d\n", pkg)) + fsys[topo+"/core_id"] = file(fmt.Sprintf("%d\n", cpu)) + fsys[topo+"/core_cpus_list"] = file(fmt.Sprintf("%d\n", cpu)) + } + pkg++ + } + + all := online.String() + + // Every node with memory of its own has normal, i.e. non-movable, memory. + // Without this a node reads as movable-only, which nothing here wants to + // describe and which leaves an allocator with no memory to hand out. + fsys["sys/devices/system/node/has_normal_memory"] = file(normal.String() + "\n") + + fsys["proc/meminfo"] = file("MemTotal: 1048576 kB\n") + fsys["sys/devices/system/cpu/online"] = file(all + "\n") + fsys["sys/devices/system/cpu/present"] = file(all + "\n") + fsys["sys/devices/system/cpu/possible"] = file(all + "\n") + + m, err := hardware.Discover(hardware.WithFS(fsys)) + if err != nil { + t.Fatalf("failed to discover the test machine: %v", err) + } + return m +} + +// oneCpuMachine is the smallest machine there is: one CPU, one node, one +// package. Asking it about any other package or node finds nothing, which is +// what the cases for hints naming hardware the machine does not have need. +func oneCpuMachine(t *testing.T) *hardware.Machine { + t.Helper() + return synthMachine(t, []synthNode{ + {cpus: "0", memKB: 1048576, distance: []int{10}}, + }) +} diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index fdaea070e..8942f9e6d 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -22,344 +22,13 @@ import ( "github.com/containers/nri-plugins/pkg/agent/podresapi" resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1" "github.com/containers/nri-plugins/pkg/cpuallocator" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cache" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" - "github.com/intel/goresctrl/pkg/sst" - idset "github.com/intel/goresctrl/pkg/utils" v1 "k8s.io/api/core/v1" ) -type mockSystemNode struct { - id idset.ID // node id - memFree uint64 - memTotal uint64 - memType sysfs.MemoryType - distance []int -} - -func (fake *mockSystemNode) MemoryInfo() (*sysfs.MemInfo, error) { - return &sysfs.MemInfo{MemFree: fake.memFree, MemTotal: fake.memTotal}, nil -} - -func (fake *mockSystemNode) PackageID() idset.ID { - return 0 -} - -func (fake *mockSystemNode) DieID() idset.ID { - return 0 -} - -func (fake *mockSystemNode) ID() idset.ID { - return fake.id -} - -func (fake *mockSystemNode) GetMemoryType() sysfs.MemoryType { - return fake.memType -} - -func (fake *mockSystemNode) HasNormalMemory() bool { - return true -} - -func (fake *mockSystemNode) CPUSet() cpuset.CPUSet { - return cpuset.New() -} - -func (fake *mockSystemNode) Distance() []int { - if len(fake.distance) == 0 { - return []int{0} - } - return fake.distance -} - -func (fake *mockSystemNode) DistanceFrom(id idset.ID) int { - return 0 -} - -func (fake *mockSystemNode) ClosestNodes() ([]idset.IDSet, []int) { - return []idset.IDSet{}, []int{} -} - -type mockCPUPackage struct { -} - -func (p *mockCPUPackage) ID() idset.ID { - return idset.ID(0) -} - -func (p *mockCPUPackage) CPUSet() cpuset.CPUSet { - return cpuset.New() -} - -func (p *mockCPUPackage) NodeIDs() []idset.ID { - return []idset.ID{} -} - -func (p *mockCPUPackage) DieIDs() []idset.ID { - return []idset.ID{0} -} - -func (p *mockCPUPackage) DieCPUSet(idset.ID) cpuset.CPUSet { - return cpuset.New() -} - -func (p *mockCPUPackage) DieNodeIDs(idset.ID) []idset.ID { - return []idset.ID{} -} - -func (p *mockCPUPackage) DieClusterIDs(idset.ID) []idset.ID { - return []idset.ID{} -} - -func (p *mockCPUPackage) DieClusterCPUSet(idset.ID, idset.ID) cpuset.CPUSet { - return cpuset.New() -} - -func (p *mockCPUPackage) LogicalDieClusterIDs(idset.ID) []idset.ID { - return []idset.ID{} -} - -func (p *mockCPUPackage) LogicalDieClusterCPUSet(idset.ID, idset.ID) cpuset.CPUSet { - return cpuset.New() -} - -func (p *mockCPUPackage) L3CacheIDs() []idset.ID { - return []idset.ID{} -} - -func (p *mockCPUPackage) L3CacheCPUSet(idset.ID) cpuset.CPUSet { - return cpuset.New() -} - -func (p *mockCPUPackage) SstInfo() *sst.PackageStatus { - return &sst.PackageStatus{} -} - -type mockCPU struct { - id idset.ID - node mockSystemNode - pkg mockCPUPackage -} - -func (c *mockCPU) BaseFrequency() uint64 { - return 0 -} -func (c *mockCPU) EPP() sysfs.EPP { - return sysfs.EPPUnknown -} -func (c *mockCPU) ID() idset.ID { - return idset.ID(0) -} -func (c *mockCPU) PackageID() idset.ID { - return c.pkg.ID() -} -func (c *mockCPU) DieID() idset.ID { - return idset.ID(0) -} -func (c *mockCPU) NodeID() idset.ID { - return c.node.ID() -} -func (c *mockCPU) CoreID() idset.ID { - return c.id -} -func (c *mockCPU) ThreadCPUSet() cpuset.CPUSet { - return cpuset.New() -} -func (c *mockCPU) FrequencyRange() sysfs.CPUFreq { - return sysfs.CPUFreq{} -} -func (c *mockCPU) Online() bool { - return true -} -func (c *mockCPU) Isolated() bool { - return false -} -func (c *mockCPU) SetFrequencyLimits(min, max uint64) error { - return nil -} - -func (c *mockCPU) SstClos() int { - return -1 -} - -func (c *mockCPU) CacheCount() int { - return 0 -} -func (c *mockCPU) GetCaches() []*sysfs.Cache { - panic("unimplemented") -} -func (c *mockCPU) GetCachesByLevel(int) []*sysfs.Cache { - panic("unimplemented") -} -func (c *mockCPU) GetCacheByIndex(int) *sysfs.Cache { - panic("unimplemented") -} -func (c *mockCPU) GetNthLevelCacheCPUSet(n int) cpuset.CPUSet { - panic("unimplemented") -} -func (c *mockCPU) GetLastLevelCaches() []*sysfs.Cache { - panic("unimplemented") -} -func (c *mockCPU) GetLastLevelCacheCPUSet() cpuset.CPUSet { - panic("unimplemented") -} - -func (c *mockCPU) ClusterID() int { - return 0 -} - -func (c *mockCPU) CoreKind() sysfs.CoreKind { - return sysfs.PerformanceCore -} - -type mockSystem struct { - isolatedCPU int - nodes []sysfs.Node - cpuCount int - packageCount int - socketCount int -} - -func (fake *mockSystem) Node(id idset.ID) sysfs.Node { - for _, node := range fake.nodes { - if node.ID() == id { - return node - } - } - return &mockSystemNode{} -} - -func (fake *mockSystem) CPU(idset.ID) sysfs.CPU { - return &mockCPU{} -} -func (fake *mockSystem) CPUCount() int { - if fake.cpuCount == 0 { - return 1 - } - return fake.cpuCount -} -func (fake *mockSystem) Discover(flags sysfs.DiscoveryFlag) error { - return nil -} -func (fake *mockSystem) Package(idset.ID) sysfs.CPUPackage { - return &mockCPUPackage{} -} -func (fake *mockSystem) PossibleCPUs() cpuset.CPUSet { - return fake.CPUSet() -} -func (fake *mockSystem) PresentCPUs() cpuset.CPUSet { - return fake.CPUSet() -} -func (fake *mockSystem) OnlineCPUs() cpuset.CPUSet { - return fake.CPUSet() -} -func (fake *mockSystem) IsolatedCPUs() cpuset.CPUSet { - return fake.Isolated() -} -func (fake *mockSystem) OfflineCPUs() cpuset.CPUSet { - return cpuset.New() -} -func (fake *mockSystem) CoreKindCPUs(sysfs.CoreKind) cpuset.CPUSet { - return cpuset.New() -} -func (fake *mockSystem) CoreKinds() []sysfs.CoreKind { - return nil -} -func (fake *mockSystem) IDSetForCPUs(cpus cpuset.CPUSet, f func(cpu sysfs.CPU) idset.ID) idset.IDSet { - panic("unimplemented") -} -func (fake *mockSystem) AllThreadsForCPUs(cpuset.CPUSet) cpuset.CPUSet { - return cpuset.New() -} -func (fake *mockSystem) SingleThreadForCPUs(cpuset.CPUSet) cpuset.CPUSet { - return cpuset.New() -} -func (fake *mockSystem) AllCPUsSharingNthLevelCacheWithCPUs(int, cpuset.CPUSet) cpuset.CPUSet { - return cpuset.New() -} -func (fake *mockSystem) Offlined() cpuset.CPUSet { - return cpuset.New() -} -func (fake *mockSystem) Isolated() cpuset.CPUSet { - if fake.isolatedCPU > 0 { - return cpuset.New(fake.isolatedCPU) - } - - return cpuset.New() -} -func (fake *mockSystem) CPUSet() cpuset.CPUSet { - return cpuset.New() -} -func (fake *mockSystem) CPUIDs() []idset.ID { - return []idset.ID{} -} -func (fake *mockSystem) PackageCount() int { - if fake.packageCount == 0 { - return 1 - } - return fake.packageCount -} -func (fake *mockSystem) SocketCount() int { - if fake.socketCount == 0 { - return 1 - } - return fake.socketCount -} -func (fake *mockSystem) NUMANodeCount() int { - return len(fake.nodes) -} -func (fake *mockSystem) MinThreadCount() int { - return 2 -} -func (fake *mockSystem) MaxThreadCount() int { - return 2 -} -func (fake *mockSystem) PackageIDs() []idset.ID { - ids := make([]idset.ID, len(fake.nodes)) - for i, node := range fake.nodes { - ids[i] = node.PackageID() - } - return ids -} -func (fake *mockSystem) NodeIDs() []idset.ID { - ids := make([]idset.ID, len(fake.nodes)) - for i, node := range fake.nodes { - ids[i] = node.ID() - } - return ids -} - -func (fake *mockSystem) FilterNodes(ids []idset.ID, filters ...sysfs.NodeFilter) idset.IDSet { - return idset.NewIDSet() -} - -func (fake *mockSystem) FilterNode(id idset.ID, filters ...sysfs.NodeFilter) bool { - return true -} - -func (fake *mockSystem) ClosestNodes(id idset.ID, filters ...sysfs.NodeFilter) ([]idset.IDSet, []int) { - return []idset.IDSet{}, []int{} -} - -func (fake *mockSystem) SetCPUFrequencyLimits(min, max uint64, cpus idset.IDSet) error { - return nil -} -func (fake *mockSystem) SetCpusOnline(online bool, cpus idset.IDSet) (idset.IDSet, error) { - return idset.NewIDSet(), nil -} -func (fake *mockSystem) NodeDistance(idset.ID, idset.ID) int { - return 10 -} -func (fake *mockSystem) NodeHintToCPUs(string) string { - return "" -} -func (fake *mockSystem) Sst() *sst.Platform { - return nil -} - type mockContainer struct { name string namespace string From 3f6e1ca0e1fe98f555cfcf10682d077d5fc6d1db Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 14:44:42 +0300 Subject: [PATCH 24/39] utils/topology: take the machine for hint scoring. NewHint took the pkg/sysfs interface to resolve a hint's NUMA nodes and CPUs against. It takes a hardware.Machine now, so the package no longer depends on that interface. Its one caller reaches the topology through the pool node it is scoring, so node gains a Machine alongside its System. The policy already has a machine in its backend options; the System goes when the policy itself stops reading it. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- cmd/plugins/topology-aware/policy/node.go | 7 +++++ .../topology-aware/policy/resources.go | 2 +- pkg/utils/topology/hints.go | 29 +++++++++++++------ 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/node.go b/cmd/plugins/topology-aware/policy/node.go index 5c662e518..cc8c6b5a1 100644 --- a/cmd/plugins/topology-aware/policy/node.go +++ b/cmd/plugins/topology-aware/policy/node.go @@ -18,6 +18,7 @@ import ( "fmt" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -122,6 +123,7 @@ type Node interface { NodeHeight() int // System returns the policy sysfs instance. System() system.System + Machine() *hardware.Machine // Policy returns the policy back pointer. Policy() *policy // GetSupply returns the full CPU at this node. @@ -393,6 +395,11 @@ func (n *node) System() system.System { return n.policy.sys } +// Machine returns the policy's machine topology. +func (n *node) Machine() *hardware.Machine { + return n.policy.options.Machine +} + // Policy returns the policy back pointer. func (n *node) Policy() *policy { return n.policy diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index f2813d24e..9bbb5cf63 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -1086,7 +1086,7 @@ func (cr *request) verifyStrictTopologyHints(g Grant) error { } for _, h := range cr.GetContainer().GetTopologyHints() { - hint := topoutil.NewHint(g.GetCPUNode().System(), h) + hint := topoutil.NewHint(g.GetCPUNode().Machine(), h) if g.SharedPortion() > 0 { if cpus := hint.MisalignedCPUSet(g.SharedCPUs()); cpus.Size() > 0 { diff --git a/pkg/utils/topology/hints.go b/pkg/utils/topology/hints.go index df8621919..30816d9e0 100644 --- a/pkg/utils/topology/hints.go +++ b/pkg/utils/topology/hints.go @@ -15,7 +15,8 @@ package topology import ( - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -24,14 +25,14 @@ import ( type TopologyHint = topology.Hint type Hint struct { - sys sysfs.System - hint *TopologyHint + machine *hardware.Machine + hint *TopologyHint } -func NewHint(sys sysfs.System, h TopologyHint) *Hint { +func NewHint(m *hardware.Machine, h TopologyHint) *Hint { return &Hint{ - sys: sys, - hint: &h, + machine: m, + hint: &h, } } @@ -43,8 +44,8 @@ func (h *Hint) CPUSetForCPUs() cpuset.CPUSet { func (h *Hint) MemsForCPUs() libmem.NodeMask { mems := libmem.NewNodeMask() cset, _ := cpuset.Parse(h.hint.CPUs) - for _, id := range h.sys.NodeIDs() { - if !h.sys.Node(id).CPUSet().Intersection(cset).IsEmpty() { + for _, id := range h.machine.MemoryNodeIDs() { + if h.machine.MemoryNode(id).CPUs().Intersects(toCpuMask(cset)) { mems.Set(id) } } @@ -55,7 +56,7 @@ func (h *Hint) CPUSetForNUMAs() cpuset.CPUSet { cset := cpuset.New() mems, _ := cpuset.Parse(h.hint.NUMAs) for _, id := range mems.UnsortedList() { - cset = cset.Union(h.sys.Node(id).CPUSet()) + cset = cset.Union(toCpuSet(h.machine.MemoryNode(id).CPUs())) } return cset } @@ -86,3 +87,13 @@ func (h *Hint) MisalignedMems(mems libmem.NodeMask) libmem.NodeMask { } return misaligned } + +// toCpuSet and toCpuMask convert between the set the hardware package speaks and +// the one these hints are expressed in. +func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { + return cpuset.New(cpus.List()...) +} + +func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { + return libcpu.NewCpuMask(cpus.List()...) +} From aeb9c14058fad284f9c905c85f87cc21218ea0a8 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 14:54:51 +0300 Subject: [PATCH 25/39] topology-aware: read the topology from hardware.Machine. The policy took its topology from the pkg/sysfs interface. It reads the hardware.Machine the resource manager discovered instead, so it no longer depends on that interface at all. The questions it asks are all about one package or one die, and the hardware package addresses dies, clusters and caches by their full coordinates, since the kernel numbers them within their package. topology.go turns the former into the latter and keeps the conversions between the two set types in one place. The node filters the interface offered become predicates on a memory node there too. Two things which had been interface values are gone from the pool node. Its system.CPUPackage is the package zone, and what it was asked for -- the NUMA nodes of the package, and of one of its dies -- is a question about which nodes are local to a set of CPUs. Its system.Node is a hardware.MemoryNode. The tests which discovered a System beside a Machine now discover only the Machine, there being nothing left to read the former. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../topology-aware/policy/coldstart_test.go | 3 +- cmd/plugins/topology-aware/policy/hint.go | 14 +- .../topology-aware/policy/hint_test.go | 5 +- .../topology-aware/policy/libmem_test.go | 10 - .../topology-aware/policy/metrics_test.go | 6 - cmd/plugins/topology-aware/policy/node.go | 49 ++-- cmd/plugins/topology-aware/policy/pools.go | 100 ++++---- .../topology-aware/policy/pools_test.go | 16 -- .../topology-aware/policy/resources.go | 8 +- .../policy/topology-aware-policy.go | 22 +- cmd/plugins/topology-aware/policy/topology.go | 242 ++++++++++++++++++ 11 files changed, 339 insertions(+), 136 deletions(-) create mode 100644 cmd/plugins/topology-aware/policy/topology.go diff --git a/cmd/plugins/topology-aware/policy/coldstart_test.go b/cmd/plugins/topology-aware/policy/coldstart_test.go index ff3930726..d9c220c5c 100644 --- a/cmd/plugins/topology-aware/policy/coldstart_test.go +++ b/cmd/plugins/topology-aware/policy/coldstart_test.go @@ -20,7 +20,6 @@ import ( "testing" "time" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" @@ -92,7 +91,7 @@ func TestColdStart(t *testing.T) { t.Skipf("Coldstart tests are disabled (can't mock enough of the system, lacks CPUs)") policy := &policy{ - sys: system.FromMachine(synthMachine(t, tc.numaNodes)), + machine: synthMachine(t, tc.numaNodes), cache: &mockCache{ returnValue1ForLookupContainer: tc.container, returnValue2ForLookupContainer: true, diff --git a/cmd/plugins/topology-aware/policy/hint.go b/cmd/plugins/topology-aware/policy/hint.go index b189c0ad8..d639017f3 100644 --- a/cmd/plugins/topology-aware/policy/hint.go +++ b/cmd/plugins/topology-aware/policy/hint.go @@ -18,7 +18,7 @@ import ( "strconv" "strings" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" @@ -55,8 +55,8 @@ func numaHintScore(hint topology.Hint, sysIDs ...idset.ID) float64 { } // Calculate the die node score of the given hint and die. -func dieHintScore(hint topology.Hint, sysID idset.ID, socket system.CPUPackage) float64 { - numaNodes := idset.NewIDSet(socket.DieNodeIDs(sysID)...) +func dieHintScore(hint topology.Hint, m *hardware.Machine, pkg, die idset.ID) float64 { + numaNodes := idset.NewIDSet(dieNodeIDs(m, pkg, die)...) for idstr := range strings.SplitSeq(hint.NUMAs, ",") { hID, err := strconv.ParseInt(idstr, 0, 0) @@ -100,8 +100,8 @@ func (cs *supply) hintCpus(h topology.Hint) cpuset.CPUSet { case h.NUMAs != "": for idstr := range strings.SplitSeq(h.NUMAs, ",") { if id, err := strconv.ParseInt(idstr, 0, 0); err == nil { - if node := cs.node.System().Node(idset.ID(id)); node != nil { - cpus = cpus.Union(node.CPUSet()) + if node := cs.node.Machine().MemoryNode(idset.ID(id)); node.Valid() { + cpus = cpus.Union(toCpuSet(node.CPUs())) } } } @@ -109,8 +109,8 @@ func (cs *supply) hintCpus(h topology.Hint) cpuset.CPUSet { case h.Sockets != "": for idstr := range strings.SplitSeq(h.Sockets, ",") { if id, err := strconv.ParseInt(idstr, 0, 0); err == nil { - if pkg := cs.node.System().Package(idset.ID(id)); pkg != nil { - cpus = cpus.Union(pkg.CPUSet()) + if pkg := packageZone(cs.node.Machine(), idset.ID(id)); pkg != nil { + cpus = cpus.Union(toCpuSet(pkg.CPUs())) } } } diff --git a/cmd/plugins/topology-aware/policy/hint_test.go b/cmd/plugins/topology-aware/policy/hint_test.go index 849ef68c4..36b2233e9 100644 --- a/cmd/plugins/topology-aware/policy/hint_test.go +++ b/cmd/plugins/topology-aware/policy/hint_test.go @@ -17,7 +17,6 @@ package topologyaware import ( "testing" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" @@ -164,7 +163,7 @@ func TestHintCpus(t *testing.T) { supply: &supply{ node: &node{ policy: &policy{ - sys: system.FromMachine(oneCpuMachine(t)), + machine: oneCpuMachine(t), }, }, }, @@ -184,7 +183,7 @@ func TestHintCpus(t *testing.T) { supply: &supply{ node: &node{ policy: &policy{ - sys: system.FromMachine(oneCpuMachine(t)), + machine: oneCpuMachine(t), }, }, }, diff --git a/cmd/plugins/topology-aware/policy/libmem_test.go b/cmd/plugins/topology-aware/policy/libmem_test.go index 710c500cc..0ec3639c2 100644 --- a/cmd/plugins/topology-aware/policy/libmem_test.go +++ b/cmd/plugins/topology-aware/policy/libmem_test.go @@ -22,7 +22,6 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/testutils" ) @@ -42,14 +41,6 @@ func setupTestPolicy(t *testing.T) (*policy, string) { } sysPath := path.Join(dir, "sysfs", "server", "sys") - sys, err := system.DiscoverSystemAt(sysPath) - if err != nil { - if rerr := os.RemoveAll(dir); rerr != nil { - t.Logf("failed to remove temp dir %q: %v", dir, rerr) - } - t.Fatalf("failed to discover system: %v", err) - } - machine, err := hardware.Discover(hardware.WithRoot(path.Dir(sysPath))) if err != nil { if rerr := os.RemoveAll(dir); rerr != nil { @@ -61,7 +52,6 @@ func setupTestPolicy(t *testing.T) (*policy, string) { p := New().(*policy) if err := p.Setup(&policyapi.BackendOptions{ Cache: &mockCache{}, - System: sys, Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{cfgapi.CPU: "750m"}, diff --git a/cmd/plugins/topology-aware/policy/metrics_test.go b/cmd/plugins/topology-aware/policy/metrics_test.go index f518384a5..546eb449f 100644 --- a/cmd/plugins/topology-aware/policy/metrics_test.go +++ b/cmd/plugins/topology-aware/policy/metrics_test.go @@ -26,7 +26,6 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/metrics" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/testutils" @@ -93,10 +92,6 @@ func newServerPolicyWithMetrics(t *testing.T) (*policy, *TopologyAwareMetrics, * // The "server" sysfs yields a multi-zone topology, which lets us assert // "one exported series per zone". - sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) - if err != nil { - t.Fatalf("failed to discover system: %v", err) - } machine, err := hardware.Discover( hardware.WithRoot(path.Join(dir, "sysfs", "server"))) if err != nil { @@ -105,7 +100,6 @@ func newServerPolicyWithMetrics(t *testing.T) (*policy, *TopologyAwareMetrics, * opts := &policyapi.BackendOptions{ Cache: &mockCache{}, - System: sys, Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ diff --git a/cmd/plugins/topology-aware/policy/node.go b/cmd/plugins/topology-aware/policy/node.go index cc8c6b5a1..050bb25e1 100644 --- a/cmd/plugins/topology-aware/policy/node.go +++ b/cmd/plugins/topology-aware/policy/node.go @@ -19,7 +19,6 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" @@ -122,7 +121,6 @@ type Node interface { // Get the height of this node (inverse of depth: tree depth - node depth). NodeHeight() int // System returns the policy sysfs instance. - System() system.System Machine() *hardware.Machine // Policy returns the policy back pointer. Policy() *policy @@ -177,23 +175,23 @@ type nodeself struct { // socketnode represents a physical CPU package/socket in the system. type socketnode struct { - node // common node data - id idset.ID // NUMA node socket id - syspkg system.CPUPackage // corresponding system.Package + node // common node data + id idset.ID // NUMA node socket id + syspkg *hardware.Zone // corresponding package zone } // dienode represents a die within a physical CPU package/socket in the system. type dienode struct { - node // common node data - id idset.ID // die id within socket - syspkg system.CPUPackage // corresponding system.Package + node // common node data + id idset.ID // die id within socket + syspkg *hardware.Zone // corresponding package zone } // numanode represents a NUMA node in the system. type numanode struct { - node // common node data - id idset.ID // NUMA node system id - sysnode system.Node // corresponding system.Node + node // common node data + id idset.ID // NUMA node system id + sysnode *hardware.MemoryNode // corresponding memory node } // l3cachenode represents an L3 cache grouping of CPUs in the system. @@ -390,14 +388,9 @@ func (n *node) BreadthFirst(fn func(Node) bool) bool { return false } -// System returns the policy System instance. -func (n *node) System() system.System { - return n.policy.sys -} - // Machine returns the policy's machine topology. func (n *node) Machine() *hardware.Machine { - return n.policy.options.Machine + return n.policy.machine } // Policy returns the policy back pointer. @@ -482,7 +475,7 @@ func (p *policy) NewNumaNode(id idset.ID, parent Node) *numanode { n.self.node = n n.init(p, fmt.Sprintf("NUMA node #%v", id), NumaNode, parent) n.id = id - n.sysnode = p.sys.Node(id) + n.sysnode = p.machine.MemoryNode(id) return n } @@ -522,7 +515,7 @@ func (n *numanode) GetMemset(mtype memoryType) idset.IDSet { func (n *numanode) HintScore(hint topology.Hint) float64 { switch { case hint.CPUs != "": - return cpuHintScore(hint, n.sysnode.CPUSet()) + return cpuHintScore(hint, toCpuSet(n.sysnode.CPUs())) case hint.NUMAs != "": return numaHintScore(hint, n.id) @@ -532,7 +525,7 @@ func (n *numanode) HintScore(hint topology.Hint) float64 { score := socketHintScore(hint, n.sysnode.PackageID()) if score > 0.0 { // penalize underfit reciprocally (inverse-proportionally) to the socket size - score /= float64(len(n.System().Package(pkgID).NodeIDs())) + score /= float64(len(packageNodeIDs(n.Machine(), pkgID))) } return score } @@ -625,7 +618,7 @@ func (p *policy) NewDieNode(id idset.ID, parent Node) *dienode { n.self.node = n n.init(p, fmt.Sprintf("die #%v/%v", pkg.id, id), DieNode, parent) n.id = id - n.syspkg = p.sys.Package(pkg.id) + n.syspkg = packageZone(p.machine, pkg.id) return n } @@ -671,16 +664,16 @@ func (n *dienode) GetMemset(mtype memoryType) idset.IDSet { func (n *dienode) HintScore(hint topology.Hint) float64 { switch { case hint.CPUs != "": - return cpuHintScore(hint, n.syspkg.CPUSet()) + return cpuHintScore(hint, toCpuSet(n.syspkg.CPUs())) case hint.NUMAs != "": - return OverfitPenalty * dieHintScore(hint, n.id, n.syspkg) + return OverfitPenalty * dieHintScore(hint, n.Machine(), n.syspkg.ID(), n.id) case hint.Sockets != "": score := socketHintScore(hint, n.syspkg.ID()) if score > 0.0 { // penalize underfit reciprocally (inverse-proportionally) to the socket size in dies - score /= float64(len(n.syspkg.DieNodeIDs(n.id))) + score /= float64(len(dieNodeIDs(n.Machine(), n.syspkg.ID(), n.id))) } return score } @@ -694,7 +687,7 @@ func (p *policy) NewSocketNode(id idset.ID, parent Node) *socketnode { n.self.node = n n.init(p, fmt.Sprintf("socket #%v", id), SocketNode, parent) n.id = id - n.syspkg = p.sys.Package(id) + n.syspkg = packageZone(p.machine, id) return n } @@ -740,10 +733,10 @@ func (n *socketnode) GetMemset(mtype memoryType) idset.IDSet { func (n *socketnode) HintScore(hint topology.Hint) float64 { switch { case hint.CPUs != "": - return cpuHintScore(hint, n.syspkg.CPUSet()) + return cpuHintScore(hint, toCpuSet(n.syspkg.CPUs())) case hint.NUMAs != "": - return OverfitPenalty * numaHintScore(hint, n.syspkg.NodeIDs()...) + return OverfitPenalty * numaHintScore(hint, packageNodeIDs(n.Machine(), n.syspkg.ID())...) case hint.Sockets != "": return socketHintScore(hint, n.id) @@ -793,7 +786,7 @@ func (n *virtualnode) HintScore(hint topology.Hint) float64 { // don't bother calculating any scores, the root should always score 1.0 switch { case hint.CPUs != "": - return cpuHintScore(hint, n.System().CPUSet()) + return cpuHintScore(hint, toCpuSet(n.Machine().PresentCPUs())) case hint.NUMAs != "": return OverfitPenalty * OverfitPenalty diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index 95fcd40cb..0e08f174a 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -20,12 +20,12 @@ import ( "sort" "strings" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/utils/cpuset" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/dra" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" @@ -91,7 +91,7 @@ func (p *policy) buildRootPool() { vroot *virtualnode ) - if p.sys.SocketCount() > 1 { + if len(p.machine.Zones(hardware.LevelPackage)) > 1 { vroot = p.NewVirtualNode("root", nilnode) p.nodes[vroot.Name()] = vroot @@ -100,14 +100,14 @@ func (p *policy) buildRootPool() { log.Infof("+ created pool %s", vroot.Name()) - cpus := p.sys.CPUSet() + cpus := toCpuSet(p.machine.PresentCPUs()) vroot.noderes, vroot.freeres = p.getCpuSupply(vroot, cpus) vroot.mem, vroot.pMem, vroot.hbm = p.getMemSupply(vroot, cpus) } else { log.Infof("- omitted virtual root pool (single socket HW)") } - for _, socketID := range p.sys.PackageIDs() { + for _, socketID := range p.machine.TopologyIndex().PackageIDs() { p.buildSocketPool(socketID, root) } } @@ -123,11 +123,11 @@ func (p *policy) buildSocketPool(socketID idset.ID, root Node) { log.Infof("+ created pool %s", socket.Name()) - cpus := p.sys.Package(socketID).CPUSet() + cpus := packageCPUs(p.machine, socketID) socket.noderes, socket.freeres = p.getCpuSupply(socket, cpus) socket.mem, socket.pMem, socket.hbm = p.getMemSupply(socket, cpus) - dieIDs := p.sys.Package(socketID).DieIDs() + dieIDs := dieIDs(p.machine, socketID) omitDies := len(dieIDs) <= 1 if omitDies { log.Infof("- omitted die pools (only one die)") @@ -135,7 +135,7 @@ func (p *policy) buildSocketPool(socketID idset.ID, root Node) { dieIsCluster := true for _, dieID := range dieIDs { - clusterIDs := p.sys.Package(socketID).DieClusterIDs(dieID) + clusterIDs := clusterIDs(p.machine, socketID, dieID) if len(clusterIDs) > 1 { dieIsCluster = false break @@ -151,14 +151,14 @@ func (p *policy) buildSocketPool(socketID idset.ID, root Node) { p.buildDiePool(socketID, dieID, socket) } } else { - if nodeIDs := p.sys.Package(socketID).NodeIDs(); len(nodeIDs) > 1 { + if nodeIDs := packageNodeIDs(p.machine, socketID); len(nodeIDs) > 1 { for _, nodeID := range nodeIDs { p.buildNumaNodePool(socketID, nodeID, socket) } } else { if l3CacheIDs := p.getL3CacheIDsForCPUs(socketID, cpus); len(l3CacheIDs) > 1 { for _, l3CacheID := range l3CacheIDs { - l3CacheCPUs := p.sys.Package(socketID).L3CacheCPUSet(l3CacheID) + l3CacheCPUs := l3CacheCPUs(p.machine, socketID, l3CacheID) p.buildL3CachePool(l3CacheID, l3CacheCPUs, socket) } } @@ -173,11 +173,11 @@ func (p *policy) buildDiePool(socketID, dieID idset.ID, socket Node) { log.Infof("+ created pool %s", die.Name()) - cpus := p.sys.Package(socketID).DieCPUSet(dieID) + cpus := dieCPUs(p.machine, socketID, dieID) die.noderes, die.freeres = p.getCpuSupply(die, cpus) die.mem, die.pMem, die.hbm = p.getMemSupply(die, cpus) - nodeIDs := p.sys.Package(socketID).DieNodeIDs(dieID) + nodeIDs := dieNodeIDs(p.machine, socketID, dieID) if len(nodeIDs) > 1 { for _, nodeID := range nodeIDs { p.buildNumaNodePool(socketID, nodeID, die) @@ -185,7 +185,7 @@ func (p *policy) buildDiePool(socketID, dieID idset.ID, socket Node) { } else { if l3CacheIDs := p.getL3CacheIDsForCPUs(socketID, cpus); len(l3CacheIDs) > 1 { for _, l3CacheID := range l3CacheIDs { - l3CacheCPUs := p.sys.Package(socketID).L3CacheCPUSet(l3CacheID) + l3CacheCPUs := l3CacheCPUs(p.machine, socketID, l3CacheID) p.buildL3CachePool(l3CacheID, l3CacheCPUs, die) } } @@ -193,7 +193,7 @@ func (p *policy) buildDiePool(socketID, dieID idset.ID, socket Node) { } func (p *policy) buildNumaNodePool(socketID, nodeID idset.ID, parent Node) { - if mi, _ := p.sys.Node(nodeID).MemoryInfo(); mi != nil && mi.MemTotal == 0 { + if info, err := p.machine.MemoryNode(nodeID).Usage(); err == nil && info.Total == 0 { // Notes: // We only get called for NUMA nodes with some CPU locality. Then // if we have no attached memory, we have here a bunch of CPUs for @@ -211,14 +211,14 @@ func (p *policy) buildNumaNodePool(socketID, nodeID idset.ID, parent Node) { log.Infof("+ created pool %s", node.Name()) - cpus := p.sys.Node(nodeID).CPUSet() + cpus := toCpuSet(p.machine.MemoryNode(nodeID).CPUs()) node.noderes, node.freeres = p.getCpuSupply(node, cpus) node.mem, node.pMem, node.hbm = p.getMemSupply(node, cpus) // Check for L3 cache groups within this NUMA node if l3CacheIDs := p.getL3CacheIDsForCPUs(socketID, cpus); len(l3CacheIDs) > 1 { for _, l3CacheID := range l3CacheIDs { - l3CacheCPUs := p.sys.Package(socketID).L3CacheCPUSet(l3CacheID) + l3CacheCPUs := l3CacheCPUs(p.machine, socketID, l3CacheID) p.buildL3CachePool(l3CacheID, l3CacheCPUs, node) } } @@ -226,15 +226,15 @@ func (p *policy) buildNumaNodePool(socketID, nodeID idset.ID, parent Node) { // getL3CacheIDsForCPUs returns L3 cache IDs that are within the given CPU set scope. func (p *policy) getL3CacheIDsForCPUs(socketID idset.ID, cpus cpuset.CPUSet) []idset.ID { - var l3CacheIDs []idset.ID - for _, l3CacheID := range p.sys.Package(socketID).L3CacheIDs() { - l3CacheCPUs := p.sys.Package(socketID).L3CacheCPUSet(l3CacheID) + var within []idset.ID + for _, l3CacheID := range l3CacheIDs(p.machine, socketID) { + cacheCPUs := l3CacheCPUs(p.machine, socketID, l3CacheID) // Check if this L3 cache is entirely within the given CPU scope - if cpus.Intersection(l3CacheCPUs).Equals(l3CacheCPUs) { - l3CacheIDs = append(l3CacheIDs, l3CacheID) + if cpus.Intersection(cacheCPUs).Equals(cacheCPUs) { + within = append(within, l3CacheID) } } - return l3CacheIDs + return within } // buildL3CachePool creates an L3 cache pool as a child of the given parent. @@ -320,9 +320,9 @@ func (p *policy) getMemSupply(node Node, cpus cpuset.CPUSet) (dram, pmem, hbm id func (p *policy) getMemsForCpus(cpus cpuset.CPUSet) idset.IDSet { mems := idset.NewIDSet() - for _, nodeID := range p.sys.NodeIDs() { - node := p.sys.Node(nodeID) - if !node.CPUSet().Intersection(cpus).IsEmpty() { + for _, nodeID := range p.machine.MemoryNodeIDs() { + node := p.machine.MemoryNode(nodeID) + if node.CPUs().Intersects(toCpuMask(cpus)) { mems.Add(nodeID) } } @@ -333,23 +333,23 @@ func (p *policy) getMemsForCpus(cpus cpuset.CPUSet) idset.IDSet { func (p *policy) getClosestSpecialMem(mems idset.IDSet) idset.IDSet { var ( special = idset.NewIDSet() - nodeIDs = p.sys.NodeIDs() + nodeIDs = p.machine.MemoryNodeIDs() - pmemNoCPU = []system.NodeFilter{ - system.NodeOfPMEMType, - system.NodeHasMemory, - system.NodeHasNoLocalCPUs, + pmemNoCPU = []nodeFilter{ + nodeOfPMEMKind, + nodeHasMemory, + nodeHasNoLocalCPUs, } - hbmNoCPU = []system.NodeFilter{ - system.NodeOfHBMType, - system.NodeHasMemory, - system.NodeHasNoLocalCPUs, + hbmNoCPU = []nodeFilter{ + nodeOfHBMKind, + nodeHasMemory, + nodeHasNoLocalCPUs, } ) - for _, id := range p.sys.FilterNodes(nodeIDs, pmemNoCPU...).Members() { - closest, _ := p.sys.ClosestNodes(id, system.NodeOfDRAMType, system.NodeHasLocalCPUs) + for _, id := range filterNodes(p.machine, nodeIDs, pmemNoCPU...).Members() { + closest, _ := closestNodes(p.machine, id, nodeOfDRAMKind, nodeHasLocalCPUs) if len(closest) > 0 { for _, cid := range closest[0].Members() { if mems.Has(cid) { @@ -359,8 +359,8 @@ func (p *policy) getClosestSpecialMem(mems idset.IDSet) idset.IDSet { } } - for _, id := range p.sys.FilterNodes(nodeIDs, hbmNoCPU...).Members() { - closest, _ := p.sys.ClosestNodes(id, system.NodeOfDRAMType, system.NodeHasLocalCPUs) + for _, id := range filterNodes(p.machine, nodeIDs, hbmNoCPU...).Members() { + closest, _ := closestNodes(p.machine, id, nodeOfDRAMKind, nodeHasLocalCPUs) if len(closest) > 0 { for _, cid := range closest[0].Members() { if mems.Has(cid) { @@ -374,21 +374,23 @@ func (p *policy) getClosestSpecialMem(mems idset.IDSet) idset.IDSet { } func (p *policy) getAllMems() idset.IDSet { - return p.sys.FilterNodes(p.sys.NodeIDs(), system.NodeHasMemory) + return filterNodes(p.machine, p.machine.MemoryNodeIDs(), nodeHasMemory) } func (p *policy) splitMemsByType(ids idset.IDSet) (dram, pmem, hbm idset.IDSet) { dram, pmem, hbm = idset.NewIDSet(), idset.NewIDSet(), idset.NewIDSet() for _, id := range ids.Members() { - node := p.sys.Node(id) - switch node.GetMemoryType() { - case system.MemoryTypeDRAM: - dram.Add(id) - case system.MemoryTypePMEM: + node := p.machine.MemoryNode(id) + switch node.Kind() { + case hardware.MemoryKindPMEM: pmem.Add(id) - case system.MemoryTypeHBM: + case hardware.MemoryKindHBM: hbm.Add(id) + default: + // DRAM, and a node the hardware package could not classify: the + // pkg/sysfs interface reported those as DRAM too. + dram.Add(id) } } @@ -398,10 +400,10 @@ func (p *policy) splitMemsByType(ids idset.IDSet) (dram, pmem, hbm idset.IDSet) // checkHWTopology verifies our otherwise implicit assumptions about the HW. func (p *policy) checkHWTopology() error { // NUMA distance matrix should be symmetric. - for _, from := range p.sys.NodeIDs() { - for _, to := range p.sys.NodeIDs() { - d1 := p.sys.NodeDistance(from, to) - d2 := p.sys.NodeDistance(to, from) + for _, from := range p.machine.MemoryNodeIDs() { + for _, to := range p.machine.MemoryNodeIDs() { + d1 := p.machine.MemoryNode(from).Distance(to) + d2 := p.machine.MemoryNode(to).Distance(from) if d1 != d2 { log.Errorf("asymmetric NUMA distance (#%d, #%d): %d != %d", from, to, d1, d2) @@ -524,7 +526,7 @@ func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated, pr hidingInfo := "" pod, ok := container.GetPod() if ok && hideHyperthreadsPreference(pod, container) { - allow = p.sys.SingleThreadForCPUs(allocated) + allow = toCpuSet(hardware.SingleThreadPerCore(p.machine, toCpuMask(allocated))) if allow.Size() != allocated.Size() { hidingInfo = fmt.Sprintf(" (hide %d hyperthreads, remaining cpuset: %s)", allocated.Size()-allow.Size(), allow) } else { diff --git a/cmd/plugins/topology-aware/policy/pools_test.go b/cmd/plugins/topology-aware/policy/pools_test.go index 2f02e8aff..b6e94f255 100644 --- a/cmd/plugins/topology-aware/policy/pools_test.go +++ b/cmd/plugins/topology-aware/policy/pools_test.go @@ -29,7 +29,6 @@ import ( policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" "github.com/containers/nri-plugins/pkg/testutils" "github.com/containers/nri-plugins/pkg/utils/cpuset" ) @@ -135,10 +134,6 @@ func TestPoolCreation(t *testing.T) { } for _, tc := range tcases { t.Run(tc.name, func(t *testing.T) { - sys, err := system.DiscoverSystemAt(tc.path) - if err != nil { - panic(err) - } machine, err := hardware.Discover(hardware.WithRoot(path.Dir(tc.path))) if err != nil { panic(err) @@ -146,7 +141,6 @@ func TestPoolCreation(t *testing.T) { policyOptions := &policyapi.BackendOptions{ Cache: &mockCache{}, - System: sys, Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ @@ -265,10 +259,6 @@ func TestWorkloadPlacement(t *testing.T) { } for _, tc := range tcases { t.Run(tc.name, func(t *testing.T) { - sys, err := system.DiscoverSystemAt(tc.path) - if err != nil { - panic(err) - } machine, err := hardware.Discover(hardware.WithRoot(path.Dir(tc.path))) if err != nil { panic(err) @@ -276,7 +266,6 @@ func TestWorkloadPlacement(t *testing.T) { policyOptions := &policyapi.BackendOptions{ Cache: &mockCache{}, - System: sys, Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ @@ -530,10 +519,6 @@ func TestAffinities(t *testing.T) { for _, tc := range tcases { t.Run(tc.name, func(t *testing.T) { - sys, err := system.DiscoverSystemAt(tc.path) - if err != nil { - panic(err) - } machine, err := hardware.Discover(hardware.WithRoot(path.Dir(tc.path))) if err != nil { panic(err) @@ -541,7 +526,6 @@ func TestAffinities(t *testing.T) { policyOptions := &policyapi.BackendOptions{ Cache: &mockCache{}, - System: sys, Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 9bbb5cf63..b3a5d9d44 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -23,7 +23,7 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/containers/nri-plugins/pkg/agent/podresapi" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -1175,7 +1175,7 @@ func (cs *supply) GetScore(req Request) Score { // calculate fractional capacity score.shared -= part - lpCPUs := cs.GetNode().System().CoreKindCPUs(sysfs.EfficientCore) + lpCPUs := toCpuSet(cs.GetNode().Machine().CoreKindCPUs(hardware.EfficientCore)) if lpCPUs.Size() == 0 { lpCPUs = p.cpuAllocator.GetCPUPriorities()[lowPrio] } @@ -1183,7 +1183,7 @@ func (cs *supply) GetScore(req Request) Score { lpCnt := lpCPUs.Size() score.prio[lowPrio] = lpCnt*1000 - (1000*full + part) - hpCPUs := cs.GetNode().System().CoreKindCPUs(sysfs.PerformanceCore) + hpCPUs := toCpuSet(cs.GetNode().Machine().CoreKindCPUs(hardware.PerformanceCore)) if hpCPUs.Size() == 0 { hpCPUs = p.cpuAllocator.GetCPUPriorities()[highPrio] } @@ -1223,7 +1223,7 @@ func (cs *supply) GetScore(req Request) Score { // calculate real hint scores hints := cr.container.GetTopologyHints() - hints.ResolvePartialHints(cs.GetNode().System().NodeHintToCPUs) + hints.ResolvePartialHints(nodeHintToCPUs(cs.GetNode().Machine())) score.hints = make(map[string]float64, len(hints)) for provider, hint := range cr.container.GetTopologyHints() { diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index 7846e7542..60ed203c9 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/containers/nri-plugins/pkg/irq" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/utils/cpuset" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" @@ -32,7 +33,6 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/events" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" ) @@ -61,7 +61,7 @@ type policy struct { options *policyapi.BackendOptions // options we were created or reconfigured with cfg *cfgapi.Config cache cache.Cache // pod/container cache - sys system.System // system/HW topology info + machine *hardware.Machine // CPU and memory topology allowed cpuset.CPUSet // bounding set of CPUs we're allowed to use reserved cpuset.CPUSet // system-/kube-reserved CPUs reserveCnt int // number of CPUs to reserve if given as resource.Quantity @@ -162,7 +162,7 @@ func (p *policy) Setup(opts *policyapi.BackendOptions) error { p.cfg = cfg p.cache = opts.Cache - p.sys = opts.System + p.machine = opts.Machine p.options = opts p.cpuAllocator = cpuallocator.NewCPUAllocator(opts.Machine) p.memAllocator, err = libmem.NewAllocator(libmem.WithMachineNodes(opts.Machine)) @@ -810,7 +810,7 @@ func (p *policy) initialize() error { opt.UnlimitedBurstable = p.findExistingTopologyLevel(opt.UnlimitedBurstable) if len(opt.CPUClasses) > 0 { - cc, err := cpuclass.New(p.options.Machine) + cc, err := cpuclass.New(p.machine) if err != nil { return policyError("failed to create CPU class handler: %w", err) } @@ -855,18 +855,18 @@ func (p *policy) checkConstraints() error { if err != nil { return fmt.Errorf("failed to parse available CPU cpuset '%s': %w", amount, err) } - p.allowed = p.sys.CPUSet().Difference(cset) + p.allowed = toCpuSet(p.machine.PresentCPUs()).Difference(cset) case cfgapi.AmountQuantity: return fmt.Errorf("can't handle CPU resources given as resource.Quantity (%v)", amount) case cfgapi.AmountAbsent: // Available CPUs not specified, default to system CPUs. - p.allowed = p.sys.CPUSet() + p.allowed = toCpuSet(p.machine.PresentCPUs()) } // Allocation of only online CPUs is allowed. - p.allowed = p.allowed.Intersection(p.sys.OnlineCPUs()) + p.allowed = p.allowed.Intersection(toCpuSet(p.machine.OnlineCPUs())) - p.isolated = p.sys.Isolated().Intersection(p.allowed) + p.isolated = toCpuSet(p.machine.IsolatedCPUs()).Intersection(p.allowed) amount, kind = p.cfg.ReservedResources.Get(cfgapi.CPU) switch kind { @@ -1113,9 +1113,9 @@ func (p *policy) reapplyDRAClaims() { } func (p *policy) checkColdstartOff() { - for _, id := range p.sys.NodeIDs() { - node := p.sys.Node(id) - if node.GetMemoryType() == system.MemoryTypePMEM { + for _, id := range p.machine.MemoryNodeIDs() { + node := p.machine.MemoryNode(id) + if node.Kind() == hardware.MemoryKindPMEM { if !node.HasNormalMemory() { coldStartOff = true log.Errorf("coldstart forced off: NUMA node #%d does not have normal memory", id) diff --git a/cmd/plugins/topology-aware/policy/topology.go b/cmd/plugins/topology-aware/policy/topology.go new file mode 100644 index 000000000..f9b082d57 --- /dev/null +++ b/cmd/plugins/topology-aware/policy/topology.go @@ -0,0 +1,242 @@ +// Copyright The NRI Plugins Authors. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package topologyaware + +import ( + "slices" + + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/utils/cpuset" + idset "github.com/intel/goresctrl/pkg/utils" +) + +// toCpuSet and toCpuMask convert between the set the hardware package speaks and +// the one this policy is written in. They are the seam left by moving the policy +// onto hardware without rewriting its pool arithmetic. +func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { + return cpuset.New(cpus.List()...) +} + +func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { + return libcpu.NewCpuMask(cpus.List()...) +} + +// +// Packages, dies, clusters and caches +// +// The hardware package addresses dies, clusters and caches by their full +// coordinates, since the kernel numbers them within their package. These turn the +// questions this policy asks -- which are all "of this package" or "of this die" +// -- into those coordinates. +// + +// packageZone returns the zone of one CPU package, or nil if the machine has no +// such package. +func packageZone(m *hardware.Machine, pkg idset.ID) *hardware.Zone { + for _, z := range m.Zones(hardware.LevelPackage) { + if z.ID() == pkg { + return z + } + } + return nil +} + +// packageCPUs returns the CPUs of one package. +func packageCPUs(m *hardware.Machine, pkg idset.ID) cpuset.CPUSet { + return toCpuSet(m.TopologyIndex().PackageCPUs(pkg)) +} + +// packageNodeIDs returns the NUMA nodes whose CPUs are in one package. +func packageNodeIDs(m *hardware.Machine, pkg idset.ID) []idset.ID { + return sortedIDs(hardware.MemoryNodesFor(m, m.TopologyIndex().PackageCPUs(pkg))) +} + +// dieIDs returns the die numbers of one package, in increasing order. +func dieIDs(m *hardware.Machine, pkg idset.ID) []idset.ID { + var ids []idset.ID + for _, die := range m.TopologyIndex().DieIDs(pkg) { + ids = append(ids, die.Die) + } + return ids +} + +// dieCPUs returns the CPUs of one die of one package. +func dieCPUs(m *hardware.Machine, pkg, die idset.ID) cpuset.CPUSet { + return toCpuSet(m.TopologyIndex().DieCPUs(hardware.DieID{ + Package: pkg, + Die: die, + })) +} + +// dieNodeIDs returns the NUMA nodes whose CPUs are on one die of one package. +func dieNodeIDs(m *hardware.Machine, pkg, die idset.ID) []idset.ID { + cpus := m.TopologyIndex().DieCPUs(hardware.DieID{Package: pkg, Die: die}) + return sortedIDs(hardware.MemoryNodesFor(m, cpus)) +} + +// clusterIDs returns the cluster numbers of one die of one package, in increasing +// order. These are the clusters the kernel reports, not the logical ones: this +// policy uses them to decide whether the cluster level says anything, and merging +// would hide a die whose every cluster is a single core. +func clusterIDs(m *hardware.Machine, pkg, die idset.ID) []idset.ID { + var ids []idset.ID + for _, cl := range m.TopologyIndex().ClusterIDs(hardware.DieID{ + Package: pkg, + Die: die, + }) { + ids = append(ids, cl.Cluster) + } + return ids +} + +// l3CacheIDs returns the ids of the level 3 caches this package's CPUs use. +func l3CacheIDs(m *hardware.Machine, pkg idset.ID) []idset.ID { + var ids []idset.ID + for _, z := range l3CacheZones(m, pkg) { + ids = append(ids, z.ID()) + } + slices.Sort(ids) + return ids +} + +// l3CacheCPUs returns every CPU sharing one level 3 cache of this package, +// including any outside the package: a cache shared across packages belongs to +// both, and the whole of its CPU set is what it groups. +func l3CacheCPUs(m *hardware.Machine, pkg, cache idset.ID) cpuset.CPUSet { + for _, z := range l3CacheZones(m, pkg) { + if z.ID() == cache { + return toCpuSet(z.CPUs()) + } + } + return cpuset.New() +} + +// l3CacheZones returns the level 3 cache zones this package's CPUs use. +func l3CacheZones(m *hardware.Machine, pkg idset.ID) []*hardware.Zone { + return hardware.ZonesOverlapping(m, hardware.LevelL3Cache, + m.TopologyIndex().PackageCPUs(pkg)) +} + +// +// NUMA nodes +// + +// nodeFilter is a predicate on a memory node, replacing the filters the pkg/sysfs +// interface offered. +type nodeFilter func(*hardware.MemoryNode) bool + +var ( + // nodeHasMemory passes a node with some memory of its own. + nodeHasMemory = func(n *hardware.MemoryNode) bool { return n.HasMemory() } + // nodeHasLocalCPUs passes a node with CPUs of its own. + nodeHasLocalCPUs = func(n *hardware.MemoryNode) bool { return !n.CPUs().IsEmpty() } + // nodeHasNoLocalCPUs passes a node with none. + nodeHasNoLocalCPUs = func(n *hardware.MemoryNode) bool { return n.CPUs().IsEmpty() } + // nodeOfPMEMKind and nodeOfHBMKind pass a node of that kind. + nodeOfPMEMKind = nodeOfKind(hardware.MemoryKindPMEM) + nodeOfHBMKind = nodeOfKind(hardware.MemoryKindHBM) + // nodeOfDRAMKind passes a node of ordinary memory. A node the hardware + // package could not classify counts as one: every node with CPUs is + // classified, so an unknown one has none, and this is only ever asked + // together with nodeHasLocalCPUs. + nodeOfDRAMKind = func(n *hardware.MemoryNode) bool { + return n.Kind() == hardware.MemoryKindDRAM || + n.Kind() == hardware.MemoryKindUnknown + } +) + +// nodeOfKind returns a filter passing nodes of the given memory kind. +func nodeOfKind(kind hardware.MemoryKind) nodeFilter { + return func(n *hardware.MemoryNode) bool { return n.Kind() == kind } +} + +// filterNodes returns those of the given nodes which pass every filter. A node +// the machine does not have passes nothing. +func filterNodes(m *hardware.Machine, ids []idset.ID, filters ...nodeFilter) idset.IDSet { + out := idset.NewIDSet() + + for _, id := range ids { + node := m.MemoryNode(id) + if !node.Valid() { + continue + } + if !nodePasses(node, filters...) { + continue + } + out.Add(id) + } + + return out +} + +// nodePasses reports whether a node passes every filter. +func nodePasses(node *hardware.MemoryNode, filters ...nodeFilter) bool { + for _, pass := range filters { + if !pass(node) { + return false + } + } + return true +} + +// closestNodes returns the nodes passing every filter grouped by how far they are +// from the given one, nearest first. +func closestNodes( + m *hardware.Machine, from idset.ID, filters ...nodeFilter, +) ([]idset.IDSet, []int) { + groups := hardware.ClosestMemoryNodes(m, from, func(n *hardware.MemoryNode) bool { + return nodePasses(n, filters...) + }) + + nodes := make([]idset.IDSet, 0, len(groups)) + distances := make([]int, 0, len(groups)) + for _, g := range groups { + nodes = append(nodes, idset.NewIDSet(g.Nodes...)) + distances = append(distances, g.Distance) + } + + return nodes, distances +} + +// sortedIDs returns ids in increasing order, never nil. +func sortedIDs(ids []idset.ID) []idset.ID { + out := slices.Clone(ids) + if out == nil { + out = []idset.ID{} + } + slices.Sort(out) + return out +} + +// nodeHintToCPUs turns a topology hint's list of NUMA nodes into the online CPUs +// of those nodes, as a cpuset string. An unparsable list yields nothing. +func nodeHintToCPUs(m *hardware.Machine) func(string) string { + return func(nodes string) string { + mems, err := cpuset.Parse(nodes) + if err != nil { + return "" + } + + cpus := cpuset.New() + for _, id := range mems.List() { + if node := m.MemoryNode(id); node.Valid() { + cpus = cpus.Union(toCpuSet(node.CPUs())) + } + } + + return cpus.Intersection(toCpuSet(m.OnlineCPUs())).String() + } +} From b72db2b3030ca7d1c9c9a285d24d52f9d4828183 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 14:56:26 +0300 Subject: [PATCH 26/39] topology-aware: test hints against a machine which has sockets. The hint tests could only check that a socket or NUMA hint naming something the machine does not have resolves to nothing. A TODO beside them asked for the other half and said why it was missing: the package could not be constructed, being a closed struct behind an interface. A machine can be described and discovered, so it can. Two sockets of two CPUs, one NUMA node each, and a hint naming either resolves to that socket's or that node's CPUs. This is the case which would have caught getting the package lookup wrong, as opposed to getting a miss wrong. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../topology-aware/policy/hint_test.go | 32 +++++++++++++++++-- .../topology-aware/policy/machine_test.go | 10 ++++++ 2 files changed, 39 insertions(+), 3 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/hint_test.go b/cmd/plugins/topology-aware/policy/hint_test.go index 36b2233e9..55cb1e219 100644 --- a/cmd/plugins/topology-aware/policy/hint_test.go +++ b/cmd/plugins/topology-aware/policy/hint_test.go @@ -159,7 +159,7 @@ func TestHintCpus(t *testing.T) { }, }, { - name: "non-zero Sockets hint and empty system.Package", + name: "Sockets hint naming a socket the machine does not have", supply: &supply{ node: &node{ policy: &policy{ @@ -179,7 +179,7 @@ func TestHintCpus(t *testing.T) { }, }, { - name: "non-zero NUMAs hint and empty system.Node", + name: "NUMAs hint naming a node the machine does not have", supply: &supply{ node: &node{ policy: &policy{ @@ -191,7 +191,33 @@ func TestHintCpus(t *testing.T) { NUMAs: "1", }, }, - // TODO(rojkov): add tests for non-empty system.Package's (can't be done while system.Package is closed struct) + { + // Two packages of two CPUs, one NUMA node each. A hint naming a + // socket resolves to that socket's CPUs, and one naming a NUMA node + // to that node's. + name: "Sockets hint resolves to the socket's CPUs", + supply: &supply{ + node: &node{ + policy: &policy{machine: twoSocketMachine(t)}, + }, + }, + hint: topology.Hint{ + Sockets: "1", + }, + expected: cpuset.New(2, 3), + }, + { + name: "NUMAs hint resolves to the node's CPUs", + supply: &supply{ + node: &node{ + policy: &policy{machine: twoSocketMachine(t)}, + }, + }, + hint: topology.Hint{ + NUMAs: "0", + }, + expected: cpuset.New(0, 1), + }, { name: "non-zero CPUs hint", supply: &supply{}, diff --git a/cmd/plugins/topology-aware/policy/machine_test.go b/cmd/plugins/topology-aware/policy/machine_test.go index 5b8cd7881..4440a459a 100644 --- a/cmd/plugins/topology-aware/policy/machine_test.go +++ b/cmd/plugins/topology-aware/policy/machine_test.go @@ -121,3 +121,13 @@ func oneCpuMachine(t *testing.T) *hardware.Machine { {cpus: "0", memKB: 1048576, distance: []int{10}}, }) } + +// twoSocketMachine has two packages of two CPUs, with one NUMA node each: cpus +// 0-1 in package 0 and node 0, cpus 2-3 in package 1 and node 1. +func twoSocketMachine(t *testing.T) *hardware.Machine { + t.Helper() + return synthMachine(t, []synthNode{ + {cpus: "0-1", memKB: 1048576, distance: []int{10, 20}}, + {cpus: "2-3", memKB: 1048576, distance: []int{20, 10}}, + }) +} From c554a82f977c8b69fa0289003e17543bc0416053 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 18:21:13 +0300 Subject: [PATCH 27/39] resmgr: collect system metrics from the hardware.Machine. The metrics collector read the NUMA nodes and CPUs it reports through the pkg/sysfs interface. It reads them from the machine instead. It is the last thing in the tree which read that interface. Its memory figures come from one read of a node's meminfo now rather than from a MemoryInfo which returned both, and a node it cannot read reports zero capacity as well as zero usage. Reporting the capacity discovery recorded beside a usage which could not be read would be stating more than is known. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/resmgr/policy/metrics.go | 34 +++++++++++++++++++--------------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/pkg/resmgr/policy/metrics.go b/pkg/resmgr/policy/metrics.go index f4fbec974..49ebab9bb 100644 --- a/pkg/resmgr/policy/metrics.go +++ b/pkg/resmgr/policy/metrics.go @@ -23,7 +23,7 @@ import ( "go.opentelemetry.io/otel/metric" v1 "k8s.io/api/core/v1" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" + "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/metrics" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -32,7 +32,7 @@ import ( type ( SystemCollector struct { cache cache.Cache - system system.System + machine *hardware.Machine Nodes map[int]*NodeMetric Cpus map[int]*CpuMetric NodeCapacity metric.Int64Gauge @@ -60,10 +60,10 @@ func (p *policy) newSystemCollector() (*SystemCollector, error) { var ( meter = metrics.Provider("policy").Meter("system", metrics.WithOmitSubsystem()) s = &SystemCollector{ - cache: p.cache, - system: p.system, - Nodes: map[int]*NodeMetric{}, - Cpus: map[int]*CpuMetric{}, + cache: p.cache, + machine: p.machine, + Nodes: map[int]*NodeMetric{}, + Cpus: map[int]*CpuMetric{}, } err error ) @@ -111,9 +111,9 @@ func (p *policy) newSystemCollector() (*SystemCollector, error) { return nil, fmt.Errorf("failed to create cpu.container.count meter: %w", err) } - for _, id := range s.system.NodeIDs() { + for _, id := range s.machine.MemoryNodeIDs() { var ( - sys = s.system.Node(id) + sys = s.machine.MemoryNode(id) capa, used = s.getMemInfo(sys) node = &NodeMetric{ Id: sys.ID(), @@ -131,13 +131,13 @@ func (p *policy) newSystemCollector() (*SystemCollector, error) { node.Capacity, metric.WithAttributes( append(node.IdLabel.ToSlice(), - attribute.String("node.type", sys.GetMemoryType().String()), + attribute.String("node.type", sys.Kind().String()), )..., ), ) } - for _, id := range s.system.CPUIDs() { + for _, id := range s.machine.CPUIDs() { cpu := &CpuMetric{ Id: id, IdLabel: attribute.NewSet( @@ -158,7 +158,7 @@ func (s *SystemCollector) Update() { } for _, n := range s.Nodes { - sys := s.system.Node(n.Id) + sys := s.machine.MemoryNode(n.Id) _, used := s.getMemInfo(sys) n.Usage = used n.ContainerCount = 0 @@ -224,10 +224,14 @@ func (s *SystemCollector) Update() { } } -func (s *SystemCollector) getMemInfo(n system.Node) (capacity, used int64) { - if n != nil { - if i, _ := n.MemoryInfo(); i != nil { - return int64(i.MemTotal), int64(i.MemUsed) +// getMemInfo reads a node's memory usage now. The capacity comes from the same +// read rather than from what discovery recorded, so that a node whose meminfo has +// become unreadable reports zero for both instead of a capacity it cannot +// corroborate. +func (s *SystemCollector) getMemInfo(n *hardware.MemoryNode) (capacity, used int64) { + if n.Valid() { + if info, err := n.Usage(); err == nil { + return info.Total, info.Used } } return 0, 0 From 2574eba6bb6ff8bf741d59bbc9a1494452f4f53e Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 18:22:36 +0300 Subject: [PATCH 28/39] resmgr: stop handing the pkg/sysfs interface to backends. BackendOptions carried the machine and, wrapped around the same machine, the pkg/sysfs interface, for backends which had not been moved over. All of them have, and the metrics collector was the last thing in the tree to read it, so the field goes, along with the policy's own copy and the sys root it needed. pkg/sysfs and the drop-in over it are now referenced by nothing but each other and their own tests. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/resmgr/policy/policy.go | 7 ------- pkg/resmgr/resource-manager.go | 7 ++----- 2 files changed, 2 insertions(+), 12 deletions(-) diff --git a/pkg/resmgr/policy/policy.go b/pkg/resmgr/policy/policy.go index f36421c92..a97a3394b 100644 --- a/pkg/resmgr/policy/policy.go +++ b/pkg/resmgr/policy/policy.go @@ -27,7 +27,6 @@ import ( "github.com/prometheus/client_golang/prometheus" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" // nrt "github.com/k8stopologyawareschedwg/noderesourcetopology-api/pkg/apis/topology/v1alpha1" ) @@ -74,9 +73,6 @@ type Options struct { type BackendOptions struct { // Machine provides system/HW/topology information. Machine *hardware.Machine - // System is Machine behind the pkg/sysfs interface, for backends which have - // not been moved over to Machine yet. It goes with the last of them. - System system.System // System state/cache Cache cache.Cache // SendEvent is the function for delivering events up to the resource manager. @@ -257,7 +253,6 @@ type policy struct { cache cache.Cache // system state cache active Backend // our active backend machine *hardware.Machine // CPU and memory topology - system system.System // the same, behind the pkg/sysfs interface scollect *SystemCollector // system metrics collector } @@ -277,7 +272,6 @@ func NewPolicy(backend Backend, cache cache.Cache, o *Options) (Policy, error) { options: *o, active: backend, machine: o.Machine, - system: system.FromMachine(o.Machine), } return p, nil @@ -297,7 +291,6 @@ func (p *policy) Start(cfg any) error { if err := p.active.Setup(&BackendOptions{ Cache: p.cache, Machine: p.machine, - System: p.system, SendEvent: p.options.SendEvent, Config: cfg, KubeClientFn: p.options.KubeClientFn, diff --git a/pkg/resmgr/resource-manager.go b/pkg/resmgr/resource-manager.go index 8f18bab4f..3702cb0b1 100644 --- a/pkg/resmgr/resource-manager.go +++ b/pkg/resmgr/resource-manager.go @@ -24,7 +24,6 @@ import ( "github.com/containers/nri-plugins/pkg/healthz" "github.com/containers/nri-plugins/pkg/instrumentation" "github.com/containers/nri-plugins/pkg/lib/hardware" - sysfs "github.com/containers/nri-plugins/pkg/lib/hardware/system" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/pidfile" "github.com/containers/nri-plugins/pkg/resmgr/cache" @@ -80,14 +79,12 @@ func NewResourceManager(backend policy.Backend, agt *agent.Agent) (ResourceManag topology.SetLogger(logger.Get(topologyLogger)) if opt.HostRoot != "" { - sysfs.SetSysRoot(opt.HostRoot) topology.SetSysRoot(opt.HostRoot) irq.SetProcRoot(opt.HostRoot) } - // The topology is discovered once here and handed down. Anything which still - // wants the pkg/sysfs interface wraps this with sysfs.FromMachine, so there is - // one discovery and one view of the hardware however it is reached. + // The topology is discovered once here and handed down, so that everything + // below sees one discovery and one view of the hardware. machine, err := hardware.Discover( hardware.WithRoot(opt.HostRoot), hardware.WithEnvOverrides(), From 272d49f6a9e8b59d375ca680812dbb852b0d6342 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 18:24:16 +0300 Subject: [PATCH 29/39] sysfs: deprecate the package. Nothing in the repository uses it any more. Mark it deprecated so that tooling says so, and keep it for a release, since anything outside the repository which uses it deserves one in which to move over. The notice names the replacement and says how the two differ, which is more useful than pointing at the new package and leaving the reader to discover that a machine is discovered once, that lookups return handles which are never nil, that dies and cores are addressed by their coordinates, and that SST is not there at all. The drop-in stays too, and its own note is updated to say why: the migration step it provided is done, but it is what compares this package against the hardware package over the recorded machines, and that comparison cannot outlive its subject. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- pkg/lib/hardware/system/doc.go | 18 +++--- pkg/lib/hardware/system/equivalence_test.go | 2 +- .../system/internal/dropin/viasysfs/api.go | 2 +- pkg/lib/hardware/system/shim_test.go | 2 +- pkg/sysfs/doc.go | 56 +++++++++++++++++++ 5 files changed, 70 insertions(+), 10 deletions(-) create mode 100644 pkg/sysfs/doc.go diff --git a/pkg/lib/hardware/system/doc.go b/pkg/lib/hardware/system/doc.go index fc2df84b8..8f54c3fcc 100644 --- a/pkg/lib/hardware/system/doc.go +++ b/pkg/lib/hardware/system/doc.go @@ -22,19 +22,23 @@ // // # Why it exists // -// It is a migration step, and a proof. pkg/sysfs stays in the tree beside it, so +// It was a migration step, and it is a proof. +// +// The step is done: nothing in this repository reads this interface any more, and +// the consumers which did were moved onto the hardware package one at a time, +// each with this in between so that the change was a single import line. +// +// The proof is why it is still here. pkg/sysfs stays in the tree beside it, so // both implementations are in one build and equivalence_test.go can run every // method of both against the same recorded sysfs trees and compare the answers. // That is a much stronger statement than a rewritten pkg/sysfs could make, where -// the only reference left would be in git history. -// -// It also splits the work into reviewable pieces. This package plus the hardware -// package underneath it change no behaviour and no caller, so they can land on -// their own. Moving consumers off pkg/sysfs, and deleting it, comes after. +// the only reference left would be in git history. It goes when pkg/sysfs goes, +// and whatever is worth keeping of it by then has to be recorded some other way +// first, because the comparison cannot outlive its subject. // // Nothing new should be built on this package. New code should use // [github.com/containers/nri-plugins/pkg/lib/hardware] directly; this is -// here to be deleted. +// here to be deleted, and sooner than pkg/sysfs is. // // # Where it is faithful, and where it cannot be // diff --git a/pkg/lib/hardware/system/equivalence_test.go b/pkg/lib/hardware/system/equivalence_test.go index 1a8505d98..ec0a14d0a 100644 --- a/pkg/lib/hardware/system/equivalence_test.go +++ b/pkg/lib/hardware/system/equivalence_test.go @@ -33,7 +33,7 @@ import ( "testing" "github.com/containers/nri-plugins/pkg/lib/hardware/system" - "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/sysfs" //nolint:staticcheck // deprecated on purpose: this is what it is compared against "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) diff --git a/pkg/lib/hardware/system/internal/dropin/viasysfs/api.go b/pkg/lib/hardware/system/internal/dropin/viasysfs/api.go index 4537c8784..601416e95 100644 --- a/pkg/lib/hardware/system/internal/dropin/viasysfs/api.go +++ b/pkg/lib/hardware/system/internal/dropin/viasysfs/api.go @@ -10,7 +10,7 @@ package viasysfs import ( - sysfs "github.com/containers/nri-plugins/pkg/sysfs" + sysfs "github.com/containers/nri-plugins/pkg/sysfs" //nolint:staticcheck // deprecated on purpose: this is what it is compared against "github.com/containers/nri-plugins/pkg/utils/cpuset" "github.com/intel/goresctrl/pkg/sst" diff --git a/pkg/lib/hardware/system/shim_test.go b/pkg/lib/hardware/system/shim_test.go index b616920de..2f46ba355 100644 --- a/pkg/lib/hardware/system/shim_test.go +++ b/pkg/lib/hardware/system/shim_test.go @@ -27,7 +27,7 @@ import ( "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/lib/hardware/system" - "github.com/containers/nri-plugins/pkg/sysfs" + "github.com/containers/nri-plugins/pkg/sysfs" //nolint:staticcheck // deprecated on purpose: this is what it is compared against "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) diff --git a/pkg/sysfs/doc.go b/pkg/sysfs/doc.go new file mode 100644 index 000000000..5902c8351 --- /dev/null +++ b/pkg/sysfs/doc.go @@ -0,0 +1,56 @@ +// Copyright 2020 Intel Corporation. All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package sysfs discovers the CPU and memory topology of a machine, and the +// details of its caches, CPU frequencies and Intel Speed Select state. +// +// Deprecated: use [github.com/containers/nri-plugins/pkg/lib/hardware] instead. +// Nothing in this repository uses this package any more, and it will be removed +// in a later release. It is still here so that anything outside the repository +// has a release in which to move over. +// +// # Moving over +// +// [github.com/containers/nri-plugins/pkg/lib/hardware] is not a renaming of this +// package. It describes the same hardware with a smaller interface, and the +// differences are deliberate: +// +// - A machine is discovered once and does not change. There is no Discover on +// an existing one, and no discovery flags: everything is read up front. +// - Lookups return concrete handles which are never nil. A CPU or a node the +// machine does not have reports Valid() == false rather than being a nil +// behind a non-nil interface. +// - Packages, dies, clusters, cores and caches are all zones, addressed by +// their full coordinates, since the kernel numbers dies and cores within +// their package. hardware.TopologyIndex is the lookup table for those. +// - CPU sets are [github.com/containers/nri-plugins/pkg/lib/cpu] masks rather +// than k8s.io/utils/cpuset sets, and the ones a machine hands out are sealed. +// - Discovery reads through an io/fs.FS rooted at the host root, which is what +// a test passes a recorded or synthetic tree through, instead of a package +// global sys root. +// - Intel Speed Select is not there. It is a property of the running platform +// rather than of its shape, and the code which wanted it now probes for +// itself. +// +// [github.com/containers/nri-plugins/pkg/lib/hardware/system] reimplements this +// package's interface on top of that one, and is what the tree used while its +// consumers were moved over one at a time. A caller which wants the migration in +// two steps rather than one can do the same, but it is going away with this +// package rather than outliving it. +// +// ParseFileEntries and GetMemoryCapacity have moved and are only forwarded from +// here: they are [github.com/containers/nri-plugins/pkg/utils/parse].FileEntries +// and [github.com/containers/nri-plugins/pkg/utils].GetMemoryCapacity now, and +// neither has anything to do with topology. +package sysfs From 4bd073423d60e68573cc6b2fb9954e37f9057fb2 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 19:06:15 +0300 Subject: [PATCH 30/39] topology-aware: enable the cold start test. The test was disabled a year ago because the mocked system it ran against had no CPUs. It builds a machine of its own now, so that reason is gone and the skip goes with it. One thing was left over from the mocks. The memory type and cold start preferences reached the policy through GetResmgrAnnotation, which is not what it reads them with; they are annotations scoped to a container, resolved by GetEffectiveAnnotation. The mock pod's fields for the old form have no users left and go too. What the test asserts is what cold start promises: the container starts on the PMEM node alone, and the DRAM node joins it once the timer expires. Both halves fail if broken deliberately, so it is worth having back. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../topology-aware/policy/coldstart_test.go | 22 +++++++++---------- .../topology-aware/policy/mocks_test.go | 5 ----- 2 files changed, 11 insertions(+), 16 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/coldstart_test.go b/cmd/plugins/topology-aware/policy/coldstart_test.go index d9c220c5c..d915819cd 100644 --- a/cmd/plugins/topology-aware/policy/coldstart_test.go +++ b/cmd/plugins/topology-aware/policy/coldstart_test.go @@ -73,10 +73,13 @@ func TestColdStart(t *testing.T) { name: "demo-coldstart-container", returnValueForGetID: "1234", pod: &mockPod{ - coldStartTimeout: 1000 * time.Millisecond, - returnValue1FotGetResmgrAnnotation: "demo-coldstart-container: pmem,dram", - returnValue2FotGetResmgrAnnotation: true, - coldStartContainerName: "demo-coldstart-container", + // The policy reads both preferences with + // GetEffectiveAnnotation, so they have to be annotations + // scoped to this container, in the form it parses them. + annotations: map[string]string{ + preferMemoryTypeKey + "/container.demo-coldstart-container": "pmem,dram", + preferColdStartKey + "/container.demo-coldstart-container": "{ duration: 1s }", + }, }, }, expectedColdStartTimeout: 1000 * time.Millisecond, @@ -88,10 +91,10 @@ func TestColdStart(t *testing.T) { } for _, tc := range tcases { t.Run(tc.name, func(t *testing.T) { - t.Skipf("Coldstart tests are disabled (can't mock enough of the system, lacks CPUs)") + m := synthMachine(t, tc.numaNodes) policy := &policy{ - machine: synthMachine(t, tc.numaNodes), + machine: m, cache: &mockCache{ returnValue1ForLookupContainer: tc.container, returnValue2ForLookupContainer: true, @@ -104,12 +107,9 @@ func TestColdStart(t *testing.T) { } policy.allocations.policy = policy policy.options.SendEvent = sendEvent - // No nodes: the allocator takes them from a hardware.Machine now, - // and the mocked system above cannot stand in for one. Moot while - // the test is skipped, which is for the same reason. - ma, err := libmem.NewAllocator() + ma, err := libmem.NewAllocator(libmem.WithMachineNodes(m)) if err != nil { - panic(err) + t.Fatalf("failed to create memory allocator: %v", err) } policy.memAllocator = ma diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index 8942f9e6d..c6e99e4ca 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -297,8 +297,6 @@ type mockPod struct { returnValueFotGetQOSClass v1.PodQOSClass returnValue1FotGetResmgrAnnotation string returnValue2FotGetResmgrAnnotation bool - coldStartTimeout time.Duration - coldStartContainerName string annotations map[string]string } @@ -336,9 +334,6 @@ func (m *mockPod) GetResmgrLabel(string) (string, bool) { panic("unimplemented") } func (m *mockPod) GetResmgrAnnotation(key string) (string, bool) { - if key == preferColdStartKey && len(m.coldStartContainerName) > 0 { - return m.coldStartContainerName + ": { duration: " + m.coldStartTimeout.String() + " }", true - } return m.returnValue1FotGetResmgrAnnotation, m.returnValue2FotGetResmgrAnnotation } func (m *mockPod) GetEffectiveAnnotation(key, container string) (string, bool) { From 30cb31f2a02d99ab20e2d94b42c30210f8ab2483 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 20:01:27 +0300 Subject: [PATCH 31/39] cpuallocator: speak libcpu.CpuMask. The allocator took and returned k8s cpuset sets, so a caller holding a mask converted on the way in and back on the way out, and the allocator converted the machine's own sets a third time to fill its topology cache. It takes masks now: the cache shares the machine's sealed sets instead of copying them, and the set algebra underneath every allocation is a good deal cheaper than the map-based sets it replaces. The from parameter loses its pointer. It was one because allocating takes the allocated CPUs out of the set, which a mask does natively, so the &-taking at the call sites goes with it. A CpuMask has no usable zero value where a cpuset.CPUSet has, so what used to read as an empty set now reads as a nil pointer: a map with no entry for a core kind the machine does not have, a struct field nobody assigned, an array of priorities with holes in it. Those read through EmptyIfNil. Nothing tested what allocating and releasing do to the set they are given, which is the whole way a caller learns what is left, so pin that. Releasing leaves the released CPUs in the set and returns the ones kept; its debug message said the opposite and now agrees with the code. The two policies still keep their own CPU sets as cpuset.CPUSet and convert where they call this. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .codespellignorelines | 2 +- .../balloons/policy/balloons-policy.go | 18 +- .../topology-aware/policy/mocks_test.go | 20 +- .../topology-aware/policy/resources.go | 13 +- .../policy/topology-aware-policy.go | 6 +- pkg/cpuallocator/allocator.go | 193 ++++++----- pkg/cpuallocator/cpuallocator_test.go | 309 ++++++++++++------ pkg/cpuallocator/topology.go | 19 +- 8 files changed, 357 insertions(+), 223 deletions(-) diff --git a/.codespellignorelines b/.codespellignorelines index 9c43195e0..050948c58 100644 --- a/.codespellignorelines +++ b/.codespellignorelines @@ -3,7 +3,7 @@ docker run -v "$(pwd):/mnt/models" "$FMBT_IMAGE" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\$sn\$as\$al' | grep -v AAL | sed -e 's/^, / /g' -e '/^STEP/! s/\(^.*\)/echo "TESTGEN: \1"/g' -e 's/^STEP\([0-9]*\)i:\(.*\)/echo "TESTGEN: STEP \1"; vm-command "date +%T.%N"; \2; vm-command "date +%T.%N"; kubectl get pods -A/g' | sed "s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" > "$OUTFILE" allocs := map[string]cpuset.CPUSet{"--:allo": currentCpus} allocName := fmt.Sprintf("%02d:allo", i+1) - preferTightestFit = func(cA, cB *cpuCluster, pkgA, pkgB, dieA, dieB int, csetA, csetB cpuset.CPUSet) (r int) { + preferTightestFit = func(cA, cB *cpuCluster, pkgA, pkgB, dieA, dieB int, csetA, csetB *libcpu.CpuMask) (r int) { if dieA >= a.cnt && dieB < a.cnt { if dieA < a.cnt && dieB >= a.cnt { if dieA >= a.cnt && dieB >= a.cnt { diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 6f46d78e2..4e4cc0df7 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -1227,8 +1227,8 @@ func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool, c cache.Contain virtDevCpusets: map[string][]cpuset.CPUSet{ virtDevReservedCpus: {p.reserved}, virtDevIsolatedCpus: {toCpuSet(p.machine.IsolatedCPUs())}, - virtDevECores: {p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityLow]}, - virtDevPCores: {p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityHigh]}, + virtDevECores: {toCpuSet(p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityLow])}, + virtDevPCores: {toCpuSet(p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityHigh])}, }, } // Pod resource hints to container's physical devices (GPUs, @@ -1294,9 +1294,11 @@ func (p *balloons) deleteBalloon(bln *Balloon) { p.balloons = remainingBalloons p.forgetCpuClass(bln) p.freeCpus = p.freeCpus.Union(bln.Cpus) - if _, err := p.cpuAllocator.ReleaseCpus(&bln.Cpus, bln.Cpus.Size(), bln.Def.AllocatorPriority.Value().Option()); err != nil { + blnCpus := toCpuMask(bln.Cpus) + if _, err := p.cpuAllocator.ReleaseCpus(blnCpus, bln.Cpus.Size(), bln.Def.AllocatorPriority.Value().Option()); err != nil { log.Warnf("failed to release CPUs %q of balloon %s[%d]: %v", bln.Cpus, bln.Def.Name, bln.Instance, err) } + bln.Cpus = toCpuSet(blnCpus) } // freeBalloon clears a balloon and deletes it if allowed. @@ -2540,10 +2542,14 @@ func (p *balloons) resizeBalloon(bln *Balloon, newMilliCpus int) error { return balloonsError("resize/inflate: failed to choose a cpuset for allocating additional %d CPUs: %w", cpuCountDelta, err) } log.Debugf("- allocating %d CPUs from %q", cpuCountDelta, addFromCpus) - newCpus, err := p.cpuAllocator.AllocateCpus(&addFromCpus, newCpuCount-oldCpuCount, bln.Def.AllocatorPriority.Value().Option()) + // The allocator takes the allocated CPUs out of the set it is given. + // Nothing here reads what is left of it, only what came back. + allocated, err := p.cpuAllocator.AllocateCpus(toCpuMask(addFromCpus), + newCpuCount-oldCpuCount, bln.Def.AllocatorPriority.Value().Option()) if err != nil { return balloonsError("resize/inflate: allocating %d CPUs for %s failed: %w", cpuCountDelta, bln, err) } + newCpus := toCpuSet(allocated) oldBlnCpus := bln.Cpus oldFreeCpus := p.freeCpus p.freeCpus = p.freeCpus.Difference(newCpus) @@ -2557,10 +2563,12 @@ func (p *balloons) resizeBalloon(bln *Balloon, newMilliCpus int) error { return balloonsError("resize/deflate: failed to choose a cpuset for releasing %d CPUs: %w", -cpuCountDelta, err) } log.Debugf("- releasing %d CPUs from cpuset %q", -cpuCountDelta, removeFromCpus) - _, err = p.cpuAllocator.ReleaseCpus(&removeFromCpus, -cpuCountDelta, bln.Def.AllocatorPriority.Value().Option()) + removeFrom := toCpuMask(removeFromCpus) + _, err = p.cpuAllocator.ReleaseCpus(removeFrom, -cpuCountDelta, bln.Def.AllocatorPriority.Value().Option()) if err != nil { return balloonsError("resize/deflate: releasing %d CPUs from %s failed: %w", -cpuCountDelta, bln, err) } + removeFromCpus = toCpuSet(removeFrom) oldBlnCpus := bln.Cpus oldFreeCpus := p.freeCpus p.freeCpus = p.freeCpus.Union(removeFromCpus) diff --git a/cmd/plugins/topology-aware/policy/mocks_test.go b/cmd/plugins/topology-aware/policy/mocks_test.go index c6e99e4ca..7aa7bd52d 100644 --- a/cmd/plugins/topology-aware/policy/mocks_test.go +++ b/cmd/plugins/topology-aware/policy/mocks_test.go @@ -22,10 +22,10 @@ import ( "github.com/containers/nri-plugins/pkg/agent/podresapi" resmgr "github.com/containers/nri-plugins/pkg/apis/resmgr/v1alpha1" "github.com/containers/nri-plugins/pkg/cpuallocator" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/resmgr/cache" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" "github.com/containers/nri-plugins/pkg/topology" - "github.com/containers/nri-plugins/pkg/utils/cpuset" v1 "k8s.io/api/core/v1" ) @@ -486,16 +486,22 @@ func (m *mockCache) WriteFile(string, string, os.FileMode, []byte) error { type mockCPUAllocator struct{} -func (m *mockCPUAllocator) AllocateCpus(from *cpuset.CPUSet, cnt int, options ...cpuallocator.Option) (cpuset.CPUSet, error) { - return cpuset.New(0), nil +func (m *mockCPUAllocator) AllocateCpus(from *libcpu.CpuMask, cnt int, options ...cpuallocator.Option) (*libcpu.CpuMask, error) { + return libcpu.NewCpuMask(0), nil } -func (m *mockCPUAllocator) ReleaseCpus(from *cpuset.CPUSet, cnt int, options ...cpuallocator.Option) (cpuset.CPUSet, error) { - return cpuset.New(0), nil +func (m *mockCPUAllocator) ReleaseCpus(from *libcpu.CpuMask, cnt int, options ...cpuallocator.Option) (*libcpu.CpuMask, error) { + return libcpu.NewCpuMask(0), nil } -func (m *mockCPUAllocator) GetCPUPriorities() map[cpuallocator.CPUPriority]cpuset.CPUSet { - return map[cpuallocator.CPUPriority]cpuset.CPUSet{} +func (m *mockCPUAllocator) GetCPUPriorities() map[cpuallocator.CPUPriority]*libcpu.CpuMask { + // An entry per priority, as the real allocator promises. A caller is entitled + // to index this without checking. + prios := map[cpuallocator.CPUPriority]*libcpu.CpuMask{} + for prio := range cpuallocator.NumCPUPriorities { + prios[prio] = libcpu.NewCpuMask() + } + return prios } var ( diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index b3a5d9d44..53c067b9c 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -721,7 +721,12 @@ func (cs *supply) Reserve(g Grant, o *libmem.Offer) (map[string]libmem.NodeMask, // takeCPUs takes up to cnt CPUs from a given CPU set to another. func (cs *supply) takeCPUs(from, to *cpuset.CPUSet, cnt int, prio cpuPrio) (cpuset.CPUSet, error) { - cset, err := cs.node.Policy().cpuAllocator.AllocateCpus(from, cnt, prio.Option()) + // The allocator speaks libcpu sets and takes what it allocated out of the + // set it is given, so hand it one and copy back what is left either way. + fromCpus := toCpuMask(*from) + allocated, err := cs.node.Policy().cpuAllocator.AllocateCpus(fromCpus, cnt, prio.Option()) + cset := toCpuSet(allocated) + *from = toCpuSet(fromCpus) if err != nil { return cset, err } @@ -1177,7 +1182,7 @@ func (cs *supply) GetScore(req Request) Score { lpCPUs := toCpuSet(cs.GetNode().Machine().CoreKindCPUs(hardware.EfficientCore)) if lpCPUs.Size() == 0 { - lpCPUs = p.cpuAllocator.GetCPUPriorities()[lowPrio] + lpCPUs = toCpuSet(p.cpuAllocator.GetCPUPriorities()[lowPrio].EmptyIfNil()) } lpCPUs = lpCPUs.Intersection(cs.SharableCPUs()) lpCnt := lpCPUs.Size() @@ -1185,13 +1190,13 @@ func (cs *supply) GetScore(req Request) Score { hpCPUs := toCpuSet(cs.GetNode().Machine().CoreKindCPUs(hardware.PerformanceCore)) if hpCPUs.Size() == 0 { - hpCPUs = p.cpuAllocator.GetCPUPriorities()[highPrio] + hpCPUs = toCpuSet(p.cpuAllocator.GetCPUPriorities()[highPrio].EmptyIfNil()) } hpCPUs = hpCPUs.Intersection(cs.SharableCPUs()) hpCnt := hpCPUs.Size() score.prio[highPrio] = hpCnt*1000 - (1000*full + part) - npCPUs := p.cpuAllocator.GetCPUPriorities()[normalPrio] + npCPUs := toCpuSet(p.cpuAllocator.GetCPUPriorities()[normalPrio].EmptyIfNil()) npCPUs = npCPUs.Intersection(cs.SharableCPUs()) npCnt := npCPUs.Size() score.prio[normalPrio] = npCnt*1000 - (1000*full + part) diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index 60ed203c9..159223b1e 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -913,12 +913,12 @@ func (p *policy) checkConstraints() error { // Use CpuAllocator to pick reserved CPUs from the allowed ones but // avoiding isolated CPUs. The picked CPUs are not removed from the // allowed set. - from := p.allowed.Difference(p.isolated) - cset, err := p.cpuAllocator.AllocateCpus(&from, p.reserveCnt, normalPrio.Option()) + from := toCpuMask(p.allowed.Difference(p.isolated)) + cset, err := p.cpuAllocator.AllocateCpus(from, p.reserveCnt, normalPrio.Option()) if err != nil { return policyError("cannot reserve %dm CPUs for ReservedResources from AvailableResources: %s", qty.MilliValue(), err) } - p.reserved = cset + p.reserved = toCpuSet(cset) } if p.reserved.IsEmpty() { diff --git a/pkg/cpuallocator/allocator.go b/pkg/cpuallocator/allocator.go index 4ffc2c456..b770ebbc2 100644 --- a/pkg/cpuallocator/allocator.go +++ b/pkg/cpuallocator/allocator.go @@ -19,8 +19,7 @@ import ( "slices" "sort" - "github.com/containers/nri-plugins/pkg/utils/cpuset" - + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/utils" @@ -53,17 +52,17 @@ type allocatorHelper struct { machine *hardware.Machine // CPU and memory topology topology topologyCache // cached topology information flags AllocFlag // allocation preferences - from cpuset.CPUSet // set of CPUs to allocate from + from *libcpu.CpuMask // set of CPUs to allocate from prefer CPUPriority // CPU priority to prefer cnt int // number of CPUs to allocate - result cpuset.CPUSet // set of CPUs allocated + result *libcpu.CpuMask // set of CPUs allocated } // CPUAllocator is an interface for a generic CPU allocator type CPUAllocator interface { - AllocateCpus(from *cpuset.CPUSet, cnt int, options ...Option) (cpuset.CPUSet, error) - ReleaseCpus(from *cpuset.CPUSet, cnt int, options ...Option) (cpuset.CPUSet, error) - GetCPUPriorities() map[CPUPriority]cpuset.CPUSet + AllocateCpus(from *libcpu.CpuMask, cnt int, options ...Option) (*libcpu.CpuMask, error) + ReleaseCpus(from *libcpu.CpuMask, cnt int, options ...Option) (*libcpu.CpuMask, error) + GetCPUPriorities() map[CPUPriority]*libcpu.CpuMask } type CPUPriority int @@ -107,23 +106,22 @@ type cpuAllocator struct { // topologyCache caches topology lookups type topologyCache struct { - pkg map[idset.ID]cpuset.CPUSet - node map[idset.ID]cpuset.CPUSet - core map[idset.ID]cpuset.CPUSet - kind map[hardware.CoreKind]cpuset.CPUSet + pkg map[idset.ID]*libcpu.CpuMask + core map[idset.ID]*libcpu.CpuMask + kind map[hardware.CoreKind]*libcpu.CpuMask cpuPriorities cpuPriorities // CPU priority mapping clusters []*cpuCluster // CPU clusters cacheGroups []*cacheGroup // CPU cache groups } -type cpuPriorities [NumCPUPriorities]cpuset.CPUSet +type cpuPriorities [NumCPUPriorities]*libcpu.CpuMask type cpuCluster struct { pkg idset.ID die idset.ID cluster idset.ID - cpus cpuset.CPUSet + cpus *libcpu.CpuMask kind hardware.CoreKind } @@ -132,7 +130,7 @@ type cacheGroup struct { pkg idset.ID die idset.ID node idset.ID - cpus cpuset.CPUSet + cpus *libcpu.CpuMask kind hardware.CoreKind } @@ -178,6 +176,10 @@ func newAllocatorHelper(m *hardware.Machine, topo topologyCache) *allocatorHelpe machine: m, topology: topo, flags: AllocDefault, + // Both sets are read and unioned into before anything assigns them, so + // they exist from the start. A CpuMask has no usable zero value. + from: libcpu.NewCpuMask(), + result: libcpu.NewCpuMask(), } return a @@ -187,7 +189,7 @@ func newAllocatorHelper(m *hardware.Machine, topo topologyCache) *allocatorHelpe func (a *allocatorHelper) takeIdlePackages() { a.Debugf("* takeIdlePackages()...") - offline := toCpuSet(a.machine.OfflineCPUs()) + offline := a.machine.OfflineCPUs() // pick idle packages pkgs := pickIds(a.machine.TopologyIndex().PackageIDs(), @@ -234,25 +236,21 @@ func (a *allocatorHelper) takeIdlePackages() { } } -var ( - emptyCPUSet = cpuset.New() -) - // Allocate full idle CPU clusters. func (a *allocatorHelper) takeIdleClusters() { var ( - offline = toCpuSet(a.machine.OfflineCPUs()) - pickIdle = func(c *cpuCluster) (bool, cpuset.CPUSet) { + offline = a.machine.OfflineCPUs() + pickIdle = func(c *cpuCluster) (bool, *libcpu.CpuMask) { if len(a.topology.kind) > 1 { // we only take E-clusters for low-prio requests if a.prefer != PriorityLow && c.kind == hardware.EfficientCore { a.Debugf(" - omit %s, CPU preference is %s", c, a.prefer) - return false, emptyCPUSet + return false, libcpu.EmptyCpuMask } // we only take P-clusters for other than low-prio requests if a.prefer == PriorityLow && c.kind == hardware.PerformanceCore { a.Debugf(" - omit %s, CPU preference is %s", c, a.prefer) - return false, emptyCPUSet + return false, libcpu.EmptyCpuMask } } @@ -261,13 +259,13 @@ func (a *allocatorHelper) takeIdleClusters() { free := cset.Intersection(a.from) if free.IsEmpty() || !free.Equals(cset) { a.Debugf(" - omit %s, %d usable CPUs (%s)", c, free.Size(), free) - return false, emptyCPUSet + return false, libcpu.EmptyCpuMask } a.Debugf(" + pick %s, %d usable CPUs (%s)", c, free.Size(), free) return true, free } - preferTightestFit = func(cA, cB *cpuCluster, pkgA, pkgB, dieA, dieB int, csetA, csetB cpuset.CPUSet) (r int) { + preferTightestFit = func(cA, cB *cpuCluster, pkgA, pkgB, dieA, dieB int, csetA, csetB *libcpu.CpuMask) (r int) { defer func() { if r < 0 { a.Debugf(" + prefer %s", cA) @@ -467,18 +465,18 @@ func (a *allocatorHelper) takeCacheGroups() { // o fragment fewest groups possible (take from small to large, preserve large groups) var ( - offline = toCpuSet(a.machine.OfflineCPUs()) - pickGroups = func(g *cacheGroup) (pickVerdict, cpuset.CPUSet) { + offline = a.machine.OfflineCPUs() + pickGroups = func(g *cacheGroup) (pickVerdict, *libcpu.CpuMask) { if len(a.topology.kind) > 1 { // only take E-groups for low-prio requests, or if we have none other if a.prefer != PriorityLow && g.kind == hardware.EfficientCore { log.Debugf(" - ignore %s (CPU preference is %s)", g, a.prefer) - return pickIgnore, emptyCPUSet + return pickIgnore, libcpu.EmptyCpuMask } // only take P-groups for other than low-prio requests, or if we have none other if a.prefer == PriorityLow && g.kind == hardware.PerformanceCore { log.Debugf(" - ignore %s (CPU preference is %s)", g, a.prefer) - return pickIgnore, emptyCPUSet + return pickIgnore, libcpu.EmptyCpuMask } } @@ -488,7 +486,7 @@ func (a *allocatorHelper) takeCacheGroups() { // ignore groups without usable CPUs if free.IsEmpty() { log.Debugf(" - ignore %s (no usable CPUs)", g) - return pickIgnore, emptyCPUSet + return pickIgnore, libcpu.EmptyCpuMask } // prefer fully usable idle groups @@ -945,7 +943,7 @@ func (a *allocatorHelper) takeCacheGroups() { func (a *allocatorHelper) takeIdleCores() { a.Debugf("* takeIdleCores()...") - offline := toCpuSet(a.machine.OfflineCPUs()) + offline := a.machine.OfflineCPUs() // pick (first id for all) idle cores cores := pickIds(a.machine.CPUIDs(), @@ -987,7 +985,7 @@ func (a *allocatorHelper) takeIdleCores() { // Allocate idle CPU hyperthreads. func (a *allocatorHelper) takeIdleThreads() { - offline := toCpuSet(a.machine.OfflineCPUs()) + offline := a.machine.OfflineCPUs() // pick all threads with free capacity cores := pickIds(a.machine.CPUIDs(), @@ -1033,8 +1031,8 @@ func (a *allocatorHelper) takeIdleThreads() { return iPkg < jPkg } - iCset := cpuset.New(int(cores[i])) - jCset := cpuset.New(int(cores[j])) + iCset := libcpu.NewCpuMask(int(cores[i])) + jCset := libcpu.NewCpuMask(int(cores[j])) if res := a.topology.cpuPriorities.cmpCPUSet(iCset, jCset, a.prefer, 0); res != 0 { return res > 0 } @@ -1060,7 +1058,7 @@ func (a *allocatorHelper) takeIdleThreads() { for _, id := range cores { cset := a.topology.core[id].Difference(offline) a.Debugf(" => considering thread %v (#%s)...", id, cset) - cset = cpuset.New(int(id)) + cset = libcpu.NewCpuMask(int(id)) a.result = a.result.Union(cset) a.from = a.from.Difference(cset) a.cnt -= cset.Size() @@ -1078,7 +1076,7 @@ func (a *allocatorHelper) takeAny() { cpus := a.from.List() if len(cpus) >= a.cnt { - cset := cpuset.New(cpus[0:a.cnt]...) + cset := libcpu.NewCpuMask(cpus[0:a.cnt]...) a.result = a.result.Union(cset) a.from = a.from.Difference(cset) a.cnt = 0 @@ -1086,7 +1084,7 @@ func (a *allocatorHelper) takeAny() { } // Perform CPU allocation. -func (a *allocatorHelper) allocate() cpuset.CPUSet { +func (a *allocatorHelper) allocate() *libcpu.CpuMask { if a.machine != nil { if (a.flags & AllocIdlePackages) != 0 { a.takeIdlePackages() @@ -1116,20 +1114,20 @@ func (a *allocatorHelper) allocate() cpuset.CPUSet { return a.result } - return cpuset.New() + return libcpu.NewCpuMask() } type clusterSorter struct { // function to pick or ignore a cluster - pick func(*cpuCluster) (bool, cpuset.CPUSet) + pick func(*cpuCluster) (bool, *libcpu.CpuMask) // function to sort slice of picked clusters - sort func(a, b *cpuCluster, pkgCntA, pkgCntB, dieCntA, dieCntB int, cpusA, cpusB cpuset.CPUSet) int + sort func(a, b *cpuCluster, pkgCntA, pkgCntB, dieCntA, dieCntB int, cpusA, cpusB *libcpu.CpuMask) int // resulting cluster, available CPU count per package and die, available CPUs per cluster clusters []*cpuCluster pkgCPUCnt map[idset.ID]int dieCPUCnt map[idset.ID]map[idset.ID]int - cpus map[*cpuCluster]cpuset.CPUSet + cpus map[*cpuCluster]*libcpu.CpuMask } func (a *allocatorHelper) sortCPUClusters(s *clusterSorter) { @@ -1137,13 +1135,13 @@ func (a *allocatorHelper) sortCPUClusters(s *clusterSorter) { clusters = []*cpuCluster{} pkgCPUCnt = map[idset.ID]int{} dieCPUCnt = map[idset.ID]map[idset.ID]int{} - cpus = map[*cpuCluster]cpuset.CPUSet{} + cpus = map[*cpuCluster]*libcpu.CpuMask{} ) a.Debugf("picking suitable clusters") for _, c := range a.topology.clusters { - var cset cpuset.CPUSet + var cset *libcpu.CpuMask // pick or ignore cluster, determine usable cluster CPUs if s.pick == nil { @@ -1208,7 +1206,7 @@ const ( type cacheGroupSorter struct { // function to pick preferred and usable cache groups - pick func(*cacheGroup) (pickVerdict, cpuset.CPUSet) + pick func(*cacheGroup) (pickVerdict, *libcpu.CpuMask) // functions for sorting picked cache groups sortPrefer func(a, b *cacheGroup, s *cacheGroupSorter) int sortUsable func(a, b *cacheGroup, s *cacheGroupSorter) int @@ -1224,7 +1222,7 @@ type cacheGroupSorter struct { usableDie map[idset.ID]map[idset.ID]int // available CPUs per group - cpus map[*cacheGroup]cpuset.CPUSet + cpus map[*cacheGroup]*libcpu.CpuMask // full and partial groups worth of requested CPUs full int @@ -1247,7 +1245,7 @@ func (s *cacheGroupSorter) usableDieCPUCount(pkg, die idset.ID) int { return s.usableDie[pkg][die] } -func (s *cacheGroupSorter) CPUSet(g *cacheGroup) cpuset.CPUSet { +func (s *cacheGroupSorter) CPUSet(g *cacheGroup) *libcpu.CpuMask { return s.cpus[g] } @@ -1258,7 +1256,7 @@ func (s *cacheGroupSorter) sortCacheGroups(a *allocatorHelper) { s.usable = []*cacheGroup{} s.usablePkg = map[idset.ID]int{} s.usableDie = map[idset.ID]map[idset.ID]int{} - s.cpus = map[*cacheGroup]cpuset.CPUSet{} + s.cpus = map[*cacheGroup]*libcpu.CpuMask{} log.Debugf("picking suitable cache groups") @@ -1350,26 +1348,33 @@ func (s *cacheGroupSorter) sortCacheGroups(a *allocatorHelper) { } } -func (ca *cpuAllocator) allocateCpus(from *cpuset.CPUSet, cnt int, options ...Option) (cpuset.CPUSet, error) { - var result cpuset.CPUSet +func (ca *cpuAllocator) allocateCpus(from *libcpu.CpuMask, cnt int, options ...Option) (*libcpu.CpuMask, error) { + var result *libcpu.CpuMask var err error switch { case from.Size() < cnt: - result, err = cpuset.New(), fmt.Errorf("cpuset %s does not have %d CPUs", from, cnt) + result, err = libcpu.NewCpuMask(), fmt.Errorf("cpuset %s does not have %d CPUs", from, cnt) case from.Size() == cnt: - result, err, *from = from.Clone(), nil, cpuset.New() + result, err = from.Clone(), nil + take(from, result) default: a := newAllocatorHelper(ca.machine, ca.topologyCache) for _, o := range options { if err := o(a); err != nil { - return cpuset.New(), err + return libcpu.NewCpuMask(), err } } a.from = from.Clone() a.cnt = cnt - result, err, *from = a.allocate(), nil, a.from.Clone() + result, err = a.allocate(), nil + + // The helper works on a copy, so what it consumed is what its copy no + // longer has. Take that out of the caller's set, which is not + // necessarily the same as the result: a helper which could not satisfy + // the request returns nothing and still leaves its copy short. + take(from, from.Difference(a.from)) a.Debugf("%d cpus from #%v (preferring #%v) => #%v", cnt, from.Union(result), a.prefer, result) } @@ -1377,26 +1382,35 @@ func (ca *cpuAllocator) allocateCpus(from *cpuset.CPUSet, cnt int, options ...Op return result, err } -// AllocateCpus allocates a number of CPUs from the given set. -func (ca *cpuAllocator) AllocateCpus(from *cpuset.CPUSet, cnt int, options ...Option) (cpuset.CPUSet, error) { +// take removes cpus from a set in place. The set is the caller's, and taking +// CPUs out of it is how a caller learns what is left. +func take(from *libcpu.CpuMask, cpus *libcpu.CpuMask) { + from.Clear(cpus.UnsortedList()...) +} + +// AllocateCpus allocates a number of CPUs from the given set, taking the ones it +// allocated out of it. +func (ca *cpuAllocator) AllocateCpus(from *libcpu.CpuMask, cnt int, options ...Option) (*libcpu.CpuMask, error) { result, err := ca.allocateCpus(from, cnt, options...) return result, err } -// ReleaseCpus releases a number of CPUs from the given set. -func (ca *cpuAllocator) ReleaseCpus(from *cpuset.CPUSet, cnt int, options ...Option) (cpuset.CPUSet, error) { +// ReleaseCpus picks a number of CPUs to release from the given set. It leaves +// those in the set and returns the ones to keep. +func (ca *cpuAllocator) ReleaseCpus(from *libcpu.CpuMask, cnt int, options ...Option) (*libcpu.CpuMask, error) { oset := from.Clone() result, err := ca.allocateCpus(from, from.Size()-cnt, options...) - ca.Debugf("ReleaseCpus(#%s, %d) => kept: #%s, released: #%s", oset, cnt, from, result) + ca.Debugf("ReleaseCpus(#%s, %d) => kept: #%s, released: #%s", oset, cnt, result, from) return result, err } -// GetCPUPriorities returns the CPUSets for the discovered priorities. -func (ca *cpuAllocator) GetCPUPriorities() map[CPUPriority]cpuset.CPUSet { - prios := make(map[CPUPriority]cpuset.CPUSet) +// GetCPUPriorities returns the CPUSets for the discovered priorities. Every +// priority has an entry, and the sets are the caller's own to modify. +func (ca *cpuAllocator) GetCPUPriorities() map[CPUPriority]*libcpu.CpuMask { + prios := make(map[CPUPriority]*libcpu.CpuMask) for prio := range NumCPUPriorities { cset := ca.topologyCache.cpuPriorities[prio] prios[prio] = cset.Clone() @@ -1406,23 +1420,29 @@ func (ca *cpuAllocator) GetCPUPriorities() map[CPUPriority]cpuset.CPUSet { func newTopologyCache(m *hardware.Machine) topologyCache { c := topologyCache{ - pkg: make(map[idset.ID]cpuset.CPUSet), - node: make(map[idset.ID]cpuset.CPUSet), - core: make(map[idset.ID]cpuset.CPUSet), + pkg: make(map[idset.ID]*libcpu.CpuMask), + core: make(map[idset.ID]*libcpu.CpuMask), } + + // The machine's own sets, not copies of them: they are sealed, so sharing + // them costs nothing and anything which tries to modify one panics rather + // than changing what the machine says. if m != nil { x := m.TopologyIndex() for _, id := range x.PackageIDs() { - c.pkg[id] = toCpuSet(x.PackageCPUs(id)) - } - for _, id := range m.MemoryNodeIDs() { - c.node[id] = toCpuSet(m.MemoryNode(id).CPUs()) + c.pkg[id] = x.PackageCPUs(id) } for _, id := range m.CPUIDs() { - c.core[id] = toCpuSet(m.CPU(id).Threads()) + c.core[id] = m.CPU(id).Threads() } } + // Every priority has a set, so that a lookup for a priority the machine has + // no CPUs at answers with an empty one rather than a nil. + for prio := range c.cpuPriorities { + c.cpuPriorities[prio] = libcpu.NewCpuMask() + } + c.discoverCPUClusters(m) c.discoverCacheGroups(m) c.discoverCPUPriorities(m) @@ -1435,6 +1455,9 @@ func (c *topologyCache) discoverCPUPriorities(m *hardware.Machine) { return } var prio cpuPriorities + for p := range prio { + prio[p] = libcpu.NewCpuMask() + } // Probe Speed Select once for the whole machine rather than per package. pkgIDs := make([]idset.ID, 0, len(c.pkg)) @@ -1452,11 +1475,11 @@ func (c *topologyCache) discoverCPUPriorities(m *hardware.Machine) { } ecores := c.kind[hardware.EfficientCore] - ocores := toCpuSet(m.OnlineCPUs()).Difference(ecores) + ocores := m.OnlineCPUs().Difference(ecores) for p, cpus := range cpuPriorities { source := map[bool]string{true: "sst", false: "cpufreq"}[sstActive] - cset := cpuset.New(cpus...) + cset := libcpu.NewCpuMask(cpus...) if p != int(PriorityLow) && ocores.Size() > 0 { cset = cset.Difference(ecores) @@ -1701,7 +1724,7 @@ func (c *topologyCache) discoverCPUClusters(m *hardware.Machine) { pkg: id, die: die.Die, cluster: first.ClusterID(), - cpus: toCpuSet(cpus), + cpus: cpus, kind: first.Kind(), }) } @@ -1716,9 +1739,9 @@ func (c *topologyCache) discoverCPUClusters(m *hardware.Machine) { } } - c.kind = map[hardware.CoreKind]cpuset.CPUSet{} + c.kind = map[hardware.CoreKind]*libcpu.CpuMask{} for _, kind := range m.CoreKinds() { - c.kind[kind] = toCpuSet(m.CoreKindCPUs(kind)) + c.kind[kind] = m.CoreKindCPUs(kind) } } @@ -1728,15 +1751,15 @@ func (c *topologyCache) pickCacheLevelForGrouping(m *hardware.Machine) int { } x := m.TopologyIndex() - online := toCpuSet(m.OnlineCPUs()) + online := m.OnlineCPUs() for _, id := range online.List() { cpu := m.CPU(id) var ( - pkgCPUs = toCpuSet(x.PackageCPUs(cpu.PackageID())) - dieCPUs = toCpuSet(x.DieCPUs(hardware.DieID{ + pkgCPUs = x.PackageCPUs(cpu.PackageID()) + dieCPUs = x.DieCPUs(hardware.DieID{ Package: cpu.PackageID(), Die: cpu.DieID(), - })) + }) ) for n := len(cpu.Caches()) - 1; n > 0; n-- { cpus := cacheCPUsAtLevel(m, id, n) @@ -1744,7 +1767,7 @@ func (c *topologyCache) pickCacheLevelForGrouping(m *hardware.Machine) int { switch { case cpus.Size() == 0 || cpus.Size() == 1: continue - case cpus.Equals(toCpuSet(cpu.Threads()).Intersection(online)): + case cpus.Equals(cpu.Threads().Intersection(online)): continue case cpus.Equals(dieCPUs.Intersection(online)): continue @@ -1773,9 +1796,9 @@ func (c *topologyCache) discoverCacheGroups(m *hardware.Machine) { log.Infof("picked cache level %d for CPU grouping", n) x := m.TopologyIndex() - online := toCpuSet(m.OnlineCPUs()) + online := m.OnlineCPUs() for _, id := range x.PackageIDs() { - pkgCPUs := toCpuSet(x.PackageCPUs(id)) + pkgCPUs := x.PackageCPUs(id) groups := []*cacheGroup{} assigned := idset.NewIDSet() @@ -1786,15 +1809,15 @@ func (c *topologyCache) discoverCacheGroups(m *hardware.Machine) { cpu := m.CPU(cpuID) cpus := cacheCPUsAtLevel(m, cpuID, n).Intersection(online) - dieCPUs := toCpuSet(x.DieCPUs(hardware.DieID{ + dieCPUs := x.DieCPUs(hardware.DieID{ Package: cpu.PackageID(), Die: cpu.DieID(), - })) + }) switch { case cpus.Size() == 0 || cpus.Size() == 1: continue - case cpus.Equals(toCpuSet(cpu.Threads()).Intersection(online)): + case cpus.Equals(cpu.Threads().Intersection(online)): continue case cpus.Equals(dieCPUs.Intersection(online)): continue @@ -1898,7 +1921,7 @@ func (p CPUPriority) String() string { // > 0 if cpuset A is preferred // < 0 if cpuset B is preferred // 0 if cpusets A and B are equal in terms of cpu priority -func (c *cpuPriorities) cmpCPUSet(csetA, csetB cpuset.CPUSet, prefer CPUPriority, cpuCnt int) int { +func (c *cpuPriorities) cmpCPUSet(csetA, csetB *libcpu.CpuMask, prefer CPUPriority, cpuCnt int) int { if prefer == PriorityNone { return 0 } diff --git a/pkg/cpuallocator/cpuallocator_test.go b/pkg/cpuallocator/cpuallocator_test.go index bb7d7f625..f77347255 100644 --- a/pkg/cpuallocator/cpuallocator_test.go +++ b/pkg/cpuallocator/cpuallocator_test.go @@ -19,10 +19,9 @@ import ( "path" "testing" - "github.com/containers/nri-plugins/pkg/testutils" - "github.com/containers/nri-plugins/pkg/utils/cpuset" - + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" + "github.com/containers/nri-plugins/pkg/testutils" logger "github.com/containers/nri-plugins/pkg/log" ) @@ -49,39 +48,39 @@ func TestAllocatorHelper(t *testing.T) { // Fake cpu priorities: 5 cores from pkg #0 as high prio // Package CPUs: #0: [0-19,40-59], #1: [20-39,60-79] - topoCache.cpuPriorities = [NumCPUPriorities]cpuset.CPUSet{ - cpuset.MustParse("2,5,8,15,17,42,45,48,55,57"), - cpuset.MustParse("20-39,60-79"), - cpuset.MustParse("0,1,3,4,6,7,9-14,16,18,19,40,41,43,44,46,47,49-54,56,58,59"), + topoCache.cpuPriorities = cpuPriorities{ + libcpu.MustParseCpuMask("2,5,8,15,17,42,45,48,55,57"), + libcpu.MustParseCpuMask("20-39,60-79"), + libcpu.MustParseCpuMask("0,1,3,4,6,7,9-14,16,18,19,40,41,43,44,46,47,49-54,56,58,59"), } tcs := []struct { description string - from cpuset.CPUSet + from *libcpu.CpuMask prefer CPUPriority cnt int - expected cpuset.CPUSet + expected *libcpu.CpuMask }{ { description: "too few available CPUs", - from: cpuset.MustParse("2,3,10-14,20"), + from: libcpu.MustParseCpuMask("2,3,10-14,20"), prefer: PriorityNormal, cnt: 9, - expected: cpuset.New(), + expected: libcpu.NewCpuMask(), }, { description: "request all available CPUs", - from: cpuset.MustParse("2,3,10-14,20"), + from: libcpu.MustParseCpuMask("2,3,10-14,20"), prefer: PriorityNormal, cnt: 8, - expected: cpuset.MustParse("2,3,10-14,20"), + expected: libcpu.MustParseCpuMask("2,3,10-14,20"), }, { description: "prefer high priority cpus", - from: cpuset.MustParse("2,3,10-25"), + from: libcpu.MustParseCpuMask("2,3,10-25"), prefer: PriorityHigh, cnt: 4, - expected: cpuset.New(2, 3, 15, 17), + expected: libcpu.NewCpuMask(2, 3, 15, 17), }, } @@ -89,7 +88,7 @@ func TestAllocatorHelper(t *testing.T) { for _, tc := range tcs { t.Run(tc.description, func(t *testing.T) { a := newAllocatorHelper(m, topoCache) - a.from = tc.from + a.from = tc.from.Clone() a.prefer = tc.prefer a.cnt = tc.cnt result := a.allocate() @@ -126,8 +125,8 @@ func TestClusteredAllocation(t *testing.T) { // Fake cpu priorities: 5 cores from pkg #0 as high prio // Package CPUs: #0: [0-19,40-59], #1: [20-39,60-79] - topoCache.cpuPriorities = [NumCPUPriorities]cpuset.CPUSet{ - cpuset.MustParse("0-79"), + topoCache.cpuPriorities = cpuPriorities{ + libcpu.MustParseCpuMask("0-79"), } topoCache.clusters = []*cpuCluster{ @@ -135,169 +134,169 @@ func TestClusteredAllocation(t *testing.T) { pkg: 0, die: 0, cluster: 0, - cpus: cpuset.MustParse("0-3"), + cpus: libcpu.MustParseCpuMask("0-3"), }, { pkg: 0, die: 0, cluster: 1, - cpus: cpuset.MustParse("4-7"), + cpus: libcpu.MustParseCpuMask("4-7"), }, { pkg: 0, die: 0, cluster: 2, - cpus: cpuset.MustParse("8-11"), + cpus: libcpu.MustParseCpuMask("8-11"), }, { pkg: 0, die: 0, cluster: 3, - cpus: cpuset.MustParse("12-15"), + cpus: libcpu.MustParseCpuMask("12-15"), }, { pkg: 0, die: 0, cluster: 4, - cpus: cpuset.MustParse("16-19"), + cpus: libcpu.MustParseCpuMask("16-19"), }, { pkg: 0, die: 0, cluster: 5, - cpus: cpuset.MustParse("40-43"), + cpus: libcpu.MustParseCpuMask("40-43"), }, { pkg: 0, die: 0, cluster: 6, - cpus: cpuset.MustParse("44-47"), + cpus: libcpu.MustParseCpuMask("44-47"), }, { pkg: 0, die: 0, cluster: 7, - cpus: cpuset.MustParse("48-51"), + cpus: libcpu.MustParseCpuMask("48-51"), }, { pkg: 0, die: 0, cluster: 8, - cpus: cpuset.MustParse("52-55"), + cpus: libcpu.MustParseCpuMask("52-55"), }, { pkg: 0, die: 0, cluster: 9, - cpus: cpuset.MustParse("56-59"), + cpus: libcpu.MustParseCpuMask("56-59"), }, { pkg: 1, die: 0, cluster: 0, - cpus: cpuset.MustParse("20,22,24,26"), + cpus: libcpu.MustParseCpuMask("20,22,24,26"), }, { pkg: 1, die: 0, cluster: 1, - cpus: cpuset.MustParse("21,23,25,27"), + cpus: libcpu.MustParseCpuMask("21,23,25,27"), }, { pkg: 1, die: 0, cluster: 2, - cpus: cpuset.MustParse("28-31"), + cpus: libcpu.MustParseCpuMask("28-31"), }, { pkg: 1, die: 0, cluster: 3, - cpus: cpuset.MustParse("32-35"), + cpus: libcpu.MustParseCpuMask("32-35"), }, { pkg: 1, die: 0, cluster: 4, - cpus: cpuset.MustParse("36-39"), + cpus: libcpu.MustParseCpuMask("36-39"), }, { pkg: 1, die: 0, cluster: 5, - cpus: cpuset.MustParse("60-63"), + cpus: libcpu.MustParseCpuMask("60-63"), }, { pkg: 1, die: 0, cluster: 6, - cpus: cpuset.MustParse("64-67"), + cpus: libcpu.MustParseCpuMask("64-67"), }, { pkg: 1, die: 0, cluster: 7, - cpus: cpuset.MustParse("68-71"), + cpus: libcpu.MustParseCpuMask("68-71"), }, { pkg: 1, die: 0, cluster: 8, - cpus: cpuset.MustParse("72-75"), + cpus: libcpu.MustParseCpuMask("72-75"), }, { pkg: 1, die: 0, cluster: 9, - cpus: cpuset.MustParse("76-79"), + cpus: libcpu.MustParseCpuMask("76-79"), }, } - pkg0 := cpuset.MustParse("0-19,40-59") - pkg1 := cpuset.MustParse("20-39,60-79") + pkg0 := libcpu.MustParseCpuMask("0-19,40-59") + pkg1 := libcpu.MustParseCpuMask("20-39,60-79") tcs := []struct { description string - from cpuset.CPUSet + from *libcpu.CpuMask cnt int - expected cpuset.CPUSet + expected *libcpu.CpuMask }{ { description: "CPU cores worth one cluster", from: pkg0, cnt: 4, - expected: cpuset.MustParse("0-3"), + expected: libcpu.MustParseCpuMask("0-3"), }, { description: "CPU cores worth 2 clusters", from: pkg0, cnt: 8, - expected: cpuset.MustParse("0-7"), + expected: libcpu.MustParseCpuMask("0-7"), }, { description: "CPU cores worth 4 clusters in a package", from: pkg0, cnt: 16, - expected: cpuset.MustParse("0-15"), + expected: libcpu.MustParseCpuMask("0-15"), }, { description: "CPU cores worth all clusters in a package", from: pkg0, cnt: 40, - expected: cpuset.MustParse("0-19,40-59"), + expected: libcpu.MustParseCpuMask("0-19,40-59"), }, { description: "CPU cores 1 cluster more than available in the 1st package", from: pkg0.Union(pkg1), cnt: 44, - expected: cpuset.MustParse("0-19,20,22,24,26,40-59"), + expected: libcpu.MustParseCpuMask("0-19,20,22,24,26,40-59"), }, { description: "CPU cores 2 clusters more than available in the 1st package", from: pkg0.Union(pkg1), cnt: 48, - expected: cpuset.MustParse("0-27,40-59"), + expected: libcpu.MustParseCpuMask("0-27,40-59"), }, } @@ -305,7 +304,7 @@ func TestClusteredAllocation(t *testing.T) { for _, tc := range tcs { t.Run(tc.description, func(t *testing.T) { a := newAllocatorHelper(m, topoCache) - a.from = tc.from + a.from = tc.from.Clone() a.cnt = tc.cnt result := a.allocate() if !result.Equals(tc.expected) { @@ -343,70 +342,70 @@ func TestClusteredCoreKindAllocation(t *testing.T) { pkg: 0, die: 0, cluster: 0, - cpus: cpuset.MustParse("0-3"), + cpus: libcpu.MustParseCpuMask("0-3"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 1, - cpus: cpuset.MustParse("4-7"), + cpus: libcpu.MustParseCpuMask("4-7"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 2, - cpus: cpuset.MustParse("8-11"), + cpus: libcpu.MustParseCpuMask("8-11"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 3, - cpus: cpuset.MustParse("12-15"), + cpus: libcpu.MustParseCpuMask("12-15"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 4, - cpus: cpuset.MustParse("16-19"), + cpus: libcpu.MustParseCpuMask("16-19"), kind: hardware.EfficientCore, }, { pkg: 0, die: 0, cluster: 5, - cpus: cpuset.MustParse("40-43"), + cpus: libcpu.MustParseCpuMask("40-43"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 6, - cpus: cpuset.MustParse("44-47"), + cpus: libcpu.MustParseCpuMask("44-47"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 7, - cpus: cpuset.MustParse("48-51"), + cpus: libcpu.MustParseCpuMask("48-51"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 8, - cpus: cpuset.MustParse("52-55"), + cpus: libcpu.MustParseCpuMask("52-55"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 9, - cpus: cpuset.MustParse("56-59"), + cpus: libcpu.MustParseCpuMask("56-59"), kind: hardware.EfficientCore, }, @@ -414,70 +413,70 @@ func TestClusteredCoreKindAllocation(t *testing.T) { pkg: 1, die: 0, cluster: 0, - cpus: cpuset.MustParse("20,22,24,26"), + cpus: libcpu.MustParseCpuMask("20,22,24,26"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 1, - cpus: cpuset.MustParse("21,23,25,27"), + cpus: libcpu.MustParseCpuMask("21,23,25,27"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 2, - cpus: cpuset.MustParse("28-31"), + cpus: libcpu.MustParseCpuMask("28-31"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 3, - cpus: cpuset.MustParse("32-35"), + cpus: libcpu.MustParseCpuMask("32-35"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 4, - cpus: cpuset.MustParse("36-39"), + cpus: libcpu.MustParseCpuMask("36-39"), kind: hardware.EfficientCore, }, { pkg: 1, die: 0, cluster: 5, - cpus: cpuset.MustParse("60-63"), + cpus: libcpu.MustParseCpuMask("60-63"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 6, - cpus: cpuset.MustParse("64-67"), + cpus: libcpu.MustParseCpuMask("64-67"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 7, - cpus: cpuset.MustParse("68-71"), + cpus: libcpu.MustParseCpuMask("68-71"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 8, - cpus: cpuset.MustParse("72-75"), + cpus: libcpu.MustParseCpuMask("72-75"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 9, - cpus: cpuset.MustParse("76-79"), + cpus: libcpu.MustParseCpuMask("76-79"), kind: hardware.EfficientCore, }, } @@ -487,70 +486,70 @@ func TestClusteredCoreKindAllocation(t *testing.T) { pkg: 0, die: 0, cluster: 0, - cpus: cpuset.MustParse("0-3"), + cpus: libcpu.MustParseCpuMask("0-3"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 1, - cpus: cpuset.MustParse("4-7"), + cpus: libcpu.MustParseCpuMask("4-7"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 2, - cpus: cpuset.MustParse("8-11"), + cpus: libcpu.MustParseCpuMask("8-11"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 3, - cpus: cpuset.MustParse("12-15"), + cpus: libcpu.MustParseCpuMask("12-15"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 4, - cpus: cpuset.MustParse("16-19"), + cpus: libcpu.MustParseCpuMask("16-19"), kind: hardware.EfficientCore, }, { pkg: 0, die: 0, cluster: 5, - cpus: cpuset.MustParse("40-43"), + cpus: libcpu.MustParseCpuMask("40-43"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 6, - cpus: cpuset.MustParse("44-47"), + cpus: libcpu.MustParseCpuMask("44-47"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 7, - cpus: cpuset.MustParse("48-51"), + cpus: libcpu.MustParseCpuMask("48-51"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 8, - cpus: cpuset.MustParse("52-55"), + cpus: libcpu.MustParseCpuMask("52-55"), kind: hardware.PerformanceCore, }, { pkg: 0, die: 0, cluster: 9, - cpus: cpuset.MustParse("56-59"), + cpus: libcpu.MustParseCpuMask("56-59"), kind: hardware.EfficientCore, }, @@ -558,92 +557,92 @@ func TestClusteredCoreKindAllocation(t *testing.T) { pkg: 1, die: 0, cluster: 0, - cpus: cpuset.MustParse("20,22,24,26"), + cpus: libcpu.MustParseCpuMask("20,22,24,26"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 1, - cpus: cpuset.MustParse("21,23,25,27"), + cpus: libcpu.MustParseCpuMask("21,23,25,27"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 2, - cpus: cpuset.MustParse("28-31"), + cpus: libcpu.MustParseCpuMask("28-31"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 3, - cpus: cpuset.MustParse("32-35"), + cpus: libcpu.MustParseCpuMask("32-35"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 4, - cpus: cpuset.MustParse("36-37"), + cpus: libcpu.MustParseCpuMask("36-37"), kind: hardware.EfficientCore, }, { pkg: 1, die: 0, cluster: 5, - cpus: cpuset.MustParse("38-39"), + cpus: libcpu.MustParseCpuMask("38-39"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 6, - cpus: cpuset.MustParse("60-63"), + cpus: libcpu.MustParseCpuMask("60-63"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 7, - cpus: cpuset.MustParse("64-67"), + cpus: libcpu.MustParseCpuMask("64-67"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 8, - cpus: cpuset.MustParse("68-71"), + cpus: libcpu.MustParseCpuMask("68-71"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 9, - cpus: cpuset.MustParse("72-75"), + cpus: libcpu.MustParseCpuMask("72-75"), kind: hardware.PerformanceCore, }, { pkg: 1, die: 0, cluster: 10, - cpus: cpuset.MustParse("76-79"), + cpus: libcpu.MustParseCpuMask("76-79"), kind: hardware.EfficientCore, }, } - pkg0 := cpuset.MustParse("0-19,40-59") - pkg1 := cpuset.MustParse("20-39,60-79") + pkg0 := libcpu.MustParseCpuMask("0-19,40-59") + pkg1 := libcpu.MustParseCpuMask("20-39,60-79") all := pkg0.Union(pkg1) tcs := []struct { description string clusters []*cpuCluster - from cpuset.CPUSet + from *libcpu.CpuMask prefer CPUPriority cnt int - expected cpuset.CPUSet + expected *libcpu.CpuMask }{ { description: "P-cores worth one cluster", @@ -651,7 +650,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityNormal, cnt: 4, - expected: cpuset.MustParse("0-3"), + expected: libcpu.MustParseCpuMask("0-3"), }, { description: "P-cores worth 2 clusters", @@ -659,7 +658,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityNormal, cnt: 8, - expected: cpuset.MustParse("0-7"), + expected: libcpu.MustParseCpuMask("0-7"), }, { description: "P-cores worth all clusters in a package", @@ -667,7 +666,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityNormal, cnt: 32, - expected: cpuset.MustParse("0-15,40-55"), + expected: libcpu.MustParseCpuMask("0-15,40-55"), }, { description: "E-cores worth 1 cluster", @@ -675,7 +674,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityLow, cnt: 4, - expected: cpuset.MustParse("16-19"), + expected: libcpu.MustParseCpuMask("16-19"), }, { description: "E-cores worth 2 clusters", @@ -683,7 +682,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityLow, cnt: 8, - expected: cpuset.MustParse("16-19,56-59"), + expected: libcpu.MustParseCpuMask("16-19,56-59"), }, { description: "P-cores worth 1 cluster more than in the 1st package", @@ -691,7 +690,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityNormal, cnt: 36, - expected: cpuset.MustParse("0-15,40-55,20,22,24,26"), + expected: libcpu.MustParseCpuMask("0-15,40-55,20,22,24,26"), }, { description: "P-cores worth 2 clusters more than in the 1st package", @@ -699,7 +698,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityNormal, cnt: 40, - expected: cpuset.MustParse("0-15,20-27,40-55"), + expected: libcpu.MustParseCpuMask("0-15,20-27,40-55"), }, { description: "E-cores worth 1 clusters, should take tighter fit", @@ -707,7 +706,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityLow, cnt: 2, - expected: cpuset.MustParse("36-37"), + expected: libcpu.MustParseCpuMask("36-37"), }, { description: "E-cores worth 2 clusters, should take tighter fit", @@ -715,7 +714,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityLow, cnt: 6, - expected: cpuset.MustParse("36-37,76-79"), + expected: libcpu.MustParseCpuMask("36-37,76-79"), }, { description: "E-cores worth 2 clusters, should take single die", @@ -723,7 +722,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { from: all, prefer: PriorityLow, cnt: 8, - expected: cpuset.MustParse("16-19,56-59"), + expected: libcpu.MustParseCpuMask("16-19,56-59"), }, } @@ -733,7 +732,7 @@ func TestClusteredCoreKindAllocation(t *testing.T) { topoCache := newTopologyCache(m) topoCache.clusters = tc.clusters a := newAllocatorHelper(m, topoCache) - a.from = tc.from + a.from = tc.from.Clone() a.prefer = tc.prefer a.cnt = tc.cnt result := a.allocate() @@ -749,3 +748,103 @@ func removeAll(t *testing.T, path string) { t.Fatalf("failed to remove %q: %v", path, err) } } + +// TestAllocateAndReleaseCpus pins what the two public entry points do to the set +// they are given, which is how every caller learns what is left. Nothing else +// here covers it: the tests above drive the helper directly. +func TestAllocateAndReleaseCpus(t *testing.T) { + tmpdir, err := os.MkdirTemp("", "nri-resource-policy-test-") + if err != nil { + t.Fatalf("failed to create tmpdir: %v", err) + } + defer removeAll(t, tmpdir) + + if err := testutils.UncompressTbz2(path.Join("testdata", "sysfs.tar.bz2"), tmpdir); err != nil { + t.Fatalf("failed to decompress testdata: %v", err) + } + + m, err := hardware.Discover( + hardware.WithRoot(path.Join(tmpdir, "sysfs", "2-socket-4-node-40-core"))) + if err != nil { + t.Fatalf("failed to discover mock system: %v", err) + } + ca := NewCPUAllocator(m) + + for _, tc := range []struct { + description string + from string + cnt int + release bool + expected string // what comes back + remaining string // what the given set holds afterwards + expectErr bool + }{ + { + description: "allocating takes the CPUs out of the set", + from: "0-7", + cnt: 2, + expected: "0-1", + remaining: "2-7", + }, + { + description: "allocating everything empties the set", + from: "0-7", + cnt: 8, + expected: "0-7", + remaining: "", + }, + { + description: "asking for more than there is leaves the set alone", + from: "0-7", + cnt: 9, + expected: "", + remaining: "0-7", + expectErr: true, + }, + { + // Note which way round this is: the set is left holding the CPUs to + // release, and the ones to keep are returned. + description: "releasing leaves the released CPUs in the set", + from: "0-7", + cnt: 2, + release: true, + expected: "0-5", + remaining: "6-7", + }, + { + description: "releasing everything keeps nothing", + from: "0-7", + cnt: 8, + release: true, + expected: "", + remaining: "0-7", + }, + } { + t.Run(tc.description, func(t *testing.T) { + var ( + from = libcpu.MustParseCpuMask(tc.from) + cpus *libcpu.CpuMask + err error + ) + + if tc.release { + cpus, err = ca.ReleaseCpus(from, tc.cnt) + } else { + cpus, err = ca.AllocateCpus(from, tc.cnt) + } + + switch { + case tc.expectErr && err == nil: + t.Error("expected an error, got none") + case !tc.expectErr && err != nil: + t.Errorf("unexpected error: %v", err) + } + if got := cpus.String(); got != tc.expected { + t.Errorf("expected %q back, got %q", tc.expected, got) + } + if got := from.String(); got != tc.remaining { + t.Errorf("expected %q left in the set, got %q", tc.remaining, got) + } + }) + } +} diff --git a/pkg/cpuallocator/topology.go b/pkg/cpuallocator/topology.go index eb9cfa7da..950f3861c 100644 --- a/pkg/cpuallocator/topology.go +++ b/pkg/cpuallocator/topology.go @@ -17,34 +17,27 @@ package cpuallocator import ( libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) -// toCpuSet converts a set the hardware package returns to the one this package -// keeps its topology in. It goes away if this package ever switches to libcpu -// sets throughout. -func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { - return cpuset.New(cpus.List()...) -} - // cacheCPUsAtLevel returns the CPUs sharing any of a CPU's caches at one level, -// or its thread siblings when it has no caches at all. +// or its thread siblings when it has no caches at all. The result is for reading +// only: a CPU with no caches is answered with the machine's own sealed set. // // Note the union: a level with a separate data and instruction cache contributes // both, which is why this does not just take hardware.CPU.Cache. -func cacheCPUsAtLevel(m *hardware.Machine, id idset.ID, level int) cpuset.CPUSet { +func cacheCPUsAtLevel(m *hardware.Machine, id idset.ID, level int) *libcpu.CpuMask { c := m.CPU(id) caches := c.Caches() if len(caches) == 0 { - return toCpuSet(c.Threads()) + return c.Threads() } - cpus := cpuset.New() + cpus := libcpu.NewCpuMask() for _, cache := range caches { if cache.Level() == level { - cpus = cpus.Union(toCpuSet(cache.CPUs())) + cpus = cpus.Union(cache.CPUs()) } else if cache.Level() > level { break } From 7612c906016fd825fa35d53052b061f6d31fcad7 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 21:36:31 +0300 Subject: [PATCH 32/39] balloons: track CPUs as libcpu.CpuMask. The policy kept every CPU set as a k8s cpuset set and converted to a mask wherever it asked the hardware package or the CPU allocator something. Its own sets are masks now, so the CPU tree the allocator walks per admission compares masks rather than maps, and the conversions at the topology and allocator edges go away. What is left of the seam is where interfaces this policy calls still take cpuset sets: the CPU class controller, the IRQ affinity helpers, libmem's CPUSetAffinity, and the configuration which parses an operator's cpuset string. Fifteen call sites, all of them cold, against internals which no longer convert at all. Nothing here modifies a set in place, so nothing aliases: every operation builds a new mask, as it built a new cpuset before. The sets which used to fall out of a zero value are now created where they belong, in New and in the balloon constructors, since a mask has no usable zero value. The two map lookups which relied on that -- a CPU tree test asking about a balloon it has not created yet, and its accumulator of allocation rounds -- read through EmptyIfNil. Metric labels are unchanged: a mask spells itself the same way. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .codespellignorelines | 2 +- .../balloons/policy/balloons-policy.go | 202 ++++++++++-------- cmd/plugins/balloons/policy/cpuclass_test.go | 7 +- cmd/plugins/balloons/policy/cputree.go | 105 ++++----- cmd/plugins/balloons/policy/cputree_test.go | 54 ++--- cmd/plugins/balloons/policy/metrics.go | 8 +- 6 files changed, 196 insertions(+), 182 deletions(-) diff --git a/.codespellignorelines b/.codespellignorelines index 050948c58..8c1c5e377 100644 --- a/.codespellignorelines +++ b/.codespellignorelines @@ -1,7 +1,7 @@ ( vm-command "kubectl delete pods -n kube-system \$(kubectl get pods -n kube-system | awk '/t[0-9]r[gb][ue]/{print \$1}')" ) || true ( vm-command "kubectl delete pods -n default \$(kubectl get pods -n default | awk '/t[0-9][rgb][ue][0-9]/{print \$1}')" ) || true docker run -v "$(pwd):/mnt/models" "$FMBT_IMAGE" sh -c 'cd /mnt/models; fmbt tmp.fuzz.fmbt.conf 2>/dev/null | fmbt-log -f STEP\$sn\$as\$al' | grep -v AAL | sed -e 's/^, / /g' -e '/^STEP/! s/\(^.*\)/echo "TESTGEN: \1"/g' -e 's/^STEP\([0-9]*\)i:\(.*\)/echo "TESTGEN: STEP \1"; vm-command "date +%T.%N"; \2; vm-command "date +%T.%N"; kubectl get pods -A/g' | sed "s/\([^a-z0-9]\)\(r\?\)\(gu\|bu\|be\)\([0-9]\)/\1t${testid}\2\3\4/g" > "$OUTFILE" - allocs := map[string]cpuset.CPUSet{"--:allo": currentCpus} + allocs := map[string]*libcpu.CpuMask{"--:allo": currentCpus} allocName := fmt.Sprintf("%02d:allo", i+1) preferTightestFit = func(cA, cB *cpuCluster, pkgA, pkgB, dieA, dieB int, csetA, csetB *libcpu.CpuMask) (r int) { if dieA >= a.cnt && dieB < a.cnt { diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 4e4cc0df7..314c825f7 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -31,6 +31,7 @@ import ( "github.com/containers/nri-plugins/pkg/cpuallocator" "github.com/containers/nri-plugins/pkg/irq" "github.com/containers/nri-plugins/pkg/kubernetes" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cache" @@ -83,10 +84,10 @@ type balloons struct { bpoptions *BalloonsOptions // balloons-specific configuration machine *hardware.Machine // CPU and memory topology cch cache.Cache // nri-resource-policy cache - allowed cpuset.CPUSet // bounding set of CPUs we're allowed to use - reserved cpuset.CPUSet // system-/kube-reserved CPUs - freeCpus cpuset.CPUSet // CPUs to be included in growing or new ballons - ifreeCpus cpuset.CPUSet // initially free CPUs before assigning any containers + allowed *libcpu.CpuMask // bounding set of CPUs we're allowed to use + reserved *libcpu.CpuMask // system-/kube-reserved CPUs + freeCpus *libcpu.CpuMask // CPUs to be included in growing or new ballons + ifreeCpus *libcpu.CpuMask // initially free CPUs before assigning any containers cpuTree *cpuTreeNode // system CPU topology reservedBalloonDef *BalloonDef // reserved balloon definition, pointer to bpoptions.BalloonDefs[x] @@ -110,14 +111,14 @@ type Balloon struct { // zero for every balloon definition. Instance int // Cpus is the set of CPUs exclusive to this balloon instance only. - Cpus cpuset.CPUSet + Cpus *libcpu.CpuMask // Mems is the set of memory nodes with minimal access delay // from CPUs. Mems idset.IDSet // SharedIdleCpus is the set of idle CPUs that workloads in a // balloon are allowed to use with workloads in other balloons // that shareIdleCpus. - SharedIdleCpus cpuset.CPUSet + SharedIdleCpus *libcpu.CpuMask // PodIDs maps pod ID to list of container IDs. // - len(PodIDs) is the number of pods in the balloon. // - len(PodIDs[podID]) is the number of containers of podID @@ -146,6 +147,26 @@ type loadClassVirtDev struct { updateOnEveryCpuAllocation bool } +// toCpuSet and toCpuMask convert between the CPU sets this policy keeps and the +// ones some of the interfaces it calls still take: the CPU class controller, +// libmem and the IRQ affinity helpers. Everything the policy does with CPUs in +// between is done with libcpu masks. +func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { + return cpuset.New(cpus.List()...) +} + +func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { + return libcpu.NewCpuMask(cpus.List()...) +} + +func toCpuMasks(sets []cpuset.CPUSet) []*libcpu.CpuMask { + masks := make([]*libcpu.CpuMask, 0, len(sets)) + for _, cpus := range sets { + masks = append(masks, toCpuMask(cpus)) + } + return masks +} + var log logger.Logger = logger.NewLogger("policy") // String is a stringer for a balloon. @@ -181,7 +202,7 @@ func (bln Balloon) AvailMilliCpus() int { return bln.Cpus.Size() * 1000 } -func (bln Balloon) MaxAvailMilliCpus(freeCpus cpuset.CPUSet) int { +func (bln Balloon) MaxAvailMilliCpus(freeCpus *libcpu.CpuMask) int { availableFreeCpus := freeCpus.Size() if len(bln.components) > 0 { // MaxCpus of component balloons can limit the size of @@ -206,7 +227,15 @@ func (bln Balloon) MaxAvailMilliCpus(freeCpus cpuset.CPUSet) int { // New creates a new uninitialized balloons policy instance. func New() policy.Backend { - return &balloons{} + // The policy's own CPU sets exist from the start, before any configuration + // lands: a CpuMask has no usable zero value, and something may ask this for + // metrics before, or instead of, configuring it. + return &balloons{ + allowed: libcpu.NewCpuMask(), + reserved: libcpu.NewCpuMask(), + freeCpus: libcpu.NewCpuMask(), + ifreeCpus: libcpu.NewCpuMask(), + } } // Setup initializes the balloons policy instance. @@ -566,7 +595,7 @@ func (p *balloons) GetTopologyZones() []*policy.TopologyZone { ctrCpusetCpus := c.GetCpusetCpus() ctrAllowedmCpu := sysmCpu if ctrCpusetCpus != "" { - ctrAllowedmCpu = 1000 * cpuset.MustParse(ctrCpusetCpus).Size() + ctrAllowedmCpu = 1000 * libcpu.MustParseCpuMask(ctrCpusetCpus).Size() } if ctrLimitmCpu == 0 || ctrLimitmCpu > ctrAllowedmCpu { ctrCapacitymCpu = ctrAllowedmCpu @@ -620,14 +649,14 @@ func (p *balloons) GetExtendedResources() map[string]*resource.Quantity { log.Warnf("ignoring publishExtendedResource on non-PCT cpuClass %q", cc.Name) continue } - held := cpuset.New() + held := libcpu.NewCpuMask() for _, bln := range p.balloons { if p.resolveCpuClassName(bln.Def.CpuClass) == cc.Name { continue } held = held.Union(bln.Cpus) } - free := max(p.cpuClasses.PctFreeClassCapacity(cc.Name, held), 0) + free := max(p.cpuClasses.PctFreeClassCapacity(cc.Name, toCpuSet(held)), 0) out["cpuclass.balloons.nri.io/"+cc.Name] = resource.NewQuantity(int64(free), resource.DecimalSI) } return out @@ -878,7 +907,7 @@ func (p *balloons) resetCpuClass() error { return nil } idle := p.resolveCpuClassName(p.bpoptions.IdleCpuClass) - if err := p.cpuClasses.UseClass(idle, p.allowed); err != nil { + if err := p.cpuClasses.UseClass(idle, toCpuSet(p.allowed)); err != nil { log.Warnf("failed to reset class of available cpus: %v", err) } else { log.Debugf("reset class of available cpus: %q to idle class %q (reserved: %q)", @@ -922,7 +951,7 @@ func (p *balloons) useCpuClass(bln *Balloon) error { } cpuClass := p.resolveCpuClassName(bln.Def.CpuClass) log.Debugf("apply CPU class %q on CPUs %q of %q", cpuClass, bln.Cpus, bln.PrettyName()) - if err := p.cpuClasses.UseClass(cpuClass, bln.Cpus); err != nil { + if err := p.cpuClasses.UseClass(cpuClass, toCpuSet(bln.Cpus)); err != nil { log.Warnf("failed to apply class %q on CPUs %q: %v", cpuClass, bln.Cpus, err) } return nil @@ -936,7 +965,7 @@ func (p *balloons) forgetCpuClass(bln *Balloon) { return } idle := p.resolveCpuClassName(p.bpoptions.IdleCpuClass) - if err := p.cpuClasses.UseClass(idle, bln.Cpus); err != nil { + if err := p.cpuClasses.UseClass(idle, toCpuSet(bln.Cpus)); err != nil { log.Warnf("failed to forget class of cpus %q (idle class %q): %v", bln.Cpus, idle, err) } else { if len(bln.components) > 0 { @@ -961,8 +990,8 @@ func (p *balloons) irqOptionsInUse() bool { // irqClaimCpus returns the union of CPUs of balloons that claim the // given interrupt. -func (p *balloons) irqClaimCpus(hwIrq *irq.Irq) cpuset.CPUSet { - claimCpus := cpuset.New() +func (p *balloons) irqClaimCpus(hwIrq *irq.Irq) *libcpu.CpuMask { + claimCpus := libcpu.NewCpuMask() for _, bln := range p.balloons { if bln.Cpus.IsEmpty() { continue @@ -985,8 +1014,8 @@ func (p *balloons) applyIrqAffinities() { log.Warnf("failed to read interrupts for IRQ affinity update: %v", err) return } - sinkCpus := cpuset.New() - isolateCpus := cpuset.New() + sinkCpus := libcpu.NewCpuMask() + isolateCpus := libcpu.NewCpuMask() for _, bln := range p.balloons { if bln.Cpus.IsEmpty() { continue @@ -1000,10 +1029,11 @@ func (p *balloons) applyIrqAffinities() { } for _, hwIrq := range hwIrqs { newCpus := p.allowed - curCpus, err := hwIrq.AffinityCpus() + affinity, err := hwIrq.AffinityCpus() if err != nil { continue } + curCpus := toCpuMask(affinity) switch claimCpus := p.irqClaimCpus(hwIrq); { case !claimCpus.IsEmpty(): newCpus = claimCpus @@ -1020,7 +1050,7 @@ func (p *balloons) applyIrqAffinities() { if curCpus.Equals(newCpus) { continue } - if err := hwIrq.SetAffinityCpus(newCpus); err != nil { + if err := hwIrq.SetAffinityCpus(toCpuSet(newCpus)); err != nil { log.Debugf("failed to set affinity of %s to %q: %v", hwIrq, newCpus, err) } else { log.Debugf("set affinity of %s to %q", hwIrq, newCpus) @@ -1043,7 +1073,7 @@ func (p *balloons) updateLoadedVirtDevsInAllocatorOptions(allocatorOptions *cpuT // Go through all balloons that share the same loaded // virtual device and collect their CPUs into vdCpus // (virtual device CPUs) - vdCpus := cpuset.New() + vdCpus := libcpu.NewCpuMask() for _, bln := range p.balloons { if _, ok := bln.LoadedVirtDevs[vdName]; !ok { continue @@ -1056,8 +1086,8 @@ func (p *balloons) updateLoadedVirtDevsInAllocatorOptions(allocatorOptions *cpuT return loadedVirtDevs } -func (p *balloons) updateLoadedVirtDev(allocatorOptions *cpuTreeAllocatorOptions, virtDev *loadClassVirtDev, vdCpus cpuset.CPUSet, overwrite bool) { - prevCpus := cpuset.New() +func (p *balloons) updateLoadedVirtDev(allocatorOptions *cpuTreeAllocatorOptions, virtDev *loadClassVirtDev, vdCpus *libcpu.CpuMask, overwrite bool) { + prevCpus := libcpu.NewCpuMask() virtDevName := virtDev.name if !overwrite && len(allocatorOptions.virtDevCpusets[virtDevName]) > 0 { prevCpus = allocatorOptions.virtDevCpusets[virtDevName][0] @@ -1066,11 +1096,11 @@ func (p *balloons) updateLoadedVirtDev(allocatorOptions *cpuTreeAllocatorOptions switch virtDev.level { case CPUTopologyLevelCore: // add all CPUs from same cores of virtual device CPUs - allocatorOptions.virtDevCpusets[virtDevName] = []cpuset.CPUSet{prevCpus.Union(toCpuSet(hardware.AllThreads(p.machine, toCpuMask(vdCpus))))} + allocatorOptions.virtDevCpusets[virtDevName] = []*libcpu.CpuMask{prevCpus.Union(hardware.AllThreads(p.machine, vdCpus))} case CPUTopologyLevelL2Cache: // add all CPUs from the same L2 cache of virtual // device CPUs - allocatorOptions.virtDevCpusets[virtDevName] = []cpuset.CPUSet{prevCpus.Union(toCpuSet(hardware.CPUsSharingCache(p.machine, 2, toCpuMask(vdCpus))))} + allocatorOptions.virtDevCpusets[virtDevName] = []*libcpu.CpuMask{prevCpus.Union(hardware.CPUsSharingCache(p.machine, 2, vdCpus))} default: log.Errorf("internal error: not implemented load level %q used in virtual device %q", virtDev.level, virtDevName) } @@ -1166,8 +1196,8 @@ func (p *balloons) newCompositeBalloon(blnDef *BalloonDef, confCpus bool, freeIn Instance: freeInstance, Groups: make(map[string]int), PodIDs: make(map[string][]string), - Cpus: cpuset.New(), - SharedIdleCpus: cpuset.New(), + Cpus: libcpu.NewCpuMask(), + SharedIdleCpus: libcpu.NewCpuMask(), LoadedVirtDevs: make(map[string]struct{}), cpuTreeAlloc: nil, // Allocator is not used for composite balloons. memTypeMask: memTypeMask, @@ -1191,7 +1221,6 @@ func (p *balloons) newCompositeBalloon(blnDef *BalloonDef, confCpus bool, freeIn } func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool, c cache.Container) (*Balloon, error) { - var cpus cpuset.CPUSet var err error blnsOfDef := p.balloonsByDef(blnDef) // Allowed to create new balloon instance from blnDef? @@ -1224,11 +1253,11 @@ func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool, c cache.Contain preferSpreadOnPhysicalCores: p.bpoptions.PreferSpreadOnPhysicalCores, preferCloseToDevices: append([]string(nil), blnDef.PreferCloseToDevices...), preferFarFromDevices: append([]string(nil), blnDef.PreferFarFromDevices...), - virtDevCpusets: map[string][]cpuset.CPUSet{ + virtDevCpusets: map[string][]*libcpu.CpuMask{ virtDevReservedCpus: {p.reserved}, - virtDevIsolatedCpus: {toCpuSet(p.machine.IsolatedCPUs())}, - virtDevECores: {toCpuSet(p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityLow])}, - virtDevPCores: {toCpuSet(p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityHigh])}, + virtDevIsolatedCpus: {p.machine.IsolatedCPUs()}, + virtDevECores: {p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityLow]}, + virtDevPCores: {p.cpuAllocator.GetCPUPriorities()[cpuallocator.PriorityHigh]}, }, } // Pod resource hints to container's physical devices (GPUs, @@ -1238,7 +1267,7 @@ func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool, c cache.Contain // allocator to choose the best alternative CPU set for the // pod resources. p.applyPodResourcesHints(&allocatorOptions, c) - p.applyCpuClassHints(&allocatorOptions, p.resolveCpuClassName(blnDef.CpuClass), cpuset.New(), 0) + p.applyCpuClassHints(&allocatorOptions, p.resolveCpuClassName(blnDef.CpuClass), libcpu.NewCpuMask(), 0) if blnDef.AllocatorTopologyBalancing != nil { allocatorOptions.topologyBalancing = *blnDef.AllocatorTopologyBalancing } @@ -1256,14 +1285,14 @@ func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool, c cache.Contain Instance: freeInstance, Groups: make(map[string]int), PodIDs: make(map[string][]string), - Cpus: cpuset.New(), - SharedIdleCpus: cpuset.New(), + Cpus: libcpu.NewCpuMask(), + SharedIdleCpus: libcpu.NewCpuMask(), LoadedVirtDevs: loadedVirtDevs, cpuTreeAlloc: cpuTreeAlloc, memTypeMask: memTypeMask, } if p.virtDevsChangeDuringCpuAllocation(blnDef.Loads) { - bln.cpuTreeAlloc.options.deviceUpdateOnEveryCpu = func(currentCpus cpuset.CPUSet) { + bln.cpuTreeAlloc.options.deviceUpdateOnEveryCpu = func(currentCpus *libcpu.CpuMask) { for _, load := range blnDef.Loads { p.updateLoadedVirtDev(&cpuTreeAlloc.options, p.loadVirtDev[load], currentCpus, false) } @@ -1275,7 +1304,7 @@ func (p *balloons) newBalloon(blnDef *BalloonDef, confCpus bool, c cache.Contain bln.Mems = p.closestMems(bln.Cpus) if confCpus { if err = p.useCpuClass(bln); err != nil { - log.Errorf("failed to apply CPU configuration to new balloon %s[%d] (cpus: %s): %v", blnDef.Name, freeInstance, cpus, err) + log.Errorf("failed to apply CPU configuration to new balloon %s[%d] (cpus: %s): %v", blnDef.Name, freeInstance, bln.Cpus, err) return nil, err } } @@ -1294,11 +1323,9 @@ func (p *balloons) deleteBalloon(bln *Balloon) { p.balloons = remainingBalloons p.forgetCpuClass(bln) p.freeCpus = p.freeCpus.Union(bln.Cpus) - blnCpus := toCpuMask(bln.Cpus) - if _, err := p.cpuAllocator.ReleaseCpus(blnCpus, bln.Cpus.Size(), bln.Def.AllocatorPriority.Value().Option()); err != nil { + if _, err := p.cpuAllocator.ReleaseCpus(bln.Cpus, bln.Cpus.Size(), bln.Def.AllocatorPriority.Value().Option()); err != nil { log.Warnf("failed to release CPUs %q of balloon %s[%d]: %v", bln.Cpus, bln.Def.Name, bln.Instance, err) } - bln.Cpus = toCpuSet(blnCpus) } // freeBalloon clears a balloon and deletes it if allowed. @@ -1659,7 +1686,7 @@ func (p *balloons) Reconfigure(newCfg any) error { if err := p.cpuClasses.Configure(cpuclass.ConfigSpec{ Classes: p.bpoptions.CPUClasses, TurboDomain: p.bpoptions.TurboDomain, - Allowed: p.allowed, + Allowed: toCpuSet(p.allowed), }); err != nil { log.Warnf("failed to reconfigure CPU class handler: %v", err) } @@ -1697,7 +1724,7 @@ func (p *balloons) Reconfigure(newCfg any) error { // applyBalloonDef creates user-defined balloons or reconfigures built-in // balloons according to the blnDef. Does not initialize balloon CPUs. -func (p *balloons) applyBalloonDef(balloons *[]*Balloon, blnDef *BalloonDef, freeCpus *cpuset.CPUSet) error { +func (p *balloons) applyBalloonDef(balloons *[]*Balloon, blnDef *BalloonDef, freeCpus **libcpu.CpuMask) error { for blnIdx := 0; blnIdx < blnDef.MinBalloons; blnIdx++ { newBln, err := p.newBalloon(blnDef, false, nil) if err != nil { @@ -1888,7 +1915,7 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { // Handle AvailableResources.cpus, if defined. // Set p.allowed: CPUs available for the policy. - var availableCpus cpuset.CPUSet + var availableCpus *libcpu.CpuMask amount, kind := bpoptions.AvailableResources.Get(cfgapi.CPU) switch kind { case cfgapi.AmountCPUSet: @@ -1896,22 +1923,23 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { if err != nil { return balloonsError("failed to parse available CPU cpuset '%s': %w", amount, err) } - availableCpus = cset + availableCpus = toCpuMask(cset) case cfgapi.AmountExcludeCPUSet: cset, err := amount.ParseCPUSet() if err != nil { return balloonsError("failed to parse available CPU cpuset '%s': %w", amount, err) } - availableCpus = toCpuSet(p.machine.PresentCPUs()).Difference(cset) + availableCpus = p.machine.PresentCPUs().Difference(toCpuMask(cset)) case cfgapi.AmountQuantity: return balloonsError("can't handle CPU resources given as resource.Quantity (%v)", amount) case cfgapi.AmountAbsent: // Available CPUs not specified, default to system CPUs. - availableCpus = toCpuSet(p.machine.PresentCPUs()) + availableCpus = p.machine.PresentCPUs() } - // Allocation of only online CPUs is allowed. - p.allowed = availableCpus.Intersection(toCpuSet(p.machine.OnlineCPUs())) + // Allocation of only online CPUs is allowed. A kind of amount not handled + // above leaves no CPUs available, which is what it did before too. + p.allowed = availableCpus.Intersection(p.machine.OnlineCPUs()) setOmittedDefaults(bpoptions) @@ -1946,7 +1974,7 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { if err := p.cpuClasses.Configure(cpuclass.ConfigSpec{ Classes: bpoptions.CPUClasses, TurboDomain: bpoptions.TurboDomain, - Allowed: p.allowed, + Allowed: toCpuSet(p.allowed), }); err != nil { return balloonsError("failed to configure CPU class handler: %w", err) } @@ -1989,7 +2017,7 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { for blnIdx, bln := range p.balloons { log.Infof("- balloon %d: %s", blnIdx, bln) } - p.updatePinning(p.shareIdleCpus(p.freeCpus, cpuset.New())...) + p.updatePinning(p.shareIdleCpus(p.freeCpus, libcpu.NewCpuMask())...) // (Re)configures all CPUs in balloons. if err := p.resetCpuClass(); err != nil { log.Warnf("failed to reset CPU class: %v", err) @@ -2068,10 +2096,11 @@ func (p *balloons) fillBuiltinBalloonDefs(bpoptions *BalloonsOptions) (*BalloonD // can still allocate CPUs first. If reserved // balloon's MinCpus is undefined, set it to catch all // (or at most MaxCpu) CPUs in the reserved cpuset. - cset, err := amount.ParseCPUSet() + parsed, err := amount.ParseCPUSet() if err != nil { return nil, nil, balloonsError("failed to parse reserved CPU cpuset '%s': %v", amount, err) } + cset := toCpuMask(parsed) if kind == cfgapi.AmountExcludeCPUSet { cset = p.allowed.Difference(cset) } @@ -2116,7 +2145,7 @@ func (p *balloons) fillBuiltinBalloonDefs(bpoptions *BalloonsOptions) (*BalloonD return nil, nil, balloonsError("mismatching reserved balloon minCpus: %d and ReservedResources cpus: %d mCPU", reservedBalloonDef.MinCpus, qty.MilliValue()) } - p.reserved = cpuset.New() + p.reserved = libcpu.NewCpuMask() } reservedBalloonDef.MinBalloons = 1 @@ -2165,14 +2194,14 @@ const cpuClassHintDevPrefix = "__cls_" // room accounting in PCT hints). // - requestedCount: number of CPUs the upcoming allocation wants, // negative when the balloon is about to release CPUs. -func (p *balloons) applyCpuClassHints(opts *cpuTreeAllocatorOptions, cpuClass string, currentCpus cpuset.CPUSet, requestedCount int) { +func (p *balloons) applyCpuClassHints(opts *cpuTreeAllocatorOptions, cpuClass string, currentCpus *libcpu.CpuMask, requestedCount int) { if p.cpuClasses == nil || opts == nil { return } mergeCpuClassHints(opts, p.cpuClasses, cpuclass.AllocationIntent{ ClassName: cpuClass, - CurrentCpus: currentCpus, - FreeCpus: p.freeCpus, + CurrentCpus: toCpuSet(currentCpus), + FreeCpus: toCpuSet(p.freeCpus), RequestedCount: requestedCount, }) } @@ -2193,7 +2222,7 @@ func mergeCpuClassHints(opts *cpuTreeAllocatorOptions, provider cpuClassHints, i return } if opts.virtDevCpusets == nil { - opts.virtDevCpusets = map[string][]cpuset.CPUSet{} + opts.virtDevCpusets = map[string][]*libcpu.CpuMask{} } opts.preferCloseToDevices = filterOutPrefixDevs(opts.preferCloseToDevices, cpuClassHintDevPrefix) opts.preferFarFromDevices = filterOutPrefixDevs(opts.preferFarFromDevices, cpuClassHintDevPrefix) @@ -2205,13 +2234,13 @@ func mergeCpuClassHints(opts *cpuTreeAllocatorOptions, provider cpuClassHints, i hints := provider.Hints(intent) for i, pref := range hints.Prefer { name := fmt.Sprintf("%spref_%d_%s", cpuClassHintDevPrefix, i, pref.Name) - opts.virtDevCpusets[name] = pref.Cpus + opts.virtDevCpusets[name] = toCpuMasks(pref.Cpus) opts.preferCloseToDevices = append(opts.preferCloseToDevices, name) log.Debugf("cpuclass hint: prefer %q -> %v", name, pref.Cpus) } for i, av := range hints.Avoid { name := fmt.Sprintf("%savoid_%d_%s", cpuClassHintDevPrefix, i, av.Name) - opts.virtDevCpusets[name] = av.Cpus + opts.virtDevCpusets[name] = toCpuMasks(av.Cpus) opts.preferFarFromDevices = append(opts.preferFarFromDevices, name) log.Debugf("cpuclass hint: avoid %q -> %v", name, av.Cpus) } @@ -2263,13 +2292,13 @@ func filterOutPrefixDevs(devs []string, prefix string) []string { // false if the pod resources are not available, if c has no such // device assigned, or if the assigned devices carry no NUMA topology // information. -func (p *balloons) containerDeviceCpus(c cache.Container, resourceName string) (cpuset.CPUSet, bool) { +func (p *balloons) containerDeviceCpus(c cache.Container, resourceName string) (*libcpu.CpuMask, bool) { if c == nil { - return emptyCpuSet, false + return libcpu.NewCpuMask(), false } ctrRes := c.GetPodResources() if ctrRes == nil { - return emptyCpuSet, false + return libcpu.NewCpuMask(), false } numas := idset.NewIDSet() for _, dev := range ctrRes.GetDevices() { @@ -2290,9 +2319,9 @@ func (p *balloons) containerDeviceCpus(c cache.Container, resourceName string) ( dev.GetResourceName(), c.PrettyName(), resourceName, devNumas, dev.GetDeviceIds()) } if numas.Size() == 0 { - return emptyCpuSet, false + return libcpu.NewCpuMask(), false } - cpus := cpuset.New() + cpus := libcpu.NewCpuMask() for _, numaID := range numas.SortedMembers() { // Check that the node exists before calling Node(): for an // unknown id Node() returns a nil *node wrapped in a non-nil @@ -2303,10 +2332,10 @@ func (p *balloons) containerDeviceCpus(c cache.Container, resourceName string) ( numaID, resourceName, c.PrettyName()) continue } - cpus = cpus.Union(toCpuSet(p.machine.MemoryNode(numaID).CPUs())) + cpus = cpus.Union(p.machine.MemoryNode(numaID).CPUs()) } if cpus.IsEmpty() { - return emptyCpuSet, false + return libcpu.NewCpuMask(), false } return cpus.Intersection(p.allowed), true } @@ -2324,7 +2353,7 @@ func (p *balloons) applyPodResourcesHints(opts *cpuTreeAllocatorOptions, c cache return } if opts.virtDevCpusets == nil { - opts.virtDevCpusets = map[string][]cpuset.CPUSet{} + opts.virtDevCpusets = map[string][]*libcpu.CpuMask{} } opts.preferCloseToDevices = p.resolvePodResourceDevs(opts.preferCloseToDevices, opts.virtDevCpusets, c) } @@ -2334,7 +2363,7 @@ func (p *balloons) applyPodResourcesHints(opts *cpuTreeAllocatorOptions, c cache // resolved CPUs in virtDevCpusets. Non-pod-resource entries are kept // unchanged in their original position. Unresolvable pod-resource // entries are dropped. -func (p *balloons) resolvePodResourceDevs(devs []string, virtDevCpusets map[string][]cpuset.CPUSet, c cache.Container) []string { +func (p *balloons) resolvePodResourceDevs(devs []string, virtDevCpusets map[string][]*libcpu.CpuMask, c cache.Container) []string { out := make([]string, 0, len(devs)) for _, dev := range devs { resourceName, isPodRes := podResourceDeviceName(dev) @@ -2353,7 +2382,7 @@ func (p *balloons) resolvePodResourceDevs(devs []string, virtDevCpusets map[stri continue } name := podResourceHintDevName(c, resourceName) - virtDevCpusets[name] = []cpuset.CPUSet{cpus} + virtDevCpusets[name] = []*libcpu.CpuMask{cpus} out = append(out, name) log.Debugf("pod-resource hint: device %q of container %s prefers CPUs %s", resourceName, c.PrettyName(), cpus) @@ -2476,16 +2505,16 @@ func memTypeMaskFromStringList(memTypes []string) (libmem.TypeMask, error) { // closestMems returns memory node IDs good for pinning containers // that run on given CPUs -func (p *balloons) closestMems(cpus cpuset.CPUSet) idset.IDSet { - return idset.NewIDSet(p.memAllocator.CPUSetAffinity(cpus).Slice()...) +func (p *balloons) closestMems(cpus *libcpu.CpuMask) idset.IDSet { + return idset.NewIDSet(p.memAllocator.CPUSetAffinity(toCpuSet(cpus)).Slice()...) } // resizeCompositeBalloon changes the CPUs allocated for all sub-components func (p *balloons) resizeCompositeBalloon(bln *Balloon, newMilliCpus int) error { origFreeCpus := p.freeCpus.Clone() - origCompBlnsCpus := []cpuset.CPUSet{} + origCompBlnsCpus := []*libcpu.CpuMask{} newMilliCpusPerComponent := newMilliCpus / len(bln.components) - blnCpus := cpuset.New() + blnCpus := libcpu.NewCpuMask() for _, compBln := range bln.components { origCompBlnsCpus = append(origCompBlnsCpus, compBln.Cpus.Clone()) if err := p.resizeBalloon(compBln, newMilliCpusPerComponent); err != nil { @@ -2544,12 +2573,11 @@ func (p *balloons) resizeBalloon(bln *Balloon, newMilliCpus int) error { log.Debugf("- allocating %d CPUs from %q", cpuCountDelta, addFromCpus) // The allocator takes the allocated CPUs out of the set it is given. // Nothing here reads what is left of it, only what came back. - allocated, err := p.cpuAllocator.AllocateCpus(toCpuMask(addFromCpus), + newCpus, err := p.cpuAllocator.AllocateCpus(addFromCpus, newCpuCount-oldCpuCount, bln.Def.AllocatorPriority.Value().Option()) if err != nil { return balloonsError("resize/inflate: allocating %d CPUs for %s failed: %w", cpuCountDelta, bln, err) } - newCpus := toCpuSet(allocated) oldBlnCpus := bln.Cpus oldFreeCpus := p.freeCpus p.freeCpus = p.freeCpus.Difference(newCpus) @@ -2563,18 +2591,16 @@ func (p *balloons) resizeBalloon(bln *Balloon, newMilliCpus int) error { return balloonsError("resize/deflate: failed to choose a cpuset for releasing %d CPUs: %w", -cpuCountDelta, err) } log.Debugf("- releasing %d CPUs from cpuset %q", -cpuCountDelta, removeFromCpus) - removeFrom := toCpuMask(removeFromCpus) - _, err = p.cpuAllocator.ReleaseCpus(removeFrom, -cpuCountDelta, bln.Def.AllocatorPriority.Value().Option()) + _, err = p.cpuAllocator.ReleaseCpus(removeFromCpus, -cpuCountDelta, bln.Def.AllocatorPriority.Value().Option()) if err != nil { return balloonsError("resize/deflate: releasing %d CPUs from %s failed: %w", -cpuCountDelta, bln, err) } - removeFromCpus = toCpuSet(removeFrom) oldBlnCpus := bln.Cpus oldFreeCpus := p.freeCpus p.freeCpus = p.freeCpus.Union(removeFromCpus) bln.Cpus = bln.Cpus.Difference(removeFromCpus) log.Debugf("- released, changed cpus: balloon from %q to %q, free from %q to %q", oldBlnCpus, bln.Cpus, oldFreeCpus, p.freeCpus) - p.updatePinning(p.shareIdleCpus(removeFromCpus, cpuset.New())...) + p.updatePinning(p.shareIdleCpus(removeFromCpus, libcpu.NewCpuMask())...) } log.Debugf("- resize successful: %s, freecpus: %s", bln, p.freeCpus) p.updatePinning(bln) @@ -2583,15 +2609,17 @@ func (p *balloons) resizeBalloon(bln *Balloon, newMilliCpus int) error { func (p *balloons) updatePinning(blns ...*Balloon) { for _, bln := range blns { - var cpusNoHt cpuset.CPUSet - var allowedCpus cpuset.CPUSet + var cpusNoHt *libcpu.CpuMask + var allowedCpus *libcpu.CpuMask pinnableCpus := bln.Cpus.Union(bln.SharedIdleCpus) bln.Mems = p.closestMems(pinnableCpus) for _, cID := range bln.ContainerIDs() { if c, ok := p.cch.LookupContainer(cID); ok { if runWithoutHyperthreads(c, bln) { - if cpusNoHt.Size() == 0 { - cpusNoHt = toCpuSet(hardware.SingleThreadPerCore(p.machine, toCpuMask(pinnableCpus))) + // nil, not empty: this is worked out once per balloon, and + // a balloon with no CPUs has an empty answer to cache. + if cpusNoHt == nil { + cpusNoHt = hardware.SingleThreadPerCore(p.machine, pinnableCpus) } allowedCpus = cpusNoHt } else { @@ -2618,7 +2646,7 @@ func runWithoutHyperthreads(c cache.Container, bln *Balloon) bool { // shareIdleCpus adds addCpus and removes removeCpus to those balloons // that whose containers are allowed to use shared idle CPUs. Returns // balloons that will need re-pinning. -func (p *balloons) shareIdleCpus(addCpus, removeCpus cpuset.CPUSet) []*Balloon { +func (p *balloons) shareIdleCpus(addCpus, removeCpus *libcpu.CpuMask) []*Balloon { updateBalloons := map[int]struct{}{} if removeCpus.Size() > 0 { for blnIdx, bln := range p.balloons { @@ -2628,14 +2656,14 @@ func (p *balloons) shareIdleCpus(addCpus, removeCpus cpuset.CPUSet) []*Balloon { } } } - addCpus = addCpus.Difference(toCpuSet(p.machine.IsolatedCPUs())) + addCpus = addCpus.Difference(p.machine.IsolatedCPUs()) if addCpus.Size() > 0 { for blnIdx, bln := range p.balloons { topoLevel := bln.Def.ShareIdleCpusInSame if topoLevel == cfgapi.CPUTopologyLevelUndefined { continue } - idleCpusInTopoLevel := cpuset.New() + idleCpusInTopoLevel := libcpu.NewCpuMask() if err := p.cpuTree.DepthFirstWalk(func(t *cpuTreeNode) error { // Dive in correct topology level. if t.level != topoLevel { @@ -2782,7 +2810,7 @@ func (p *balloons) dismissContainer(c cache.Container, bln *Balloon) { } // pinCpuMem pins container to CPUs and memory nodes if flagged -func (p *balloons) pinCpuMem(c cache.Container, cpus cpuset.CPUSet, mems idset.IDSet, memTypeMask libmem.TypeMask, blnDefPinMemory *bool) { +func (p *balloons) pinCpuMem(c cache.Container, cpus *libcpu.CpuMask, mems idset.IDSet, memTypeMask libmem.TypeMask, blnDefPinMemory *bool) { if p.bpoptions.PinCPU == nil || *p.bpoptions.PinCPU { log.Debugf(" - pinning %s to cpuset: %s", c.PrettyName(), cpus) c.SetCpusetCpus(cpus.String()) @@ -2880,7 +2908,7 @@ func (p *balloons) allocMem(c cache.Container, mems idset.IDSet, types libmem.Ty } func parseIDSet(mems string) (idset.IDSet, error) { - cset, err := cpuset.Parse(mems) + cset, err := cpuset.Parse(mems) // memory nodes, not CPUs if err != nil { return idset.NewIDSet(), err } diff --git a/cmd/plugins/balloons/policy/cpuclass_test.go b/cmd/plugins/balloons/policy/cpuclass_test.go index b29a60f0b..f9ac2c542 100644 --- a/cmd/plugins/balloons/policy/cpuclass_test.go +++ b/cmd/plugins/balloons/policy/cpuclass_test.go @@ -15,6 +15,7 @@ package balloons import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "strings" "testing" @@ -49,7 +50,7 @@ func countHintDevs(devs []string) int { return n } -func countHintMapKeys(m map[string][]cpuset.CPUSet) int { +func countHintMapKeys(m map[string][]*libcpu.CpuMask) int { n := 0 for k := range m { if strings.HasPrefix(k, cpuClassHintDevPrefix) { @@ -99,7 +100,7 @@ func TestMergeCpuClassHintsNoAccumulation(t *testing.T) { opts := &cpuTreeAllocatorOptions{ preferCloseToDevices: []string{"user-dev-A", "user-dev-B"}, preferFarFromDevices: []string{"user-far"}, - virtDevCpusets: map[string][]cpuset.CPUSet{}, + virtDevCpusets: map[string][]*libcpu.CpuMask{}, } for round := 1; round <= 3; round++ { @@ -152,7 +153,7 @@ func userDevs(devs []string) []string { return out } -func mapKeys(m map[string][]cpuset.CPUSet) []string { +func mapKeys(m map[string][]*libcpu.CpuMask) []string { out := make([]string, 0, len(m)) for k := range m { out = append(out, k) diff --git a/cmd/plugins/balloons/policy/cputree.go b/cmd/plugins/balloons/policy/cputree.go index ef9271d2f..e7e7d4c90 100644 --- a/cmd/plugins/balloons/policy/cputree.go +++ b/cmd/plugins/balloons/policy/cputree.go @@ -24,7 +24,6 @@ import ( libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // cpuTreeNode is a node in the CPU tree. @@ -33,7 +32,7 @@ type cpuTreeNode struct { level CPUTopologyLevel parent *cpuTreeNode children []*cpuTreeNode - cpus cpuset.CPUSet // union of CPUs of child nodes + cpus *libcpu.CpuMask // union of CPUs of child nodes } // cpuTreeNodeAttributes contains various attributes of a CPU tree @@ -46,8 +45,8 @@ type cpuTreeNode struct { type cpuTreeNodeAttributes struct { t *cpuTreeNode depth int - currentCpus cpuset.CPUSet - freeCpus cpuset.CPUSet + currentCpus *libcpu.CpuMask + freeCpus *libcpu.CpuMask currentCpuCount int currentCpuCounts []int freeCpuCount int @@ -59,7 +58,7 @@ type cpuTreeNodeAttributes struct { type cpuTreeAllocator struct { options cpuTreeAllocatorOptions root *cpuTreeNode - cacheCloseCpuSets map[string][]cpuset.CPUSet + cacheCloseCpuSets map[string][]*libcpu.CpuMask } // cpuTreeAllocatorOptions contains parameters for the CPU allocator @@ -72,22 +71,8 @@ type cpuTreeAllocatorOptions struct { preferSpreadOnPhysicalCores bool preferCloseToDevices []string preferFarFromDevices []string - virtDevCpusets map[string][]cpuset.CPUSet - deviceUpdateOnEveryCpu func(cpuset.CPUSet) -} - -var emptyCpuSet = cpuset.New() - -// toCpuSet and toCpuMask convert between the set the hardware package speaks and -// the one this policy is written in. They are the seam left by moving the policy -// onto hardware without rewriting its allocation logic; they disappear if the -// policy ever switches to libcpu sets throughout. -func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { - return cpuset.New(cpus.List()...) -} - -func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { - return libcpu.NewCpuMask(cpus.List()...) + virtDevCpusets map[string][]*libcpu.CpuMask + deviceUpdateOnEveryCpu func(*libcpu.CpuMask) } // String returns string representation of a CPU tree node. @@ -124,7 +109,7 @@ func (tna cpuTreeNodeAttributes) String() string { func NewCpuTree(name string) *cpuTreeNode { return &cpuTreeNode{ name: name, - cpus: cpuset.New(), + cpus: libcpu.NewCpuMask(), } } @@ -172,7 +157,7 @@ func (t *cpuTreeNode) AddChild(child *cpuTreeNode) { } // AddCpus adds CPUs to a CPU tree node and all its parents. -func (t *cpuTreeNode) AddCpus(cpus cpuset.CPUSet) { +func (t *cpuTreeNode) AddCpus(cpus *libcpu.CpuMask) { t.cpus = t.cpus.Union(cpus) if t.parent != nil { t.parent.AddCpus(cpus) @@ -180,7 +165,7 @@ func (t *cpuTreeNode) AddCpus(cpus cpuset.CPUSet) { } // Cpus returns CPUs of a CPU tree node. -func (t *cpuTreeNode) Cpus() cpuset.CPUSet { +func (t *cpuTreeNode) Cpus() *libcpu.CpuMask { return t.cpus } @@ -250,7 +235,7 @@ func (t *cpuTreeNode) DepthFirstWalk(handler func(*cpuTreeNode) error) error { // CpuLocations returns a slice where each element contains names of // topology elements over which a set of CPUs spans. Example: // systemNode.CpuLocations(cpuset:0,99) = [["system"],["p0", "p1"], ["p0d0", "p1d0"], ...] -func (t *cpuTreeNode) CpuLocations(cpus cpuset.CPUSet) [][]string { +func (t *cpuTreeNode) CpuLocations(cpus *libcpu.CpuMask) [][]string { tLeafDepth := t.LeafDepth() names := make([][]string, tLeafDepth+1) if err := t.DepthFirstWalk(func(tn *cpuTreeNode) error { @@ -319,7 +304,7 @@ func NewCpuTreeFromMachine(m *hardware.Machine) *cpuTreeNode { threadTree := NewCpuTree(fmt.Sprintf("%st%d", coreTree.name, threadID)) threadTree.level = CPUTopologyLevelThread coreTree.AddChild(threadTree) - threadTree.AddCpus(cpuset.New(threadID)) + threadTree.AddCpus(libcpu.NewCpuMask(threadID)) } } } @@ -336,7 +321,7 @@ func NewCpuTreeFromMachine(m *hardware.Machine) *cpuTreeNode { // - freeCpus is the set of CPUs that can be allocated in coming operation // - filter(tna) returns false if the node can be ignored func (t *cpuTreeNode) ToAttributedSlice( - currentCpus, freeCpus cpuset.CPUSet, + currentCpus, freeCpus *libcpu.CpuMask, filter func(*cpuTreeNodeAttributes) bool) []cpuTreeNodeAttributes { tnas := []cpuTreeNodeAttributes{} currentCpuCounts := []int{} @@ -346,7 +331,7 @@ func (t *cpuTreeNode) ToAttributedSlice( } func (t *cpuTreeNode) toAttributedSlice( - currentCpus, freeCpus cpuset.CPUSet, + currentCpus, freeCpus *libcpu.CpuMask, filter func(*cpuTreeNodeAttributes) bool, tnas *[]cpuTreeNodeAttributes, depth int, @@ -408,7 +393,7 @@ func (t *cpuTreeNode) SplitLevel(splitLevel CPUTopologyLevel, cpuClassifier func tn.children = make([]*cpuTreeNode, 0, len(classCpus)) // Add new child corresponding each class. for class, cpus := range classCpus { - cpuMask := cpuset.New(cpus...) + cpuMask := libcpu.NewCpuMask(cpus...) newNode := NewCpuTree(fmt.Sprintf("%sclass%d", tn.name, class)) tn.AddChild(newNode) newNode.cpus = tn.cpus.Intersection(cpuMask) @@ -453,7 +438,7 @@ func (t *cpuTreeNode) NewAllocator(options cpuTreeAllocatorOptions) *cpuTreeAllo options: options, } if options.virtDevCpusets == nil { - ta.cacheCloseCpuSets = map[string][]cpuset.CPUSet{} + ta.cacheCloseCpuSets = map[string][]*libcpu.CpuMask{} } else { ta.cacheCloseCpuSets = options.virtDevCpusets } @@ -560,7 +545,7 @@ func (ta *cpuTreeAllocator) sorterRelease(tnas []cpuTreeNodeAttributes) func(int // these CPUs. // - removeFromCpus contains CPUs in currentCpus set from which // abs(delta) CPUs can be freed. -func (ta *cpuTreeAllocator) ResizeCpus(currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) ResizeCpus(currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { resizers := []cpuResizerFunc{ ta.resizeCpusOnlyIfNecessary, ta.resizeCpusWithDynamicDeviceHints, @@ -571,9 +556,9 @@ func (ta *cpuTreeAllocator) ResizeCpus(currentCpus, freeCpus cpuset.CPUSet, delt return ta.nextCpuResizer(resizers, currentCpus, freeCpus, delta) } -type cpuResizerFunc func(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) +type cpuResizerFunc func(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) -func (ta *cpuTreeAllocator) nextCpuResizer(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) nextCpuResizer(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { if len(resizers) == 0 { return freeCpus, currentCpus, fmt.Errorf("internal error: a CPU resizer consulted next resizer but there was no one left") } @@ -586,30 +571,30 @@ func (ta *cpuTreeAllocator) nextCpuResizer(resizers []cpuResizerFunc, currentCpu // resizeCpusNow does not call next resizer. Instead it keeps all CPU // allocations from freeCpus and CPU releases from currentCpus equally // good. This is the terminal block of resizers chain. -func (ta *cpuTreeAllocator) resizeCpusNow(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) resizeCpusNow(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { return freeCpus, currentCpus, nil } // resizeCpusOnlyIfNecessary is the fast path for making trivial // reservations and to fail if resizing is not possible. -func (ta *cpuTreeAllocator) resizeCpusOnlyIfNecessary(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) resizeCpusOnlyIfNecessary(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { switch { case delta == 0: // Nothing to do. - return emptyCpuSet, emptyCpuSet, nil + return libcpu.NewCpuMask(), libcpu.NewCpuMask(), nil case delta > 0: if freeCpus.Size() < delta { - return freeCpus, emptyCpuSet, fmt.Errorf("not enough free CPUs (%d) to resize current CPU set from %d to %d CPUs", freeCpus.Size(), currentCpus.Size(), currentCpus.Size()+delta) + return freeCpus, libcpu.NewCpuMask(), fmt.Errorf("not enough free CPUs (%d) to resize current CPU set from %d to %d CPUs", freeCpus.Size(), currentCpus.Size(), currentCpus.Size()+delta) } else if freeCpus.Size() == delta { // Allocate all the remaining free CPUs. - return freeCpus, emptyCpuSet, nil + return freeCpus, libcpu.NewCpuMask(), nil } case delta < 0: if currentCpus.Size() < -delta { - return emptyCpuSet, currentCpus, fmt.Errorf("not enough current CPUs (%d) to release %d CPUs", currentCpus.Size(), -delta) + return libcpu.NewCpuMask(), currentCpus, fmt.Errorf("not enough current CPUs (%d) to release %d CPUs", currentCpus.Size(), -delta) } else if currentCpus.Size() == -delta { // Free all allocated CPUs. - return emptyCpuSet, currentCpus, nil + return libcpu.NewCpuMask(), currentCpus, nil } } return ta.nextCpuResizer(resizers, currentCpus, freeCpus, delta) @@ -618,7 +603,7 @@ func (ta *cpuTreeAllocator) resizeCpusOnlyIfNecessary(resizers []cpuResizerFunc, // resizeCpusWithDynamicDeviceHints handles allocating CPUs in // scenarios where each selected CPU may change the set of CPUs are // good to be selected next. -func (ta *cpuTreeAllocator) resizeCpusWithDynamicDeviceHints(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) resizeCpusWithDynamicDeviceHints(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { // If the deviceUpdateOnEveryCpu callback is set, call it // after each CPU allocation to update the state of virtual // devices. If not set or if CPUs are released instead of @@ -641,14 +626,14 @@ func (ta *cpuTreeAllocator) resizeCpusWithDynamicDeviceHints(resizers []cpuResiz if err != nil || addFrom.Size() < delta { return addFrom, removeFrom, err } - addedCpus := cpuset.New() + addedCpus := libcpu.NewCpuMask() for { addedCpu := addFrom.List()[0] - addedCpus = addedCpus.Union(cpuset.New(addedCpu)) + addedCpus = addedCpus.Union(libcpu.NewCpuMask(addedCpu)) if addedCpus.Size() >= delta { break } - currentCpus = currentCpus.Union(cpuset.New(addedCpu)) + currentCpus = currentCpus.Union(libcpu.NewCpuMask(addedCpu)) freeCpus = freeCpus.Difference(currentCpus) ta.options.deviceUpdateOnEveryCpu(currentCpus) addFrom, removeFrom, err = ta.nextCpuResizer(resizers, currentCpus, freeCpus, 1) @@ -662,11 +647,11 @@ func (ta *cpuTreeAllocator) resizeCpusWithDynamicDeviceHints(resizers []cpuResiz // resizeCpusWithDevices prefers allocating CPUs from those freeCpus // that are topologically close to preferred devices, and releasing // those currentCpus that are not. -func (ta *cpuTreeAllocator) resizeCpusWithDevices(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) resizeCpusWithDevices(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { // allCloseCpuSets contains cpusets in the order of priority. // Applying the first cpusets in it are prioritized over ones // after them. - allCloseCpuSets := [][]cpuset.CPUSet{} + allCloseCpuSets := [][]*libcpu.CpuMask{} for _, devPath := range ta.options.preferCloseToDevices { if closeCpuSets := ta.topologyHintCpus(devPath); len(closeCpuSets) > 0 { log.Debugf(" - prepare: close to %q, prefer cpusets: %v", devPath, closeCpuSets) @@ -676,7 +661,7 @@ func (ta *cpuTreeAllocator) resizeCpusWithDevices(resizers []cpuResizerFunc, cur for _, devPath := range ta.options.preferFarFromDevices { for _, farCpuSet := range ta.topologyHintCpus(devPath) { log.Debugf(" - prepare: far from %q, prefer cpusets: %s", devPath, freeCpus.Difference(farCpuSet)) - allCloseCpuSets = append(allCloseCpuSets, []cpuset.CPUSet{freeCpus.Difference(farCpuSet)}) + allCloseCpuSets = append(allCloseCpuSets, []*libcpu.CpuMask{freeCpus.Difference(farCpuSet)}) } } if len(allCloseCpuSets) == 0 { @@ -727,13 +712,13 @@ func (ta *cpuTreeAllocator) resizeCpusWithDevices(resizers []cpuResizerFunc, cur return currentCpuHints[leastHintedCpus[i]] < currentCpuHints[leastHintedCpus[j]] }) maxHints := currentCpuHints[leastHintedCpus[-delta]] - currentToFreeForSure := cpuset.New() - currentToFreeMaybe := cpuset.New() + currentToFreeForSure := libcpu.NewCpuMask() + currentToFreeMaybe := libcpu.NewCpuMask() for i := 0; i < len(leastHintedCpus) && currentCpuHints[leastHintedCpus[i]] <= maxHints; i++ { if currentCpuHints[leastHintedCpus[i]] < maxHints { - currentToFreeForSure = currentToFreeForSure.Union(cpuset.New(leastHintedCpus[i])) + currentToFreeForSure = currentToFreeForSure.Union(libcpu.NewCpuMask(leastHintedCpus[i])) } else { - currentToFreeMaybe = currentToFreeMaybe.Union(cpuset.New(leastHintedCpus[i])) + currentToFreeMaybe = currentToFreeMaybe.Union(libcpu.NewCpuMask(leastHintedCpus[i])) } } remainingDelta := delta + currentToFreeForSure.Size() @@ -747,7 +732,7 @@ func (ta *cpuTreeAllocator) resizeCpusWithDevices(resizers []cpuResizerFunc, cur if currentToFreeForSure.Size() >= -delta { break } - currentToFreeForSure = currentToFreeForSure.Union(cpuset.New(cpu)) + currentToFreeForSure = currentToFreeForSure.Union(libcpu.NewCpuMask(cpu)) } return freeCpus, currentToFreeForSure, err } @@ -755,23 +740,23 @@ func (ta *cpuTreeAllocator) resizeCpusWithDevices(resizers []cpuResizerFunc, cur } // Fetch cached topology hint, return error only once per bad dev -func (ta *cpuTreeAllocator) topologyHintCpus(dev string) []cpuset.CPUSet { +func (ta *cpuTreeAllocator) topologyHintCpus(dev string) []*libcpu.CpuMask { if closeCpuSets, ok := ta.cacheCloseCpuSets[dev]; ok { return closeCpuSets } topologyHints, err := topology.NewTopologyHints(dev) if err != nil { log.Errorf("failed to find topology of device %q: %v", dev, err) - ta.cacheCloseCpuSets[dev] = []cpuset.CPUSet{} + ta.cacheCloseCpuSets[dev] = []*libcpu.CpuMask{} } else { for _, topologyHint := range topologyHints { - ta.cacheCloseCpuSets[dev] = append(ta.cacheCloseCpuSets[dev], cpuset.MustParse(topologyHint.CPUs)) + ta.cacheCloseCpuSets[dev] = append(ta.cacheCloseCpuSets[dev], libcpu.MustParseCpuMask(topologyHint.CPUs)) } } return ta.cacheCloseCpuSets[dev] } -func (ta *cpuTreeAllocator) resizeCpusOneAtATime(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) resizeCpusOneAtATime(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { if delta > 0 { addFromSuperset, removeFromSuperset, err := ta.nextCpuResizer(resizers, currentCpus, freeCpus, delta) if !ta.options.preferSpreadOnPhysicalCores || addFromSuperset.Size() == delta { @@ -783,7 +768,7 @@ func (ta *cpuTreeAllocator) resizeCpusOneAtATime(resizers []cpuResizerFunc, curr // of these does not result in equally good // result. Therefore, in this case, construct addFrom // set by adding one CPU at a time. - addFrom := cpuset.New() + addFrom := libcpu.NewCpuMask() for n := range delta { addSingleFrom, _, err := ta.nextCpuResizer(resizers, currentCpus, freeCpus, 1) if err != nil { @@ -807,8 +792,8 @@ func (ta *cpuTreeAllocator) resizeCpusOneAtATime(resizers []cpuResizerFunc, curr // In multi-CPU removal, remove CPUs one by one instead of // trying to find a single topology element from which all of // them could be removed. - removeFrom := cpuset.New() - addFrom := cpuset.New() + removeFrom := libcpu.NewCpuMask() + addFrom := libcpu.NewCpuMask() for n := 0; n < -delta; n++ { _, removeSingleFrom, err := ta.nextCpuResizer(resizers, currentCpus, freeCpus, -1) if err != nil { @@ -833,7 +818,7 @@ func (ta *cpuTreeAllocator) resizeCpusOneAtATime(resizers []cpuResizerFunc, curr return addFrom, removeFrom, nil } -func (ta *cpuTreeAllocator) resizeCpusMaxLocalSet(resizers []cpuResizerFunc, currentCpus, freeCpus cpuset.CPUSet, delta int) (cpuset.CPUSet, cpuset.CPUSet, error) { +func (ta *cpuTreeAllocator) resizeCpusMaxLocalSet(resizers []cpuResizerFunc, currentCpus, freeCpus *libcpu.CpuMask, delta int) (*libcpu.CpuMask, *libcpu.CpuMask, error) { tnas := ta.root.ToAttributedSlice(currentCpus, freeCpus, func(tna *cpuTreeNodeAttributes) bool { // filter out branches with insufficient cpus diff --git a/cmd/plugins/balloons/policy/cputree_test.go b/cmd/plugins/balloons/policy/cputree_test.go index ebbe0b07e..96c78aa53 100644 --- a/cmd/plugins/balloons/policy/cputree_test.go +++ b/cmd/plugins/balloons/policy/cputree_test.go @@ -16,12 +16,11 @@ package balloons import ( "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "slices" "sort" "strings" "testing" - - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) type cpuInTopology struct { @@ -47,7 +46,7 @@ func (cit cpuInTopology) TopoName(topoLevel string) string { panic("invalid topoLevel") } -func (csit cpusInTopology) dumps(nameCpus map[string]cpuset.CPUSet) string { +func (csit cpusInTopology) dumps(nameCpus map[string]*libcpu.CpuMask) string { lines := []string{} names := make([]string, 0, len(nameCpus)) for name := range nameCpus { @@ -96,7 +95,7 @@ func newCpuTreeFromInt5(pdnct [5]int) (*cpuTreeNode, cpusInTopology) { threadTree := NewCpuTree(fmt.Sprintf("p%dd%dn%dc%02dt%d", packageID, dieID, numaID, coreID, threadID)) threadTree.level = CPUTopologyLevelThread coreTree.AddChild(threadTree) - threadTree.AddCpus(cpuset.New(cpuID)) + threadTree.AddCpus(libcpu.NewCpuMask(cpuID)) csit[cpuID] = cpuInTopology{ packageID, dieID, numaID, coreID, threadID, cpuID, packageTree.name, dieTree.name, numaTree.name, coreTree.name, threadTree.name, @@ -111,7 +110,7 @@ func newCpuTreeFromInt5(pdnct [5]int) (*cpuTreeNode, cpusInTopology) { return sysTree, csit } -func verifyOn(t *testing.T, nameContents string, cpus cpuset.CPUSet, csit cpusInTopology) { +func verifyOn(t *testing.T, nameContents string, cpus *libcpu.CpuMask, csit cpusInTopology) { for _, cpuID := range cpus.List() { name := csit[cpuID].threadName if !strings.Contains(name, nameContents) { @@ -120,7 +119,7 @@ func verifyOn(t *testing.T, nameContents string, cpus cpuset.CPUSet, csit cpusIn } } -func verifyNotOn(t *testing.T, nameContents string, cpus cpuset.CPUSet, csit cpusInTopology) { +func verifyNotOn(t *testing.T, nameContents string, cpus *libcpu.CpuMask, csit cpusInTopology) { for _, cpuID := range cpus.List() { name := csit[cpuID].threadName if strings.Contains(name, nameContents) { @@ -129,7 +128,7 @@ func verifyNotOn(t *testing.T, nameContents string, cpus cpuset.CPUSet, csit cpu } } -func doVerifySame(t *testing.T, topoLevel string, cpus cpuset.CPUSet, csit cpusInTopology, inversed bool) { +func doVerifySame(t *testing.T, topoLevel string, cpus *libcpu.CpuMask, csit cpusInTopology, inversed bool) { seenName := "" seenCpuID := -1 for _, cpuID := range cpus.List() { @@ -159,15 +158,15 @@ func doVerifySame(t *testing.T, topoLevel string, cpus cpuset.CPUSet, csit cpusI } } -func verifySame(t *testing.T, topoLevel string, cpus cpuset.CPUSet, csit cpusInTopology) { +func verifySame(t *testing.T, topoLevel string, cpus *libcpu.CpuMask, csit cpusInTopology) { doVerifySame(t, topoLevel, cpus, csit, false) } -func verifyNotSame(t *testing.T, topoLevel string, cpus cpuset.CPUSet, csit cpusInTopology) { +func verifyNotSame(t *testing.T, topoLevel string, cpus *libcpu.CpuMask, csit cpusInTopology) { doVerifySame(t, topoLevel, cpus, csit, true) } -func (csit cpusInTopology) getElements(topoLevel string, cpus cpuset.CPUSet) []string { +func (csit cpusInTopology) getElements(topoLevel string, cpus *libcpu.CpuMask) []string { elts := []string{} for _, cpuID := range cpus.List() { elts = append(elts, csit[cpuID].TopoName(topoLevel)) @@ -175,7 +174,7 @@ func (csit cpusInTopology) getElements(topoLevel string, cpus cpuset.CPUSet) []s return elts } -func (csit cpusInTopology) verifyDisjoint(t *testing.T, topoLevel string, cpusA cpuset.CPUSet, cpusB cpuset.CPUSet) { +func (csit cpusInTopology) verifyDisjoint(t *testing.T, topoLevel string, cpusA *libcpu.CpuMask, cpusB *libcpu.CpuMask) { eltsA := csit.getElements(topoLevel, cpusA) eltsB := csit.getElements(topoLevel, cpusB) for _, eltA := range eltsA { @@ -538,20 +537,21 @@ func TestResizeCpus(t *testing.T) { preferFarFromDevices: tc.allocatorPFfD, }) for _, dev := range append(tc.allocatorPCtD, tc.allocatorPFfD...) { - treeA.cacheCloseCpuSets[dev] = []cpuset.CPUSet{ - cpuset.MustParse(dev[len("/sys/cpus:"):]), + treeA.cacheCloseCpuSets[dev] = []*libcpu.CpuMask{ + libcpu.MustParseCpuMask(dev[len("/sys/cpus:"):]), } } - currentCpus := cpuset.New() + currentCpus := libcpu.NewCpuMask() freeCpus := tree.Cpus() if len(tc.allocations) > 0 { - currentCpus = currentCpus.Union(cpuset.New(tc.allocations...)) - freeCpus = freeCpus.Difference(cpuset.New(tc.allocations...)) + currentCpus = currentCpus.Union(libcpu.NewCpuMask(tc.allocations...)) + freeCpus = freeCpus.Difference(libcpu.NewCpuMask(tc.allocations...)) } - ccidCurrentCpus := map[int]cpuset.CPUSet{0: currentCpus} - allocs := map[string]cpuset.CPUSet{"--:allo": currentCpus} + ccidCurrentCpus := map[int]*libcpu.CpuMask{0: currentCpus} + allocs := map[string]*libcpu.CpuMask{"--:allo": currentCpus} for i, delta := range tc.deltas { if i < len(tc.operateOnCcid) && tc.operateOnCcid[i] > 0 { + // A ccid this case has not operated on yet starts empty. currentCpus = ccidCurrentCpus[tc.operateOnCcid[i]] } t.Logf("ResizeCpus(current=%s; free=%s; delta=%d)", currentCpus, freeCpus, delta) @@ -576,26 +576,26 @@ func TestResizeCpus(t *testing.T) { } if tc.allocate { allocName := fmt.Sprintf("%02d:allo", i+1) - allocs[allocName] = cpuset.New() + allocs[allocName] = libcpu.NewCpuMask() for n, cpuID := range addFrom.List() { if n >= delta { break } - freeCpus = freeCpus.Difference(cpuset.New(cpuID)) - currentCpus = currentCpus.Union(cpuset.New(cpuID)) - allocs[allocName] = allocs[allocName].Union(cpuset.New(cpuID)) + freeCpus = freeCpus.Difference(libcpu.NewCpuMask(cpuID)) + currentCpus = currentCpus.Union(libcpu.NewCpuMask(cpuID)) + allocs[allocName] = allocs[allocName].Union(libcpu.NewCpuMask(cpuID)) } allocName = fmt.Sprintf("%02d:free", i+1) for n, cpuID := range removeFrom.List() { if n >= -delta { break } - freeCpus = freeCpus.Union(cpuset.New(cpuID)) + freeCpus = freeCpus.Union(libcpu.NewCpuMask(cpuID)) if i < len(tc.operateOnCcid) && tc.operateOnCcid[i] > 0 { - currentCpus = currentCpus.Difference(cpuset.New(cpuID)) + currentCpus = currentCpus.Difference(libcpu.NewCpuMask(cpuID)) } - allocs[allocName] = allocs[allocName].Union(cpuset.New(cpuID)) + allocs[allocName] = allocs[allocName].Union(libcpu.NewCpuMask(cpuID)) } if i < len(tc.operateOnCcid) && tc.operateOnCcid[i] > 0 { ccidCurrentCpus[tc.operateOnCcid[i]] = currentCpus @@ -616,7 +616,7 @@ func TestResizeCpus(t *testing.T) { verifyNotOn(t, tc.expectCurrentNotOn[i], currentCpus, csit) } if i < len(tc.expectAllOnSame) && tc.expectAllOnSame[i] != "" { - allCpus := cpuset.New() + allCpus := libcpu.NewCpuMask() for _, cpus := range ccidCurrentCpus { allCpus = allCpus.Union(cpus) } @@ -710,7 +710,7 @@ func TestWalk(t *testing.T) { func TestCpuLocations(t *testing.T) { tree, _ := newCpuTreeFromInt5([5]int{2, 2, 2, 4, 2}) - cpus := cpuset.New(0, 1, 3, 4, 16) + cpus := libcpu.NewCpuMask(0, 1, 3, 4, 16) systemlocations := tree.CpuLocations(cpus) package1locations := tree.children[1].CpuLocations(cpus) p0d1locations := tree.children[0].children[1].CpuLocations(cpus) diff --git a/cmd/plugins/balloons/policy/metrics.go b/cmd/plugins/balloons/policy/metrics.go index c0d0015e0..349ba13a4 100644 --- a/cmd/plugins/balloons/policy/metrics.go +++ b/cmd/plugins/balloons/policy/metrics.go @@ -16,6 +16,7 @@ package balloons import ( "context" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "sort" "strconv" "strings" @@ -24,7 +25,6 @@ import ( "go.opentelemetry.io/otel/metric" "github.com/containers/nri-plugins/pkg/metrics" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // Metrics defines the balloons-specific metric instruments. @@ -44,7 +44,7 @@ type BalloonMetrics struct { // Balloon instance metrics PrettyName string Groups string - Cpus cpuset.CPUSet + Cpus *libcpu.CpuMask CpusCount int Numas []string NumasCount int @@ -52,9 +52,9 @@ type BalloonMetrics struct { DiesCount int Packages []string PackagesCount int - SharedIdleCpus cpuset.CPUSet + SharedIdleCpus *libcpu.CpuMask SharedIdleCpusCount int - CpusAllowed cpuset.CPUSet + CpusAllowed *libcpu.CpuMask CpusAllowedCount int Mems string ContainerNames string From 1f576adb3630341a643a2826946afdc794b1464b Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Thu, 10 Sep 2026 21:58:02 +0300 Subject: [PATCH 33/39] topology-aware: track CPUs as libcpu.CpuMask. The policy kept its CPU sets as k8s cpuset sets and converted wherever it asked the hardware package or the CPU allocator something. Its own sets are masks now: the pool supplies, the grants, the hint scoring, the isolated and sliceable sets. Two things fall out of the type. takeCPUs took a destination set which all ten of its callers passed as nil, and a pointer to a source set so that it could write back what was left; the destination goes, and the allocator takes what it allocated out of the mask it is handed. A dry run, which works on a copy so that the supply is left alone, now says that rather than pointing at one. pkg/utils/topology comes along. It scores hints for this policy and has no other caller, so leaving it in cpuset sets would have meant converting into it and back three times per check. kubernetes.ShortCPUSet takes the libcpu interface, since all it ever wanted was to spell a set. Its test turned out to compare the unshortened form, so the shortening was never checked; with the comparison fixed the function needed fixing too, as it emitted segments out of order, and two of the expectations it had been measured against were wrong themselves. What is left of the seam is the CPU class controller, the IRQ affinity helpers, and the configuration which parses an operator's cpuset string. The sets which used to fall out of a zero value are created where they belong, in New. The tests which build a policy field by field, or leave a set out of a table, say so through EmptyIfNil. Saved state is unaffected: a grant's CPUs are persisted as the string they always were. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- cmd/plugins/topology-aware/policy/cache.go | 6 +- .../topology-aware/policy/cache_test.go | 7 +- .../topology-aware/policy/coldstart_test.go | 6 + .../topology-aware/policy/cpu-class.go | 8 +- cmd/plugins/topology-aware/policy/hint.go | 16 +- .../topology-aware/policy/hint_test.go | 14 +- .../topology-aware/policy/irq-affinity.go | 11 +- cmd/plugins/topology-aware/policy/metrics.go | 6 +- .../topology-aware/policy/metrics_test.go | 4 +- cmd/plugins/topology-aware/policy/node.go | 18 +- cmd/plugins/topology-aware/policy/pools.go | 42 ++--- .../topology-aware/policy/resources.go | 162 ++++++++---------- .../policy/topology-aware-policy.go | 36 ++-- cmd/plugins/topology-aware/policy/topology.go | 30 ++-- pkg/kubernetes/cpuset.go | 103 +++++------ pkg/kubernetes/cpuset_test.go | 17 +- pkg/utils/topology/hints.go | 36 ++-- 17 files changed, 265 insertions(+), 257 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/cache.go b/cmd/plugins/topology-aware/policy/cache.go index 8abf4966c..a62a3f67a 100644 --- a/cmd/plugins/topology-aware/policy/cache.go +++ b/cmd/plugins/topology-aware/policy/cache.go @@ -17,12 +17,12 @@ package topologyaware import ( "encoding/json" "errors" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "maps" "time" "github.com/containers/nri-plugins/pkg/resmgr/cache" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) const ( @@ -185,7 +185,7 @@ func (ccg *cachedGrant) ToGrant(policy *policy) (Grant, error) { container, ccg.CPUType, ccg.CPUClass, - cpuset.MustParse(ccg.Exclusive), + libcpu.MustParseCpuMask(ccg.Exclusive), ccg.Part, ccg.MemType, ccg.Irqs, @@ -209,7 +209,7 @@ func (cg *grant) UnmarshalJSON(data []byte) error { return policyError("failed to restore grant: %v", err) } - cg.exclusive = cpuset.MustParse(ccg.Exclusive) + cg.exclusive = libcpu.MustParseCpuMask(ccg.Exclusive) return nil } diff --git a/cmd/plugins/topology-aware/policy/cache_test.go b/cmd/plugins/topology-aware/policy/cache_test.go index 57ba09e7c..f9bfe5793 100644 --- a/cmd/plugins/topology-aware/policy/cache_test.go +++ b/cmd/plugins/topology-aware/policy/cache_test.go @@ -16,9 +16,8 @@ package topologyaware import ( "bytes" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "testing" - - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func TestToGrant(t *testing.T) { @@ -104,8 +103,8 @@ func TestAllocationMarshalling(t *testing.T) { node: node{ name: "testnode", kind: UnknownNode, - noderes: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0), - freeres: newSupply(&node{}, cpuset.New(), cpuset.New(), cpuset.New(), 0, 0), + noderes: newSupply(&node{}, libcpu.NewCpuMask(), libcpu.NewCpuMask(), libcpu.NewCpuMask(), 0, 0), + freeres: newSupply(&node{}, libcpu.NewCpuMask(), libcpu.NewCpuMask(), libcpu.NewCpuMask(), 0, 0), }, }, }, diff --git a/cmd/plugins/topology-aware/policy/coldstart_test.go b/cmd/plugins/topology-aware/policy/coldstart_test.go index d915819cd..6af31d2bc 100644 --- a/cmd/plugins/topology-aware/policy/coldstart_test.go +++ b/cmd/plugins/topology-aware/policy/coldstart_test.go @@ -20,6 +20,7 @@ import ( "testing" "time" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/events" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" @@ -95,6 +96,11 @@ func TestColdStart(t *testing.T) { policy := &policy{ machine: m, + // A configured policy always has these; one built field by + // field has to say so, empty being what it had before. + allowed: libcpu.NewCpuMask(), + reserved: libcpu.NewCpuMask(), + isolated: libcpu.NewCpuMask(), cache: &mockCache{ returnValue1ForLookupContainer: tc.container, returnValue2ForLookupContainer: true, diff --git a/cmd/plugins/topology-aware/policy/cpu-class.go b/cmd/plugins/topology-aware/policy/cpu-class.go index f3a5e5b97..2d95c5443 100644 --- a/cmd/plugins/topology-aware/policy/cpu-class.go +++ b/cmd/plugins/topology-aware/policy/cpu-class.go @@ -16,9 +16,9 @@ package topologyaware import ( "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/resmgr/cache" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func (p *policy) validateCpuClasses() error { @@ -33,7 +33,7 @@ func (p *policy) setReservedPoolCpuClass() { if opt.ReservedPoolCpuClass == "" { return } - if err := p.cpuClasses.UseClass(opt.ReservedPoolCpuClass, p.reserved); err != nil { + if err := p.cpuClasses.UseClass(opt.ReservedPoolCpuClass, toCpuSet(p.reserved)); err != nil { log.Errorf("failed to set reserved pool CPU class for %s: %v", p.reserved, err) } } @@ -59,14 +59,14 @@ func (p *policy) resolveCpuClass(ctr cache.Container) (string, bool, error) { return class, isCtrScoped, nil } -func (p *policy) resetCpuClass(subject string, cpus cpuset.CPUSet) { +func (p *policy) resetCpuClass(subject string, cpus *libcpu.CpuMask) { if p.cpuClasses == nil { return } if opt.SharedPoolCpuClass == "" { return } - if err := p.cpuClasses.UseClass(opt.SharedPoolCpuClass, cpus); err != nil { + if err := p.cpuClasses.UseClass(opt.SharedPoolCpuClass, toCpuSet(cpus)); err != nil { log.Errorf("%s: failed to reset CPU class for %s: %v", subject, cpus, err) } } diff --git a/cmd/plugins/topology-aware/policy/hint.go b/cmd/plugins/topology-aware/policy/hint.go index d639017f3..2f9b4eee1 100644 --- a/cmd/plugins/topology-aware/policy/hint.go +++ b/cmd/plugins/topology-aware/policy/hint.go @@ -18,15 +18,15 @@ import ( "strconv" "strings" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" - "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) // Calculate the hint score of the given hint and CPUSet. -func cpuHintScore(hint topology.Hint, CPUs cpuset.CPUSet) float64 { - hCPUs, err := cpuset.Parse(hint.CPUs) +func cpuHintScore(hint topology.Hint, CPUs *libcpu.CpuMask) float64 { + hCPUs, err := libcpu.ParseCpuMask(hint.CPUs) if err != nil { log.Warnf("invalid hint CPUs '%s' from %s", hint.CPUs, hint.Provider) return 0.0 @@ -90,18 +90,18 @@ func socketHintScore(hint topology.Hint, sysID idset.ID) float64 { } // return the cpuset for the CPU, NUMA or socket hints, preferred in this particular order. -func (cs *supply) hintCpus(h topology.Hint) cpuset.CPUSet { - var cpus cpuset.CPUSet +func (cs *supply) hintCpus(h topology.Hint) *libcpu.CpuMask { + cpus := libcpu.NewCpuMask() switch { case h.CPUs != "": - cpus = cpuset.MustParse(h.CPUs) + cpus = libcpu.MustParseCpuMask(h.CPUs) case h.NUMAs != "": for idstr := range strings.SplitSeq(h.NUMAs, ",") { if id, err := strconv.ParseInt(idstr, 0, 0); err == nil { if node := cs.node.Machine().MemoryNode(idset.ID(id)); node.Valid() { - cpus = cpus.Union(toCpuSet(node.CPUs())) + cpus = cpus.Union(node.CPUs()) } } } @@ -110,7 +110,7 @@ func (cs *supply) hintCpus(h topology.Hint) cpuset.CPUSet { for idstr := range strings.SplitSeq(h.Sockets, ",") { if id, err := strconv.ParseInt(idstr, 0, 0); err == nil { if pkg := packageZone(cs.node.Machine(), idset.ID(id)); pkg != nil { - cpus = cpus.Union(toCpuSet(pkg.CPUs())) + cpus = cpus.Union(pkg.CPUs()) } } } diff --git a/cmd/plugins/topology-aware/policy/hint_test.go b/cmd/plugins/topology-aware/policy/hint_test.go index 55cb1e219..46fbc9f98 100644 --- a/cmd/plugins/topology-aware/policy/hint_test.go +++ b/cmd/plugins/topology-aware/policy/hint_test.go @@ -15,10 +15,10 @@ package topologyaware import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "testing" "github.com/containers/nri-plugins/pkg/topology" - "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -27,7 +27,7 @@ func TestCpuHintScore(t *testing.T) { name string expected float64 hint topology.Hint - cpus cpuset.CPUSet + cpus *libcpu.CpuMask disabled bool // TODO(rojkov): remove this field when the code is fixed. }{ { @@ -51,7 +51,7 @@ func TestCpuHintScore(t *testing.T) { hint: topology.Hint{ CPUs: "1,2", }, - cpus: cpuset.New(1), + cpus: libcpu.NewCpuMask(1), expected: 0.5, }, } @@ -149,7 +149,7 @@ func TestHintCpus(t *testing.T) { name string supply *supply hint topology.Hint - expected cpuset.CPUSet + expected *libcpu.CpuMask }{ { name: "handle unparsable Sockets gracefully", @@ -204,7 +204,7 @@ func TestHintCpus(t *testing.T) { hint: topology.Hint{ Sockets: "1", }, - expected: cpuset.New(2, 3), + expected: libcpu.NewCpuMask(2, 3), }, { name: "NUMAs hint resolves to the node's CPUs", @@ -216,7 +216,7 @@ func TestHintCpus(t *testing.T) { hint: topology.Hint{ NUMAs: "0", }, - expected: cpuset.New(0, 1), + expected: libcpu.NewCpuMask(0, 1), }, { name: "non-zero CPUs hint", @@ -224,7 +224,7 @@ func TestHintCpus(t *testing.T) { hint: topology.Hint{ CPUs: "1", }, - expected: cpuset.New(1), + expected: libcpu.NewCpuMask(1), }, } for _, tc := range tcases { diff --git a/cmd/plugins/topology-aware/policy/irq-affinity.go b/cmd/plugins/topology-aware/policy/irq-affinity.go index c09bd099a..623b3d71b 100644 --- a/cmd/plugins/topology-aware/policy/irq-affinity.go +++ b/cmd/plugins/topology-aware/policy/irq-affinity.go @@ -16,11 +16,11 @@ package topologyaware import ( "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "strconv" "github.com/containers/nri-plugins/pkg/irq" "github.com/containers/nri-plugins/pkg/topology" - "github.com/containers/nri-plugins/pkg/utils/cpuset" "sigs.k8s.io/yaml" ) @@ -106,8 +106,8 @@ func addIrqAffinityForHints(a *IrqAffinity, hints topology.Hints) error { return nil } -func (p *policy) irqCpus(hwIrq *irq.Irq) (preMask, claim, mask cpuset.CPUSet) { - preMask, claim, mask = cpuset.New(), cpuset.New(), cpuset.New() +func (p *policy) irqCpus(hwIrq *irq.Irq) (preMask, claim, mask *libcpu.CpuMask) { + preMask, claim, mask = libcpu.NewCpuMask(), libcpu.NewCpuMask(), libcpu.NewCpuMask() for _, g := range p.allocations.grants { irqs := g.IrqAffinity() switch { @@ -156,11 +156,12 @@ func (p *policy) applyIrqAffinity(user string) { } for _, hwIrq := range hwIrqs { - current, err := hwIrq.AffinityCpus() + affinity, err := hwIrq.AffinityCpus() if err != nil { log.Errorf("%s: failed to read affinity: %v", hwIrq.String(), err) continue } + current := toCpuMask(affinity) preMask, claim, mask := p.irqCpus(hwIrq) @@ -182,7 +183,7 @@ func (p *policy) applyIrqAffinity(user string) { continue } - if err := hwIrq.SetAffinityCpus(cpus); err != nil { + if err := hwIrq.SetAffinityCpus(toCpuSet(cpus)); err != nil { log.Errorf("%s: failed to set affinity to cpus %s (for %s): %v", hwIrq.String(), cpus.String(), user, err) } diff --git a/cmd/plugins/topology-aware/policy/metrics.go b/cmd/plugins/topology-aware/policy/metrics.go index 83c7dfb04..bc1ea6237 100644 --- a/cmd/plugins/topology-aware/policy/metrics.go +++ b/cmd/plugins/topology-aware/policy/metrics.go @@ -17,6 +17,7 @@ package topologyaware import ( "context" "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "slices" "strings" @@ -25,7 +26,6 @@ import ( "github.com/containers/nri-plugins/pkg/metrics" libmem "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) type TopologyAwareMetrics struct { @@ -45,9 +45,9 @@ type TopologyAwareMetrics struct { type Zone struct { Name string - Cpus cpuset.CPUSet + Cpus *libcpu.CpuMask Mems libmem.NodeMask - SharedPool cpuset.CPUSet + SharedPool *libcpu.CpuMask SharedAssigned int SharedAvailable int MemCapacity int64 diff --git a/cmd/plugins/topology-aware/policy/metrics_test.go b/cmd/plugins/topology-aware/policy/metrics_test.go index 546eb449f..1dcc8f295 100644 --- a/cmd/plugins/topology-aware/policy/metrics_test.go +++ b/cmd/plugins/topology-aware/policy/metrics_test.go @@ -25,11 +25,11 @@ import ( "go.opentelemetry.io/otel/sdk/metric/metricdata" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/metrics" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/testutils" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // TestMetricsUpdateNilReceiver verifies that Update() on a nil @@ -273,7 +273,7 @@ func TestSharedPoolMetricsDoNotLeakSeriesOnCpusetChange(t *testing.T) { if len(list) == 0 { continue } - s.sharable = s.sharable.Difference(cpuset.New(list[0])) + s.sharable = s.sharable.Difference(libcpu.NewCpuMask(list[0])) changed = true } if !changed { diff --git a/cmd/plugins/topology-aware/policy/node.go b/cmd/plugins/topology-aware/policy/node.go index 050bb25e1..46de9df17 100644 --- a/cmd/plugins/topology-aware/policy/node.go +++ b/cmd/plugins/topology-aware/policy/node.go @@ -18,9 +18,9 @@ import ( "fmt" cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" - "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -196,9 +196,9 @@ type numanode struct { // l3cachenode represents an L3 cache grouping of CPUs in the system. type l3cachenode struct { - node // common node data - id idset.ID // L3 cache id from sysfs - cpus cpuset.CPUSet // CPUs in this L3 cache group + node // common node data + id idset.ID // L3 cache id from sysfs + cpus *libcpu.CpuMask // CPUs in this L3 cache group } // virtualnode represents a virtual node (ATM only the root in a multi-socket system). @@ -515,7 +515,7 @@ func (n *numanode) GetMemset(mtype memoryType) idset.IDSet { func (n *numanode) HintScore(hint topology.Hint) float64 { switch { case hint.CPUs != "": - return cpuHintScore(hint, toCpuSet(n.sysnode.CPUs())) + return cpuHintScore(hint, n.sysnode.CPUs()) case hint.NUMAs != "": return numaHintScore(hint, n.id) @@ -534,7 +534,7 @@ func (n *numanode) HintScore(hint topology.Hint) float64 { } // NewL3CacheNode creates a node for an L3 cache group. -func (p *policy) NewL3CacheNode(id idset.ID, cpus cpuset.CPUSet, parent Node) *l3cachenode { +func (p *policy) NewL3CacheNode(id idset.ID, cpus *libcpu.CpuMask, parent Node) *l3cachenode { n := &l3cachenode{} n.self.node = n n.init(p, fmt.Sprintf("%s/L3 cache #%v", parent.Name(), id), L3CacheNode, parent) @@ -664,7 +664,7 @@ func (n *dienode) GetMemset(mtype memoryType) idset.IDSet { func (n *dienode) HintScore(hint topology.Hint) float64 { switch { case hint.CPUs != "": - return cpuHintScore(hint, toCpuSet(n.syspkg.CPUs())) + return cpuHintScore(hint, n.syspkg.CPUs()) case hint.NUMAs != "": return OverfitPenalty * dieHintScore(hint, n.Machine(), n.syspkg.ID(), n.id) @@ -733,7 +733,7 @@ func (n *socketnode) GetMemset(mtype memoryType) idset.IDSet { func (n *socketnode) HintScore(hint topology.Hint) float64 { switch { case hint.CPUs != "": - return cpuHintScore(hint, toCpuSet(n.syspkg.CPUs())) + return cpuHintScore(hint, n.syspkg.CPUs()) case hint.NUMAs != "": return OverfitPenalty * numaHintScore(hint, packageNodeIDs(n.Machine(), n.syspkg.ID())...) @@ -786,7 +786,7 @@ func (n *virtualnode) HintScore(hint topology.Hint) float64 { // don't bother calculating any scores, the root should always score 1.0 switch { case hint.CPUs != "": - return cpuHintScore(hint, toCpuSet(n.Machine().PresentCPUs())) + return cpuHintScore(hint, n.Machine().PresentCPUs()) case hint.NUMAs != "": return OverfitPenalty * OverfitPenalty diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index 0e08f174a..a92300cd8 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -20,8 +20,8 @@ import ( "sort" "strings" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/utils/cpuset" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/types" @@ -100,7 +100,7 @@ func (p *policy) buildRootPool() { log.Infof("+ created pool %s", vroot.Name()) - cpus := toCpuSet(p.machine.PresentCPUs()) + cpus := p.machine.PresentCPUs() vroot.noderes, vroot.freeres = p.getCpuSupply(vroot, cpus) vroot.mem, vroot.pMem, vroot.hbm = p.getMemSupply(vroot, cpus) } else { @@ -211,7 +211,7 @@ func (p *policy) buildNumaNodePool(socketID, nodeID idset.ID, parent Node) { log.Infof("+ created pool %s", node.Name()) - cpus := toCpuSet(p.machine.MemoryNode(nodeID).CPUs()) + cpus := p.machine.MemoryNode(nodeID).CPUs() node.noderes, node.freeres = p.getCpuSupply(node, cpus) node.mem, node.pMem, node.hbm = p.getMemSupply(node, cpus) @@ -225,7 +225,7 @@ func (p *policy) buildNumaNodePool(socketID, nodeID idset.ID, parent Node) { } // getL3CacheIDsForCPUs returns L3 cache IDs that are within the given CPU set scope. -func (p *policy) getL3CacheIDsForCPUs(socketID idset.ID, cpus cpuset.CPUSet) []idset.ID { +func (p *policy) getL3CacheIDsForCPUs(socketID idset.ID, cpus *libcpu.CpuMask) []idset.ID { var within []idset.ID for _, l3CacheID := range l3CacheIDs(p.machine, socketID) { cacheCPUs := l3CacheCPUs(p.machine, socketID, l3CacheID) @@ -238,7 +238,7 @@ func (p *policy) getL3CacheIDsForCPUs(socketID idset.ID, cpus cpuset.CPUSet) []i } // buildL3CachePool creates an L3 cache pool as a child of the given parent. -func (p *policy) buildL3CachePool(id idset.ID, cpus cpuset.CPUSet, parent Node) { +func (p *policy) buildL3CachePool(id idset.ID, cpus *libcpu.CpuMask, parent Node) { l3CacheNode := p.NewL3CacheNode(id, cpus, parent) p.nodes[l3CacheNode.Name()] = l3CacheNode l3CacheNode.depth = l3CacheNode.RootDistance() @@ -249,7 +249,7 @@ func (p *policy) buildL3CachePool(id idset.ID, cpus cpuset.CPUSet, parent Node) l3CacheNode.mem, l3CacheNode.pMem, l3CacheNode.hbm = p.getMemSupply(l3CacheNode, cpus) } -func (p *policy) getCpuSupply(node Node, cpus cpuset.CPUSet) (Supply, Supply) { +func (p *policy) getCpuSupply(node Node, cpus *libcpu.CpuMask) (Supply, Supply) { var ( allowed = cpus.Intersection(p.allowed) isolated = allowed.Intersection(p.isolated) @@ -264,7 +264,7 @@ func (p *policy) getCpuSupply(node Node, cpus cpuset.CPUSet) (Supply, Supply) { return s, newSupply(node, isolated, reserved, sharable, 0, 0) } -func (p *policy) getMemSupply(node Node, cpus cpuset.CPUSet) (dram, pmem, hbm idset.IDSet) { +func (p *policy) getMemSupply(node Node, cpus *libcpu.CpuMask) (dram, pmem, hbm idset.IDSet) { if p.root == node { dram, pmem, hbm = p.splitMemsByType(p.getAllMems()) if dram.Size() > 0 { @@ -317,12 +317,12 @@ func (p *policy) getMemSupply(node Node, cpus cpuset.CPUSet) (dram, pmem, hbm id return dram, pmem, hbm } -func (p *policy) getMemsForCpus(cpus cpuset.CPUSet) idset.IDSet { +func (p *policy) getMemsForCpus(cpus *libcpu.CpuMask) idset.IDSet { mems := idset.NewIDSet() for _, nodeID := range p.machine.MemoryNodeIDs() { node := p.machine.MemoryNode(nodeID) - if node.CPUs().Intersects(toCpuMask(cpus)) { + if node.CPUs().Intersects(cpus) { mems.Add(nodeID) } } @@ -521,12 +521,12 @@ func (p *policy) allocatePool(container cache.Container, poolHint string) (Grant // allocated for it, taking into account if the container should run // with hyperthreads hidden. CPUs in preserve are always included in the // final cpuset regardless of hide-hyperthreads filtering (e.g. DRA claimed CPUs). -func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated, preserve cpuset.CPUSet, info string) { +func (p *policy) setPreferredCpusetCpus(container cache.Container, allocated, preserve *libcpu.CpuMask, info string) { allow := allocated hidingInfo := "" pod, ok := container.GetPod() if ok && hideHyperthreadsPreference(pod, container) { - allow = toCpuSet(hardware.SingleThreadPerCore(p.machine, toCpuMask(allocated))) + allow = hardware.SingleThreadPerCore(p.machine, allocated) if allow.Size() != allocated.Size() { hidingInfo = fmt.Sprintf(" (hide %d hyperthreads, remaining cpuset: %s)", allocated.Size()-allow.Size(), allow) } else { @@ -549,7 +549,7 @@ func (p *policy) applyGrant(grant Grant) { shared := grant.SharedCPUs() cpuPortion := grant.SharedPortion() - cpus := cpuset.New() + cpus := libcpu.NewCpuMask() kind := "" switch cpuType { case cpuNormal: @@ -639,7 +639,7 @@ func (p *policy) applyGrant(grant Grant) { container.SetCPUShares(int64(cache.MilliCPUToShares(int64(milliCPU)))) if exclusive.Size() > 0 && grant.CPUClass() != "" { - if err := p.cpuClasses.UseClass(grant.CPUClass(), exclusive); err != nil { + if err := p.cpuClasses.UseClass(grant.CPUClass(), toCpuSet(exclusive)); err != nil { log.Errorf("%s: failed to apply CPU class to cpuset %s: %v", container.PrettyName(), exclusive, err) } @@ -880,9 +880,11 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco if request.FullCPUs() > 0 { log.Debugf(" %s: free %s, CPU class hints: %+v, class hinted %s", node1.Name(), - score1.Supply().SharableCPUs(), score1.CpuClassHints(), score1.CpuClassCpus()) + score1.Supply().SharableCPUs(), score1.CpuClassHints(), + score1.CpuClassCpus()) log.Debugf(" %s: free %s, CPU class hints: %+v, class hinted %s", node2.Name(), - score2.Supply().SharableCPUs(), score2.CpuClassHints(), score2.CpuClassCpus()) + score2.Supply().SharableCPUs(), score2.CpuClassHints(), + score2.CpuClassCpus()) } // @@ -1146,12 +1148,12 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco // for cpuClasses the sole node that can fufill the request wins if score1.CpuClassHints() != nil && score2.CpuClassHints() != nil { offer1, offer2 := score1.CPUOffer(), score2.CPUOffer() - hcpus1, hcpus2 := cpuset.New(), cpuset.New() + hcpus1, hcpus2 := libcpu.NewCpuMask(), libcpu.NewCpuMask() for _, h := range score1.CpuClassHints().Prefer { for _, hinted := range h.Cpus { - if offer1.Intersection(hinted).Equals(offer1) { - hcpus1 = hinted + if offer1.Intersection(toCpuMask(hinted)).Equals(offer1) { + hcpus1 = toCpuMask(hinted) break } } @@ -1161,8 +1163,8 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco } for _, h := range score2.CpuClassHints().Prefer { for _, hinted := range h.Cpus { - if offer2.Intersection(hinted).Equals(offer2) { - hcpus2 = hinted + if offer2.Intersection(toCpuMask(hinted)).Equals(offer2) { + hcpus2 = toCpuMask(hinted) break } } diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 53c067b9c..3c5a6c0f9 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -23,6 +23,7 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/containers/nri-plugins/pkg/agent/podresapi" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" "github.com/containers/nri-plugins/pkg/utils/cpuset" @@ -64,11 +65,11 @@ type Supply interface { // Clone creates a copy of this supply. Clone() Supply // IsolatedCPUs returns the isolated cpuset in this supply. - IsolatedCPUs() cpuset.CPUSet + IsolatedCPUs() *libcpu.CpuMask // ReservedCPUs returns the reserved cpuset in this supply. - ReservedCPUs() cpuset.CPUSet + ReservedCPUs() *libcpu.CpuMask // SharableCPUs returns the sharable cpuset in this supply. - SharableCPUs() cpuset.CPUSet + SharableCPUs() *libcpu.CpuMask // GrantedReserved returns the locally granted reserved CPU capacity in this supply. GrantedReserved() int // GrantedShared returns the locally granted shared CPU capacity in this supply. @@ -92,14 +93,14 @@ type Supply interface { // AllocatableSharedCPU calculates the allocatable amount of shared CPU of this supply. AllocatableSharedCPU(...bool) int // SliceableCPUs calculates the shared cpuset we can slice exclusive CPUs off of. - SliceableCPUs() (cpuset.CPUSet, error) + SliceableCPUs() (*libcpu.CpuMask, error) // Allocate allocates a grant from the supply. Allocate(Request, *libmem.Offer) (Grant, map[string]libmem.NodeMask, error) // ReleaseCPU releases a previously allocated CPU grant from this supply. ReleaseCPU(Grant) // GetCPUOffer returns the exclusive CPUs that would be allocated for the given request. - GetCPUOffer(Request) (cpuset.CPUSet, error) + GetCPUOffer(Request) (*libcpu.CpuMask, error) // Reserve accounts for CPU grants after reloading cached allocations. Reserve(Grant, *libmem.Offer) (map[string]libmem.NodeMask, error) @@ -169,17 +170,17 @@ type Grant interface { // CPUPortion() == ReservedPortion() + SharedPortion(). CPUPortion() int // ExclusiveCPUs returns the exclusively granted non-isolated cpuset. - ExclusiveCPUs() cpuset.CPUSet + ExclusiveCPUs() *libcpu.CpuMask // ReservedCPUs returns the reserved granted cpuset. - ReservedCPUs() cpuset.CPUSet + ReservedCPUs() *libcpu.CpuMask // ReservedPortion() returns the amount of CPUs in milli-CPU granted. ReservedPortion() int // SharedCPUs returns the shared granted cpuset. - SharedCPUs() cpuset.CPUSet + SharedCPUs() *libcpu.CpuMask // SharedPortion returns the amount of CPUs in milli-CPU granted. SharedPortion() int // IsolatedCpus returns the exclusively granted isolated cpuset. - IsolatedCPUs() cpuset.CPUSet + IsolatedCPUs() *libcpu.CpuMask // MemoryType returns the type(s) of granted memory. MemoryType() memoryType // SetMemoryType sets the memory type for this grant. @@ -230,28 +231,28 @@ type Score interface { HintScores() map[string]float64 PrioCapacity(cpuPrio) int CpuClassHints() *cpuclass.AllocationHints - CpuClassCpus() cpuset.CPUSet + CpuClassCpus() *libcpu.CpuMask MemOffer() *libmem.Offer - CPUOffer() cpuset.CPUSet + CPUOffer() *libcpu.CpuMask String() string } // supply implements our Supply interface. type supply struct { - node Node // node supplying CPUs and memory - isolated cpuset.CPUSet // isolated CPUs at this node - reserved cpuset.CPUSet // reserved CPUs at this node - sharable cpuset.CPUSet // sharable CPUs at this node - grantedReserved int // amount of reserved CPUs allocated - grantedShared int // amount of shareable CPUs allocated + node Node // node supplying CPUs and memory + isolated *libcpu.CpuMask // isolated CPUs at this node + reserved *libcpu.CpuMask // reserved CPUs at this node + sharable *libcpu.CpuMask // sharable CPUs at this node + grantedReserved int // amount of reserved CPUs allocated + grantedShared int // amount of shareable CPUs allocated // claimRefs tracks, per DRA claim UID, the CPUs subtracted from isolated/ // sharable capacity of this supply on behalf of that claim. Marking is // tree-wide: the same UID is (re)marked in every ancestor supply too (see // ClaimCPUs/UnclaimCPUs). - claimRefs map[types.UID]cpuset.CPUSet + claimRefs map[types.UID]*libcpu.CpuMask // cloned indicates that this supply is a Clone() copy and should not // propagate ClaimCPUs/UnclaimCPUs to real ancestor nodes. @@ -290,7 +291,7 @@ var _ Request = &request{} type grant struct { container cache.Container // container CPU is granted to node Node // node CPU is supplied from - exclusive cpuset.CPUSet // exclusive CPUs + exclusive *libcpu.CpuMask // exclusive CPUs cpuType cpuType // type of CPUs (normal, reserved, ...) cpuPortion int // milliCPUs granted from CPUs of cpuType memType memoryType // requested types of memory @@ -309,7 +310,7 @@ type score struct { supply Supply // CPU supply (node) req Request // CPU request (container) mem *libmem.Offer // possible memory allocation - cpu cpuset.CPUSet // CPUs offered by this supply for the request + cpu *libcpu.CpuMask // CPUs offered by this supply for the request isolated int // remaining isolated CPUs reserved int // remaining reserved CPUs shared int // remaining shared capacity @@ -317,14 +318,14 @@ type score struct { colocated int // number of colocated containers hints map[string]float64 // hint scores ccHints *cpuclass.AllocationHints // CPU class hints - ccCpus cpuset.CPUSet // CPU class-hinted CPUs for the offered CPUs + ccCpus *libcpu.CpuMask // CPU class-hinted CPUs for the offered CPUs } var _ Score = &score{} // newSupply creates CPU supply for the given node, cpusets and existing grant. -func newSupply(n Node, isolated, reserved, sharable cpuset.CPUSet, grantedReserved int, grantedShared int) Supply { +func newSupply(n Node, isolated, reserved, sharable *libcpu.CpuMask, grantedReserved int, grantedShared int) Supply { return &supply{ node: n, isolated: isolated.Clone(), @@ -354,17 +355,17 @@ func (cs *supply) Clone() Supply { } // IsolatedCpus returns the isolated CPUSet of this supply. -func (cs *supply) IsolatedCPUs() cpuset.CPUSet { +func (cs *supply) IsolatedCPUs() *libcpu.CpuMask { return cs.isolated.Clone() } // ReservedCpus returns the reserved CPUSet of this supply. -func (cs *supply) ReservedCPUs() cpuset.CPUSet { +func (cs *supply) ReservedCPUs() *libcpu.CpuMask { return cs.reserved.Clone() } // SharableCpus returns the sharable CPUSet of this supply. -func (cs *supply) SharableCPUs() cpuset.CPUSet { +func (cs *supply) SharableCPUs() *libcpu.CpuMask { return cs.sharable.Clone() } @@ -511,7 +512,7 @@ func (cs *supply) Allocate(r Request, o *libmem.Offer) (Grant, map[string]libmem // AllocateCPU allocates CPU for a grant from the supply. func (cs *supply) AllocateCPU(r Request) (Grant, error) { var ( - exclusive cpuset.CPUSet + exclusive *libcpu.CpuMask err error ) @@ -561,45 +562,44 @@ func (cs *supply) AllocateCPU(r Request) (Grant, error) { return grant, nil } -func (cs *supply) GetCPUOffer(r Request) (cpuset.CPUSet, error) { +func (cs *supply) GetCPUOffer(r Request) (*libcpu.CpuMask, error) { return cs.pickExclusiveCPUs(r, r.FullCPUs(), true) } -func (cs *supply) pickExclusiveCPUs(r Request, cnt int, dryRun bool) (cpuset.CPUSet, error) { +func (cs *supply) pickExclusiveCPUs(r Request, cnt int, dryRun bool) (*libcpu.CpuMask, error) { var ( cr = r.(*request) - none = cpuset.New() + none = libcpu.NewCpuMask() ) switch { case cr.isolate && cs.isolated.Size() >= cnt: var ( - from = &cs.isolated - pick cpuset.CPUSet + from = cs.isolated + pick *libcpu.CpuMask err error ) if dryRun { - copy := from.Clone() - from = © + from = from.Clone() } if cr.PickByHints() { pick, err = cs.takeCPUsByHints(from, cr.GetContainer().GetTopologyHints(), cnt, cr.CPUPrio()) } else { - pick, err = cs.takeCPUs(from, nil, cnt, cr.CPUPrio()) + pick, err = cs.takeCPUs(from, cnt, cr.CPUPrio()) } if err != nil { return none, policyError("internal error: "+ "%s: can't take %d exclusive isolated CPUs from %s: %v", - cs.node.Name(), cnt, *from, err) + cs.node.Name(), cnt, from, err) } return pick, nil case cs.AllocatableSharedCPU() >= 1000*cnt: var ( slice, err = cs.SliceableCPUs() - pick cpuset.CPUSet + pick *libcpu.CpuMask ) if err != nil { @@ -611,9 +611,9 @@ func (cs *supply) pickExclusiveCPUs(r Request, cnt int, dryRun bool) (cpuset.CPU log.Debugf("%s: sliceable cpuset is %s", cs.node.Name(), slice) if cr.PickByHints() { - pick, err = cs.takeCPUsByHints(&slice, cr.GetContainer().GetTopologyHints(), cnt, cr.CPUPrio()) + pick, err = cs.takeCPUsByHints(slice, cr.GetContainer().GetTopologyHints(), cnt, cr.CPUPrio()) } else { - pick, err = cs.takeCPUs(&slice, nil, cnt, cr.CPUPrio()) + pick, err = cs.takeCPUs(slice, cnt, cr.CPUPrio()) } if err != nil { return none, policyError("internal error: "+ @@ -719,27 +719,14 @@ func (cs *supply) Reserve(g Grant, o *libmem.Offer) (map[string]libmem.NodeMask, return updates, nil } -// takeCPUs takes up to cnt CPUs from a given CPU set to another. -func (cs *supply) takeCPUs(from, to *cpuset.CPUSet, cnt int, prio cpuPrio) (cpuset.CPUSet, error) { - // The allocator speaks libcpu sets and takes what it allocated out of the - // set it is given, so hand it one and copy back what is left either way. - fromCpus := toCpuMask(*from) - allocated, err := cs.node.Policy().cpuAllocator.AllocateCpus(fromCpus, cnt, prio.Option()) - cset := toCpuSet(allocated) - *from = toCpuSet(fromCpus) - if err != nil { - return cset, err - } - - if to != nil { - *to = to.Union(cset) - } - - return cset, err +// takeCPUs takes up to cnt CPUs out of the given set, which the allocator +// removes them from. +func (cs *supply) takeCPUs(from *libcpu.CpuMask, cnt int, prio cpuPrio) (*libcpu.CpuMask, error) { + return cs.node.Policy().cpuAllocator.AllocateCpus(from, cnt, prio.Option()) } // takeCPUsByHints tries to allocate isolated or exclusive CPUs by topology hints. -func (cs *supply) takeCPUsByHints(from *cpuset.CPUSet, all topology.Hints, cnt int, prio cpuPrio) (cpuset.CPUSet, error) { +func (cs *supply) takeCPUsByHints(from *libcpu.CpuMask, all topology.Hints, cnt int, prio cpuPrio) (*libcpu.CpuMask, error) { hints := []*topology.Hint{} for provider, h := range all { if podresapi.IsPodResourceHint(provider) { @@ -747,7 +734,7 @@ func (cs *supply) takeCPUsByHints(from *cpuset.CPUSet, all topology.Hints, cnt i } } if len(hints) == 0 || len(hints) > cnt { - return cs.takeCPUs(from, nil, cnt, prio) + return cs.takeCPUs(from, cnt, prio) } total := cnt @@ -756,14 +743,14 @@ func (cs *supply) takeCPUsByHints(from *cpuset.CPUSet, all topology.Hints, cnt i perHint = total / len(hints) } - free := (*from).Clone() - cpus := cpuset.New() + free := from.Clone() + cpus := libcpu.NewCpuMask() for _, h := range hints { - cset := free.Intersection(cpuset.MustParse(h.CPUs)) - pick, err := cs.takeCPUs(&cset, nil, perHint, prio) + cset := free.Intersection(libcpu.MustParseCpuMask(h.CPUs)) + pick, err := cs.takeCPUs(cset, perHint, prio) if err != nil { log.Errorf("failed to allocate CPUs by topology hints: %v", err) - return cs.takeCPUs(from, nil, cnt, prio) + return cs.takeCPUs(from, cnt, prio) } cpus = cpus.Union(pick) free = free.Difference(pick) @@ -771,16 +758,17 @@ func (cs *supply) takeCPUsByHints(from *cpuset.CPUSet, all topology.Hints, cnt i } if total > 0 { - pick, err := cs.takeCPUs(&free, nil, total, prio) + pick, err := cs.takeCPUs(free, total, prio) if err != nil { log.Errorf("failed to allocate CPUs by topology hints: %v", err) - return cs.takeCPUs(from, nil, cnt, prio) + return cs.takeCPUs(from, cnt, prio) } cpus = cpus.Union(pick) - free = free.Difference(pick) } - *from = free + // Everything picked came out of a clone, so take it out of the caller's set + // now: what is left is what the clone has. + from.Clear(cpus.UnsortedList()...) return cpus, nil } @@ -1180,23 +1168,23 @@ func (cs *supply) GetScore(req Request) Score { // calculate fractional capacity score.shared -= part - lpCPUs := toCpuSet(cs.GetNode().Machine().CoreKindCPUs(hardware.EfficientCore)) + lpCPUs := cs.GetNode().Machine().CoreKindCPUs(hardware.EfficientCore) if lpCPUs.Size() == 0 { - lpCPUs = toCpuSet(p.cpuAllocator.GetCPUPriorities()[lowPrio].EmptyIfNil()) + lpCPUs = p.cpuAllocator.GetCPUPriorities()[lowPrio] } lpCPUs = lpCPUs.Intersection(cs.SharableCPUs()) lpCnt := lpCPUs.Size() score.prio[lowPrio] = lpCnt*1000 - (1000*full + part) - hpCPUs := toCpuSet(cs.GetNode().Machine().CoreKindCPUs(hardware.PerformanceCore)) + hpCPUs := cs.GetNode().Machine().CoreKindCPUs(hardware.PerformanceCore) if hpCPUs.Size() == 0 { - hpCPUs = toCpuSet(p.cpuAllocator.GetCPUPriorities()[highPrio].EmptyIfNil()) + hpCPUs = p.cpuAllocator.GetCPUPriorities()[highPrio] } hpCPUs = hpCPUs.Intersection(cs.SharableCPUs()) hpCnt := hpCPUs.Size() score.prio[highPrio] = hpCnt*1000 - (1000*full + part) - npCPUs := toCpuSet(p.cpuAllocator.GetCPUPriorities()[normalPrio].EmptyIfNil()) + npCPUs := p.cpuAllocator.GetCPUPriorities()[normalPrio] npCPUs = npCPUs.Intersection(cs.SharableCPUs()) npCnt := npCPUs.Size() score.prio[normalPrio] = npCnt*1000 - (1000*full + part) @@ -1211,7 +1199,7 @@ func (cs *supply) GetScore(req Request) Score { hints := p.cpuClasses.Hints(cpuclass.AllocationIntent{ ClassName: cr.cpuClass, CurrentCpus: cpuset.New(), - FreeCpus: cpus, + FreeCpus: toCpuSet(cpus), RequestedCount: cr.full, }) score.ccHints = &hints @@ -1335,9 +1323,9 @@ func (cs *supply) AllocatableSharedCPU(quiet ...bool) int { } // SliceableCPUs calculates the shared cpuset we can slice exclusive CPUs off of. -func (cs *supply) SliceableCPUs() (cpuset.CPUSet, error) { +func (cs *supply) SliceableCPUs() (*libcpu.CpuMask, error) { var ( - sliceable = cpuset.New() + sliceable = libcpu.NewCpuMask() errs []error ) @@ -1364,7 +1352,7 @@ func (cs *supply) SliceableCPUs() (cpuset.CPUSet, error) { // priority preference of any ongoing allocation, trying to slice // CPUs with a matching preference. We don't do that ATM. - cset, err := cs.takeCPUs(&cpus, nil, free, nonePrio) + cset, err := cs.takeCPUs(cpus, free, nonePrio) if err != nil { errs = append(errs, err) return false @@ -1375,7 +1363,7 @@ func (cs *supply) SliceableCPUs() (cpuset.CPUSet, error) { }) if len(errs) > 0 { - return cpuset.New(), errors.Join(errs...) + return libcpu.NewCpuMask(), errors.Join(errs...) } return sliceable, nil @@ -1422,7 +1410,7 @@ func (score *score) CpuClassHints() *cpuclass.AllocationHints { return score.ccHints } -func (score *score) CpuClassCpus() cpuset.CPUSet { +func (score *score) CpuClassCpus() *libcpu.CpuMask { return score.ccCpus } @@ -1430,7 +1418,7 @@ func (score *score) MemOffer() *libmem.Offer { return score.mem } -func (score *score) CPUOffer() cpuset.CPUSet { +func (score *score) CPUOffer() *libcpu.CpuMask { return score.cpu } @@ -1440,7 +1428,7 @@ func (score *score) String() string { } // newGrant creates a CPU grant from the given node for the container. -func newGrant(n Node, c cache.Container, cpuType cpuType, cpuCls string, exclusive cpuset.CPUSet, cpuPortion int, mt memoryType, irqs *IrqAffinity, coldstart time.Duration) Grant { +func newGrant(n Node, c cache.Container, cpuType cpuType, cpuCls string, exclusive *libcpu.CpuMask, cpuPortion int, mt memoryType, irqs *IrqAffinity, coldstart time.Duration) Grant { grant := &grant{ node: n, container: c, @@ -1488,9 +1476,11 @@ func (cg *grant) SetColdstart(period time.Duration) { // Clone creates a copy of this grant. func (cg *grant) Clone() Grant { return &grant{ - node: cg.GetCPUNode(), - container: cg.GetContainer(), - exclusive: cg.ExclusiveCPUs(), + node: cg.GetCPUNode(), + container: cg.GetContainer(), + // the clone's own set: ExclusiveCPUs hands out the mask itself, which + // was a copy back when these sets were values + exclusive: cg.ExclusiveCPUs().Clone(), cpuType: cg.CPUType(), cpuClass: cg.CPUClass(), cpuPortion: cg.SharedPortion(), @@ -1552,12 +1542,12 @@ func (cg *grant) CPUPortion() int { } // ExclusiveCPUs returns the non-isolated exclusive CPUSet in this grant. -func (cg *grant) ExclusiveCPUs() cpuset.CPUSet { +func (cg *grant) ExclusiveCPUs() *libcpu.CpuMask { return cg.exclusive } // ReservedCPUs returns the reserved CPUSet in the supply of this grant. -func (cg *grant) ReservedCPUs() cpuset.CPUSet { +func (cg *grant) ReservedCPUs() *libcpu.CpuMask { return cg.node.GetSupply().ReservedCPUs() } @@ -1570,7 +1560,7 @@ func (cg *grant) ReservedPortion() int { } // SharedCPUs returns the shared CPUSet in the supply of this grant. -func (cg *grant) SharedCPUs() cpuset.CPUSet { +func (cg *grant) SharedCPUs() *libcpu.CpuMask { return cg.node.FreeSupply().SharableCPUs() } @@ -1583,7 +1573,7 @@ func (cg *grant) SharedPortion() int { } // ExclusiveCPUs returns the isolated exclusive CPUSet in this grant. -func (cg *grant) IsolatedCPUs() cpuset.CPUSet { +func (cg *grant) IsolatedCPUs() *libcpu.CpuMask { return cg.node.GetSupply().IsolatedCPUs().Intersection(cg.exclusive) } diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index 159223b1e..0c178a930 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -20,6 +20,7 @@ import ( "fmt" "github.com/containers/nri-plugins/pkg/irq" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/utils/cpuset" "k8s.io/apimachinery/pkg/api/resource" @@ -62,10 +63,10 @@ type policy struct { cfg *cfgapi.Config cache cache.Cache // pod/container cache machine *hardware.Machine // CPU and memory topology - allowed cpuset.CPUSet // bounding set of CPUs we're allowed to use - reserved cpuset.CPUSet // system-/kube-reserved CPUs + allowed *libcpu.CpuMask // bounding set of CPUs we're allowed to use + reserved *libcpu.CpuMask // system-/kube-reserved CPUs reserveCnt int // number of CPUs to reserve if given as resource.Quantity - isolated cpuset.CPUSet // (our allowed set of) isolated CPUs + isolated *libcpu.CpuMask // (our allowed set of) isolated CPUs nodes map[string]Node // pool nodes by name pools []Node // pre-populated node slice for scoring, etc... root Node // root of our pool/partition tree @@ -143,7 +144,14 @@ var coldStartOff bool // New creates a new uninitialized topology-aware policy instance. func New() policyapi.Backend { - return &policy{} + // The policy's own CPU sets exist from the start, before a configuration + // lands: a CpuMask has no usable zero value, and something may ask this for + // metrics before, or instead of, configuring it. + return &policy{ + allowed: libcpu.NewCpuMask(), + reserved: libcpu.NewCpuMask(), + isolated: libcpu.NewCpuMask(), + } } // Setup initializes the topology-aware policy instance. @@ -819,7 +827,7 @@ func (p *policy) initialize() error { if err := p.cpuClasses.Configure(cpuclass.ConfigSpec{ Classes: opt.CPUClasses, TurboDomain: "package", - Allowed: p.allowed, + Allowed: toCpuSet(p.allowed), }); err != nil { return policyError("failed to configure CPU class handler: %w", err) } @@ -848,25 +856,25 @@ func (p *policy) checkConstraints() error { if err != nil { return fmt.Errorf("failed to parse available CPU cpuset '%s': %w", amount, err) } - p.allowed = cset + p.allowed = toCpuMask(cset) case cfgapi.AmountExcludeCPUSet: cset, err := amount.ParseCPUSet() if err != nil { return fmt.Errorf("failed to parse available CPU cpuset '%s': %w", amount, err) } - p.allowed = toCpuSet(p.machine.PresentCPUs()).Difference(cset) + p.allowed = p.machine.PresentCPUs().Difference(toCpuMask(cset)) case cfgapi.AmountQuantity: return fmt.Errorf("can't handle CPU resources given as resource.Quantity (%v)", amount) case cfgapi.AmountAbsent: // Available CPUs not specified, default to system CPUs. - p.allowed = toCpuSet(p.machine.PresentCPUs()) + p.allowed = p.machine.PresentCPUs() } // Allocation of only online CPUs is allowed. - p.allowed = p.allowed.Intersection(toCpuSet(p.machine.OnlineCPUs())) + p.allowed = p.allowed.Intersection(p.machine.OnlineCPUs()) - p.isolated = toCpuSet(p.machine.IsolatedCPUs()).Intersection(p.allowed) + p.isolated = p.machine.IsolatedCPUs().Intersection(p.allowed) amount, kind = p.cfg.ReservedResources.Get(cfgapi.CPU) switch kind { @@ -879,9 +887,9 @@ func (p *policy) checkConstraints() error { return fmt.Errorf("failed to parse reserved CPU cpuset '%s': %w", amount, err) } if kind == cfgapi.AmountExcludeCPUSet { - p.reserved = p.allowed.Difference(cset) + p.reserved = p.allowed.Difference(toCpuMask(cset)) } else { - p.reserved = cset + p.reserved = toCpuMask(cset) } // check that all reserved CPUs are in the allowed set @@ -913,12 +921,12 @@ func (p *policy) checkConstraints() error { // Use CpuAllocator to pick reserved CPUs from the allowed ones but // avoiding isolated CPUs. The picked CPUs are not removed from the // allowed set. - from := toCpuMask(p.allowed.Difference(p.isolated)) + from := p.allowed.Difference(p.isolated) cset, err := p.cpuAllocator.AllocateCpus(from, p.reserveCnt, normalPrio.Option()) if err != nil { return policyError("cannot reserve %dm CPUs for ReservedResources from AvailableResources: %s", qty.MilliValue(), err) } - p.reserved = toCpuSet(cset) + p.reserved = cset } if p.reserved.IsEmpty() { diff --git a/cmd/plugins/topology-aware/policy/topology.go b/cmd/plugins/topology-aware/policy/topology.go index f9b082d57..965157641 100644 --- a/cmd/plugins/topology-aware/policy/topology.go +++ b/cmd/plugins/topology-aware/policy/topology.go @@ -23,9 +23,9 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" ) -// toCpuSet and toCpuMask convert between the set the hardware package speaks and -// the one this policy is written in. They are the seam left by moving the policy -// onto hardware without rewriting its pool arithmetic. +// toCpuSet and toCpuMask convert between the CPU sets this policy keeps and the +// ones some of the interfaces it calls still take: the CPU class controller, the +// IRQ affinity helpers, topology hints and libmem. func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { return cpuset.New(cpus.List()...) } @@ -55,8 +55,8 @@ func packageZone(m *hardware.Machine, pkg idset.ID) *hardware.Zone { } // packageCPUs returns the CPUs of one package. -func packageCPUs(m *hardware.Machine, pkg idset.ID) cpuset.CPUSet { - return toCpuSet(m.TopologyIndex().PackageCPUs(pkg)) +func packageCPUs(m *hardware.Machine, pkg idset.ID) *libcpu.CpuMask { + return m.TopologyIndex().PackageCPUs(pkg) } // packageNodeIDs returns the NUMA nodes whose CPUs are in one package. @@ -74,11 +74,11 @@ func dieIDs(m *hardware.Machine, pkg idset.ID) []idset.ID { } // dieCPUs returns the CPUs of one die of one package. -func dieCPUs(m *hardware.Machine, pkg, die idset.ID) cpuset.CPUSet { - return toCpuSet(m.TopologyIndex().DieCPUs(hardware.DieID{ +func dieCPUs(m *hardware.Machine, pkg, die idset.ID) *libcpu.CpuMask { + return m.TopologyIndex().DieCPUs(hardware.DieID{ Package: pkg, Die: die, - })) + }) } // dieNodeIDs returns the NUMA nodes whose CPUs are on one die of one package. @@ -115,13 +115,13 @@ func l3CacheIDs(m *hardware.Machine, pkg idset.ID) []idset.ID { // l3CacheCPUs returns every CPU sharing one level 3 cache of this package, // including any outside the package: a cache shared across packages belongs to // both, and the whole of its CPU set is what it groups. -func l3CacheCPUs(m *hardware.Machine, pkg, cache idset.ID) cpuset.CPUSet { +func l3CacheCPUs(m *hardware.Machine, pkg, cache idset.ID) *libcpu.CpuMask { for _, z := range l3CacheZones(m, pkg) { if z.ID() == cache { - return toCpuSet(z.CPUs()) + return z.CPUs() } } - return cpuset.New() + return libcpu.NewCpuMask() } // l3CacheZones returns the level 3 cache zones this package's CPUs use. @@ -225,18 +225,18 @@ func sortedIDs(ids []idset.ID) []idset.ID { // of those nodes, as a cpuset string. An unparsable list yields nothing. func nodeHintToCPUs(m *hardware.Machine) func(string) string { return func(nodes string) string { - mems, err := cpuset.Parse(nodes) + mems, err := cpuset.Parse(nodes) // NUMA nodes, not CPUs if err != nil { return "" } - cpus := cpuset.New() + cpus := libcpu.NewCpuMask() for _, id := range mems.List() { if node := m.MemoryNode(id); node.Valid() { - cpus = cpus.Union(toCpuSet(node.CPUs())) + cpus = cpus.Union(node.CPUs()) } } - return cpus.Intersection(toCpuSet(m.OnlineCPUs())).String() + return cpus.Intersection(m.OnlineCPUs()).String() } } diff --git a/pkg/kubernetes/cpuset.go b/pkg/kubernetes/cpuset.go index ba373267b..255e10aa0 100644 --- a/pkg/kubernetes/cpuset.go +++ b/pkg/kubernetes/cpuset.go @@ -18,67 +18,74 @@ import ( "strconv" "strings" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) -// ShortCPUSet prints the cpuset as a string, trying to further shorten compared to .String(). -func ShortCPUSet(cset cpuset.CPUSet) string { - str, sep := "", "" +type segment struct { + beg, end, step int +} + +func (s segment) String() string { + if s.beg < 0 { + return "" + } + if s.end < 0 { + return strconv.FormatInt(int64(s.beg), 10) + } + if s.step == 1 { + return strconv.FormatInt(int64(s.beg), 10) + "-" + strconv.FormatInt(int64(s.end), 10) + } + return strconv.FormatInt(int64(s.beg), 10) + "-" + strconv.FormatInt(int64(s.end), 10) + ":" + strconv.FormatInt(int64(s.step), 10) +} - beg, end, step := -1, -1, -1 - for cpu := range strings.SplitSeq(cset.String(), ",") { - if strings.Contains(cpu, "-") { - str += sep + cpu - sep = "," +// ShortCPUSet prints the cpuset as a string, trying to further shorten compared to .String(). +func ShortCPUSet(cset libcpu.CPUSet) string { + segments := []segment{{beg: -1}} + for _, part := range strings.Split(cset.String(), ",") { + if part == "" { continue } - i, err := strconv.ParseInt(cpu, 10, 0) - if err != nil { - return cset.String() + curr := len(segments) - 1 + if strings.Contains(part, "-") { + parts := strings.SplitN(part, "-", 2) + beg, _ := strconv.Atoi(parts[0]) + end, _ := strconv.Atoi(parts[1]) + seg := segment{beg: beg, end: end, step: 1} + if segments[curr].beg < 0 { + segments[curr] = seg + } else { + segments = append(segments, seg) + } + segments = append(segments, segment{beg: -1}) + continue } - id := int(i) - if beg < 0 { - beg, end = id, id + cpu, _ := strconv.Atoi(part) + if segments[curr].beg < 0 { + segments[curr] = segment{beg: cpu, end: -1} continue } - if step < 0 { - end = id - step = end - beg + if segments[curr].end < 0 { + segments[curr].end = cpu + segments[curr].step = segments[curr].end - segments[curr].beg continue } - if id-end == step { - end = id + if cpu-segments[curr].end == segments[curr].step { + segments[curr].end = cpu continue } - str += sep + mkRange(beg, end, step) - sep = "," - beg, end = id, id - step = -1 - } - - if beg >= 0 { - str += sep + mkRange(beg, end, step) - } - - return str -} - -func mkRange(beg, end, step int) string { - if beg < 0 { - return "" - } - if beg == end { - return strconv.FormatInt(int64(beg), 10) + segments = append(segments, segment{beg: cpu, end: -1}) } - b, e := strconv.FormatInt(int64(beg), 10), strconv.FormatInt(int64(end), 10) - if step == 1 { - return b + "-" + e - } - if beg+step == end { - return b + "," + e + str := strings.Builder{} + sep := "" + for _, seg := range segments { + part := seg.String() + if part == "" { + continue + } + str.WriteString(sep) + str.WriteString(part) + sep = "," } - - s := strconv.FormatInt(int64(step), 10) - return b + "-" + e + ":" + s + return str.String() } diff --git a/pkg/kubernetes/cpuset_test.go b/pkg/kubernetes/cpuset_test.go index 6b6fcf6ab..b5d3c31d2 100644 --- a/pkg/kubernetes/cpuset_test.go +++ b/pkg/kubernetes/cpuset_test.go @@ -17,7 +17,7 @@ package kubernetes import ( "testing" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) func TestShortCPUSet(t *testing.T) { @@ -28,25 +28,26 @@ func TestShortCPUSet(t *testing.T) { }{ {source: "", native: "", short: ""}, {source: "1", native: "1", short: "1"}, - {source: "1,2", native: "1-2", short: "1,2"}, + {source: "1,2", native: "1-2", short: "1-2"}, + {source: "1-2", native: "1-2", short: "1-2"}, {source: "1,2,3,4,5,6,7", native: "1-7", short: "1-7"}, {source: "1,3,5,7,9,11", native: "1,3,5,7,9,11", short: "1-11:2"}, - {source: "1,3,5,7,8,10,12,14,16", native: "1,3,5,7-8,10,12,14,16", short: "1-7:2,10-16:2"}, + {source: "1,3,5,7,8,10,12,14,16", native: "1,3,5,7-8,10,12,14,16", short: "1-5:2,7-8,10-16:2"}, { - source: "0,2,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110", - native: "0,2,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110", - short: "0-110:2", + source: "0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110", + native: "0,2,4,6,8,10,12,14,16,18,20,22,24,26,28,30,32,34,36,38,40,42,44,46,48,50,52,54,56,58,64,66,68,70,72,74,76,78,80,82,84,86,88,90,92,94,96,98,100,102,104,106,108,110", + short: "0-58:2,64-110:2", }, } for _, tc := range tcases { - cset := cpuset.MustParse(tc.source) + cset := libcpu.MustParseCpuMask(tc.source) native := cset.String() if native != tc.native { t.Errorf("incorrect native CPUSet for %q, expected %q, got %q", tc.source, tc.native, native) } short := ShortCPUSet(cset) - if native != tc.native { + if short != tc.short { t.Errorf("incorrect shortened CPUSet for %q, expected %q, got %q", tc.source, tc.short, short) } diff --git a/pkg/utils/topology/hints.go b/pkg/utils/topology/hints.go index 30816d9e0..9a8bb99fd 100644 --- a/pkg/utils/topology/hints.go +++ b/pkg/utils/topology/hints.go @@ -36,29 +36,33 @@ func NewHint(m *hardware.Machine, h TopologyHint) *Hint { } } -func (h *Hint) CPUSetForCPUs() cpuset.CPUSet { - cset, _ := cpuset.Parse(h.hint.CPUs) - return cset +func (h *Hint) CPUSetForCPUs() *libcpu.CpuMask { + cpus, err := libcpu.ParseCpuMask(h.hint.CPUs) + if err != nil { + return libcpu.NewCpuMask() + } + return cpus } func (h *Hint) MemsForCPUs() libmem.NodeMask { mems := libmem.NewNodeMask() - cset, _ := cpuset.Parse(h.hint.CPUs) + cpus := h.CPUSetForCPUs() for _, id := range h.machine.MemoryNodeIDs() { - if h.machine.MemoryNode(id).CPUs().Intersects(toCpuMask(cset)) { + if h.machine.MemoryNode(id).CPUs().Intersects(cpus) { mems.Set(id) } } return mems } -func (h *Hint) CPUSetForNUMAs() cpuset.CPUSet { - cset := cpuset.New() +func (h *Hint) CPUSetForNUMAs() *libcpu.CpuMask { + cpus := libcpu.NewCpuMask() + // a NUMA node list, in the same syntax a CPU list uses mems, _ := cpuset.Parse(h.hint.NUMAs) for _, id := range mems.UnsortedList() { - cset = cset.Union(toCpuSet(h.machine.MemoryNode(id).CPUs())) + cpus = cpus.Union(h.machine.MemoryNode(id).CPUs()) } - return cset + return cpus } func (h *Hint) MemsForNUMAs() libmem.NodeMask { @@ -66,8 +70,8 @@ func (h *Hint) MemsForNUMAs() libmem.NodeMask { return mems } -func (h *Hint) MisalignedCPUSet(cpus cpuset.CPUSet) cpuset.CPUSet { - misaligned := cpuset.New() +func (h *Hint) MisalignedCPUSet(cpus *libcpu.CpuMask) *libcpu.CpuMask { + misaligned := libcpu.NewCpuMask() if aligned := h.CPUSetForCPUs(); !aligned.IsEmpty() { misaligned = misaligned.Union(cpus.Difference(aligned)) } @@ -87,13 +91,3 @@ func (h *Hint) MisalignedMems(mems libmem.NodeMask) libmem.NodeMask { } return misaligned } - -// toCpuSet and toCpuMask convert between the set the hardware package speaks and -// the one these hints are expressed in. -func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { - return cpuset.New(cpus.List()...) -} - -func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { - return libcpu.NewCpuMask(cpus.List()...) -} From a5ea003d4c9020399a3bd56aaae522bb5908d702 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 11 Sep 2026 00:46:12 +0300 Subject: [PATCH 34/39] irq: speak libcpu.CpuMask. An interrupt's affinity went in and out of this package as a k8s cpuset set, so both policies converted at the call and back at the return, having nothing but masks on either side. Nothing about what reaches procfs changes: the cache still writes the set's String() to smp_affinity_list and parses what it reads back, and a mask spells itself the same way. The affinity maps keep telling "no entry" from "empty set" with the two-value form, which is what they always did. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 5 ++-- .../topology-aware/policy/irq-affinity.go | 5 ++-- pkg/irq/irq-cache.go | 28 +++++++++---------- pkg/irq/irq-cache_test.go | 16 +++++------ pkg/irq/irq.go | 6 ++-- pkg/irq/irq_test.go | 8 +++--- 6 files changed, 33 insertions(+), 35 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 314c825f7..056fffca6 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -1029,11 +1029,10 @@ func (p *balloons) applyIrqAffinities() { } for _, hwIrq := range hwIrqs { newCpus := p.allowed - affinity, err := hwIrq.AffinityCpus() + curCpus, err := hwIrq.AffinityCpus() if err != nil { continue } - curCpus := toCpuMask(affinity) switch claimCpus := p.irqClaimCpus(hwIrq); { case !claimCpus.IsEmpty(): newCpus = claimCpus @@ -1050,7 +1049,7 @@ func (p *balloons) applyIrqAffinities() { if curCpus.Equals(newCpus) { continue } - if err := hwIrq.SetAffinityCpus(toCpuSet(newCpus)); err != nil { + if err := hwIrq.SetAffinityCpus(newCpus); err != nil { log.Debugf("failed to set affinity of %s to %q: %v", hwIrq, newCpus, err) } else { log.Debugf("set affinity of %s to %q", hwIrq, newCpus) diff --git a/cmd/plugins/topology-aware/policy/irq-affinity.go b/cmd/plugins/topology-aware/policy/irq-affinity.go index 623b3d71b..8496212d9 100644 --- a/cmd/plugins/topology-aware/policy/irq-affinity.go +++ b/cmd/plugins/topology-aware/policy/irq-affinity.go @@ -156,12 +156,11 @@ func (p *policy) applyIrqAffinity(user string) { } for _, hwIrq := range hwIrqs { - affinity, err := hwIrq.AffinityCpus() + current, err := hwIrq.AffinityCpus() if err != nil { log.Errorf("%s: failed to read affinity: %v", hwIrq.String(), err) continue } - current := toCpuMask(affinity) preMask, claim, mask := p.irqCpus(hwIrq) @@ -183,7 +182,7 @@ func (p *policy) applyIrqAffinity(user string) { continue } - if err := hwIrq.SetAffinityCpus(toCpuSet(cpus)); err != nil { + if err := hwIrq.SetAffinityCpus(cpus); err != nil { log.Errorf("%s: failed to set affinity to cpus %s (for %s): %v", hwIrq.String(), cpus.String(), user, err) } diff --git a/pkg/irq/irq-cache.go b/pkg/irq/irq-cache.go index 05cd25310..5a6baeaa2 100644 --- a/pkg/irq/irq-cache.go +++ b/pkg/irq/irq-cache.go @@ -35,7 +35,7 @@ import ( "sync" "syscall" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) // irqInfo is a numbered interrupt as parsed from the interrupts file, @@ -58,10 +58,10 @@ type irqCache struct { proc string // procfs mountpoint, all paths are built from this ops fileOps // file operations to use - info map[int]irqInfo // cached numbered interrupts, nil if not read - cpus map[int]cpuset.CPUSet // affinities read from or set through the cache - pending map[int]cpuset.CPUSet // affinities not yet written to procfs, empty if unblocked - readOnly map[int]bool // interrupts whose affinity cannot be written + info map[int]irqInfo // cached numbered interrupts, nil if not read + cpus map[int]*libcpu.CpuMask // affinities read from or set through the cache + pending map[int]*libcpu.CpuMask // affinities not yet written to procfs, empty if unblocked + readOnly map[int]bool // interrupts whose affinity cannot be written writeBlock int // writes to affinities are only buffered while positive } @@ -78,8 +78,8 @@ func newIrqCache() *irqCache { readFile: os.ReadFile, writeFile: os.WriteFile, }, - cpus: map[int]cpuset.CPUSet{}, - pending: map[int]cpuset.CPUSet{}, + cpus: map[int]*libcpu.CpuMask{}, + pending: map[int]*libcpu.CpuMask{}, readOnly: map[int]bool{}, } } @@ -220,7 +220,7 @@ func (c *irqCache) irqByNum(num int, allow []string) (*Irq, error) { // The affinity is read from procfs only until it is known, and // affinities set through the cache are visible before they have been // written to procfs. -func (c *irqCache) affinityOf(num int) (cpuset.CPUSet, error) { +func (c *irqCache) affinityOf(num int) (*libcpu.CpuMask, error) { c.mu.Lock() defer c.mu.Unlock() @@ -230,11 +230,11 @@ func (c *irqCache) affinityOf(num int) (cpuset.CPUSet, error) { data, err := c.ops.readFile(smpAffinityListPath(c.proc, num)) if err != nil { - return cpuset.New(), fmt.Errorf("failed to read affinity of irq %d: %w", num, err) + return libcpu.NewCpuMask(), fmt.Errorf("failed to read affinity of irq %d: %w", num, err) } - cpus, err := cpuset.Parse(strings.TrimSpace(string(data))) + cpus, err := libcpu.ParseCpuMask(strings.TrimSpace(string(data))) if err != nil { - return cpuset.New(), fmt.Errorf("failed to parse affinity of irq %d: %w", num, err) + return libcpu.NewCpuMask(), fmt.Errorf("failed to parse affinity of irq %d: %w", num, err) } c.cpus[num] = cpus @@ -244,7 +244,7 @@ func (c *irqCache) affinityOf(num int) (cpuset.CPUSet, error) { // setAffinity sets the CPUs in the affinity of the given interrupt. // While writes are blocked, the affinity is only buffered and no error // is returned. Otherwise it is written to procfs immediately. -func (c *irqCache) setAffinity(num int, cpus cpuset.CPUSet) error { +func (c *irqCache) setAffinity(num int, cpus *libcpu.CpuMask) error { c.mu.Lock() if c.readOnly[num] { // Unwritable, do not even try again. @@ -347,7 +347,7 @@ func (c *irqCache) reset(procDir string) { defer c.mu.Unlock() c.proc = procDir c.info = nil - c.cpus = map[int]cpuset.CPUSet{} - c.pending = map[int]cpuset.CPUSet{} + c.cpus = map[int]*libcpu.CpuMask{} + c.pending = map[int]*libcpu.CpuMask{} c.readOnly = map[int]bool{} } diff --git a/pkg/irq/irq-cache_test.go b/pkg/irq/irq-cache_test.go index b22a220dc..07badf4ac 100644 --- a/pkg/irq/irq-cache_test.go +++ b/pkg/irq/irq-cache_test.go @@ -23,7 +23,7 @@ import ( "syscall" "testing" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) // fakeProc is a fake procfs which counts reads and writes. @@ -170,7 +170,7 @@ func affinityPath(num int) string { // setAffinity sets the affinity of the given interrupt. func setAffinity(t *testing.T, num int, cpus string) { t.Helper() - if err := (&Irq{num: num}).SetAffinityCpus(cpuset.MustParse(cpus)); err != nil { + if err := (&Irq{num: num}).SetAffinityCpus(libcpu.MustParseCpuMask(cpus)); err != nil { t.Fatalf("SetAffinityCpus(%q) on irq %d failed: %v", cpus, num, err) } } @@ -372,7 +372,7 @@ func TestUnblockedWriteIsImmediate(t *testing.T) { // Write errors are returned when writes are not blocked. fake.failWrites(errors.New("test write error")) - if err := (&Irq{num: 42}).SetAffinityCpus(cpuset.MustParse("2")); err == nil { + if err := (&Irq{num: 42}).SetAffinityCpus(libcpu.MustParseCpuMask("2")); err == nil { t.Errorf("SetAffinityCpus() succeeded despite a failing write") } } @@ -425,7 +425,7 @@ func TestFailedWriteForgottenAndRetried(t *testing.T) { if got := getAffinity(t, 42); got != "0-3" { t.Fatalf("AffinityCpus() = %q, want 0-3", got) } - if err := (&Irq{num: 42}).SetAffinityCpus(cpuset.MustParse("1,3")); err == nil { + if err := (&Irq{num: 42}).SetAffinityCpus(libcpu.MustParseCpuMask("1,3")); err == nil { t.Errorf("SetAffinityCpus() succeeded despite a failing write") } if got := fake.writeCount(path); got != 1 { @@ -584,7 +584,7 @@ func TestCallbackReentrancy(t *testing.T) { if err != nil { return err } - return irq.SetAffinityCpus(cpus.Union(cpuset.MustParse("4"))) + return irq.SetAffinityCpus(cpus.Union(libcpu.MustParseCpuMask("4"))) }) if err != nil { t.Fatalf("ForEachInterrupt() failed: %v", err) @@ -602,11 +602,11 @@ func TestAffinityErrorsNeverWrite(t *testing.T) { fake := setupCache(t) fake.set(affinityPath(42), "0-3") - if err := (&Irq{num: 42}).SetAffinityCpus(cpuset.New()); err == nil { + if err := (&Irq{num: 42}).SetAffinityCpus(libcpu.NewCpuMask()); err == nil { t.Errorf("SetAffinityCpus(empty) should fail") } - err := (&Irq{num: 42, denied: true}).SetAffinityCpus(cpuset.MustParse("1")) + err := (&Irq{num: 42, denied: true}).SetAffinityCpus(libcpu.MustParseCpuMask("1")) if !errors.Is(err, ErrDeniedInterrupt) { t.Errorf("SetAffinityCpus() on a denied irq: %v, want %v", err, ErrDeniedInterrupt) } @@ -707,7 +707,7 @@ func TestReadWriteAmplification(t *testing.T) { } // Every pass sets a different affinity, just like a policy // which resizes its CPU pools while allocating containers. - newCpus := cpuset.MustParse(fmt.Sprintf("%d", pass%4)) + newCpus := libcpu.MustParseCpuMask(fmt.Sprintf("%d", pass%4)) for _, irq := range irqs { if _, err := irq.AffinityCpus(); err != nil { t.Fatalf("pass %d: AffinityCpus() failed: %v", pass, err) diff --git a/pkg/irq/irq.go b/pkg/irq/irq.go index 786cfb3f0..485dcecb7 100644 --- a/pkg/irq/irq.go +++ b/pkg/irq/irq.go @@ -30,8 +30,8 @@ import ( "path/filepath" "strconv" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" logger "github.com/containers/nri-plugins/pkg/log" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) var ( @@ -287,14 +287,14 @@ func (irq *Irq) IsAllowed() bool { // AffinityCpus returns the CPUs in the affinity of the interrupt. The // returned CPUs are the ones set last through this package, even if // they have not reached procfs yet. -func (irq *Irq) AffinityCpus() (cpuset.CPUSet, error) { +func (irq *Irq) AffinityCpus() (*libcpu.CpuMask, error) { return cache.affinityOf(irq.num) } // SetAffinityCpus sets the CPUs in the affinity of the interrupt. // While writes are blocked, the affinity is only buffered and write // errors are logged instead of being returned. -func (irq *Irq) SetAffinityCpus(cpus cpuset.CPUSet) error { +func (irq *Irq) SetAffinityCpus(cpus *libcpu.CpuMask) error { if !irq.IsAllowed() { return fmt.Errorf("%w: refusing to set affinity of irq %d", ErrDeniedInterrupt, irq.num) } diff --git a/pkg/irq/irq_test.go b/pkg/irq/irq_test.go index 27eb41fb9..a1871e4cd 100644 --- a/pkg/irq/irq_test.go +++ b/pkg/irq/irq_test.go @@ -20,7 +20,7 @@ import ( "path/filepath" "testing" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) const sampleInterrupts = ` CPU0 CPU1 CPU2 CPU3 @@ -156,11 +156,11 @@ func TestAffinityReadWrite(t *testing.T) { if err != nil { t.Fatalf("AffinityCpus() failed: %v", err) } - if !cpus.Equals(cpuset.MustParse("0-3")) { + if !cpus.Equals(libcpu.MustParseCpuMask("0-3")) { t.Errorf("AffinityCpus() = %q, want 0-3", cpus) } - if err := irq.SetAffinityCpus(cpuset.MustParse("1,3")); err != nil { + if err := irq.SetAffinityCpus(libcpu.MustParseCpuMask("1,3")); err != nil { t.Fatalf("SetAffinityCpus() failed: %v", err) } data, err := os.ReadFile(filepath.Join(irqDir, "smp_affinity_list")) @@ -171,7 +171,7 @@ func TestAffinityReadWrite(t *testing.T) { t.Errorf("written affinity = %q, want 1,3", string(data)) } - if err := irq.SetAffinityCpus(cpuset.New()); err == nil { + if err := irq.SetAffinityCpus(libcpu.NewCpuMask()); err == nil { t.Errorf("SetAffinityCpus(empty) should fail") } } From 4f48ce235dc9166c7bbcf031b59201babc48828e Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 11 Sep 2026 00:50:49 +0300 Subject: [PATCH 35/39] cpuclass: speak libcpu.CpuMask. The CPU class controller took k8s cpuset sets throughout: the allowed set it is configured with, the CPUs a class is applied to, the allocation intent it scores hints against, and the candidate sets those hints hand back. Both policies had masks on their side of every one of those calls. Its own state and its inward edges go the same way. The sets pct keeps per package, the ones cpufreq keeps per domain, and the CPUs it reads out of goresctrl's SST types are all masks now, so nothing is converted anywhere inside either. That empties the seam. What is left of it in the policies is libmem's CPUSetAffinity, and the configuration, which parses an operator's cpuset string; balloons keeps one converter for each and topology-aware one. The k8s type is still used to parse a list of NUMA nodes in two places, which is what it is, and not a set of CPUs. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 33 ++-- cmd/plugins/balloons/policy/cpuclass_test.go | 21 ++- .../topology-aware/policy/cpu-class.go | 4 +- cmd/plugins/topology-aware/policy/pools.go | 10 +- .../topology-aware/policy/resources.go | 5 +- .../policy/topology-aware-policy.go | 5 +- cmd/plugins/topology-aware/policy/topology.go | 9 +- pkg/resmgr/cpuclass/cpuclass.go | 14 +- pkg/resmgr/cpuclass/handler_commit_test.go | 4 +- .../cpuclass/internal/cpufreq/cpufreq.go | 24 +-- pkg/resmgr/cpuclass/internal/pct/pct.go | 54 +++---- pkg/resmgr/cpuclass/internal/pct/pct_sst.go | 4 +- .../internal/pct/pct_sst_goresctrl.go | 4 +- .../cpuclass/internal/pct/pct_sst_mock.go | 6 +- pkg/resmgr/cpuclass/internal/pct/pct_test.go | 148 +++++++++--------- pkg/resmgr/cpuclass/internal/types/types.go | 14 +- 16 files changed, 171 insertions(+), 188 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 056fffca6..5cf49395e 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -148,9 +148,8 @@ type loadClassVirtDev struct { } // toCpuSet and toCpuMask convert between the CPU sets this policy keeps and the -// ones some of the interfaces it calls still take: the CPU class controller, -// libmem and the IRQ affinity helpers. Everything the policy does with CPUs in -// between is done with libcpu masks. +// two interfaces which still take the k8s ones: libmem's CPUSetAffinity, and the +// configuration, which parses an operator's cpuset string. func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { return cpuset.New(cpus.List()...) } @@ -159,14 +158,6 @@ func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { return libcpu.NewCpuMask(cpus.List()...) } -func toCpuMasks(sets []cpuset.CPUSet) []*libcpu.CpuMask { - masks := make([]*libcpu.CpuMask, 0, len(sets)) - for _, cpus := range sets { - masks = append(masks, toCpuMask(cpus)) - } - return masks -} - var log logger.Logger = logger.NewLogger("policy") // String is a stringer for a balloon. @@ -656,7 +647,7 @@ func (p *balloons) GetExtendedResources() map[string]*resource.Quantity { } held = held.Union(bln.Cpus) } - free := max(p.cpuClasses.PctFreeClassCapacity(cc.Name, toCpuSet(held)), 0) + free := max(p.cpuClasses.PctFreeClassCapacity(cc.Name, held), 0) out["cpuclass.balloons.nri.io/"+cc.Name] = resource.NewQuantity(int64(free), resource.DecimalSI) } return out @@ -907,7 +898,7 @@ func (p *balloons) resetCpuClass() error { return nil } idle := p.resolveCpuClassName(p.bpoptions.IdleCpuClass) - if err := p.cpuClasses.UseClass(idle, toCpuSet(p.allowed)); err != nil { + if err := p.cpuClasses.UseClass(idle, p.allowed); err != nil { log.Warnf("failed to reset class of available cpus: %v", err) } else { log.Debugf("reset class of available cpus: %q to idle class %q (reserved: %q)", @@ -951,7 +942,7 @@ func (p *balloons) useCpuClass(bln *Balloon) error { } cpuClass := p.resolveCpuClassName(bln.Def.CpuClass) log.Debugf("apply CPU class %q on CPUs %q of %q", cpuClass, bln.Cpus, bln.PrettyName()) - if err := p.cpuClasses.UseClass(cpuClass, toCpuSet(bln.Cpus)); err != nil { + if err := p.cpuClasses.UseClass(cpuClass, bln.Cpus); err != nil { log.Warnf("failed to apply class %q on CPUs %q: %v", cpuClass, bln.Cpus, err) } return nil @@ -965,7 +956,7 @@ func (p *balloons) forgetCpuClass(bln *Balloon) { return } idle := p.resolveCpuClassName(p.bpoptions.IdleCpuClass) - if err := p.cpuClasses.UseClass(idle, toCpuSet(bln.Cpus)); err != nil { + if err := p.cpuClasses.UseClass(idle, bln.Cpus); err != nil { log.Warnf("failed to forget class of cpus %q (idle class %q): %v", bln.Cpus, idle, err) } else { if len(bln.components) > 0 { @@ -1685,7 +1676,7 @@ func (p *balloons) Reconfigure(newCfg any) error { if err := p.cpuClasses.Configure(cpuclass.ConfigSpec{ Classes: p.bpoptions.CPUClasses, TurboDomain: p.bpoptions.TurboDomain, - Allowed: toCpuSet(p.allowed), + Allowed: p.allowed, }); err != nil { log.Warnf("failed to reconfigure CPU class handler: %v", err) } @@ -1973,7 +1964,7 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { if err := p.cpuClasses.Configure(cpuclass.ConfigSpec{ Classes: bpoptions.CPUClasses, TurboDomain: bpoptions.TurboDomain, - Allowed: toCpuSet(p.allowed), + Allowed: p.allowed, }); err != nil { return balloonsError("failed to configure CPU class handler: %w", err) } @@ -2199,8 +2190,8 @@ func (p *balloons) applyCpuClassHints(opts *cpuTreeAllocatorOptions, cpuClass st } mergeCpuClassHints(opts, p.cpuClasses, cpuclass.AllocationIntent{ ClassName: cpuClass, - CurrentCpus: toCpuSet(currentCpus), - FreeCpus: toCpuSet(p.freeCpus), + CurrentCpus: currentCpus, + FreeCpus: p.freeCpus, RequestedCount: requestedCount, }) } @@ -2233,13 +2224,13 @@ func mergeCpuClassHints(opts *cpuTreeAllocatorOptions, provider cpuClassHints, i hints := provider.Hints(intent) for i, pref := range hints.Prefer { name := fmt.Sprintf("%spref_%d_%s", cpuClassHintDevPrefix, i, pref.Name) - opts.virtDevCpusets[name] = toCpuMasks(pref.Cpus) + opts.virtDevCpusets[name] = pref.Cpus opts.preferCloseToDevices = append(opts.preferCloseToDevices, name) log.Debugf("cpuclass hint: prefer %q -> %v", name, pref.Cpus) } for i, av := range hints.Avoid { name := fmt.Sprintf("%savoid_%d_%s", cpuClassHintDevPrefix, i, av.Name) - opts.virtDevCpusets[name] = toCpuMasks(av.Cpus) + opts.virtDevCpusets[name] = av.Cpus opts.preferFarFromDevices = append(opts.preferFarFromDevices, name) log.Debugf("cpuclass hint: avoid %q -> %v", name, av.Cpus) } diff --git a/cmd/plugins/balloons/policy/cpuclass_test.go b/cmd/plugins/balloons/policy/cpuclass_test.go index f9ac2c542..a17baf2a7 100644 --- a/cmd/plugins/balloons/policy/cpuclass_test.go +++ b/cmd/plugins/balloons/policy/cpuclass_test.go @@ -20,7 +20,6 @@ import ( "testing" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // fakeHintProvider returns a scripted sequence of cpuclass.AllocationHints, @@ -66,32 +65,32 @@ func countHintMapKeys(m map[string][]*libcpu.CpuMask) int { // the hint count reported by the provider on that round, regardless // of how many earlier rounds added different hints. func TestMergeCpuClassHintsNoAccumulation(t *testing.T) { - cpusA := cpuset.MustParse("2-3") - cpusB := cpuset.MustParse("4-5") - cpusC := cpuset.MustParse("6-7") - cpusAvoid := cpuset.MustParse("0-1") + cpusA := libcpu.MustParseCpuMask("2-3") + cpusB := libcpu.MustParseCpuMask("4-5") + cpusC := libcpu.MustParseCpuMask("6-7") + cpusAvoid := libcpu.MustParseCpuMask("0-1") provider := &fakeHintProvider{ script: []cpuclass.AllocationHints{ // Round 1: one prefer (A), one avoid. { - Prefer: []cpuclass.CpuPreference{{Name: "hp-reserve", Cpus: []cpuset.CPUSet{cpusA}}}, - Avoid: []cpuclass.CpuPreference{{Name: "lp-clos", Cpus: []cpuset.CPUSet{cpusAvoid}}}, + Prefer: []cpuclass.CpuPreference{{Name: "hp-reserve", Cpus: []*libcpu.CpuMask{cpusA}}}, + Avoid: []cpuclass.CpuPreference{{Name: "lp-clos", Cpus: []*libcpu.CpuMask{cpusAvoid}}}, }, // Round 2: two prefers (A, B) - different name at index 1 // so the slot-0 name stays stable, slot-1 is new. { Prefer: []cpuclass.CpuPreference{ - {Name: "hp-reserve", Cpus: []cpuset.CPUSet{cpusA}}, - {Name: "extra", Cpus: []cpuset.CPUSet{cpusB}}, + {Name: "hp-reserve", Cpus: []*libcpu.CpuMask{cpusA}}, + {Name: "extra", Cpus: []*libcpu.CpuMask{cpusB}}, }, - Avoid: []cpuclass.CpuPreference{{Name: "lp-clos", Cpus: []cpuset.CPUSet{cpusAvoid}}}, + Avoid: []cpuclass.CpuPreference{{Name: "lp-clos", Cpus: []*libcpu.CpuMask{cpusAvoid}}}, }, // Round 3: name at slot 0 CHANGES to C - without proper // cleanup the stale "__cls_pref_0_hp-reserve" map key from // rounds 1+2 would survive into round 3. { - Prefer: []cpuclass.CpuPreference{{Name: "third", Cpus: []cpuset.CPUSet{cpusC}}}, + Prefer: []cpuclass.CpuPreference{{Name: "third", Cpus: []*libcpu.CpuMask{cpusC}}}, Avoid: nil, }, }, diff --git a/cmd/plugins/topology-aware/policy/cpu-class.go b/cmd/plugins/topology-aware/policy/cpu-class.go index 2d95c5443..71859a1e1 100644 --- a/cmd/plugins/topology-aware/policy/cpu-class.go +++ b/cmd/plugins/topology-aware/policy/cpu-class.go @@ -33,7 +33,7 @@ func (p *policy) setReservedPoolCpuClass() { if opt.ReservedPoolCpuClass == "" { return } - if err := p.cpuClasses.UseClass(opt.ReservedPoolCpuClass, toCpuSet(p.reserved)); err != nil { + if err := p.cpuClasses.UseClass(opt.ReservedPoolCpuClass, p.reserved); err != nil { log.Errorf("failed to set reserved pool CPU class for %s: %v", p.reserved, err) } } @@ -66,7 +66,7 @@ func (p *policy) resetCpuClass(subject string, cpus *libcpu.CpuMask) { if opt.SharedPoolCpuClass == "" { return } - if err := p.cpuClasses.UseClass(opt.SharedPoolCpuClass, toCpuSet(cpus)); err != nil { + if err := p.cpuClasses.UseClass(opt.SharedPoolCpuClass, cpus); err != nil { log.Errorf("%s: failed to reset CPU class for %s: %v", subject, cpus, err) } } diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index a92300cd8..f3b95f90d 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -639,7 +639,7 @@ func (p *policy) applyGrant(grant Grant) { container.SetCPUShares(int64(cache.MilliCPUToShares(int64(milliCPU)))) if exclusive.Size() > 0 && grant.CPUClass() != "" { - if err := p.cpuClasses.UseClass(grant.CPUClass(), toCpuSet(exclusive)); err != nil { + if err := p.cpuClasses.UseClass(grant.CPUClass(), exclusive); err != nil { log.Errorf("%s: failed to apply CPU class to cpuset %s: %v", container.PrettyName(), exclusive, err) } @@ -1152,8 +1152,8 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco for _, h := range score1.CpuClassHints().Prefer { for _, hinted := range h.Cpus { - if offer1.Intersection(toCpuMask(hinted)).Equals(offer1) { - hcpus1 = toCpuMask(hinted) + if offer1.Intersection(hinted).Equals(offer1) { + hcpus1 = hinted break } } @@ -1163,8 +1163,8 @@ func (p *policy) compareScores(request Request, pools []Node, scores map[int]Sco } for _, h := range score2.CpuClassHints().Prefer { for _, hinted := range h.Cpus { - if offer2.Intersection(toCpuMask(hinted)).Equals(offer2) { - hcpus2 = toCpuMask(hinted) + if offer2.Intersection(hinted).Equals(offer2) { + hcpus2 = hinted break } } diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 3c5a6c0f9..45c9e528c 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -26,7 +26,6 @@ import ( libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/topology" - "github.com/containers/nri-plugins/pkg/utils/cpuset" "github.com/containers/nri-plugins/pkg/cpuallocator" "github.com/containers/nri-plugins/pkg/kubernetes" @@ -1198,8 +1197,8 @@ func (cs *supply) GetScore(req Request) Score { score.cpu = cpus hints := p.cpuClasses.Hints(cpuclass.AllocationIntent{ ClassName: cr.cpuClass, - CurrentCpus: cpuset.New(), - FreeCpus: toCpuSet(cpus), + CurrentCpus: libcpu.NewCpuMask(), + FreeCpus: cpus, RequestedCount: cr.full, }) score.ccHints = &hints diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index 0c178a930..60b07e6b9 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -22,7 +22,6 @@ import ( "github.com/containers/nri-plugins/pkg/irq" libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/utils/cpuset" "k8s.io/apimachinery/pkg/api/resource" "k8s.io/apimachinery/pkg/types" @@ -646,7 +645,7 @@ func (p *policy) GetExtendedResources() map[string]*resource.Quantity { log.Warnf("ignoring publishExtendedResource on non-PCT cpuClass %q", cc.Name) continue } - free := max(p.cpuClasses.PctFreeClassCapacity(cc.Name, cpuset.New()), 0) + free := max(p.cpuClasses.PctFreeClassCapacity(cc.Name, libcpu.NewCpuMask()), 0) out[CpuClassResourceDomain+"/"+cc.Name] = resource.NewQuantity(int64(free), resource.DecimalSI) } return out @@ -827,7 +826,7 @@ func (p *policy) initialize() error { if err := p.cpuClasses.Configure(cpuclass.ConfigSpec{ Classes: opt.CPUClasses, TurboDomain: "package", - Allowed: toCpuSet(p.allowed), + Allowed: p.allowed, }); err != nil { return policyError("failed to configure CPU class handler: %w", err) } diff --git a/cmd/plugins/topology-aware/policy/topology.go b/cmd/plugins/topology-aware/policy/topology.go index 965157641..949877965 100644 --- a/cmd/plugins/topology-aware/policy/topology.go +++ b/cmd/plugins/topology-aware/policy/topology.go @@ -23,13 +23,8 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" ) -// toCpuSet and toCpuMask convert between the CPU sets this policy keeps and the -// ones some of the interfaces it calls still take: the CPU class controller, the -// IRQ affinity helpers, topology hints and libmem. -func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { - return cpuset.New(cpus.List()...) -} - +// toCpuMask converts a set the configuration parsed out of an operator's cpuset +// string into the ones this policy keeps. It is the last of the seam. func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { return libcpu.NewCpuMask(cpus.List()...) } diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index a0f2d2c91..a123cea5a 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -30,6 +30,7 @@ import ( "sort" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpufreq" @@ -37,7 +38,6 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/uncorefreq" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) var log = logger.NewLogger("cpuclass") @@ -72,7 +72,7 @@ type ConfigSpec struct { TurboDomain string // Allowed bounds every cpuclass operation. CPUs outside this // set are silently dropped by Configure, UseClass and Hints. - Allowed cpuset.CPUSet + Allowed *libcpu.CpuMask } // Handler is the sole cpuclass entry point for policy code. It owns @@ -80,7 +80,7 @@ type ConfigSpec struct { // (cpufreq, pct) and writers (cpufreq, cpuidle, uncorefreq). type Handler struct { machine *hardware.Machine - allowed cpuset.CPUSet + allowed *libcpu.CpuMask cpufreq *cpufreq.Allocator pct *pct.Allocator @@ -138,7 +138,7 @@ func New(m *hardware.Machine) (*Handler, error) { // node, given that 'held' lists CPUs already consumed by some // balloon belonging to any other cpuClass. Returns 0 if PCT is // inactive or the class has no PCT plan. -func (h *Handler) PctFreeClassCapacity(className string, held cpuset.CPUSet) int { +func (h *Handler) PctFreeClassCapacity(className string, held *libcpu.CpuMask) int { if h == nil || h.pct == nil { return 0 } @@ -339,7 +339,7 @@ func (h *Handler) Commit() error { // UseClass applies className to the given CPUs across every internal // allocator. An empty className means "no class". CPUs outside the // configured Allowed set are silently dropped. -func (h *Handler) UseClass(className string, cpus cpuset.CPUSet) error { +func (h *Handler) UseClass(className string, cpus *libcpu.CpuMask) error { if err := h.cpufreq.UseClass(className, cpus); err != nil { log.Warnf("cpuclass: cpufreq failed to apply class %q on CPUs %s: %v", className, cpus, err) } @@ -373,12 +373,12 @@ func (h *Handler) Shutdown() error { // candidate set constrained to the given bound. Empty candidate sets // are dropped; a preference is dropped only when all of its candidate // sets become empty. Candidate order is preserved. -func intersectHints(hints AllocationHints, bound cpuset.CPUSet) AllocationHints { +func intersectHints(hints AllocationHints, bound *libcpu.CpuMask) AllocationHints { out := AllocationHints{} clip := func(prefs []CpuPreference) []CpuPreference { var res []CpuPreference for _, p := range prefs { - sets := make([]cpuset.CPUSet, 0, len(p.Cpus)) + sets := make([]*libcpu.CpuMask, 0, len(p.Cpus)) for _, c := range p.Cpus { s := c.Intersection(bound) if s.IsEmpty() { diff --git a/pkg/resmgr/cpuclass/handler_commit_test.go b/pkg/resmgr/cpuclass/handler_commit_test.go index 51632a4ff..2af50b846 100644 --- a/pkg/resmgr/cpuclass/handler_commit_test.go +++ b/pkg/resmgr/cpuclass/handler_commit_test.go @@ -21,12 +21,12 @@ import ( "testing" "testing/fstest" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpufreq" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/cpuidle" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/uncorefreq" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // dieFakeCpu specifies the (pkg, die) location of a single CPU when building a @@ -52,7 +52,7 @@ func newDieMachine(t *testing.T, cpus map[int]dieFakeCpu) *hardware.Machine { } sort.Ints(ids) - all := cpuset.New(ids...).String() + all := libcpu.NewCpuMask(ids...).String() fsys := fstest.MapFS{ "proc/meminfo": file("MemTotal: 1048576 kB\n"), "sys/devices/system/cpu/online": file(all + "\n"), diff --git a/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go b/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go index a3849208c..526a9dec8 100644 --- a/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go +++ b/pkg/resmgr/cpuclass/internal/cpufreq/cpufreq.go @@ -24,10 +24,10 @@ import ( "slices" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) var log = logger.NewLogger("cpuclass") @@ -49,14 +49,14 @@ type Allocator struct { classByName map[string]*policyapi.CPUClass turboDomain string turboInfo *platformTurboInfo - allowed cpuset.CPUSet + allowed *libcpu.CpuMask cpuDomain map[int]domainID domains []domainID // activeCpus[d][className] is the set of CPUs in turbo domain d // currently assigned to className. - activeCpus map[domainID]map[string]cpuset.CPUSet + activeCpus map[domainID]map[string]*libcpu.CpuMask // winnerPrio[d] is the highest TurboPriority among classes that // had any active CPUs in domain d the last time @@ -87,7 +87,7 @@ func New(m *hardware.Machine, sink Sink) (*Allocator, error) { a := &Allocator{ machine: m, sink: sink, - activeCpus: map[domainID]map[string]cpuset.CPUSet{}, + activeCpus: map[domainID]map[string]*libcpu.CpuMask{}, winnerPrio: map[domainID]int{}, } a.discoverPlatformInfo() @@ -97,7 +97,7 @@ func New(m *hardware.Machine, sink Sink) (*Allocator, error) { // Configure replaces the CPU class set, turbo domain mode and the // set of allowed CPUs. Resets per-domain turbo winners and // re-publishes class definitions to the sink. -func (a *Allocator) Configure(classes []*policyapi.CPUClass, turboDomain string, allowed cpuset.CPUSet) error { +func (a *Allocator) Configure(classes []*policyapi.CPUClass, turboDomain string, allowed *libcpu.CpuMask) error { a.classes = classes a.classByName = make(map[string]*policyapi.CPUClass, len(classes)) for _, cc := range classes { @@ -112,7 +112,7 @@ func (a *Allocator) Configure(classes []*policyapi.CPUClass, turboDomain string, } a.allowed = allowed a.buildCpuDomains() - a.activeCpus = map[domainID]map[string]cpuset.CPUSet{} + a.activeCpus = map[domainID]map[string]*libcpu.CpuMask{} a.winnerPrio = map[domainID]int{} a.pushInitialClassDefinitions() return nil @@ -142,7 +142,7 @@ func (a *Allocator) resolveClassName(name string) string { // recalculates the turbo winner of every affected turbo domain, then // publishes per-CPU assignments to the sink. CPUs outside the // configured Allowed set are silently dropped. -func (a *Allocator) UseClass(className string, cpus cpuset.CPUSet) error { +func (a *Allocator) UseClass(className string, cpus *libcpu.CpuMask) error { if a.allowed.Size() > 0 { cpus = cpus.Intersection(a.allowed) } @@ -155,7 +155,7 @@ func (a *Allocator) UseClass(className string, cpus cpuset.CPUSet) error { if className != "" { for d, dc := range byDomain { if a.activeCpus[d] == nil { - a.activeCpus[d] = map[string]cpuset.CPUSet{} + a.activeCpus[d] = map[string]*libcpu.CpuMask{} } a.activeCpus[d][className] = a.activeCpus[d][className].Union(dc) } @@ -172,7 +172,7 @@ func (a *Allocator) UseClass(className string, cpus cpuset.CPUSet) error { // removeCpusFromAllClasses removes the given CPUs from every active // class set, in every turbo domain. -func (a *Allocator) removeCpusFromAllClasses(cpus cpuset.CPUSet) { +func (a *Allocator) removeCpusFromAllClasses(cpus *libcpu.CpuMask) { for d, perClass := range a.activeCpus { for name, set := range perClass { newSet := set.Difference(cpus) @@ -188,14 +188,14 @@ func (a *Allocator) removeCpusFromAllClasses(cpus cpuset.CPUSet) { } } -func (a *Allocator) cpusByDomain(cpus cpuset.CPUSet) map[domainID]cpuset.CPUSet { - out := map[domainID]cpuset.CPUSet{} +func (a *Allocator) cpusByDomain(cpus *libcpu.CpuMask) map[domainID]*libcpu.CpuMask { + out := map[domainID]*libcpu.CpuMask{} for _, cpu := range cpus.UnsortedList() { d, ok := a.cpuDomain[cpu] if !ok { d = systemDomainID } - out[d] = out[d].Union(cpuset.New(cpu)) + out[d] = out[d].Union(libcpu.NewCpuMask(cpu)) } return out } diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index ed796bbf7..71136fb29 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -21,10 +21,10 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" logger "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) var log = logger.NewLogger("cpuclass") @@ -76,7 +76,7 @@ type Allocator struct { // so we use it here too. This is a hardware-level concept, // not a user-visible "idle". fallbackClos int - allowed cpuset.CPUSet + allowed *libcpu.CpuMask // hpClasses holds the names of cpuClasses currently // classified as high priority. In managed mode this is every // class with pctPriority=high. In assoc-only mode it is @@ -96,11 +96,11 @@ type Allocator struct { punitByCpu map[int]int // hpUsed[i] is the set of CPUs currently held by HP-class // workloads on punits[i] via the non-DRA (hint-driven) path. - hpUsed map[int]cpuset.CPUSet + hpUsed map[int]*libcpu.CpuMask // hpDRAUsed[i] is the set of CPUs currently held by DRA claims // on punits[i]. Separate from hpUsed so that clearHpUsage (called // from the non-DRA UseClass path) can never evict DRA holds. - hpDRAUsed map[int]cpuset.CPUSet + hpDRAUsed map[int]*libcpu.CpuMask // hpEligiblePunit[i] reports whether punits[i] can actually // host HP-class CPUs at top turbo. Populated at Configure(). // In managed mode every punit becomes eligible (the plugin @@ -133,15 +133,15 @@ func NewAllocator(sys Sys) (*Allocator, error) { // // - classes: cpuClass definitions to inspect for PCT fields. // - allowed: CPUs the allocator may configure. -func (a *Allocator) Configure(classes []*policyapi.CPUClass, allowed cpuset.CPUSet) error { +func (a *Allocator) Configure(classes []*policyapi.CPUClass, allowed *libcpu.CpuMask) error { a.classByName = make(map[string]*policyapi.CPUClass, len(classes)) for _, cc := range classes { a.classByName[cc.Name] = cc } a.fallbackClos = pctDefaultHpClos // CLOS 0 == default-after-reset a.allowed = allowed - a.hpUsed = map[int]cpuset.CPUSet{} - a.hpDRAUsed = map[int]cpuset.CPUSet{} + a.hpUsed = map[int]*libcpu.CpuMask{} + a.hpDRAUsed = map[int]*libcpu.CpuMask{} a.hpClasses = map[string]bool{} a.hpEligiblePunit = map[int]bool{} a.punits = nil @@ -489,7 +489,7 @@ func (a *Allocator) Active() bool { // // Returns 0 for classes that have no PCT plan or when PCT is not // active. Negative intermediate counts are clamped to 0. -func (a *Allocator) FreeClassCapacity(className string, held cpuset.CPUSet) int { +func (a *Allocator) FreeClassCapacity(className string, held *libcpu.CpuMask) int { if !a.Active() { return 0 } @@ -752,7 +752,7 @@ func (a *Allocator) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error // associated to the fallback CLOS. In assoc-only mode such CPUs are // left unchanged. CPUs outside the configured Allowed set are silently // dropped. -func (a *Allocator) UseClass(className string, cpus cpuset.CPUSet) error { +func (a *Allocator) UseClass(className string, cpus *libcpu.CpuMask) error { if !a.Active() { return nil } @@ -779,7 +779,7 @@ func (a *Allocator) UseClass(className string, cpus cpuset.CPUSet) error { // (e.g. outside Allowed at Configure time) are ignored: they // cannot affect HP placement and tracking them would only confuse // hpInUseCpus. -func (a *Allocator) trackHpUsage(className string, cpus cpuset.CPUSet) { +func (a *Allocator) trackHpUsage(className string, cpus *libcpu.CpuMask) { if !a.hpHintsActive() { return } @@ -796,7 +796,7 @@ func (a *Allocator) trackHpUsage(className string, cpus cpuset.CPUSet) { perPunit[idx] = append(perPunit[idx], cpu) } for idx, list := range perPunit { - set := a.hpUsed[idx].Union(cpuset.New(list...)) + set := a.hpUsed[idx].Union(libcpu.NewCpuMask(list...)) if dra := a.hpDRAUsed[idx]; !dra.IsEmpty() { set = set.Difference(dra) } @@ -805,7 +805,7 @@ func (a *Allocator) trackHpUsage(className string, cpus cpuset.CPUSet) { } // clearHpUsage removes cpus from per-punit HP bookkeeping. -func (a *Allocator) clearHpUsage(cpus cpuset.CPUSet) { +func (a *Allocator) clearHpUsage(cpus *libcpu.CpuMask) { if !a.hpHintsActive() { return } @@ -816,7 +816,7 @@ func (a *Allocator) clearHpUsage(cpus cpuset.CPUSet) { } } -func (a *Allocator) associate(cpus cpuset.CPUSet, clos int) error { +func (a *Allocator) associate(cpus *libcpu.CpuMask, clos int) error { list := cpus.UnsortedList() sort.Ints(list) assocs := make([]pctClosAssoc, 0, len(list)) @@ -885,9 +885,9 @@ func (a *Allocator) hpHintsActive() bool { // closCpus returns the subset of Allowed CPUs that are currently // associated to CLOS closID. -func (a *Allocator) closCpus(closID int) cpuset.CPUSet { +func (a *Allocator) closCpus(closID int) *libcpu.CpuMask { if !a.Active() { - return cpuset.New() + return libcpu.NewCpuMask() } out := []int{} for _, cpu := range a.allowed.UnsortedList() { @@ -899,7 +899,7 @@ func (a *Allocator) closCpus(closID int) cpuset.CPUSet { out = append(out, cpu) } } - return cpuset.New(out...) + return libcpu.NewCpuMask(out...) } // hpInUseCpus returns the union of CPUs of every punit currently @@ -907,11 +907,11 @@ func (a *Allocator) closCpus(closID int) cpuset.CPUSet { // HP usage to whole-punit (rather than whole-package) granularity // keeps the Avoid hint for non-HP classes from being unnecessarily // broad on TPMI-class platforms with multiple punits per package. -func (a *Allocator) hpInUseCpus() cpuset.CPUSet { +func (a *Allocator) hpInUseCpus() *libcpu.CpuMask { if !a.hpHintsActive() { - return cpuset.New() + return libcpu.NewCpuMask() } - out := cpuset.New() + out := libcpu.NewCpuMask() // Range over punits rather than hpUsed so that DRA-only punits // (present in hpDRAUsed but absent from hpUsed) are not skipped. for idx := range a.punits { @@ -966,7 +966,7 @@ func (a *Allocator) hpInUseCpus() cpuset.CPUSet { // - requested: number of CPUs the upcoming allocation wants. // 0 means "unknown" (initial priming before the count is // known); Tier A is used. -func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, requested int) []cpuset.CPUSet { +func (a *Allocator) hpReserveCpus(free *libcpu.CpuMask, excludeBln *libcpu.CpuMask, requested int) []*libcpu.CpuMask { if !a.hpHintsActive() { return nil } @@ -978,7 +978,7 @@ func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, } type punitState struct { - free cpuset.CPUSet + free *libcpu.CpuMask room int } states := make([]punitState, len(a.punits)) @@ -1036,7 +1036,7 @@ func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, } return a.punits[ix].PunitID < a.punits[iy].PunitID }) - reserve := make([]cpuset.CPUSet, 0, len(tierA)) + reserve := make([]*libcpu.CpuMask, 0, len(tierA)) for _, i := range tierA { reserve = append(reserve, states[i].free) log.Debugf("pct: hpReserveCpus tier=A punit=%d/%d room=%d free=%s", @@ -1051,7 +1051,7 @@ func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, if requested > 0 { type pkgAgg struct { room int - free cpuset.CPUSet + free *libcpu.CpuMask freeN int } agg := map[int]*pkgAgg{} @@ -1061,7 +1061,7 @@ func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, } e, ok := agg[pu.PkgID] if !ok { - e = &pkgAgg{free: cpuset.New()} + e = &pkgAgg{free: libcpu.NewCpuMask()} agg[pu.PkgID] = e } e.room += states[i].room @@ -1086,7 +1086,7 @@ func (a *Allocator) hpReserveCpus(free cpuset.CPUSet, excludeBln cpuset.CPUSet, return pkgIDs[x] < pkgIDs[y] }) if len(pkgIDs) > 0 { - reserve := make([]cpuset.CPUSet, 0, len(pkgIDs)) + reserve := make([]*libcpu.CpuMask, 0, len(pkgIDs)) for _, id := range pkgIDs { reserve = append(reserve, agg[id].free) log.Debugf("pct: hpReserveCpus tier=B pkg=%d room=%d free=%s", @@ -1159,7 +1159,7 @@ func (a *Allocator) Hints(intent types.AllocationIntent) types.AllocationHints { if freeClosCpus.Size() >= intent.RequestedCount { out.Prefer = append(out.Prefer, types.CpuPreference{ Name: virtDevSstClosHint(closID), - Cpus: []cpuset.CPUSet{freeClosCpus}, + Cpus: []*libcpu.CpuMask{freeClosCpus}, }) } } @@ -1180,7 +1180,7 @@ func (a *Allocator) Hints(intent types.AllocationIntent) types.AllocationHints { if !inUse.IsEmpty() { out.Avoid = append(out.Avoid, types.CpuPreference{ Name: virtDevSstHpInUseHint, - Cpus: []cpuset.CPUSet{inUse}, + Cpus: []*libcpu.CpuMask{inUse}, }) } } diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_sst.go b/pkg/resmgr/cpuclass/internal/pct/pct_sst.go index 30ed1a97b..a5fc08f4d 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_sst.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_sst.go @@ -17,7 +17,7 @@ package pct import ( "os" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) // pctClosConfig describes one CLOS configuration that the @@ -45,7 +45,7 @@ type pctClosAssoc struct { type pctPunit struct { PkgID int PunitID int - CPUs cpuset.CPUSet + CPUs *libcpu.CpuMask MaxHpCpus int // GuaranteedHpCpus is the count of HP CPUs on this punit that // can simultaneously sustain the highest turbo frequency the diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_sst_goresctrl.go b/pkg/resmgr/cpuclass/internal/pct/pct_sst_goresctrl.go index 03adebf58..a468a9cdc 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_sst_goresctrl.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_sst_goresctrl.go @@ -21,7 +21,7 @@ import ( gosst "github.com/intel/goresctrl/pkg/sst" "github.com/intel/goresctrl/pkg/utils" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) // sstGoresctrl is the real-hardware sst backed by @@ -100,7 +100,7 @@ func discoverPunits(plat *gosst.Platform) []pctPunit { sort.Ints(punitIDs) for _, pid := range punitIDs { pu := st.Punits[utils.ID(pid)] - cpus := cpuset.New(pu.CPUs.Members()...) + cpus := libcpu.NewCpuMask(pu.CPUs.Members()...) max := 0 gtd := 0 if pi, ok := info[utils.ID(pid)]; ok { diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_sst_mock.go b/pkg/resmgr/cpuclass/internal/pct/pct_sst_mock.go index 0d8bee0e9..096b7e4eb 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_sst_mock.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_sst_mock.go @@ -22,7 +22,7 @@ import ( "sort" "strings" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) // sstOverrideEnvVar holds JSON seeding the in-memory SST mock. @@ -260,7 +260,7 @@ func (b *sstMock) Punits() []pctPunit { out = append(out, pctPunit{ PkgID: pkg.ID, PunitID: 0, - CPUs: cpuset.New(cpus...), + CPUs: libcpu.NewCpuMask(cpus...), MaxHpCpus: pkg.MaxHpCpus, GuaranteedHpCpus: pkg.MaxHpCpus, }) @@ -277,7 +277,7 @@ func (b *sstMock) Punits() []pctPunit { out = append(out, pctPunit{ PkgID: pkg.ID, PunitID: pu.ID, - CPUs: cpuset.New(cpus...), + CPUs: libcpu.NewCpuMask(cpus...), MaxHpCpus: pu.MaxHpCpus, GuaranteedHpCpus: gtd, }) diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index 1a88c8594..4e59ff6a0 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -23,9 +23,9 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/types" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) var errFakeSstNoClos = errors.New("fakeSst: no CLOS for CPU") @@ -49,7 +49,7 @@ type fakeSst struct { supported bool cpuClos map[int]int // cpu -> CLOS id maxHp map[int]int // pkgID -> max HP CPUs (missing = "unknown") - pkgCpus map[int]cpuset.CPUSet + pkgCpus map[int]*libcpu.CpuMask // punits, when non-nil, overrides the synthesized one-punit-per-package // Punits() output. Use to exercise multi-punit-per-package layouts. punits []pctPunit @@ -98,9 +98,9 @@ func (s *fakeSst) Punits() []pctPunit { // Derive a default cpu range matching newTwoPackageFakeSys layout. switch id { case 0: - cpus = cpuset.MustParse("0-3") + cpus = libcpu.MustParseCpuMask("0-3") case 1: - cpus = cpuset.MustParse("4-7") + cpus = libcpu.MustParseCpuMask("4-7") } } out = append(out, pctPunit{ @@ -140,7 +140,7 @@ func (s *fakeSst) TFStatus() (map[pctPunitID]bool, error) { // --- helpers to construct a hand-wired Allocator ----------------- func newManagedPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans map[string]*pctClassPlan, - allowed cpuset.CPUSet, sys *fakeSys, sst *fakeSst) *Allocator { + allowed *libcpu.CpuMask, sys *fakeSys, sst *fakeSst) *Allocator { t.Helper() a := &Allocator{ sys: sys, @@ -149,8 +149,8 @@ func newManagedPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans map classByName: map[string]*policyapi.CPUClass{}, classPlan: plans, allowed: allowed, - hpUsed: map[int]cpuset.CPUSet{}, - hpDRAUsed: map[int]cpuset.CPUSet{}, + hpUsed: map[int]*libcpu.CpuMask{}, + hpDRAUsed: map[int]*libcpu.CpuMask{}, hpClasses: map[string]bool{}, } for _, cc := range classes { @@ -231,7 +231,7 @@ func TestPctHintsNoClassNoOp(t *testing.T) { // "anyHighPriorityClassDefined" gate must be false so no Avoid. a2 := newManagedPctForTest(t, classes, map[string]*pctClassPlan{"lp": {ClosID: 3}}, - cpuset.MustParse("0-7"), sys, sst) + libcpu.MustParseCpuMask("0-7"), sys, sst) got = a2.Hints(types.AllocationIntent{ClassName: "unknown-class", RequestedCount: 1}) if len(got.Avoid) != 0 { t.Errorf("no HP class: Avoid=%+v, want empty", got.Avoid) @@ -254,15 +254,15 @@ func TestPctHintsAssocOnlyPreferClosCpus(t *testing.T) { mode: pctModeAssocOnly, classByName: map[string]*policyapi.CPUClass{"c1": {Name: "c1"}}, classPlan: map[string]*pctClassPlan{"c1": {ClosID: 1}}, - allowed: cpuset.MustParse("0-7"), - hpUsed: map[int]cpuset.CPUSet{}, + allowed: libcpu.MustParseCpuMask("0-7"), + hpUsed: map[int]*libcpu.CpuMask{}, } pctTestWirePunits(a) got := a.Hints(types.AllocationIntent{ ClassName: "c1", // cpu 1 is on CLOS 1 but already taken by someone // else, so it must not show up in the hint. - FreeCpus: cpuset.MustParse("2-7"), + FreeCpus: libcpu.MustParseCpuMask("2-7"), RequestedCount: 1, }) if len(got.Prefer) != 1 { @@ -271,7 +271,7 @@ func TestPctHintsAssocOnlyPreferClosCpus(t *testing.T) { if got.Prefer[0].Name != virtDevSstClosHint(1) { t.Errorf("Prefer[0].Name = %q, want %q", got.Prefer[0].Name, virtDevSstClosHint(1)) } - want := cpuset.MustParse("2-3") + want := libcpu.MustParseCpuMask("2-3") if !got.Prefer[0].Cpus[0].Equals(want) { t.Errorf("Prefer[0].Cpus = %v, want %s", got.Prefer[0].Cpus, want) } @@ -300,17 +300,17 @@ func TestPctHintsHighPriorityReserveAndClosCpus(t *testing.T) { "hp": {Name: "hp", PctPriority: "high"}, }, classPlan: map[string]*pctClassPlan{"hp": {ClosID: 0}}, - allowed: cpuset.MustParse("0-7"), + allowed: libcpu.MustParseCpuMask("0-7"), // pkg0 has 1 HP cpu already used (cpu 0). - hpUsed: map[int]cpuset.CPUSet{0: cpuset.MustParse("0")}, + hpUsed: map[int]*libcpu.CpuMask{0: libcpu.MustParseCpuMask("0")}, } pctTestWirePunits(a) // Free pool excludes the already-used cpu 0. - free := cpuset.MustParse("1-7") + free := libcpu.MustParseCpuMask("1-7") got := a.Hints(types.AllocationIntent{ ClassName: "hp", - CurrentCpus: cpuset.New(), + CurrentCpus: libcpu.NewCpuMask(), FreeCpus: free, RequestedCount: 1, }) @@ -325,7 +325,7 @@ func TestPctHintsHighPriorityReserveAndClosCpus(t *testing.T) { if got.Prefer[0].Name != virtDevSstClosHint(0) { t.Errorf("Prefer[0].Name = %q, want %q", got.Prefer[0].Name, virtDevSstClosHint(0)) } - wantClos := cpuset.MustParse("1") + wantClos := libcpu.MustParseCpuMask("1") if !got.Prefer[0].Cpus[0].Equals(wantClos) { t.Errorf("Prefer[0].Cpus = %v, want %s (cpu 0 on CLOS 0 but not free)", got.Prefer[0].Cpus, wantClos) @@ -333,7 +333,7 @@ func TestPctHintsHighPriorityReserveAndClosCpus(t *testing.T) { if got.Prefer[1].Name != virtDevSstHpReserveHint { t.Errorf("Prefer[1].Name = %q, want %q", got.Prefer[1].Name, virtDevSstHpReserveHint) } - wantReserve := cpuset.MustParse("4-7") + wantReserve := libcpu.MustParseCpuMask("4-7") if !got.Prefer[1].Cpus[0].Equals(wantReserve) { t.Errorf("HP reserve = %v, want %s (largest-room package)", got.Prefer[1].Cpus, wantReserve) } @@ -366,14 +366,14 @@ func TestPctHintsManagedNonHpAvoidsHpInUse(t *testing.T) { "hp": {ClosID: 0}, "lp": {ClosID: 3}, }, - allowed: cpuset.MustParse("0-7"), + allowed: libcpu.MustParseCpuMask("0-7"), // pkg0 hosts HP cpu 1. - hpUsed: map[int]cpuset.CPUSet{0: cpuset.MustParse("1")}, + hpUsed: map[int]*libcpu.CpuMask{0: libcpu.MustParseCpuMask("1")}, } pctTestWirePunits(a) got := a.Hints(types.AllocationIntent{ ClassName: "lp", - FreeCpus: cpuset.MustParse("2-7"), + FreeCpus: libcpu.MustParseCpuMask("2-7"), RequestedCount: 1, }) @@ -392,7 +392,7 @@ func TestPctHintsManagedNonHpAvoidsHpInUse(t *testing.T) { if got.Avoid[0].Name != virtDevSstHpInUseHint { t.Errorf("Avoid[0].Name = %q, want %q", got.Avoid[0].Name, virtDevSstHpInUseHint) } - wantAvoid := cpuset.MustParse("0-3") // entire pkg0 + wantAvoid := libcpu.MustParseCpuMask("0-3") // entire pkg0 if !got.Avoid[0].Cpus[0].Equals(wantAvoid) { t.Errorf("Avoid[0].Cpus = %v, want %s (pkg0 == HP-in-use package)", got.Avoid[0].Cpus, wantAvoid) } @@ -418,16 +418,16 @@ func TestPctHintsAllowedBoundsResults(t *testing.T) { }, classPlan: map[string]*pctClassPlan{"hp": {ClosID: 0}}, // allowed restricts to pkg0 only. - allowed: cpuset.MustParse("0-3"), - hpUsed: map[int]cpuset.CPUSet{ - 0: cpuset.MustParse("0"), - 1: cpuset.MustParse("4"), // outside allowed + allowed: libcpu.MustParseCpuMask("0-3"), + hpUsed: map[int]*libcpu.CpuMask{ + 0: libcpu.MustParseCpuMask("0"), + 1: libcpu.MustParseCpuMask("4"), // outside allowed }, } pctTestWirePunits(a) got := a.Hints(types.AllocationIntent{ ClassName: "hp", - FreeCpus: cpuset.MustParse("1-3"), + FreeCpus: libcpu.MustParseCpuMask("1-3"), RequestedCount: 1, }) // closCpus walks a.allowed, so cpu 4 is excluded automatically. @@ -435,13 +435,13 @@ func TestPctHintsAllowedBoundsResults(t *testing.T) { if len(got.Prefer) == 0 { t.Fatalf("Prefer empty, want at least closCpus hint") } - if !got.Prefer[0].Cpus[0].Equals(cpuset.MustParse("1")) { + if !got.Prefer[0].Cpus[0].Equals(libcpu.MustParseCpuMask("1")) { t.Errorf("Prefer[0].Cpus = %v, want {1} (cpu 4 outside allowed)", got.Prefer[0].Cpus) } // HP reserve must come from a package whose free CPUs are // inside allowed; only pkg0 qualifies. if len(got.Prefer) >= 2 { - want := cpuset.MustParse("1-3") + want := libcpu.MustParseCpuMask("1-3") if !got.Prefer[1].Cpus[0].Equals(want) { t.Errorf("HP reserve = %v, want %s (pkg0 free cpus inside allowed)", got.Prefer[1].Cpus, want) } @@ -454,10 +454,10 @@ func TestPctHintsAllowedBoundsResults(t *testing.T) { // newTwoPunitFakeSys, with the given MaxHpCpus per punit. func makeTwoPunitsPerPkg(hp0, hp1, hp2, hp3 int) []pctPunit { return []pctPunit{ - {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), MaxHpCpus: hp0}, - {PkgID: 0, PunitID: 1, CPUs: cpuset.MustParse("4-7"), MaxHpCpus: hp1}, - {PkgID: 1, PunitID: 2, CPUs: cpuset.MustParse("8-11"), MaxHpCpus: hp2}, - {PkgID: 1, PunitID: 3, CPUs: cpuset.MustParse("12-15"), MaxHpCpus: hp3}, + {PkgID: 0, PunitID: 0, CPUs: libcpu.MustParseCpuMask("0-3"), MaxHpCpus: hp0}, + {PkgID: 0, PunitID: 1, CPUs: libcpu.MustParseCpuMask("4-7"), MaxHpCpus: hp1}, + {PkgID: 1, PunitID: 2, CPUs: libcpu.MustParseCpuMask("8-11"), MaxHpCpus: hp2}, + {PkgID: 1, PunitID: 3, CPUs: libcpu.MustParseCpuMask("12-15"), MaxHpCpus: hp3}, } } @@ -476,20 +476,20 @@ func TestPctHints_HpRoomTierAPunitWins(t *testing.T) { mode: pctModeManaged, classByName: map[string]*policyapi.CPUClass{"hp": {Name: "hp", PctPriority: "high"}}, classPlan: map[string]*pctClassPlan{"hp": {ClosID: 0}}, - allowed: cpuset.MustParse("0-15"), + allowed: libcpu.MustParseCpuMask("0-15"), // Punit-0 fully booked with HP (cpus 0,1 take both HP slots). - hpUsed: map[int]cpuset.CPUSet{0: cpuset.MustParse("0-1")}, + hpUsed: map[int]*libcpu.CpuMask{0: libcpu.MustParseCpuMask("0-1")}, } pctTestWirePunits(a) got := a.Hints(types.AllocationIntent{ ClassName: "hp", - FreeCpus: cpuset.MustParse("2-15"), + FreeCpus: libcpu.MustParseCpuMask("2-15"), RequestedCount: 1, }) // Find HP reserve hint. - var reserve cpuset.CPUSet + var reserve *libcpu.CpuMask for _, p := range got.Prefer { if p.Name == virtDevSstHpReserveHint { reserve = p.Cpus[0] @@ -504,7 +504,7 @@ func TestPctHints_HpRoomTierAPunitWins(t *testing.T) { // Actually both punit-1 (room=2), punit-2 (room=2), punit-3 // (room=2) tie; tie-break by free-CPU count (all 4) and then // by iteration order (slice index 1 first). So expect punit-1. - want := cpuset.MustParse("4-7") + want := libcpu.MustParseCpuMask("4-7") if !reserve.Equals(want) { t.Errorf("Tier A HP reserve = %s, want %s (punit-1)", reserve, want) } @@ -526,21 +526,21 @@ func TestPctHints_HpRoomTierBSamePackage(t *testing.T) { mode: pctModeManaged, classByName: map[string]*policyapi.CPUClass{"hp": {Name: "hp", PctPriority: "high"}}, classPlan: map[string]*pctClassPlan{"hp": {ClosID: 0}}, - allowed: cpuset.MustParse("0-15"), + allowed: libcpu.MustParseCpuMask("0-15"), // Both pkg0 punits already host 1 HP CPU each, leaving room=1 in each. - hpUsed: map[int]cpuset.CPUSet{ - 0: cpuset.MustParse("0"), // punit-0 idx 0 - 1: cpuset.MustParse("4"), // punit-1 idx 1 + hpUsed: map[int]*libcpu.CpuMask{ + 0: libcpu.MustParseCpuMask("0"), // punit-0 idx 0 + 1: libcpu.MustParseCpuMask("4"), // punit-1 idx 1 }, } pctTestWirePunits(a) got := a.Hints(types.AllocationIntent{ ClassName: "hp", - FreeCpus: cpuset.MustParse("1-3,5-15"), + FreeCpus: libcpu.MustParseCpuMask("1-3,5-15"), RequestedCount: 2, }) - var reserve cpuset.CPUSet + var reserve *libcpu.CpuMask for _, p := range got.Prefer { if p.Name == virtDevSstHpReserveHint { reserve = p.Cpus[0] @@ -552,7 +552,7 @@ func TestPctHints_HpRoomTierBSamePackage(t *testing.T) { // Tier A is impossible (no single punit has room>=2 in pkg0, // and pkg1 punit-2 has 1 cpu only). Tier B: pkg0 sum-room=2 // >= 2, pkg1 sum-room=1 < 2. Reserve = pkg0 free CPUs. - want := cpuset.MustParse("1-3,5-7") + want := libcpu.MustParseCpuMask("1-3,5-7") if !reserve.Equals(want) { t.Errorf("Tier B HP reserve = %s, want %s (pkg0 union)", reserve, want) } @@ -575,14 +575,14 @@ func TestPctHints_HpRoomTierCNoCrossPackage(t *testing.T) { mode: pctModeManaged, classByName: map[string]*policyapi.CPUClass{"hp": {Name: "hp", PctPriority: "high"}}, classPlan: map[string]*pctClassPlan{"hp": {ClosID: 0}}, - allowed: cpuset.MustParse("0-15"), - hpUsed: map[int]cpuset.CPUSet{}, + allowed: libcpu.MustParseCpuMask("0-15"), + hpUsed: map[int]*libcpu.CpuMask{}, } pctTestWirePunits(a) got := a.Hints(types.AllocationIntent{ ClassName: "hp", - FreeCpus: cpuset.MustParse("0-15"), + FreeCpus: libcpu.MustParseCpuMask("0-15"), RequestedCount: 3, // > any single package's HP capacity (2) }) for _, p := range got.Prefer { @@ -614,22 +614,22 @@ func TestPctHints_HpInUseIsPunitGranular(t *testing.T) { "hp": {ClosID: 0}, "lp": {ClosID: 3}, }, - allowed: cpuset.MustParse("0-15"), + allowed: libcpu.MustParseCpuMask("0-15"), // HP work on punit-0 only (pkg0). - hpUsed: map[int]cpuset.CPUSet{0: cpuset.MustParse("0")}, + hpUsed: map[int]*libcpu.CpuMask{0: libcpu.MustParseCpuMask("0")}, } pctTestWirePunits(a) got := a.Hints(types.AllocationIntent{ ClassName: "lp", - FreeCpus: cpuset.MustParse("1-15"), + FreeCpus: libcpu.MustParseCpuMask("1-15"), RequestedCount: 1, }) if len(got.Avoid) != 1 { t.Fatalf("Avoid count = %d, want 1: got=%+v", len(got.Avoid), got.Avoid) } // Must be punit-0 (cpus 0-3) ONLY, not all of pkg0 (0-7). - want := cpuset.MustParse("0-3") + want := libcpu.MustParseCpuMask("0-3") if !got.Avoid[0].Cpus[0].Equals(want) { t.Errorf("Avoid = %v, want %s (punit-0 only, not full pkg0)", got.Avoid[0].Cpus, want) } @@ -830,7 +830,7 @@ func TestPctPunitGuaranteedHpCpus_NeitherSupported(t *testing.T) { // hpEligiblePunit must be set up by the caller after the helper // returns to keep the test intent explicit. func newAssocOnlyPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans map[string]*pctClassPlan, - allowed cpuset.CPUSet, sys *fakeSys, sst *fakeSst) *Allocator { + allowed *libcpu.CpuMask, sys *fakeSys, sst *fakeSst) *Allocator { t.Helper() a := &Allocator{ sys: sys, @@ -839,7 +839,7 @@ func newAssocOnlyPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans m classByName: map[string]*policyapi.CPUClass{}, classPlan: plans, allowed: allowed, - hpUsed: map[int]cpuset.CPUSet{}, + hpUsed: map[int]*libcpu.CpuMask{}, hpClasses: map[string]bool{}, hpEligiblePunit: map[int]bool{}, } @@ -869,8 +869,8 @@ func TestFreeClassCapacity_AssocOnlyHpFromFallbackCLOS(t *testing.T) { }, // Two punits (one per package); each guarantees 2 HP CPUs at top turbo. punits: []pctPunit{ - {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), GuaranteedHpCpus: 2}, - {PkgID: 1, PunitID: 0, CPUs: cpuset.MustParse("4-7"), GuaranteedHpCpus: 2}, + {PkgID: 0, PunitID: 0, CPUs: libcpu.MustParseCpuMask("0-3"), GuaranteedHpCpus: 2}, + {PkgID: 1, PunitID: 0, CPUs: libcpu.MustParseCpuMask("4-7"), GuaranteedHpCpus: 2}, }, } classes := []*policyapi.CPUClass{ @@ -879,11 +879,11 @@ func TestFreeClassCapacity_AssocOnlyHpFromFallbackCLOS(t *testing.T) { } a := newAssocOnlyPctForTest(t, classes, map[string]*pctClassPlan{"hp": {ClosID: 0}, "lp": {ClosID: 3}}, - cpuset.MustParse("0-7"), sys, sst) + libcpu.MustParseCpuMask("0-7"), sys, sst) a.hpClasses["hp"] = true // simulate classifyAssocOnlyHP result // Held by some non-HP balloon: 2 CPUs (one per punit). - held := cpuset.MustParse("3,7") + held := libcpu.MustParseCpuMask("3,7") gotHp := a.FreeClassCapacity("hp", held) wantHp := 2 + 2 // both punits: min(2, |{0,1,2}|=3)=2 and min(2, |{4,5,6}|=3)=2 @@ -909,20 +909,20 @@ func TestFreeClassCapacity_AssocOnlyHpTFDisabledPunitExcluded(t *testing.T) { sst := &fakeSst{ supported: true, punits: []pctPunit{ - {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), GuaranteedHpCpus: 2}, - {PkgID: 1, PunitID: 0, CPUs: cpuset.MustParse("4-7"), GuaranteedHpCpus: 2}, + {PkgID: 0, PunitID: 0, CPUs: libcpu.MustParseCpuMask("0-3"), GuaranteedHpCpus: 2}, + {PkgID: 1, PunitID: 0, CPUs: libcpu.MustParseCpuMask("4-7"), GuaranteedHpCpus: 2}, }, } a := newAssocOnlyPctForTest(t, []*policyapi.CPUClass{{Name: "hp"}}, map[string]*pctClassPlan{"hp": {ClosID: 0}}, - cpuset.MustParse("0-7"), sys, sst) + libcpu.MustParseCpuMask("0-7"), sys, sst) a.hpClasses["hp"] = true // pctTestWirePunits marked both eligible; flip pkg1 punit to // TF-disabled to model the assoc-only "operator did not enable // SST-TF on this punit" case. a.hpEligiblePunit[1] = false - got := a.FreeClassCapacity("hp", cpuset.New()) + got := a.FreeClassCapacity("hp", libcpu.NewCpuMask()) want := 2 // only pkg0 contributes if got != want { t.Errorf("HP capacity with one TF-disabled punit = %d, want %d", got, want) @@ -937,16 +937,16 @@ func TestFreeClassCapacity_AssocOnlyNoHpClassification(t *testing.T) { sst := &fakeSst{ supported: true, punits: []pctPunit{ - {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), GuaranteedHpCpus: 2}, - {PkgID: 1, PunitID: 0, CPUs: cpuset.MustParse("4-7"), GuaranteedHpCpus: 2}, + {PkgID: 0, PunitID: 0, CPUs: libcpu.MustParseCpuMask("0-3"), GuaranteedHpCpus: 2}, + {PkgID: 1, PunitID: 0, CPUs: libcpu.MustParseCpuMask("4-7"), GuaranteedHpCpus: 2}, }, } a := newAssocOnlyPctForTest(t, []*policyapi.CPUClass{{Name: "c1"}}, map[string]*pctClassPlan{"c1": {ClosID: 1}}, - cpuset.MustParse("0-7"), sys, sst) + libcpu.MustParseCpuMask("0-7"), sys, sst) // Intentionally no entries in a.hpClasses. - got := a.FreeClassCapacity("c1", cpuset.MustParse("1,5")) + got := a.FreeClassCapacity("c1", libcpu.MustParseCpuMask("1,5")) want := 8 - 2 if got != want { t.Errorf("non-HP assoc-only capacity = %d, want %d", got, want) @@ -962,8 +962,8 @@ func TestFreeClassCapacity_ManagedHpRespectsEligibility(t *testing.T) { sst := &fakeSst{ supported: true, punits: []pctPunit{ - {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), GuaranteedHpCpus: 2}, - {PkgID: 1, PunitID: 0, CPUs: cpuset.MustParse("4-7"), GuaranteedHpCpus: 2}, + {PkgID: 0, PunitID: 0, CPUs: libcpu.MustParseCpuMask("0-3"), GuaranteedHpCpus: 2}, + {PkgID: 1, PunitID: 0, CPUs: libcpu.MustParseCpuMask("4-7"), GuaranteedHpCpus: 2}, }, } classes := []*policyapi.CPUClass{ @@ -972,21 +972,21 @@ func TestFreeClassCapacity_ManagedHpRespectsEligibility(t *testing.T) { } a := newManagedPctForTest(t, classes, map[string]*pctClassPlan{"hp": {ClosID: 0}, "lp": {ClosID: 3}}, - cpuset.MustParse("0-7"), sys, sst) + libcpu.MustParseCpuMask("0-7"), sys, sst) - gotHp := a.FreeClassCapacity("hp", cpuset.MustParse("3")) + gotHp := a.FreeClassCapacity("hp", libcpu.MustParseCpuMask("3")) wantHp := 2 + 2 // pkg0: min(2, 3)=2; pkg1: min(2, 4)=2 if gotHp != wantHp { t.Errorf("managed HP capacity = %d, want %d", gotHp, wantHp) } - gotLp := a.FreeClassCapacity("lp", cpuset.MustParse("3")) + gotLp := a.FreeClassCapacity("lp", libcpu.MustParseCpuMask("3")) wantLp := 8 - 1 if gotLp != wantLp { t.Errorf("managed LP capacity = %d, want %d", gotLp, wantLp) } // Squeeze pkg0: hold 3 of its 4 CPUs => pkg0 contributes min(2,1)=1. - gotHp = a.FreeClassCapacity("hp", cpuset.MustParse("0-2")) + gotHp = a.FreeClassCapacity("hp", libcpu.MustParseCpuMask("0-2")) wantHp = 1 + 2 if gotHp != wantHp { t.Errorf("managed HP capacity with squeezed pkg0 = %d, want %d", gotHp, wantHp) @@ -1000,8 +1000,8 @@ func TestFreeClassCapacity_UnknownClassReturnsZero(t *testing.T) { sst := &fakeSst{supported: true} a := newManagedPctForTest(t, []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, map[string]*pctClassPlan{"hp": {ClosID: 0}}, - cpuset.MustParse("0-7"), sys, sst) - if got := a.FreeClassCapacity("nope", cpuset.New()); got != 0 { + libcpu.MustParseCpuMask("0-7"), sys, sst) + if got := a.FreeClassCapacity("nope", libcpu.NewCpuMask()); got != 0 { t.Errorf("unknown class capacity = %d, want 0", got) } } diff --git a/pkg/resmgr/cpuclass/internal/types/types.go b/pkg/resmgr/cpuclass/internal/types/types.go index a6e397165..06da663a5 100644 --- a/pkg/resmgr/cpuclass/internal/types/types.go +++ b/pkg/resmgr/cpuclass/internal/types/types.go @@ -20,7 +20,7 @@ package types import ( - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) // ClassDef is the resolved, platform-aware definition of a CPU class @@ -68,10 +68,10 @@ type AllocationIntent struct { // ClassName is the cpuClass the upcoming allocation will use. ClassName string // CurrentCpus are the CPUs the caller already owns. - CurrentCpus cpuset.CPUSet + CurrentCpus *libcpu.CpuMask // FreeCpus are the CPUs the allocation can pick from. Prefer // hints never contain CPUs outside this set. - FreeCpus cpuset.CPUSet + FreeCpus *libcpu.CpuMask // RequestedCount is the number of CPUs the allocation wants. // A negative count means releasing that many CPUs from // CurrentCpus. Zero is reserved and unspecified. @@ -85,7 +85,7 @@ type AllocationIntent struct { // the first candidate it can and ignore others. type CpuPreference struct { Name string - Cpus []cpuset.CPUSet + Cpus []*libcpu.CpuMask } // AllocationHints carries technology-agnostic placement preferences @@ -96,6 +96,6 @@ type AllocationHints struct { Avoid []CpuPreference } -// CPUSet aliases cpuset.CPUSet for callers that want to refer to it -// via this package without re-importing pkg/utils/cpuset. -type CPUSet = cpuset.CPUSet +// CPUSet aliases *libcpu.CpuMask for callers that want to refer to it +// via this package without importing pkg/lib/cpu. +type CPUSet = *libcpu.CpuMask From 0594773a3d0c042b588123d4b0f1377eb0e32370 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 11 Sep 2026 09:13:15 +0300 Subject: [PATCH 36/39] dra: speak libcpu.CpuMask. The DRA plugin, the CPU class controller's DRA paths and the policy-side adapter between them all took and returned k8s cpuset sets. Everything on either side of those calls is a mask now, so they take masks too. That is the plugin's interfaces in deps.go, its claim state and CDI writer, PickHpCpus, ReleaseHpCpus and AccountHpCpus down to the punit sets pct keeps for DRA holds, the adapter which routes between them, and the CPU sets the policy parses out of claim attributes and container cpusets. Nothing crosses the kubelet boundary as a set: a device is published by name and a claim identified by UID, so what the plugin says to the outside world does not change. This is here rather than in the DRA work itself because the type it converts to arrives with the commits below. Should the two be reordered, this is the piece which moves. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- cmd/plugins/topology-aware/policy/dra.go | 4 +-- .../topology-aware/policy/dra_adapter.go | 8 ++--- cmd/plugins/topology-aware/policy/pools.go | 36 +++++++++---------- .../topology-aware/policy/resources.go | 8 ++--- .../policy/topology-aware-policy.go | 12 +++---- pkg/resmgr/cpuclass/cpuclass.go | 8 ++--- pkg/resmgr/cpuclass/internal/pct/pct.go | 16 ++++----- pkg/resmgr/dra/deps.go | 12 +++---- pkg/resmgr/dra/plugin.go | 18 +++++----- pkg/resmgr/dra/state.go | 2 +- 10 files changed, 62 insertions(+), 62 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/dra.go b/cmd/plugins/topology-aware/policy/dra.go index 23d960106..03daef91a 100644 --- a/cmd/plugins/topology-aware/policy/dra.go +++ b/cmd/plugins/topology-aware/policy/dra.go @@ -15,11 +15,11 @@ package topologyaware import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" "github.com/containers/nri-plugins/pkg/resmgr/dra" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // buildDRAPlugin constructs p.draPlugin from the current policy @@ -81,7 +81,7 @@ func (p *policy) buildDRAPlugin(opts *policyapi.BackendOptions) error { // entire package while leaf pools are NUMA or L3 nodes) is rejected // at Prepare time instead of being persisted and always failing // later when its container is created. - ValidateCPUsInPool: func(cpus cpuset.CPUSet) error { + ValidateCPUsInPool: func(cpus *libcpu.CpuMask) error { _, err := p.poolForCPUs(cpus) return err }, diff --git a/cmd/plugins/topology-aware/policy/dra_adapter.go b/cmd/plugins/topology-aware/policy/dra_adapter.go index d4ed7e639..c166b64f7 100644 --- a/cmd/plugins/topology-aware/policy/dra_adapter.go +++ b/cmd/plugins/topology-aware/policy/dra_adapter.go @@ -15,10 +15,10 @@ package topologyaware import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" resourceapi "k8s.io/api/resource/v1" "github.com/containers/nri-plugins/pkg/resmgr/dra" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // DRADriverName is the DRA driver name used to publish and identify CPU @@ -50,17 +50,17 @@ var ( ) // PickHpCpus routes to the current p.cpuClasses handler's PickHpCpus. -func (a *policyDRAAdapter) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { +func (a *policyDRAAdapter) PickHpCpus(pkgID, punitID, n int, held *libcpu.CpuMask) (*libcpu.CpuMask, error) { return a.p.cpuClasses.PickHpCpus(pkgID, punitID, n, held) } // ReleaseHpCpus routes to the current p.cpuClasses handler's ReleaseHpCpus. -func (a *policyDRAAdapter) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { +func (a *policyDRAAdapter) ReleaseHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) { a.p.cpuClasses.ReleaseHpCpus(pkgID, punitID, cpus) } // AccountHpCpus routes to the current p.cpuClasses handler's AccountHpCpus. -func (a *policyDRAAdapter) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { +func (a *policyDRAAdapter) AccountHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) error { return a.p.cpuClasses.AccountHpCpus(pkgID, punitID, cpus) } diff --git a/cmd/plugins/topology-aware/policy/pools.go b/cmd/plugins/topology-aware/policy/pools.go index f3b95f90d..5d6b1b407 100644 --- a/cmd/plugins/topology-aware/policy/pools.go +++ b/cmd/plugins/topology-aware/policy/pools.go @@ -591,7 +591,7 @@ func (p *policy) applyGrant(grant Grant) { if opt.PinCPU { if cpuType == cpuPreserve { if !claimed.IsEmpty() { - preserved, err := cpuset.Parse(container.GetCpusetCpus()) + preserved, err := libcpu.ParseCpuMask(container.GetCpusetCpus()) if err != nil { log.Errorf(" => failed to parse %s cpuset %q while adding DRA claim %s: %v", container.PrettyName(), container.GetCpusetCpus(), claimed, err) @@ -1448,12 +1448,12 @@ type claimLister interface { // // Allocs whose CPUs field fails to parse are logged and skipped; they // contribute to neither the returned union nor the per-class grouping. -func classifyClaimCPUs(uid types.UID, allocs []dra.ResultAlloc) (cpuset.CPUSet, map[string]cpuset.CPUSet) { - cpus := cpuset.New() - classCPUs := map[string]cpuset.CPUSet{} +func classifyClaimCPUs(uid types.UID, allocs []dra.ResultAlloc) (*libcpu.CpuMask, map[string]*libcpu.CpuMask) { + cpus := libcpu.NewCpuMask() + classCPUs := map[string]*libcpu.CpuMask{} for _, a := range allocs { - parsed, err := cpuset.Parse(a.CPUs) + parsed, err := libcpu.ParseCpuMask(a.CPUs) if err != nil { log.Warnf("dra: claim %s: failed to parse allocated CPUs %q: %v", uid, a.CPUs, err) continue @@ -1475,8 +1475,8 @@ func classifyClaimCPUs(uid types.UID, allocs []dra.ResultAlloc) (cpuset.CPUSet, // more than one class). type containerClaim struct { UID types.UID - CPUs cpuset.CPUSet - ClassCPUs map[string]cpuset.CPUSet + CPUs *libcpu.CpuMask + ClassCPUs map[string]*libcpu.CpuMask } // claimCPUsFromContainer looks for CDI device names on c that identify live @@ -1553,7 +1553,7 @@ func claimCPUsFromContainer(c cache.Container, plugin claimLister) []containerCl // children, so marking the CPUs claimed at the ancestor would fail to // exclude them from either leaf's own FreeSupply() — a double-booking gap // this restriction closes. -func (p *policy) poolForCPUs(cpus cpuset.CPUSet) (Node, error) { +func (p *policy) poolForCPUs(cpus *libcpu.CpuMask) (Node, error) { var ( best Node bestDepth = -1 @@ -1594,7 +1594,7 @@ func (p *policy) poolForCPUs(cpus cpuset.CPUSet) (Node, error) { // physical class must be applied per subset — applying one class to the // claim's entire unioned CPU set would silently mis-apply it to part of the // claim. -func (p *policy) applyClassCPUs(verb string, uid types.UID, classCPUs map[string]cpuset.CPUSet) error { +func (p *policy) applyClassCPUs(verb string, uid types.UID, classCPUs map[string]*libcpu.CpuMask) error { if p.cpuClasses == nil { return nil } @@ -1621,7 +1621,7 @@ func (p *policy) applyClassCPUs(verb string, uid types.UID, classCPUs map[string // subsequent calls just bump the per-claim container refcount so that // releaseClaim knows to keep the CPUs marked until the last referencing // container is released. -func (p *policy) allocateClaim(uid types.UID, cpus cpuset.CPUSet, classCPUs map[string]cpuset.CPUSet) error { +func (p *policy) allocateClaim(uid types.UID, cpus *libcpu.CpuMask, classCPUs map[string]*libcpu.CpuMask) error { if cpus.IsEmpty() { return policyError("cannot allocate DRA claim %s: empty CPU set", uid) } @@ -1655,7 +1655,7 @@ func (p *policy) allocateClaim(uid types.UID, cpus cpuset.CPUSet, classCPUs map[ // Roll back the supply mark so pool accounting stays consistent. pool.FreeSupply().UnclaimCPUs(uid) p.resetCpuClass(fmt.Sprintf("dra: rollback claim %s", uid), cpus) - if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, libcpu.NewCpuMask(), uid); reallocErr != nil { log.Errorf("dra: claim %s: failed to restore evicted grants during CPU class rollback: %v", uid, reallocErr) } return policyError("dra: claim %s: failed to apply CPU class: %v", uid, err) @@ -1675,7 +1675,7 @@ func (p *policy) allocateClaim(uid types.UID, cpus cpuset.CPUSet, classCPUs map[ pool.FreeSupply().UnclaimCPUs(uid) p.resetCpuClass(fmt.Sprintf("dra: rollback claim %s", uid), cpus) p.updateSharedAllocations(nil) - if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, libcpu.NewCpuMask(), uid); reallocErr != nil { log.Errorf("dra: claim %s: failed to restore evicted grants during rollback: %v", uid, reallocErr) } return policyError("dra: claim %s: evicted %d container(s) to free CPUs %s but failed "+ @@ -1693,7 +1693,7 @@ func (p *policy) allocateClaim(uid types.UID, cpus cpuset.CPUSet, classCPUs map[ // a snapshot of their cgroup cpuset.cpus (as it was right before eviction) — // the latter is needed by reallocateEvicted's safety net if reallocation // later fails for one of them. reason is used only for logging. -func (p *policy) evictOverlappingGrants(cpus cpuset.CPUSet, reason string) ([]cache.Container, map[string]string) { +func (p *policy) evictOverlappingGrants(cpus *libcpu.CpuMask, reason string) ([]cache.Container, map[string]string) { var evicted []cache.Container evictedCpusets := map[string]string{} for _, g := range p.allocations.grants { @@ -1722,7 +1722,7 @@ func (p *policy) evictOverlappingGrants(cpus cpuset.CPUSet, reason string) ([]ca // owns — that would let two workloads run on the same physical CPUs // simultaneously. Returns the (possibly partial-reallocation) error from // reallocateResources, or nil if evicted is empty or reallocation succeeded. -func (p *policy) reallocateEvicted(evicted []cache.Container, evictedCpusets map[string]string, cpus cpuset.CPUSet, uid types.UID) error { +func (p *policy) reallocateEvicted(evicted []cache.Container, evictedCpusets map[string]string, cpus *libcpu.CpuMask, uid types.UID) error { if len(evicted) == 0 { return nil } @@ -1735,7 +1735,7 @@ func (p *policy) reallocateEvicted(evicted []cache.Container, evictedCpusets map if _, ok := p.allocations.getGrant(c.GetID()); ok { continue } - prev, perr := cpuset.Parse(evictedCpusets[c.GetID()]) + prev, perr := libcpu.ParseCpuMask(evictedCpusets[c.GetID()]) if perr != nil { log.Errorf("dra: claim %s: cannot safely re-pin %s off claimed CPUs %s: %v", uid, c.PrettyName(), cpus, perr) @@ -1756,7 +1756,7 @@ func (p *policy) reallocateEvicted(evicted []cache.Container, evictedCpusets map uid, c.PrettyName(), cpus, prev) continue } - safe = cpuset.New(fallback.List()[0]) + safe = libcpu.NewCpuMask(fallback.List()[0]) } log.Warnf("dra: claim %s: %s could not be reallocated after eviction; "+ "forcing cpuset from %s to %s to avoid overlap with claimed CPUs %s", @@ -1776,7 +1776,7 @@ func (p *policy) reallocateEvicted(evicted []cache.Container, evictedCpusets map // that allocateClaim was never called for, or that has already been fully // released — ReleaseResources may run for containers the policy never saw // AllocateResources for (e.g. across a restart). -func (p *policy) releaseClaim(uid types.UID, cpus cpuset.CPUSet) error { +func (p *policy) releaseClaim(uid types.UID, cpus *libcpu.CpuMask) error { if p.claimContainerRefs == nil || p.claimContainerRefs[uid] == 0 { return nil } @@ -1851,7 +1851,7 @@ func (p *policy) unprepareDRAClaim(uid types.UID, allocs []dra.ResultAlloc) { // skipped (matching UnprepareResourceClaims' own parse-error policy). func (p *policy) releaseHpCPUsForAllocs(uid types.UID, allocs []dra.ResultAlloc) { for _, alloc := range allocs { - cpus, err := cpuset.Parse(alloc.CPUs) + cpus, err := libcpu.ParseCpuMask(alloc.CPUs) if err != nil { log.Warnf("dra: release claim %s device %s: parse CPUs %q: %v (skipping HP release)", uid, alloc.Device, alloc.CPUs, err) continue diff --git a/cmd/plugins/topology-aware/policy/resources.go b/cmd/plugins/topology-aware/policy/resources.go index 45c9e528c..a7495a3ce 100644 --- a/cmd/plugins/topology-aware/policy/resources.go +++ b/cmd/plugins/topology-aware/policy/resources.go @@ -83,7 +83,7 @@ type Supply interface { // them from isolated/sharable capacity in this supply and, tree-wide, in // every ancestor supply. Idempotent per uid: a second call for the same // uid replaces (does not stack on top of) the previous marking. - ClaimCPUs(uid types.UID, cpus cpuset.CPUSet) + ClaimCPUs(uid types.UID, cpus *libcpu.CpuMask) // UnclaimCPUs reverses a previous ClaimCPUs marking for uid, restoring the // claimed CPUs tree-wide. A no-op if uid is unknown. UnclaimCPUs(uid types.UID) @@ -344,7 +344,7 @@ func (cs *supply) GetNode() Node { func (cs *supply) Clone() Supply { clone := newSupply(cs.node, cs.isolated, cs.reserved, cs.sharable, cs.grantedReserved, cs.grantedShared).(*supply) if len(cs.claimRefs) > 0 { - clone.claimRefs = make(map[types.UID]cpuset.CPUSet, len(cs.claimRefs)) + clone.claimRefs = make(map[types.UID]*libcpu.CpuMask, len(cs.claimRefs)) for uid, cpus := range cs.claimRefs { clone.claimRefs[uid] = cpus.Clone() } @@ -439,7 +439,7 @@ func (cs *supply) AccountReleaseCPU(g Grant) { // top of) the previous marking at each level: the old cpuset for uid is first // restored, then the new one is subtracted. This makes re-applying claim // marks after a policy rebuild (Reconfigure/restart) idempotent. -func (cs *supply) ClaimCPUs(uid types.UID, cpus cpuset.CPUSet) { +func (cs *supply) ClaimCPUs(uid types.UID, cpus *libcpu.CpuMask) { if old, ok := cs.claimRefs[uid]; ok { full := cs.node.GetSupply() cs.isolated = cs.isolated.Union(old.Intersection(full.IsolatedCPUs())) @@ -448,7 +448,7 @@ func (cs *supply) ClaimCPUs(uid types.UID, cpus cpuset.CPUSet) { } if cs.claimRefs == nil { - cs.claimRefs = make(map[types.UID]cpuset.CPUSet) + cs.claimRefs = make(map[types.UID]*libcpu.CpuMask) } cs.claimRefs[uid] = cpus.Clone() cs.isolated = cs.isolated.Difference(cpus) diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index 60b07e6b9..de3ad60e7 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -103,7 +103,7 @@ type policy struct { // (computed independently, without regard to its claimed CPUs) doesn't // end up excluding the CPUs its CDI-injected NRI_CPU env vars claim // it has. - claimedCPUsByContainer map[string]cpuset.CPUSet + claimedCPUsByContainer map[string]*libcpu.CpuMask // draClaimsByContainer keeps the claims consumed by each container // independently of the plugin's live-claim map, which may lose a claim @@ -412,12 +412,12 @@ func (p *policy) AllocateResources(container cache.Container) error { // Reconfigure (see its own comment for why that repopulation alone isn't // sufficient there). func (p *policy) setClaimedCPUs(container cache.Container, marked []containerClaim) { - union := cpuset.New() + union := libcpu.NewCpuMask() for _, cl := range marked { union = union.Union(cl.CPUs) } if p.claimedCPUsByContainer == nil { - p.claimedCPUsByContainer = map[string]cpuset.CPUSet{} + p.claimedCPUsByContainer = map[string]*libcpu.CpuMask{} } p.claimedCPUsByContainer[container.GetID()] = union } @@ -1001,7 +1001,7 @@ func (p *policy) restoreCache() error { // allowed CPU's class back to the shared-pool default // (resetCpuClass("initialize", p.allowed)), which would otherwise silently // strip the SST-CP/EPP/governor settings a live DRA claim depends on. -func (p *policy) remarkClaimInSupply(uid types.UID, cpus cpuset.CPUSet, classCPUs map[string]cpuset.CPUSet) error { +func (p *policy) remarkClaimInSupply(uid types.UID, cpus *libcpu.CpuMask, classCPUs map[string]*libcpu.CpuMask) error { if cpus.IsEmpty() { return policyError("cannot remark DRA claim %s: empty CPU set", uid) } @@ -1068,7 +1068,7 @@ func (p *policy) reapplyDRAClaims() { if err := p.remarkClaimInSupply(uid, cpus, classCPUs); err != nil { log.Errorf("dra: reapplyDRAClaims: %v", err) - if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, cpuset.New(), uid); reallocErr != nil { + if reallocErr := p.reallocateEvicted(evicted, evictedCpusets, libcpu.NewCpuMask(), uid); reallocErr != nil { log.Errorf("dra: reapplyDRAClaims: failed to restore grants after claim %s could not be remarked: %v", uid, reallocErr) } continue @@ -1104,7 +1104,7 @@ func (p *policy) reapplyDRAClaims() { p.applyGrant(grant) } else if opt.PinCPU { union := p.claimedCPUsByContainer[c.GetID()] - p.setPreferredCpusetCpus(c, cpuset.New(), union, + p.setPreferredCpusetCpus(c, libcpu.NewCpuMask(), union, fmt.Sprintf(" => re-pinning %s to claimed cpuset %s (no regular grant)", c.PrettyName(), union)) } } diff --git a/pkg/resmgr/cpuclass/cpuclass.go b/pkg/resmgr/cpuclass/cpuclass.go index a123cea5a..da64b5e6a 100644 --- a/pkg/resmgr/cpuclass/cpuclass.go +++ b/pkg/resmgr/cpuclass/cpuclass.go @@ -156,9 +156,9 @@ func (h *Handler) PctActive() bool { // error when the handler or its PCT allocator is nil, or when the // underlying pick fails (inactive allocator, punit not found, or // insufficient HP capacity). -func (h *Handler) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { +func (h *Handler) PickHpCpus(pkgID, punitID, n int, held *libcpu.CpuMask) (*libcpu.CpuMask, error) { if h == nil || h.pct == nil { - return cpuset.New(), fmt.Errorf("cpuclass: PickHpCpus: pct allocator not initialized") + return libcpu.NewCpuMask(), fmt.Errorf("cpuclass: PickHpCpus: pct allocator not initialized") } return h.pct.PickHpCpus(pkgID, punitID, n, held) } @@ -167,7 +167,7 @@ func (h *Handler) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset. // identified by (pkgID, punitID). Delegates to the PCT allocator. // No-op when the handler or its PCT allocator is nil, or when the // punit is unknown (idempotent). -func (h *Handler) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { +func (h *Handler) ReleaseHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) { if h == nil || h.pct == nil { return } @@ -180,7 +180,7 @@ func (h *Handler) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { // Delegates to the PCT allocator. Returns an error when the handler // or its PCT allocator is nil, or when accounting fails (inactive // allocator, punit not found, or HP-ineligible punit). -func (h *Handler) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { +func (h *Handler) AccountHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) error { if h == nil || h.pct == nil { return fmt.Errorf("cpuclass: AccountHpCpus: pct allocator not initialized: %w", ErrAllocatorInactive) } diff --git a/pkg/resmgr/cpuclass/internal/pct/pct.go b/pkg/resmgr/cpuclass/internal/pct/pct.go index 71136fb29..a9445fbf2 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct.go @@ -642,16 +642,16 @@ func (a *Allocator) punitNonHPCapacity(idx int) int { // so clearHpUsage called from the non-DRA path cannot evict DRA holds). // Returns an error when the allocator is inactive, the punit is not // found or not HP-eligible, or fewer than n CPUs are available. -func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) { +func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held *libcpu.CpuMask) (*libcpu.CpuMask, error) { if !a.Active() { - return cpuset.New(), fmt.Errorf("pct: PickHpCpus: allocator not active") + return libcpu.NewCpuMask(), fmt.Errorf("pct: PickHpCpus: allocator not active") } idx := a.punitIdxByID(pkgID, punitID) if idx < 0 { - return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) not found", pkgID, punitID) + return libcpu.NewCpuMask(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) not found", pkgID, punitID) } if !a.hpEligiblePunit[idx] { - return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) is not HP-eligible", pkgID, punitID) + return libcpu.NewCpuMask(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) is not HP-eligible", pkgID, punitID) } pu := a.punits[idx] avail := pu.CPUs @@ -672,12 +672,12 @@ func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuse if hpRoom < available { available = hpRoom } - return cpuset.New(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) has %d available HP CPUs (room=%d, free=%d), need %d", + return libcpu.NewCpuMask(), fmt.Errorf("pct: PickHpCpus: punit (pkg=%d, punit=%d) has %d available HP CPUs (room=%d, free=%d), need %d", pkgID, punitID, available, hpRoom, avail.Size(), n) } // Sort for deterministic selection; take first n. list := avail.List() - picked := cpuset.New(list[:n]...) + picked := libcpu.NewCpuMask(list[:n]...) // hpDRAUsed is always non-nil here: Configure() initialises it // unconditionally, and Active() (checked above) is true only after // a successful Configure(). @@ -688,7 +688,7 @@ func (a *Allocator) PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuse // ReleaseHpCpus removes cpus from hpDRAUsed[punitIdx] for the punit // identified by (pkgID, punitID). Silently ignores unknown punits and // CPUs not present in hpDRAUsed (idempotent). -func (a *Allocator) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { +func (a *Allocator) ReleaseHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) { if !a.Active() { return } @@ -717,7 +717,7 @@ func (a *Allocator) ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) { // hpUsed > GuaranteedHpCpus) is permitted — the container may already // be running; a warning is logged but no error is returned. Union // semantics make repeated calls with the same CPUs idempotent. -func (a *Allocator) AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error { +func (a *Allocator) AccountHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) error { if !a.Active() { return fmt.Errorf("pct: AccountHpCpus: allocator not active") } diff --git a/pkg/resmgr/dra/deps.go b/pkg/resmgr/dra/deps.go index f887cf3bb..350129162 100644 --- a/pkg/resmgr/dra/deps.go +++ b/pkg/resmgr/dra/deps.go @@ -17,12 +17,12 @@ limitations under the License. package dra import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" resourceapi "k8s.io/api/resource/v1" "k8s.io/apimachinery/pkg/types" "k8s.io/client-go/kubernetes" "github.com/containers/nri-plugins/pkg/log" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // DeviceLister provides the DRA device list for a given driver name. @@ -39,7 +39,7 @@ type CDIDevice struct { // ClassName is the nri/cpuClass attribute value for the allocated device. ClassName string // CPUs is the set of CPUs allocated for this device. - CPUs cpuset.CPUSet + CPUs *libcpu.CpuMask } // ClaimAllocator provides HP CPU pick/release/account operations needed @@ -49,13 +49,13 @@ type ClaimAllocator interface { // PickHpCpus selects n HP-eligible CPUs from the punit identified by // (pkgID, punitID), excluding CPUs in held and those already tracked // in internal accounting. - PickHpCpus(pkgID, punitID, n int, held cpuset.CPUSet) (cpuset.CPUSet, error) + PickHpCpus(pkgID, punitID, n int, held *libcpu.CpuMask) (*libcpu.CpuMask, error) // ReleaseHpCpus removes cpus from DRA HP accounting on the given punit. - ReleaseHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) + ReleaseHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) // AccountHpCpus records cpus as DRA HP-held on the given punit. Used // during restart reconciliation to rebuild HP accounting without // re-allocating CPUs. - AccountHpCpus(pkgID, punitID int, cpus cpuset.CPUSet) error + AccountHpCpus(pkgID, punitID int, cpus *libcpu.CpuMask) error // IsHPClass reports whether className is currently classified as PCT // high priority. IsHPClass(className string) bool @@ -114,7 +114,7 @@ type Deps struct { // if the CPUs span multiple leaf pools (or otherwise don't fit within // one), so that a claim which would later fail at container-creation // time is instead rejected at Prepare time. Must not be nil. - ValidateCPUsInPool func(cpus cpuset.CPUSet) error + ValidateCPUsInPool func(cpus *libcpu.CpuMask) error // DeviceLister returns the list of DRA devices to publish. DeviceLister DeviceLister // ClaimAllocator provides HP CPU pick/release/account operations. diff --git a/pkg/resmgr/dra/plugin.go b/pkg/resmgr/dra/plugin.go index af6c355a2..f58f8b459 100644 --- a/pkg/resmgr/dra/plugin.go +++ b/pkg/resmgr/dra/plugin.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "os" "path/filepath" "sync" @@ -33,7 +34,6 @@ import ( "k8s.io/dynamic-resource-allocation/resourceslice" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" - "github.com/containers/nri-plugins/pkg/utils/cpuset" "tags.cncf.io/container-device-interface/pkg/parser" ) @@ -153,11 +153,11 @@ func (p *Plugin) deviceIndex() (map[string]deviceInfo, error) { // allClaimedCPUs returns the union of all CPUs currently tracked in p.claims. // Must be called inside deps.WithLock. -func (p *Plugin) allClaimedCPUs() cpuset.CPUSet { - result := cpuset.New() +func (p *Plugin) allClaimedCPUs() *libcpu.CpuMask { + result := libcpu.NewCpuMask() for _, cs := range p.claims { for _, alloc := range cs.Allocs { - parsed, err := cpuset.Parse(alloc.CPUs) + parsed, err := libcpu.ParseCpuMask(alloc.CPUs) if err == nil { result = result.Union(parsed) } @@ -230,7 +230,7 @@ func (p *Plugin) PrepareResourceClaims(_ context.Context, claims []*resourceapi. heldCPUs := p.allClaimedCPUs() var pickedAllocs []ResultAlloc var cdiDevices []CDIDevice - claimCPUs := cpuset.New() + claimCPUs := libcpu.NewCpuMask() var punit *deviceInfo for i, r := range filtered { @@ -357,7 +357,7 @@ func (p *Plugin) cdiDevicesFromClaims(uid types.UID, filtered []resourceapi.Devi } r := filtered[i] name := CDIDeviceName(uid, r.Request, r.Device, i) - cpus, err := cpuset.Parse(alloc.CPUs) + cpus, err := libcpu.ParseCpuMask(alloc.CPUs) if err != nil { continue } @@ -374,7 +374,7 @@ func (p *Plugin) cdiDevicesFromClaims(uid types.UID, filtered []resourceapi.Devi // encountered an error mid-way through allocation. func (p *Plugin) rollbackPicks(allocs []ResultAlloc) { for _, a := range allocs { - cs, err := cpuset.Parse(a.CPUs) + cs, err := libcpu.ParseCpuMask(a.CPUs) if err != nil { continue } @@ -466,7 +466,7 @@ func (p *Plugin) RestoreClaimsLocked() error { var errs []error for _, cs := range p.claims { for _, alloc := range cs.Allocs { - cpus, err := cpuset.Parse(alloc.CPUs) + cpus, err := libcpu.ParseCpuMask(alloc.CPUs) if err != nil { p.deps.Logger.Warnf("dra plugin: RestoreClaimsLocked: claim %s device %s: parse CPUs %q: %v (skipping)", cs.UID, alloc.Device, alloc.CPUs, err) continue @@ -550,7 +550,7 @@ func (p *Plugin) Start(ctx context.Context) error { startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: cpuClass %q is no longer a valid HP class (removed or renamed); cannot restore claim", uid, alloc.Device, alloc.ClassName) return } - cpus, err := cpuset.Parse(alloc.CPUs) + cpus, err := libcpu.ParseCpuMask(alloc.CPUs) if err != nil { startErr = fmt.Errorf("dra plugin: Start: claim %s device %s: parse CPUs %q: %w", uid, alloc.Device, alloc.CPUs, err) return diff --git a/pkg/resmgr/dra/state.go b/pkg/resmgr/dra/state.go index e5dea6921..13e2c7042 100644 --- a/pkg/resmgr/dra/state.go +++ b/pkg/resmgr/dra/state.go @@ -46,7 +46,7 @@ type ResultAlloc struct { PkgID int `json:"PkgID"` // PunitID is the value of the nri/punitID attribute. PunitID int `json:"PunitID"` - // CPUs is the cpuset.CPUSet.String() representation of the allocated CPUs. + // CPUs is the *libcpu.CpuMask.String() representation of the allocated CPUs. CPUs string `json:"CPUs"` } From 634d3bbc467a0af472340318a11379145e3beb74 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 11 Sep 2026 09:18:55 +0300 Subject: [PATCH 37/39] resmgr,plugins: build machines in the tests. The tests which needed a system faked one by embedding the sysfs.System interface. A hardware.Machine is a concrete type and cannot be faked, so they describe the machine they want and read it back through real discovery, which is what the rest of these suites already do. pct is the interesting one. Its Sys is two methods, both of which a Machine has, so the fake goes entirely and the tests hand it real machines with the package layouts the old fake described. That turned out to be worth less than it looks: the only thing pct reads out of a system is discoverTurboInfo, which no test entered, so nothing depended on the layout at all. TestDiscover- TurboInfo now does, which takes that function from no coverage to most of it and gives the machines something to be right about. The DRA tests take a machine each: the policy's own use oneCpuMachine, which is already there, and the CPU class ones are in an external test package and get a small machine of their own. Both are minimal on purpose, since those tests drive the PCT allocator through the Speed Select mock. The tests which built a policy backend hand it a machine now rather than a system, and the four which built one with neither get the smallest machine there is, since NewPolicy refuses to make a policy without one. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../topology-aware/policy/dra_adapter_test.go | 39 +-- cmd/plugins/topology-aware/policy/dra_test.go | 20 +- .../topology-aware/policy/machine_test.go | 12 +- .../topology-aware/policy/pools_test.go | 50 ++-- .../topology-aware/policy/resources_test.go | 40 +-- .../policy/topology-aware-policy_test.go | 68 ++--- pkg/resmgr/cpuclass/cpuclass_dra_test.go | 76 +++--- pkg/resmgr/cpuclass/dra_test.go | 8 +- pkg/resmgr/cpuclass/internal/pct/pct_test.go | 248 ++++++++++++++---- pkg/resmgr/dra/cdi_test.go | 15 +- pkg/resmgr/dra/plugin_test.go | 60 ++--- pkg/resmgr/dra/state_test.go | 22 +- pkg/resmgr/policy/policy_test.go | 32 ++- 13 files changed, 431 insertions(+), 259 deletions(-) diff --git a/cmd/plugins/topology-aware/policy/dra_adapter_test.go b/cmd/plugins/topology-aware/policy/dra_adapter_test.go index 8d1803971..25a90fb7a 100644 --- a/cmd/plugins/topology-aware/policy/dra_adapter_test.go +++ b/cmd/plugins/topology-aware/policy/dra_adapter_test.go @@ -15,26 +15,13 @@ package topologyaware import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "testing" - idset "github.com/intel/goresctrl/pkg/utils" - cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" - "github.com/containers/nri-plugins/pkg/sysfs" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) -// adapterTestSys is a minimal sysfs.System implementation sufficient for -// cpuclass.New()/Configure(). Only CPUIDs is overridden; every other method -// is delegated to the embedded nil interface, which panics if called (and is -// never called by cpuclass.New/Configure in practice). -type adapterTestSys struct { - sysfs.System -} - -func (s *adapterTestSys) CPUIDs() []idset.ID { return nil } - // newActiveClassHandler builds a *cpuclass.Handler with an active managed // PCT allocator (via the goresctrl SST in-memory mock), one HP class named // "hp". Mirrors cpuclass.newConfiguredHandler (internal, unexported), kept @@ -44,13 +31,13 @@ func newActiveClassHandler(t *testing.T) *cpuclass.Handler { t.Helper() t.Setenv("OVERRIDE_SST", `{"supported":true,"clos_count":4,"packages":[{"id":0,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"punits":[{"id":0,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":4}]}]}`) t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) - h, err := cpuclass.New(&adapterTestSys{}) + h, err := cpuclass.New(oneCpuMachine(t)) if err != nil { t.Fatalf("cpuclass.New() failed: %v", err) } if err := h.Configure(cpuclass.ConfigSpec{ Classes: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, - Allowed: cpuset.MustParse("0-7"), + Allowed: libcpu.MustParseCpuMask("0-7"), }); err != nil { t.Fatalf("Configure() failed: %v", err) } @@ -63,13 +50,13 @@ func newActiveClassHandler(t *testing.T) *cpuclass.Handler { func newInactiveClassHandler(t *testing.T) *cpuclass.Handler { t.Helper() t.Setenv("OVERRIDE_SST", "") - h, err := cpuclass.New(&adapterTestSys{}) + h, err := cpuclass.New(oneCpuMachine(t)) if err != nil { t.Fatalf("cpuclass.New() failed: %v", err) } _ = h.Configure(cpuclass.ConfigSpec{ Classes: []*cfgapi.CPUClass{{Name: "hp", PctPriority: "high"}}, - Allowed: cpuset.MustParse("0-7"), + Allowed: libcpu.MustParseCpuMask("0-7"), }) return h } @@ -89,14 +76,14 @@ func TestPolicyDRAAdapterRoutesToCurrentHandler(t *testing.T) { if a.IsHPClass("hp") { t.Error("IsHPClass with nil cpuClasses: got true, want false") } - if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + if _, err := a.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()); err == nil { t.Error("PickHpCpus with nil cpuClasses: got nil error, want error") } if devs, err := a.DRADevices(DRADriverName); err != nil || len(devs) != 0 { t.Errorf("DRADevices with nil cpuClasses: got (%v, %v), want (empty, nil)", devs, err) } - a.ReleaseHpCpus(0, 0, cpuset.New()) // must not panic - if err := a.AccountHpCpus(0, 0, cpuset.New()); err == nil { + a.ReleaseHpCpus(0, 0, libcpu.NewCpuMask()) // must not panic + if err := a.AccountHpCpus(0, 0, libcpu.NewCpuMask()); err == nil { t.Error("AccountHpCpus with nil cpuClasses: got nil error, want error") } @@ -106,7 +93,7 @@ func TestPolicyDRAAdapterRoutesToCurrentHandler(t *testing.T) { if a.IsHPClass("hp") { t.Error("IsHPClass with inactive handler: got true, want false") } - if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + if _, err := a.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()); err == nil { t.Error("PickHpCpus with inactive handler: got nil error, want error") } @@ -117,7 +104,7 @@ func TestPolicyDRAAdapterRoutesToCurrentHandler(t *testing.T) { if !a.IsHPClass("hp") { t.Error("IsHPClass with active handler: got false, want true") } - cpus, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + cpus, err := a.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus with active handler: %v", err) } @@ -154,9 +141,9 @@ func TestPolicyDRAAdapterNilCpuClassesNoPanic(t *testing.T) { } }() - _, _ = a.PickHpCpus(0, 0, 1, cpuset.New()) - a.ReleaseHpCpus(0, 0, cpuset.New()) - _ = a.AccountHpCpus(0, 0, cpuset.New()) + _, _ = a.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()) + a.ReleaseHpCpus(0, 0, libcpu.NewCpuMask()) + _ = a.AccountHpCpus(0, 0, libcpu.NewCpuMask()) _ = a.IsHPClass("hp") _, _ = a.DRADevices(DRADriverName) } diff --git a/cmd/plugins/topology-aware/policy/dra_test.go b/cmd/plugins/topology-aware/policy/dra_test.go index 7b97e3084..6d90d0683 100644 --- a/cmd/plugins/topology-aware/policy/dra_test.go +++ b/cmd/plugins/topology-aware/policy/dra_test.go @@ -16,6 +16,8 @@ package topologyaware import ( "context" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" "os" "path" "strings" @@ -27,9 +29,7 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/testutils" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // setupDRATestPolicy builds a real *policy from the "server" sysfs test @@ -57,9 +57,9 @@ func setupDRATestPolicy( t.Fatalf("failed to uncompress test sysfs data: %v", err) } - sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + machine, err := hardware.Discover(hardware.WithRoot(path.Join(dir, "sysfs", "server"))) if err != nil { - t.Fatalf("failed to discover test system: %v", err) + t.Fatalf("failed to discover test machine: %v", err) } cfg := &cfgapi.Config{ @@ -72,9 +72,9 @@ func setupDRATestPolicy( } opts := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, - Config: cfg, + Cache: &mockCache{}, + Machine: machine, + Config: cfg, } if mutateOpts != nil { mutateOpts(opts) @@ -266,7 +266,7 @@ func TestSetupDRAEnabledCDIWriterFailureReturnsError(t *testing.T) { // dra.Plugin.Stop are documented as idempotent). func TestStopCancelsContextAndStopsDRAPlugin(t *testing.T) { p := &policy{} - p.draPlugin = newTestDRAPlugin(t, cpuset.New(0), "dev0") + p.draPlugin = newTestDRAPlugin(t, libcpu.NewCpuMask(0), "dev0") ctx, cancel := context.WithCancel(context.Background()) p.draCtxCancel = cancel @@ -292,7 +292,7 @@ func TestStopCancelsContextAndStopsDRAPlugin(t *testing.T) { func newConflictingTierClassHandler(t *testing.T) *cpuclass.Handler { t.Helper() t.Setenv("OVERRIDE_SST", "") - h, err := cpuclass.New(&adapterTestSys{}) + h, err := cpuclass.New(oneCpuMachine(t)) if err != nil { t.Fatalf("cpuclass.New() failed: %v", err) } @@ -302,7 +302,7 @@ func newConflictingTierClassHandler(t *testing.T) *cpuclass.Handler { } if err := h.Configure(cpuclass.ConfigSpec{ Classes: classes, - Allowed: cpuset.MustParse("0-7"), + Allowed: libcpu.MustParseCpuMask("0-7"), }); err != nil { t.Fatalf("Configure() failed: %v", err) } diff --git a/cmd/plugins/topology-aware/policy/machine_test.go b/cmd/plugins/topology-aware/policy/machine_test.go index 4440a459a..cf20c99da 100644 --- a/cmd/plugins/topology-aware/policy/machine_test.go +++ b/cmd/plugins/topology-aware/policy/machine_test.go @@ -16,12 +16,12 @@ package topologyaware import ( "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "strings" "testing" "testing/fstest" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // synthNode describes one NUMA node of a machine to build for a test. @@ -51,17 +51,17 @@ func synthMachine(t *testing.T, nodes []synthNode) *hardware.Machine { fsys := fstest.MapFS{} var ( - online = cpuset.New() - normal = cpuset.New() + online = libcpu.NewCpuMask() + normal = libcpu.NewCpuMask() pkg = 0 ) for id, node := range nodes { dir := fmt.Sprintf("sys/devices/system/node/node%d", id) - cpus := cpuset.New() + cpus := libcpu.NewCpuMask() if node.cpus != "" { var err error - if cpus, err = cpuset.Parse(node.cpus); err != nil { + if cpus, err = libcpu.ParseCpuMask(node.cpus); err != nil { t.Fatalf("node%d: bad cpulist %q: %v", id, node.cpus, err) } } @@ -72,7 +72,7 @@ func synthMachine(t *testing.T, nodes []synthNode) *hardware.Machine { fmt.Sprintf("Node %d MemTotal: %d kB\n", id, node.memKB)) if node.memKB > 0 { - normal = normal.Union(cpuset.New(id)) + normal = normal.Union(libcpu.NewCpuMask(id)) } dist := make([]string, 0, len(node.distance)) diff --git a/cmd/plugins/topology-aware/policy/pools_test.go b/cmd/plugins/topology-aware/policy/pools_test.go index b6e94f255..5663ed59e 100644 --- a/cmd/plugins/topology-aware/policy/pools_test.go +++ b/cmd/plugins/topology-aware/policy/pools_test.go @@ -16,6 +16,7 @@ package topologyaware import ( "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "os" "path" "strings" @@ -30,7 +31,6 @@ import ( "github.com/containers/nri-plugins/pkg/lib/hardware" "github.com/containers/nri-plugins/pkg/testutils" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func findNodeWithName(name string, nodes []Node) Node { @@ -672,13 +672,13 @@ func TestClaimCPUsFromContainer(t *testing.T) { if got.UID != uid { t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) } - if want := cpuset.MustParse("0-1"); !got.ClassCPUs["gold"].Equals(want) { + if want := libcpu.MustParseCpuMask("0-1"); !got.ClassCPUs["gold"].Equals(want) { t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) } if len(got.ClassCPUs) != 1 { t.Errorf("claimCPUsFromContainer() classCPUs = %v, want exactly one class", got.ClassCPUs) } - if want := cpuset.MustParse("0-1"); !got.CPUs.Equals(want) { + if want := libcpu.MustParseCpuMask("0-1"); !got.CPUs.Equals(want) { t.Errorf("claimCPUsFromContainer() cpus = %s, want %s", got.CPUs, want) } } @@ -707,10 +707,10 @@ func TestClaimCPUsFromContainerDashedUIDAndDashedDeviceName(t *testing.T) { if got.UID != uid { t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) } - if want := cpuset.MustParse("4-5"); !got.ClassCPUs["gold"].Equals(want) { + if want := libcpu.MustParseCpuMask("4-5"); !got.ClassCPUs["gold"].Equals(want) { t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) } - if want := cpuset.MustParse("4-5"); !got.CPUs.Equals(want) { + if want := libcpu.MustParseCpuMask("4-5"); !got.CPUs.Equals(want) { t.Errorf("claimCPUsFromContainer() cpus = %s, want %s", got.CPUs, want) } } @@ -738,10 +738,10 @@ func TestClaimCPUsFromContainerUsesOnlyConsumedAllocs(t *testing.T) { t.Fatalf("claimCPUsFromContainer() returned %d claim(s), want 1", len(claims)) } got := claims[0] - if want := cpuset.MustParse("0-1"); !got.CPUs.Equals(want) { + if want := libcpu.MustParseCpuMask("0-1"); !got.CPUs.Equals(want) { t.Errorf("claimCPUsFromContainer() cpus = %s, want %s (only the consumed allocation)", got.CPUs, want) } - if want := cpuset.MustParse("0-1"); len(got.ClassCPUs) != 1 || !got.ClassCPUs["gold"].Equals(want) { + if want := libcpu.MustParseCpuMask("0-1"); len(got.ClassCPUs) != 1 || !got.ClassCPUs["gold"].Equals(want) { t.Errorf("claimCPUsFromContainer() classCPUs = %v, want {gold: %s}", got.ClassCPUs, want) } } @@ -778,16 +778,16 @@ func TestClaimCPUsFromContainerMultipleClasses(t *testing.T) { if got.UID != uid { t.Errorf("claimCPUsFromContainer() uid = %q, want %q", got.UID, uid) } - if want := cpuset.MustParse("0-3"); !got.CPUs.Equals(want) { + if want := libcpu.MustParseCpuMask("0-3"); !got.CPUs.Equals(want) { t.Errorf("claimCPUsFromContainer() cpus = %s, want %s (union across classes)", got.CPUs, want) } if len(got.ClassCPUs) != 2 { t.Fatalf("claimCPUsFromContainer() classCPUs = %v, want two distinct classes", got.ClassCPUs) } - if want := cpuset.MustParse("0-1"); !got.ClassCPUs["gold"].Equals(want) { + if want := libcpu.MustParseCpuMask("0-1"); !got.ClassCPUs["gold"].Equals(want) { t.Errorf("claimCPUsFromContainer() classCPUs[gold] = %s, want %s", got.ClassCPUs["gold"], want) } - if want := cpuset.MustParse("2-3"); !got.ClassCPUs["silver"].Equals(want) { + if want := libcpu.MustParseCpuMask("2-3"); !got.ClassCPUs["silver"].Equals(want) { t.Errorf("claimCPUsFromContainer() classCPUs[silver] = %s, want %s", got.ClassCPUs["silver"], want) } } @@ -844,10 +844,10 @@ func TestClaimCPUsFromContainerMultipleDistinctClaims(t *testing.T) { for _, cl := range claims { byUID[cl.UID] = cl } - if cl, ok := byUID[uid1]; !ok || !cl.CPUs.Equals(cpuset.MustParse("0-1")) { + if cl, ok := byUID[uid1]; !ok || !cl.CPUs.Equals(libcpu.MustParseCpuMask("0-1")) { t.Errorf("claim %s CPUs = %v, want 0-1", uid1, cl.CPUs) } - if cl, ok := byUID[uid2]; !ok || !cl.CPUs.Equals(cpuset.MustParse("2-3")) { + if cl, ok := byUID[uid2]; !ok || !cl.CPUs.Equals(libcpu.MustParseCpuMask("2-3")) { t.Errorf("claim %s CPUs = %v, want 2-3", uid2, cl.CPUs) } } @@ -855,8 +855,8 @@ func TestClaimCPUsFromContainerMultipleDistinctClaims(t *testing.T) { // goldClassCPUs wraps cpus as the single-class classCPUs allocateClaim/ // remarkClaimInSupply expect, for tests that don't care about the // multi-class case (see TestAllocateClaimAppliesPerAllocClass for that). -func goldClassCPUs(cpus cpuset.CPUSet) map[string]cpuset.CPUSet { - return map[string]cpuset.CPUSet{"gold": cpus} +func goldClassCPUs(cpus *libcpu.CpuMask) map[string]*libcpu.CpuMask { + return map[string]*libcpu.CpuMask{"gold": cpus} } // addTestGrant hands out an exclusive grant for container from pool's @@ -867,7 +867,7 @@ func goldClassCPUs(cpus cpuset.CPUSet) map[string]cpuset.CPUSet { // code path without going through the full container-annotation-driven // request/offer pipeline (which coldstart_test.go notes is impractical to // mock with a bare container). -func addTestGrant(t *testing.T, p *policy, pool Node, container cache.Container, exclusive cpuset.CPUSet) Grant { +func addTestGrant(t *testing.T, p *policy, pool Node, container cache.Container, exclusive *libcpu.CpuMask) Grant { t.Helper() g := newGrant(pool, container, cpuNormal, "", exclusive, 0, memoryDRAM, nil, 0) @@ -893,7 +893,7 @@ func TestAllocateClaimMarksTightestPool(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - cpus := cpuset.New(sharable[0], sharable[1]) + cpus := libcpu.NewCpuMask(sharable[0], sharable[1]) uid := types.UID("claim-mark") if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { @@ -926,7 +926,7 @@ func TestAllocateClaimOutsideAllowedReturnsError(t *testing.T) { // CPU 99999 does not exist on the test system at all, so it can't be a // subset of any pool's (including root's) statically assigned range. - cpus := cpuset.New(99999) + cpus := libcpu.NewCpuMask(99999) if err := p.allocateClaim(types.UID("claim-outside"), cpus, goldClassCPUs(cpus)); err == nil { t.Fatalf("allocateClaim() with CPUs outside the allowed set: got nil error, want a descriptive error") @@ -955,7 +955,7 @@ func TestAllocateClaimSpanningNoPoolReturnsError(t *testing.T) { if len(cpuA) < 1 || len(cpuB) < 1 { t.Fatalf("expected at least 1 CPU on both %q and %q", leafA.Name(), leafB.Name()) } - spanning := cpuset.New(cpuA[0], cpuB[0]) + spanning := libcpu.NewCpuMask(cpuA[0], cpuB[0]) err := p.allocateClaim(types.UID("claim-spanning"), spanning, goldClassCPUs(spanning)) if err == nil { @@ -976,7 +976,7 @@ func TestAllocateClaimRefcountsMultipleContainers(t *testing.T) { if len(sharable) < 1 { t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) } - cpus := cpuset.New(sharable[0]) + cpus := libcpu.NewCpuMask(sharable[0]) uid := types.UID("claim-shared") if err := p.allocateClaim(uid, cpus, goldClassCPUs(cpus)); err != nil { @@ -1020,7 +1020,7 @@ func TestReleaseClaimUnknownUIDNoop(t *testing.T) { leaf := findPoolNode(t, p, "NUMA node #0") before := leaf.FreeSupply().SharableCPUs() - if err := p.releaseClaim(types.UID("never-claimed"), cpuset.New(before.List()[0])); err != nil { + if err := p.releaseClaim(types.UID("never-claimed"), libcpu.NewCpuMask(before.List()[0])); err != nil { t.Errorf("releaseClaim() for an unknown uid: got error %v, want nil (idempotent)", err) } if got := leaf.FreeSupply().SharableCPUs(); !got.Equals(before) { @@ -1043,7 +1043,7 @@ func TestReleaseClaimResetsCpuClass(t *testing.T) { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } cpu := sharable[0] - claimed := cpuset.New(cpu) + claimed := libcpu.NewCpuMask(cpu) // A sibling CPU never touched by the claim: its class reflects whatever // initialize() applied via resetCpuClass("initialize", p.allowed) — the @@ -1083,12 +1083,12 @@ func TestAllocateClaimAppliesPerAllocClass(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - goldCPU := cpuset.New(sharable[0]) - silverCPU := cpuset.New(sharable[1]) + goldCPU := libcpu.NewCpuMask(sharable[0]) + silverCPU := libcpu.NewCpuMask(sharable[1]) claimed := goldCPU.Union(silverCPU) uid := types.UID("claim-multiclass") - classCPUs := map[string]cpuset.CPUSet{ + classCPUs := map[string]*libcpu.CpuMask{ "gold": goldCPU, "silver": silverCPU, } @@ -1127,7 +1127,7 @@ func TestAllocateClaimEvictsOverlappingExclusiveGrant(t *testing.T) { if len(sharable) < 1 { t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0]) + claimed := libcpu.NewCpuMask(sharable[0]) victim := &mockContainer{returnValueForGetID: "victim"} addTestGrant(t, p, leaf, victim, claimed) diff --git a/cmd/plugins/topology-aware/policy/resources_test.go b/cmd/plugins/topology-aware/policy/resources_test.go index ff1035fc7..f3a014393 100644 --- a/cmd/plugins/topology-aware/policy/resources_test.go +++ b/cmd/plugins/topology-aware/policy/resources_test.go @@ -15,6 +15,8 @@ package topologyaware import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" "os" "path" "testing" @@ -23,9 +25,7 @@ import ( cfgapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy/topologyaware" policyapi "github.com/containers/nri-plugins/pkg/resmgr/policy" - system "github.com/containers/nri-plugins/pkg/sysfs" "github.com/containers/nri-plugins/pkg/testutils" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // newDRATestPolicy builds a real policy (with a real, multi-level node tree: @@ -56,14 +56,14 @@ func newDRATestPolicyWithLock(t *testing.T, withLock func(func())) *policy { t.Fatalf("failed to uncompress test sysfs data: %v", err) } - sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + machine, err := hardware.Discover(hardware.WithRoot(path.Join(dir, "sysfs", "server"))) if err != nil { - t.Fatalf("failed to discover test system: %v", err) + t.Fatalf("failed to discover test machine: %v", err) } policyOptions := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, + Cache: &mockCache{}, + Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ cfgapi.CPU: "750m", @@ -101,9 +101,9 @@ func newDRATestPolicyWithCPUClasses(t *testing.T, sharedClass string, claimClass t.Fatalf("failed to uncompress test sysfs data: %v", err) } - sys, err := system.DiscoverSystemAt(path.Join(dir, "sysfs", "server", "sys")) + machine, err := hardware.Discover(hardware.WithRoot(path.Join(dir, "sysfs", "server"))) if err != nil { - t.Fatalf("failed to discover test system: %v", err) + t.Fatalf("failed to discover test machine: %v", err) } cpuClasses := []*cfgapi.CPUClass{{Name: sharedClass}} @@ -112,8 +112,8 @@ func newDRATestPolicyWithCPUClasses(t *testing.T, sharedClass string, claimClass } policyOptions := &policyapi.BackendOptions{ - Cache: &mockCache{}, - System: sys, + Cache: &mockCache{}, + Machine: machine, Config: &cfgapi.Config{ ReservedResources: cfgapi.Constraints{ cfgapi.CPU: "750m", @@ -177,13 +177,13 @@ func TestSupplyClaimCPUsTreeWide(t *testing.T) { } // Take exactly two CPUs from the leaf's sharable set. claimedList := claimed.List() - cpus := cpuset.New(claimedList[0], claimedList[1]) + cpus := libcpu.NewCpuMask(claimedList[0], claimedList[1]) uid := types.UID("claim-uid-1") leafSharableBefore := leaf.FreeSupply().SharableCPUs() leafAllocatableBefore := leaf.FreeSupply().AllocatableSharedCPU() - ancestorSharableBefore := make(map[string]cpuset.CPUSet, len(ancestors)) + ancestorSharableBefore := make(map[string]*libcpu.CpuMask, len(ancestors)) for _, a := range ancestors { ancestorSharableBefore[a.Name()] = a.FreeSupply().SharableCPUs() } @@ -263,7 +263,7 @@ func TestSupplyClaimCPUsReservedPartition(t *testing.T) { } reserved := reservedSupply.ReservedCPUs() - cpus := cpuset.New(reserved.List()[0]) + cpus := libcpu.NewCpuMask(reserved.List()[0]) uid := types.UID("claim-uid-reserved") reservedAllocatableBefore := reservedSupply.AllocatableReservedCPU() @@ -309,8 +309,8 @@ func TestSupplyClaimCPUsIdempotentReplace(t *testing.T) { } uid := types.UID("claim-uid-replace") - cpusA := cpuset.New(sharable[0]) - cpusB := cpuset.New(sharable[1]) + cpusA := libcpu.NewCpuMask(sharable[0]) + cpusB := libcpu.NewCpuMask(sharable[1]) before := leaf.FreeSupply().SharableCPUs() @@ -350,10 +350,10 @@ func TestSupplyUnclaimCPUsRestores(t *testing.T) { } uid := types.UID("claim-uid-unclaim") - cpus := cpuset.New(sharable[0]) + cpus := libcpu.NewCpuMask(sharable[0]) leafBefore := leaf.FreeSupply().SharableCPUs() - ancestorBefore := make(map[string]cpuset.CPUSet, len(ancestors)) + ancestorBefore := make(map[string]*libcpu.CpuMask, len(ancestors)) for _, a := range ancestors { ancestorBefore[a.Name()] = a.FreeSupply().SharableCPUs() } @@ -401,7 +401,7 @@ func TestSupplyCloneCarriesClaimRefs(t *testing.T) { } uid := types.UID("claim-uid-clone") - cpus := cpuset.New(sharable[0]) + cpus := libcpu.NewCpuMask(sharable[0]) leaf.FreeSupply().ClaimCPUs(uid, cpus) @@ -443,8 +443,8 @@ func TestSupplyClaimCPUsAncestorNotDoubleSubtracted(t *testing.T) { leafA.Name(), leafB.Name(), ancestor.Name(), leafA.Parent().Name(), leafB.Parent().Name()) } - cpusA := cpuset.New(leafA.FreeSupply().SharableCPUs().List()[0]) - cpusB := cpuset.New(leafB.FreeSupply().SharableCPUs().List()[0]) + cpusA := libcpu.NewCpuMask(leafA.FreeSupply().SharableCPUs().List()[0]) + cpusB := libcpu.NewCpuMask(leafB.FreeSupply().SharableCPUs().List()[0]) if cpusA.Intersection(cpusB).Size() != 0 { t.Fatalf("test setup error: cpusA and cpusB must be disjoint, got %s and %s", cpusA, cpusB) } diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go b/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go index 8305f91a9..78bf2cad2 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy_test.go @@ -17,6 +17,7 @@ package topologyaware import ( "context" "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "sync" "testing" @@ -29,7 +30,6 @@ import ( "github.com/containers/nri-plugins/pkg/resmgr/cache" "github.com/containers/nri-plugins/pkg/resmgr/dra" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // test helpers: a minimal real *dra.Plugin seeded with a live @@ -54,15 +54,15 @@ func (f *fakeDRADeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) // package cares about (Supply.ClaimCPUs/UnclaimCPUs) are exercised // separately via allocateClaim/releaseClaim, not via this allocator. type fakeDRAClaimAllocator struct { - pick cpuset.CPUSet + pick *libcpu.CpuMask } -func (f *fakeDRAClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { +func (f *fakeDRAClaimAllocator) PickHpCpus(_, _, _ int, _ *libcpu.CpuMask) (*libcpu.CpuMask, error) { return f.pick, nil } -func (f *fakeDRAClaimAllocator) ReleaseHpCpus(_, _ int, _ cpuset.CPUSet) {} -func (f *fakeDRAClaimAllocator) AccountHpCpus(_, _ int, _ cpuset.CPUSet) error { return nil } -func (f *fakeDRAClaimAllocator) IsHPClass(_ string) bool { return true } +func (f *fakeDRAClaimAllocator) ReleaseHpCpus(_, _ int, _ *libcpu.CpuMask) {} +func (f *fakeDRAClaimAllocator) AccountHpCpus(_, _ int, _ *libcpu.CpuMask) error { return nil } +func (f *fakeDRAClaimAllocator) IsHPClass(_ string) bool { return true } // fakeDRACDIWriter is a dra.CDIWriter that tracks per-UID "written" state // in memory instead of touching disk. Stateful (rather than a fixed @@ -104,7 +104,7 @@ func (*fakeDRAClaimStore) Load() (map[types.UID]*dra.ClaimState, error) { return // newTestDRAPlugin builds a real *dra.Plugin backed entirely by fakes, ready // for PrepareResourceClaims calls. pick is the CPUSet the fake allocator // hands out for every PickHpCpus call. -func newTestDRAPlugin(t *testing.T, pick cpuset.CPUSet, deviceName string) *dra.Plugin { +func newTestDRAPlugin(t *testing.T, pick *libcpu.CpuMask, deviceName string) *dra.Plugin { t.Helper() return newTestDRAPluginWithLock(t, pick, deviceName, func(f func()) { f() }) } @@ -113,7 +113,7 @@ func newTestDRAPlugin(t *testing.T, pick cpuset.CPUSet, deviceName string) *dra. // letting lock-contract tests share a single non-reentrant stub between the // DRA plugin's deps.WithLock and the policy's options.WithLock (both are // backed by the same resmgr write lock in production). -func newTestDRAPluginWithLock(t *testing.T, pick cpuset.CPUSet, deviceName string, withLock func(func())) *dra.Plugin { +func newTestDRAPluginWithLock(t *testing.T, pick *libcpu.CpuMask, deviceName string, withLock func(func())) *dra.Plugin { t.Helper() className := "gold" @@ -132,7 +132,7 @@ func newTestDRAPluginWithLock(t *testing.T, pick cpuset.CPUSet, deviceName strin RegistrarDir: t.TempDir(), PluginDataDir: t.TempDir(), ValidateClasses: func() error { return nil }, - ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + ValidateCPUsInPool: func(_ *libcpu.CpuMask) error { return nil }, DeviceLister: &fakeDRADeviceLister{devices: []resourceapi.Device{device}}, ClaimAllocator: &fakeDRAClaimAllocator{pick: pick}, CDIWriter: &fakeDRACDIWriter{}, @@ -206,7 +206,7 @@ func TestAllocateResourcesWithTAClaimCallsAllocateClaim(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0], sharable[1]) + claimed := libcpu.NewCpuMask(sharable[0], sharable[1]) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-alloc-1") @@ -239,7 +239,7 @@ func TestReleaseResourcesWithTAClaimCallsReleaseClaim(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0], sharable[1]) + claimed := libcpu.NewCpuMask(sharable[0], sharable[1]) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-release-1") @@ -281,7 +281,7 @@ func TestAllocateResourcesRollsBackClaimOnPoolAllocationFailure(t *testing.T) { if len(sharable) < 1 { t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0]) + claimed := libcpu.NewCpuMask(sharable[0]) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-rollback-1") @@ -335,7 +335,7 @@ func TestApplyGrantUnionsClaimedCPUsIntoContainerCpuset(t *testing.T) { t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) } claimedCPU := sharable[0] - claimed := cpuset.New(claimedCPU) + claimed := libcpu.NewCpuMask(claimedCPU) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-cpuset-union-1") @@ -355,7 +355,7 @@ func TestApplyGrantUnionsClaimedCPUsIntoContainerCpuset(t *testing.T) { t.Fatalf("AllocateResources() unexpected error: %v", err) } - gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + gotCpus, err := libcpu.ParseCpuMask(container.GetCpusetCpus()) if err != nil { t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) } @@ -387,7 +387,7 @@ func TestAllocateResourcesRollbackClearsClaimedCPUsByContainer(t *testing.T) { if len(sharable) < 1 { t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0]) + claimed := libcpu.NewCpuMask(sharable[0]) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-rollback-clear-1") @@ -429,15 +429,15 @@ func TestApplyGrantEmptyCpusUsesClaimedCPUsInsteadOfBlanking(t *testing.T) { } container := &mockContainer{returnValueForGetID: "empty-grant-claim-1"} - claimed := cpuset.New(leaf.FreeSupply().SharableCPUs().List()[0]) - p.claimedCPUsByContainer = map[string]cpuset.CPUSet{ + claimed := libcpu.NewCpuMask(leaf.FreeSupply().SharableCPUs().List()[0]) + p.claimedCPUsByContainer = map[string]*libcpu.CpuMask{ container.GetID(): claimed, } - g := newGrant(leaf, container, cpuReserved, "", cpuset.New(), 0, memoryDRAM, nil, 0) + g := newGrant(leaf, container, cpuReserved, "", libcpu.NewCpuMask(), 0, memoryDRAM, nil, 0) p.applyGrant(g) - gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + gotCpus, err := libcpu.ParseCpuMask(container.GetCpusetCpus()) if err != nil { t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) } @@ -466,19 +466,19 @@ func TestUpdateSharedAllocationsPreservesClaimedCPUsOnRepin(t *testing.T) { containerA := &mockContainer{returnValueForGetID: "shared-alloc-a"} claimACPU := sharable[0] - p.claimedCPUsByContainer = map[string]cpuset.CPUSet{ - containerA.GetID(): cpuset.New(claimACPU), + p.claimedCPUsByContainer = map[string]*libcpu.CpuMask{ + containerA.GetID(): libcpu.NewCpuMask(claimACPU), } - grantA := newGrant(leaf, containerA, cpuNormal, "", cpuset.New(), 100, memoryDRAM, nil, 0) + grantA := newGrant(leaf, containerA, cpuNormal, "", libcpu.NewCpuMask(), 100, memoryDRAM, nil, 0) p.allocations.addGrant(grantA) claimBCPU := sharable[1] uidB := types.UID("claim-shared-repin-b") - if err := p.allocateClaim(uidB, cpuset.New(claimBCPU), goldClassCPUs(cpuset.New(claimBCPU))); err != nil { + if err := p.allocateClaim(uidB, libcpu.NewCpuMask(claimBCPU), goldClassCPUs(libcpu.NewCpuMask(claimBCPU))); err != nil { t.Fatalf("allocateClaim() failed: %v", err) } - gotCpus, err := cpuset.Parse(containerA.GetCpusetCpus()) + gotCpus, err := libcpu.ParseCpuMask(containerA.GetCpusetCpus()) if err != nil { t.Fatalf("failed to parse container A's cpuset %q: %v", containerA.GetCpusetCpus(), err) } @@ -505,10 +505,10 @@ func TestReapplyDRAClaimsRepinsContainerCpusetWithGrant(t *testing.T) { } claimedCPU := sharable[0] grantCPU := sharable[1] - claimed := cpuset.New(claimedCPU) + claimed := libcpu.NewCpuMask(claimedCPU) container := &mockContainer{returnValueForGetID: "reapply-repin-1"} - addTestGrant(t, p, leaf, container, cpuset.New(grantCPU)) + addTestGrant(t, p, leaf, container, libcpu.NewCpuMask(grantCPU)) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-reapply-repin-1") @@ -524,11 +524,11 @@ func TestReapplyDRAClaimsRepinsContainerCpusetWithGrant(t *testing.T) { p.reapplyDRAClaims() - gotCpus, err := cpuset.Parse(container.GetCpusetCpus()) + gotCpus, err := libcpu.ParseCpuMask(container.GetCpusetCpus()) if err != nil { t.Fatalf("failed to parse container cpuset %q: %v", container.GetCpusetCpus(), err) } - want := cpuset.New(claimedCPU, grantCPU) + want := libcpu.NewCpuMask(claimedCPU, grantCPU) if !gotCpus.Equals(want) { t.Errorf("container cpuset after reapplyDRAClaims() = %s, want %s (claimed + grant, re-pinned without waiting for the next NRI resync)", gotCpus, want) } @@ -540,7 +540,7 @@ func TestReapplyDRAClaimsRepinsContainerCpusetWithGrant(t *testing.T) { // did before Step 8. func TestAllocateResourcesNoCDIDevicesUnaffected(t *testing.T) { p := newDRATestPolicy(t) - p.draPlugin = newTestDRAPlugin(t, cpuset.New(), "dev0") // no live claims seeded + p.draPlugin = newTestDRAPlugin(t, libcpu.NewCpuMask(), "dev0") // no live claims seeded container := &mockContainer{returnValueForGetID: "c1"} @@ -587,7 +587,7 @@ func TestStartMarksLiveDRAClaimsInPoolSupply(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0], sharable[1]) + claimed := libcpu.NewCpuMask(sharable[0], sharable[1]) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-start-1") @@ -617,7 +617,7 @@ func TestStartReappliesDRAClaimsAfterRestoreCache(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0], sharable[1]) + claimed := libcpu.NewCpuMask(sharable[0], sharable[1]) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-start-order-1") @@ -687,7 +687,7 @@ func TestStartReapplyDRAClaimsHoldsWriteLockNotReentrant(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0], sharable[1]) + claimed := libcpu.NewCpuMask(sharable[0], sharable[1]) plugin := newTestDRAPluginWithLock(t, claimed, "dev0", stub.run) uid := types.UID("claim-lock-contract-1") @@ -739,7 +739,7 @@ func TestClaimContainerRefsRebuiltAfterStartResync(t *testing.T) { if len(sharable) < 2 { t.Fatalf("expected at least 2 sharable CPUs on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0], sharable[1]) + claimed := libcpu.NewCpuMask(sharable[0], sharable[1]) plugin := newTestDRAPlugin(t, claimed, "dev0") uid := types.UID("claim-restart-resync-1") @@ -809,7 +809,7 @@ func TestReapplyDRAClaimsEvictsOverlappingRestoredGrant(t *testing.T) { if len(sharable) < 1 { t.Fatalf("expected at least 1 sharable CPU on %q", leaf.Name()) } - claimed := cpuset.New(sharable[0]) + claimed := libcpu.NewCpuMask(sharable[0]) // Simulate restoreCache()/restoreAllocations() having already reinstated // (or freshly reallocated) a regular grant that happens to overlap the diff --git a/pkg/resmgr/cpuclass/cpuclass_dra_test.go b/pkg/resmgr/cpuclass/cpuclass_dra_test.go index aad99d883..d1ea0360e 100644 --- a/pkg/resmgr/cpuclass/cpuclass_dra_test.go +++ b/pkg/resmgr/cpuclass/cpuclass_dra_test.go @@ -17,30 +17,19 @@ package cpuclass_test import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" + "github.com/containers/nri-plugins/pkg/lib/hardware" "testing" - - idset "github.com/intel/goresctrl/pkg/utils" + "testing/fstest" policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" "github.com/containers/nri-plugins/pkg/resmgr/dra" - "github.com/containers/nri-plugins/pkg/sysfs" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // Compile-time assertion: *Handler must satisfy dra.ClaimAllocator. var _ dra.ClaimAllocator = (*cpuclass.Handler)(nil) -// draTestSys is a minimal sysfs.System implementation for DRA pass-through -// tests. Only CPUIDs is overridden; all other methods are delegated to the -// embedded nil interface, which panics if called. In practice, only CPUIDs -// is invoked during Handler.New() (by cpufreq platform discovery). -type draTestSys struct { - sysfs.System -} - -func (s *draTestSys) CPUIDs() []idset.ID { return nil } - // newConfiguredHandler creates a Handler with an active managed PCT // allocator using the SST in-memory mock (OVERRIDE_SST). The mock is // seeded with one package (ID 0), one punit (ID 0), CPUs 0-7, and @@ -49,13 +38,13 @@ func newConfiguredHandler(t *testing.T) *cpuclass.Handler { t.Helper() t.Setenv("OVERRIDE_SST", `{"supported":true,"clos_count":4,"packages":[{"id":0,"cpus":"0-7","tf_supported":true,"tf_enabled":true,"cp_supported":true,"cp_enabled":false,"punits":[{"id":0,"cpus":"0-7","max_hp_cpus":4,"guaranteed_hp_cpus":4}]}]}`) t.Setenv("OVERRIDE_SST_STATE_DIR", t.TempDir()) - h, err := cpuclass.New(&draTestSys{}) + h, err := cpuclass.New(draTestMachine(t)) if err != nil { t.Fatalf("New() failed: %v", err) } if err := h.Configure(cpuclass.ConfigSpec{ Classes: []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, - Allowed: cpuset.MustParse("0-7"), + Allowed: libcpu.MustParseCpuMask("0-7"), }); err != nil { t.Fatalf("Configure() failed: %v", err) } @@ -70,14 +59,14 @@ func newInactiveHandler(t *testing.T) *cpuclass.Handler { t.Helper() // Ensure OVERRIDE_SST is unset (t.Setenv restores original value). t.Setenv("OVERRIDE_SST", "") - h, err := cpuclass.New(&draTestSys{}) + h, err := cpuclass.New(draTestMachine(t)) if err != nil { t.Fatalf("New() failed: %v", err) } // Configure with a PCT class; SST unsupported → pct stays disabled. _ = h.Configure(cpuclass.ConfigSpec{ Classes: []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, - Allowed: cpuset.MustParse("0-7"), + Allowed: libcpu.MustParseCpuMask("0-7"), }) return h } @@ -88,7 +77,7 @@ func newInactiveHandler(t *testing.T) *cpuclass.Handler { func TestHandlerPickHpCpus_NilHandler(t *testing.T) { var h *cpuclass.Handler - _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + _, err := h.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()) if err == nil { t.Fatal("expected error from nil handler, got nil") } @@ -96,7 +85,7 @@ func TestHandlerPickHpCpus_NilHandler(t *testing.T) { func TestHandlerPickHpCpus_NilPct(t *testing.T) { h := &cpuclass.Handler{} // pct is nil - _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + _, err := h.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()) if err == nil { t.Fatal("expected error from nil pct, got nil") } @@ -104,7 +93,7 @@ func TestHandlerPickHpCpus_NilPct(t *testing.T) { func TestHandlerPickHpCpus_InactivePct(t *testing.T) { h := newInactiveHandler(t) - _, err := h.PickHpCpus(0, 0, 1, cpuset.New()) + _, err := h.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()) if err == nil { t.Fatal("expected error from inactive PCT, got nil") } @@ -112,7 +101,7 @@ func TestHandlerPickHpCpus_InactivePct(t *testing.T) { func TestHandlerPickHpCpus_ActiveDelegates(t *testing.T) { h := newConfiguredHandler(t) - got, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + got, err := h.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus(0,0,2) = %v, want nil error", err) } @@ -120,7 +109,7 @@ func TestHandlerPickHpCpus_ActiveDelegates(t *testing.T) { t.Errorf("PickHpCpus(0,0,2) returned %d CPUs, want 2", got.Size()) } // Requesting more than GuaranteedHpCpus (4) must error. - if _, err := h.PickHpCpus(0, 0, 5, cpuset.New()); err == nil { + if _, err := h.PickHpCpus(0, 0, 5, libcpu.NewCpuMask()); err == nil { t.Error("PickHpCpus(0,0,5) with capacity=4: expected error, got nil") } } @@ -132,25 +121,25 @@ func TestHandlerPickHpCpus_ActiveDelegates(t *testing.T) { func TestHandlerReleaseHpCpus_NilHandler(t *testing.T) { var h *cpuclass.Handler // must not panic - h.ReleaseHpCpus(0, 0, cpuset.New()) + h.ReleaseHpCpus(0, 0, libcpu.NewCpuMask()) } func TestHandlerReleaseHpCpus_NilPct(t *testing.T) { h := &cpuclass.Handler{} // must not panic - h.ReleaseHpCpus(0, 0, cpuset.New()) + h.ReleaseHpCpus(0, 0, libcpu.NewCpuMask()) } func TestHandlerReleaseHpCpus_ActiveDelegates(t *testing.T) { h := newConfiguredHandler(t) // Pick 2 CPUs, then release them. - cpus, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + cpus, err := h.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus setup: %v", err) } h.ReleaseHpCpus(0, 0, cpus) // After release, the full 4-CPU capacity must be available again. - got, err := h.PickHpCpus(0, 0, 4, cpuset.New()) + got, err := h.PickHpCpus(0, 0, 4, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus after release: %v", err) } @@ -165,21 +154,21 @@ func TestHandlerReleaseHpCpus_ActiveDelegates(t *testing.T) { func TestHandlerAccountHpCpus_NilHandler(t *testing.T) { var h *cpuclass.Handler - if err := h.AccountHpCpus(0, 0, cpuset.New()); err == nil { + if err := h.AccountHpCpus(0, 0, libcpu.NewCpuMask()); err == nil { t.Fatal("expected error from nil handler, got nil") } } func TestHandlerAccountHpCpus_NilPct(t *testing.T) { h := &cpuclass.Handler{} - if err := h.AccountHpCpus(0, 0, cpuset.New()); err == nil { + if err := h.AccountHpCpus(0, 0, libcpu.NewCpuMask()); err == nil { t.Fatal("expected error from nil pct, got nil") } } func TestHandlerAccountHpCpus_InactivePct(t *testing.T) { h := newInactiveHandler(t) - if err := h.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + if err := h.AccountHpCpus(0, 0, libcpu.MustParseCpuMask("0")); err == nil { t.Fatal("expected error from inactive PCT, got nil") } } @@ -188,7 +177,7 @@ func TestHandlerAccountHpCpus_ActiveDelegates(t *testing.T) { h := newConfiguredHandler(t) // AccountHpCpus simulates restart reconciliation (union semantics, // no allocation): pick 2, release, then re-account. - cpus, err := h.PickHpCpus(0, 0, 2, cpuset.New()) + cpus, err := h.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus setup: %v", err) } @@ -242,3 +231,28 @@ func TestHandlerIsHPClass_ActiveDelegates(t *testing.T) { t.Error("IsHPClass(\"\") = true, want false") } } + +// draTestMachine is the smallest machine cpuclass.New will take. These tests +// drive the PCT allocator through the SST mock, so the machine's shape does not +// matter; what matters is that it is a real one, since *hardware.Machine is a +// concrete type and cannot be faked. +func draTestMachine(t *testing.T) *hardware.Machine { + t.Helper() + + file := func(s string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(s)} } + fsys := fstest.MapFS{ + "proc/meminfo": file("MemTotal: 1048576 kB\n"), + "sys/devices/system/cpu/online": file("0\n"), + "sys/devices/system/cpu/present": file("0\n"), + "sys/devices/system/cpu/possible": file("0\n"), + "sys/devices/system/cpu/cpu0/topology/physical_package_id": file("0\n"), + "sys/devices/system/cpu/cpu0/topology/core_id": file("0\n"), + "sys/devices/system/cpu/cpu0/topology/core_cpus_list": file("0\n"), + } + + m, err := hardware.Discover(hardware.WithFS(fsys)) + if err != nil { + t.Fatalf("failed to discover the test machine: %v", err) + } + return m +} diff --git a/pkg/resmgr/cpuclass/dra_test.go b/pkg/resmgr/cpuclass/dra_test.go index 99e0e9569..8592e2726 100644 --- a/pkg/resmgr/cpuclass/dra_test.go +++ b/pkg/resmgr/cpuclass/dra_test.go @@ -15,6 +15,7 @@ package cpuclass import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "regexp" "strings" "testing" @@ -24,7 +25,6 @@ import ( policyapi "github.com/containers/nri-plugins/pkg/apis/config/v1alpha1/resmgr/policy" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass/internal/pct" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func ptr[T any](v T) *T { return &v } @@ -788,7 +788,7 @@ func TestDRADevices(t *testing.T) { if err != nil { t.Fatalf("pct.NewAllocator: %v", err) } - if err := pctA.Configure(nil, cpuset.New()); err != nil { + if err := pctA.Configure(nil, libcpu.NewCpuMask()); err != nil { t.Fatalf("pct.Configure: %v", err) } h := &Handler{pct: pctA} @@ -813,7 +813,7 @@ func TestDRADevices(t *testing.T) { t.Fatalf("pct.NewAllocator: %v", err) } classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} - if err := pctA.Configure(classes, cpuset.New()); err != nil { + if err := pctA.Configure(classes, libcpu.NewCpuMask()); err != nil { t.Fatalf("pct.Configure: %v", err) } h := &Handler{pct: pctA, classes: nil} // classes not set @@ -840,7 +840,7 @@ func TestDRADevices(t *testing.T) { t.Fatalf("pct.NewAllocator: %v", err) } classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} - if err := pctA.Configure(classes, cpuset.New()); err != nil { + if err := pctA.Configure(classes, libcpu.NewCpuMask()); err != nil { t.Fatalf("pct.Configure: %v", err) } if !pctA.Active() { diff --git a/pkg/resmgr/cpuclass/internal/pct/pct_test.go b/pkg/resmgr/cpuclass/internal/pct/pct_test.go index 4e59ff6a0..0b53f3e74 100644 --- a/pkg/resmgr/cpuclass/internal/pct/pct_test.go +++ b/pkg/resmgr/cpuclass/internal/pct/pct_test.go @@ -16,8 +16,10 @@ package pct import ( "errors" + "fmt" "sort" "testing" + "testing/fstest" gosst "github.com/intel/goresctrl/pkg/sst" idset "github.com/intel/goresctrl/pkg/utils" @@ -36,10 +38,77 @@ var errFakeSstNoClos = errors.New("fakeSst: no CLOS for CPU") // frequency range and nothing else, so the tests below -- which are about CLOS // planning and association -- have no topology to provide. Turbo info then stays // nil, which is the same as on a platform whose CPUs expose no frequency data. -type fakeSys struct{} +// newMachine builds a machine with the given packages and the CPUs in each, by +// writing the layout out as the kernel would present it in sysfs and reading it +// back through real discovery. +// +// pct takes a Sys, which is two methods of *hardware.Machine, and a Machine is a +// concrete type which cannot be faked. So these tests describe the machine they +// want rather than standing something in for it, and the CPUs pct reads out of +// one are the CPUs it would read on the real thing. +func newMachine(t *testing.T, pkgCpus map[int]string, freqs ...map[int][3]uint64) *hardware.Machine { + t.Helper() + + file := func(s string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(s)} } + fsys := fstest.MapFS{"proc/meminfo": file("MemTotal: 1048576 kB\n")} + + all := libcpu.NewCpuMask() + for pkg, list := range pkgCpus { + cpus, err := libcpu.ParseCpuMask(list) + if err != nil { + t.Fatalf("package %d: bad cpu list %q: %v", pkg, list, err) + } + all = all.Union(cpus) + for _, cpu := range cpus.List() { + dir := fmt.Sprintf("sys/devices/system/cpu/cpu%d/topology", cpu) + fsys[dir+"/physical_package_id"] = file(fmt.Sprintf("%d\n", pkg)) + fsys[dir+"/core_id"] = file(fmt.Sprintf("%d\n", cpu)) + fsys[dir+"/core_cpus_list"] = file(fmt.Sprintf("%d\n", cpu)) + + // base, min and max, as cpufreq would report them + for _, f := range freqs { + khz, ok := f[cpu] + if !ok { + continue + } + cf := fmt.Sprintf("sys/devices/system/cpu/cpu%d/cpufreq", cpu) + for name, v := range map[string]uint64{ + "base_frequency": khz[0], + "cpuinfo_min_freq": khz[1], + "cpuinfo_max_freq": khz[2], + } { + if v != 0 { + fsys[cf+"/"+name] = file(fmt.Sprintf("%d\n", v)) + } + } + } + } + } + + list := all.String() + "\n" + fsys["sys/devices/system/cpu/online"] = file(list) + fsys["sys/devices/system/cpu/present"] = file(list) + fsys["sys/devices/system/cpu/possible"] = file(list) -func (*fakeSys) CPUIDs() []idset.ID { return nil } -func (*fakeSys) CPU(idset.ID) *hardware.CPU { return nil } + m, err := hardware.Discover(hardware.WithFS(fsys)) + if err != nil { + t.Fatalf("failed to discover the test machine: %v", err) + } + return m +} + +// newTwoPackageMachine has two packages of four CPUs each. +func newTwoPackageMachine(t *testing.T) *hardware.Machine { + t.Helper() + return newMachine(t, map[int]string{0: "0-3", 1: "4-7"}) +} + +// newTwoPunitMachine has two packages of eight CPUs each, for the layouts where +// each package holds two power units. +func newTwoPunitMachine(t *testing.T) *hardware.Machine { + t.Helper() + return newMachine(t, map[int]string{0: "0-7", 1: "8-15"}) +} // --- minimal sst fake ------------------------------------------------ @@ -140,7 +209,7 @@ func (s *fakeSst) TFStatus() (map[pctPunitID]bool, error) { // --- helpers to construct a hand-wired Allocator ----------------- func newManagedPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans map[string]*pctClassPlan, - allowed *libcpu.CpuMask, sys *fakeSys, sst *fakeSst) *Allocator { + allowed *libcpu.CpuMask, sys Sys, sst *fakeSst) *Allocator { t.Helper() a := &Allocator{ sys: sys, @@ -214,7 +283,7 @@ func pctTestWirePunits(a *Allocator) { // TestPctHintsNoClassNoOp covers the "no plan and not managed-with-HP" // branch where hints() must return an empty types.AllocationHints. func TestPctHintsNoClassNoOp(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{supported: true} // disabled allocator: hints must short-circuit to empty. @@ -242,7 +311,7 @@ func TestPctHintsNoClassNoOp(t *testing.T) { // branch in assoc-only mode: hints prefer free CPUs already // associated to the class's CLOS, enabling bin packing. func TestPctHintsAssocOnlyPreferClosCpus(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, // cpus 1, 2 and 3 already on CLOS 1, others on default CLOS 0. @@ -284,7 +353,7 @@ func TestPctHintsAssocOnlyPreferClosCpus(t *testing.T) { // branch: hints contain (a) free CPUs already on the HP CLOS for bin // packing and (b) the HP-reserve preference (largest-room package). func TestPctHintsHighPriorityReserveAndClosCpus(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, // cpus 0 and 1 already on CLOS 0 (HP), cpu 0 in use. @@ -348,7 +417,7 @@ func TestPctHintsHighPriorityReserveAndClosCpus(t *testing.T) { // hosting HP-class CPUs, so non-HP classes do not steal HP turbo // budget. THIS BRANCH IS NOT COVERED IN test19 e2e. func TestPctHintsManagedNonHpAvoidsHpInUse(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, cpuClos: map[int]int{}, @@ -403,7 +472,7 @@ func TestPctHintsManagedNonHpAvoidsHpInUse(t *testing.T) { // Allowed (via the handler-level intersectHints + pct-internal // allowed intersections). func TestPctHintsAllowedBoundsResults(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, cpuClos: map[int]int{1: 0, 4: 0}, // HP cpus on both packages @@ -465,7 +534,7 @@ func makeTwoPunitsPerPkg(hp0, hp1, hp2, hp3 int) []pctPunit { // HP work, punit-1 in the same package has full HP room. A request // for 1 HP CPU must steer to punit-1 (Tier A), not to pkg1. func TestPctHints_HpRoomTierAPunitWins(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPunitMachine(t) sst := &fakeSst{ supported: true, punits: makeTwoPunitsPerPkg(2, 2, 2, 2), @@ -515,7 +584,7 @@ func TestPctHints_HpRoomTierAPunitWins(t *testing.T) { // enough for the request. Pkg1 has only 1 HP slot in total. The // Tier-B aggregate must steer to pkg0 (free CPUs of both punits). func TestPctHints_HpRoomTierBSamePackage(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPunitMachine(t) sst := &fakeSst{ supported: true, punits: makeTwoPunitsPerPkg(2, 2, 1, 0), @@ -563,7 +632,7 @@ func TestPctHints_HpRoomTierBSamePackage(t *testing.T) { // allocator must return no HP-reserve hint so the caller falls back // to topology-only placement on the same socket. func TestPctHints_HpRoomTierCNoCrossPackage(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPunitMachine(t) sst := &fakeSst{ supported: true, // pkg0 has 2 HP CPUs total, pkg1 has 2 HP CPUs total. @@ -597,7 +666,7 @@ func TestPctHints_HpRoomTierCNoCrossPackage(t *testing.T) { // entire package. This is a regression guard for the punit-keyed // rewrite of hpInUseCpus. func TestPctHints_HpInUseIsPunitGranular(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPunitMachine(t) sst := &fakeSst{ supported: true, punits: makeTwoPunitsPerPkg(2, 2, 2, 2), @@ -830,7 +899,7 @@ func TestPctPunitGuaranteedHpCpus_NeitherSupported(t *testing.T) { // hpEligiblePunit must be set up by the caller after the helper // returns to keep the test intent explicit. func newAssocOnlyPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans map[string]*pctClassPlan, - allowed *libcpu.CpuMask, sys *fakeSys, sst *fakeSst) *Allocator { + allowed *libcpu.CpuMask, sys Sys, sst *fakeSst) *Allocator { t.Helper() a := &Allocator{ sys: sys, @@ -858,7 +927,7 @@ func newAssocOnlyPctForTest(t *testing.T, classes []*policyapi.CPUClass, plans m // -- not zero. (Pre-fix the result was 0 because closCpus(HP CLOS) // was empty.) func TestFreeClassCapacity_AssocOnlyHpFromFallbackCLOS(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, // All CPUs are on CLOS 3 (the LP/fallback CLOS). The HP @@ -905,7 +974,7 @@ func TestFreeClassCapacity_AssocOnlyHpFromFallbackCLOS(t *testing.T) { // GuaranteedHpCpus is non-zero. Prevents over-publishing HP // capacity on nodes that cannot actually deliver top turbo. func TestFreeClassCapacity_AssocOnlyHpTFDisabledPunitExcluded(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, punits: []pctPunit{ @@ -933,7 +1002,7 @@ func TestFreeClassCapacity_AssocOnlyHpTFDisabledPunitExcluded(t *testing.T) { // where no class was classified HP (e.g. no CLOS has a programmed // MaxFreq) falls through to the non-HP formula |Allowed \ held|. func TestFreeClassCapacity_AssocOnlyNoHpClassification(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, punits: []pctPunit{ @@ -958,7 +1027,7 @@ func TestFreeClassCapacity_AssocOnlyNoHpClassification(t *testing.T) { // (PrepareManagedMode enables SST-TF) and the result is the // guaranteed-top-turbo sum, capped by per-punit free CPUs. func TestFreeClassCapacity_ManagedHpRespectsEligibility(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{ supported: true, punits: []pctPunit{ @@ -996,7 +1065,7 @@ func TestFreeClassCapacity_ManagedHpRespectsEligibility(t *testing.T) { // TestFreeClassCapacity_UnknownClassReturnsZero: unknown class // (no PCT plan) yields 0 regardless of mode. func TestFreeClassCapacity_UnknownClassReturnsZero(t *testing.T) { - sys := &fakeSys{} + sys := newTwoPackageMachine(t) sst := &fakeSst{supported: true} a := newManagedPctForTest(t, []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}}, map[string]*pctClassPlan{"hp": {ClosID: 0}}, @@ -1010,19 +1079,19 @@ func TestFreeClassCapacity_UnknownClassReturnsZero(t *testing.T) { // given MaxHpCpus and GuaranteedHpCpus values. func makePunitsWithGtdHp(maxHp0, gtdHp0, maxHp1, gtdHp1 int) []pctPunit { return []pctPunit{ - {PkgID: 0, PunitID: 0, CPUs: cpuset.MustParse("0-3"), MaxHpCpus: maxHp0, GuaranteedHpCpus: gtdHp0}, - {PkgID: 0, PunitID: 1, CPUs: cpuset.MustParse("4-7"), MaxHpCpus: maxHp1, GuaranteedHpCpus: gtdHp1}, + {PkgID: 0, PunitID: 0, CPUs: libcpu.MustParseCpuMask("0-3"), MaxHpCpus: maxHp0, GuaranteedHpCpus: gtdHp0}, + {PkgID: 0, PunitID: 1, CPUs: libcpu.MustParseCpuMask("4-7"), MaxHpCpus: maxHp1, GuaranteedHpCpus: gtdHp1}, } } // newPickAllocator returns an Allocator pre-wired for PickHpCpus / ReleaseHpCpus tests. func newPickAllocator(t *testing.T, punits []pctPunit) *Allocator { t.Helper() - sys := newTwoPunitFakeSys() + sys := newTwoPunitMachine(t) sst := &fakeSst{supported: true, punits: punits} classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} plans := map[string]*pctClassPlan{"hp": {ClosID: 0}} - a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-7"), sys, sst) + a := newManagedPctForTest(t, classes, plans, libcpu.MustParseCpuMask("0-7"), sys, sst) return a } @@ -1065,7 +1134,7 @@ func TestPunitHPCapacity(t *testing.T) { // supply once some of the punit's CPUs are excluded (e.g. offline or // outside the reserved/shared pool). func TestPunitHPCapacity_CappedByAllowedIntersection(t *testing.T) { - sys := newTwoPunitFakeSys() + sys := newTwoPunitMachine(t) // Raw punit 0 spans CPUs 0-3 (4 CPUs) with GuaranteedHpCpus=3. // Restricting "allowed" to CPUs 0-1,4-7 leaves punit 0 with only // CPUs 0-1 (2 CPUs) after intersection -- less than its raw @@ -1073,7 +1142,7 @@ func TestPunitHPCapacity_CappedByAllowedIntersection(t *testing.T) { sst := &fakeSst{supported: true, punits: makePunitsWithGtdHp(4, 3, 4, 1)} classes := []*policyapi.CPUClass{{Name: "hp", PctPriority: "high"}} plans := map[string]*pctClassPlan{"hp": {ClosID: 0}} - a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-1,4-7"), sys, sst) + a := newManagedPctForTest(t, classes, plans, libcpu.MustParseCpuMask("0-1,4-7"), sys, sst) if got := a.punitHPCapacity(0); got != 2 { t.Errorf("punitHPCapacity(0) = %d, want 2 (capped by allowed intersection, not raw GuaranteedHpCpus=3)", got) @@ -1163,7 +1232,7 @@ func TestAllocatorPunits(t *testing.T) { func TestAllocatorPunits_NonDRAHpUsageReducesCapacity(t *testing.T) { a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 1)) // Simulate the NRI path having already claimed 1 HP CPU on punit 0. - a.hpUsed[0] = cpuset.MustParse("0") + a.hpUsed[0] = libcpu.MustParseCpuMask("0") pi := a.Punits() want := []PunitInfo{ @@ -1180,14 +1249,14 @@ func TestAllocatorPunits_NonDRAHpUsageReducesCapacity(t *testing.T) { func TestPickHpCpus(t *testing.T) { // Active()==false inactiveA := &Allocator{} - if _, err := inactiveA.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + if _, err := inactiveA.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()); err == nil { t.Error("PickHpCpus on inactive allocator: expected error, got nil") } a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) // Success: pick 2 CPUs from punit 0 (PkgID=0, PunitID=0, CPUs 0-3). - got, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + got, err := a.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus success case: %v", err) } @@ -1203,27 +1272,27 @@ func TestPickHpCpus(t *testing.T) { } // Exhaustion: already 2 DRA-held + 0 available for another pick. - if _, err := a.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + if _, err := a.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()); err == nil { t.Error("PickHpCpus exhaustion: expected error, got nil") } // HP-ineligible punit. a2 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) a2.hpEligiblePunit[0] = false - if _, err := a2.PickHpCpus(0, 0, 1, cpuset.New()); err == nil { + if _, err := a2.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()); err == nil { t.Error("PickHpCpus ineligible punit: expected error, got nil") } // (PkgID, PunitID) not found. a3 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) - if _, err := a3.PickHpCpus(99, 99, 1, cpuset.New()); err == nil { + if _, err := a3.PickHpCpus(99, 99, 1, libcpu.NewCpuMask()); err == nil { t.Error("PickHpCpus not-found: expected error, got nil") } // held exclusion: hold CPUs 0,1 → pick of 2 from a 4-CPU punit // must return 2,3 (the remaining ones). a4 := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) - held := cpuset.MustParse("0-1") + held := libcpu.MustParseCpuMask("0-1") got4, err := a4.PickHpCpus(0, 0, 2, held) if err != nil { t.Fatalf("PickHpCpus held-exclusion: %v", err) @@ -1237,7 +1306,7 @@ func TestReleaseHpCpus(t *testing.T) { a := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) // Pick 2 CPUs then release them. - picked, _ := a.PickHpCpus(0, 0, 2, cpuset.New()) + picked, _ := a.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) a.ReleaseHpCpus(0, 0, picked) if a.hpDRAUsed[0].Size() != 0 { t.Errorf("hpDRAUsed[0] after full release = %v, want empty", a.hpDRAUsed[0]) @@ -1248,14 +1317,14 @@ func TestReleaseHpCpus(t *testing.T) { } // Release CPUs not held — no-op. - a.ReleaseHpCpus(0, 0, cpuset.MustParse("0-1")) + a.ReleaseHpCpus(0, 0, libcpu.MustParseCpuMask("0-1")) // Out-of-range (not found) — no-op, no panic. - a.ReleaseHpCpus(99, 99, cpuset.MustParse("0")) + a.ReleaseHpCpus(99, 99, libcpu.MustParseCpuMask("0")) // Partial release. - picked2, _ := a.PickHpCpus(0, 0, 2, cpuset.New()) - first := cpuset.New(picked2.UnsortedList()[0]) + picked2, _ := a.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) + first := libcpu.NewCpuMask(picked2.UnsortedList()[0]) a.ReleaseHpCpus(0, 0, first) if a.hpDRAUsed[0].Size() != 1 { t.Errorf("hpDRAUsed[0] after partial release size = %d, want 1", a.hpDRAUsed[0].Size()) @@ -1265,7 +1334,7 @@ func TestReleaseHpCpus(t *testing.T) { func TestHpDRAUsedIsolation(t *testing.T) { // Build an allocator with both HP and LP classes to test that UseClass // on DRA-held CPUs does not corrupt the hpDRAUsed/hpUsed separation. - sys := newTwoPunitFakeSys() + sys := newTwoPunitMachine(t) sst := &fakeSst{supported: true, punits: makePunitsWithGtdHp(4, 2, 4, 2)} classes := []*policyapi.CPUClass{ {Name: "hp", PctPriority: "high"}, @@ -1275,10 +1344,10 @@ func TestHpDRAUsedIsolation(t *testing.T) { "hp": {ClosID: 0}, "lp": {ClosID: 3}, } - a := newManagedPctForTest(t, classes, plans, cpuset.MustParse("0-7"), sys, sst) + a := newManagedPctForTest(t, classes, plans, libcpu.MustParseCpuMask("0-7"), sys, sst) // DRA holds 2 CPUs on punit 0; hpUsed[0] is empty. - draHeld, err := a.PickHpCpus(0, 0, 2, cpuset.New()) + draHeld, err := a.PickHpCpus(0, 0, 2, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus: %v", err) } @@ -1312,7 +1381,7 @@ func TestHpDRAUsedIsolation(t *testing.T) { // non-HP class returns false; unknown class returns false; inactive allocator // returns false. func TestIsHPClass(t *testing.T) { - sys := newTwoPackageFakeSys() + sys := newTwoPackageMachine(t) sst := &fakeSst{supported: true, maxHp: map[int]int{0: 2, 1: 2}} classes := []*policyapi.CPUClass{ {Name: "hp", PctPriority: "high"}, @@ -1320,7 +1389,7 @@ func TestIsHPClass(t *testing.T) { } a := newManagedPctForTest(t, classes, map[string]*pctClassPlan{"hp": {ClosID: 0}, "lp": {ClosID: 3}}, - cpuset.MustParse("0-7"), sys, sst) + libcpu.MustParseCpuMask("0-7"), sys, sst) // HP class must return true. if !a.IsHPClass("hp") { @@ -1347,26 +1416,26 @@ func TestIsHPClass(t *testing.T) { func TestAccountHpCpus(t *testing.T) { // Inactive allocator must return an error. inactiveA := &Allocator{} - if err := inactiveA.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + if err := inactiveA.AccountHpCpus(0, 0, libcpu.MustParseCpuMask("0")); err == nil { t.Error("AccountHpCpus on inactive allocator: expected error, got nil") } // HP-ineligible punit must return an error. aInelig := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) aInelig.hpEligiblePunit[0] = false - if err := aInelig.AccountHpCpus(0, 0, cpuset.MustParse("0")); err == nil { + if err := aInelig.AccountHpCpus(0, 0, libcpu.MustParseCpuMask("0")); err == nil { t.Error("AccountHpCpus on HP-ineligible punit: expected error, got nil") } // Unknown punit must return an error. aUnknown := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) - if err := aUnknown.AccountHpCpus(99, 99, cpuset.MustParse("0")); err == nil { + if err := aUnknown.AccountHpCpus(99, 99, libcpu.MustParseCpuMask("0")); err == nil { t.Error("AccountHpCpus unknown punit: expected error, got nil") } // Success: account CPUs on an HP-eligible punit. aOK := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) - cpus := cpuset.MustParse("0-1") + cpus := libcpu.MustParseCpuMask("0-1") if err := aOK.AccountHpCpus(0, 0, cpus); err != nil { t.Fatalf("AccountHpCpus success case: %v", err) } @@ -1390,7 +1459,7 @@ func TestAccountHpCpus(t *testing.T) { // Must NOT return an error (container may already be running), and // hpDRAUsed must include all accounted CPUs. aOver := newPickAllocator(t, makePunitsWithGtdHp(4, 2, 4, 2)) // GuaranteedHpCpus=2 - overCommit := cpuset.MustParse("0-3") // 4 CPUs > GuaranteedHpCpus=2 + overCommit := libcpu.MustParseCpuMask("0-3") // 4 CPUs > GuaranteedHpCpus=2 if err := aOver.AccountHpCpus(0, 0, overCommit); err != nil { t.Fatalf("AccountHpCpus over-capacity: expected no error, got %v", err) } @@ -1406,14 +1475,14 @@ func TestHpReserveRoomWithDRAHolds(t *testing.T) { // Before any holds, room on punit 0 should be 2. // hpReserveCpus returns Tier-A candidate sets; if room>=requested we // get a candidate set back. Requesting 2 CPUs from punit 0 should succeed. - free := cpuset.MustParse("0-7") - before := a.hpReserveCpus(free, cpuset.New(), 2) + free := libcpu.MustParseCpuMask("0-7") + before := a.hpReserveCpus(free, libcpu.NewCpuMask(), 2) if len(before) == 0 { t.Fatal("hpReserveCpus before DRA holds: expected at least one candidate, got none") } // DRA picks 1 CPU on punit 0. - _, err := a.PickHpCpus(0, 0, 1, cpuset.New()) + _, err := a.PickHpCpus(0, 0, 1, libcpu.NewCpuMask()) if err != nil { t.Fatalf("PickHpCpus: %v", err) } @@ -1421,12 +1490,87 @@ func TestHpReserveRoomWithDRAHolds(t *testing.T) { // Now request 2 CPUs from punit 0: room is 1 (2 - 1 DRA hold), so // hpReserveCpus should not return punit 0 as a single-punit Tier-A // candidate for a request of 2. It may return punit 1 (unaffected). - after := a.hpReserveCpus(free, cpuset.New(), 2) + after := a.hpReserveCpus(free, libcpu.NewCpuMask(), 2) for _, candidate := range after { // No candidate set should include the DRA-held CPUs as "free" HP room // for a 2-CPU request on punit 0 alone. - if candidate.Intersection(cpuset.MustParse("0-3")).Size() > 1 { + if candidate.Intersection(libcpu.MustParseCpuMask("0-3")).Size() > 1 { t.Errorf("hpReserveCpus candidate includes punit 0 CPUs despite DRA hold reducing room to 1") } } } + +// TestDiscoverTurboInfo covers the one thing pct reads out of the system it is +// given. Everything else these tests need comes from the Speed Select fake, so +// this is what makes handing pct a machine worth anything. +// +// Its "no CPUs" and "nil CPU" branches are not covered, and cannot be through a +// machine: discovery refuses to build one without CPUs, and a machine never +// hands out a nil CPU. Those branches guard against a Sys which is not a +// machine. +func TestDiscoverTurboInfo(t *testing.T) { + for _, tc := range []struct { + name string + pkgCpus map[int]string + freqs map[int][3]uint64 // cpu -> {base, min, max}, in kHz + expect *turboInfo + expErr bool + }{ + { + name: "CPUs, but cpufreq says nothing", + pkgCpus: map[int]string{0: "0-1"}, + expErr: true, + }, + { + name: "base, min and max as reported", + pkgCpus: map[int]string{0: "0-1"}, + freqs: map[int][3]uint64{0: {2400000, 800000, 3600000}}, + expect: &turboInfo{ + baseFreqKHz: 2400000, + minFreqKHz: 800000, + maxTurboFreqKHz: 3600000, + }, + }, + { + // A CPU with no base_frequency file: the max stands in for it, + // which is what a machine without Speed Select looks like. + name: "no base frequency falls back to the maximum", + pkgCpus: map[int]string{0: "0-1"}, + freqs: map[int][3]uint64{0: {0, 800000, 3600000}}, + expect: &turboInfo{ + baseFreqKHz: 3600000, + minFreqKHz: 800000, + maxTurboFreqKHz: 3600000, + }, + }, + { + // The first CPU says nothing, so the second one answers. + name: "the first CPU with frequencies wins", + pkgCpus: map[int]string{0: "0-1"}, + freqs: map[int][3]uint64{1: {2000000, 1000000, 3000000}}, + expect: &turboInfo{ + baseFreqKHz: 2000000, + minFreqKHz: 1000000, + maxTurboFreqKHz: 3000000, + }, + }, + } { + t.Run(tc.name, func(t *testing.T) { + m := newMachine(t, tc.pkgCpus, tc.freqs) + + got, err := discoverTurboInfo(m) + switch { + case tc.expErr && err == nil: + t.Fatalf("expected an error, got %+v", got) + case !tc.expErr && err != nil: + t.Fatalf("unexpected error: %v", err) + case tc.expErr: + return + } + + if *got != *tc.expect { + t.Errorf("expected %+v, got %+v", *tc.expect, *got) + } + }) + } +} diff --git a/pkg/resmgr/dra/cdi_test.go b/pkg/resmgr/dra/cdi_test.go index 5626270a6..edbdd8b6f 100644 --- a/pkg/resmgr/dra/cdi_test.go +++ b/pkg/resmgr/dra/cdi_test.go @@ -17,6 +17,7 @@ limitations under the License. package dra import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "os" "path/filepath" "strings" @@ -25,8 +26,6 @@ import ( "k8s.io/apimachinery/pkg/types" "tags.cncf.io/container-device-interface/pkg/parser" - - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // TestCDIDeviceName_BasicCase verifies the basic format of a CDI device name. @@ -108,7 +107,7 @@ func TestWriteClaim_EnvVarsOnDisk(t *testing.T) { } uid := types.UID("test-claim-uid-0001") - cpus, _ := cpuset.Parse("0,2,4") + cpus, _ := libcpu.ParseCpuMask("0,2,4") devices := []CDIDevice{ {Name: "claim-test-claim-uid-0001-myreq-punit-0-0-0", ClassName: "gold", CPUs: cpus}, } @@ -165,7 +164,7 @@ func TestRemoveClaim_Removes(t *testing.T) { } uid := types.UID("test-uid-remove-0001") - cpus, _ := cpuset.Parse("0") + cpus, _ := libcpu.ParseCpuMask("0") devs := []CDIDevice{{Name: "claim-test-uid-remove-0001-req-dev-0", ClassName: "silver", CPUs: cpus}} if err := w.WriteClaim(uid, devs); err != nil { t.Fatalf("WriteClaim() unexpected error: %v", err) @@ -209,7 +208,7 @@ func TestClaimSpecExists_TrueAndFalse(t *testing.T) { t.Error("ClaimSpecExists() = true before WriteClaim, want false") } - cpus, _ := cpuset.Parse("1") + cpus, _ := libcpu.ParseCpuMask("1") devs := []CDIDevice{{Name: "claim-test-uid-exists-0001-req-dev-0", ClassName: "gold", CPUs: cpus}} if err := w.WriteClaim(uid, devs); err != nil { t.Fatalf("WriteClaim() unexpected error: %v", err) @@ -230,7 +229,7 @@ func TestListClaims_TwoClaims(t *testing.T) { uid1 := types.UID("test-uid-list-0001") uid2 := types.UID("test-uid-list-0002") - cpus, _ := cpuset.Parse("0") + cpus, _ := libcpu.ParseCpuMask("0") for _, uid := range []types.UID{uid1, uid2} { devs := []CDIDevice{{Name: "claim-" + string(uid) + "-req-dev-0", ClassName: "gold", CPUs: cpus}} @@ -274,7 +273,7 @@ func TestListClaims_ForeignSpecSurvives(t *testing.T) { // Write a valid claim. uid := types.UID("test-uid-foreign-0001") - cpus, _ := cpuset.Parse("0") + cpus, _ := libcpu.ParseCpuMask("0") devs := []CDIDevice{{Name: "claim-test-uid-foreign-0001-req-dev-0", ClassName: "gold", CPUs: cpus}} if err := w.WriteClaim(uid, devs); err != nil { t.Fatalf("WriteClaim() unexpected error: %v", err) @@ -314,7 +313,7 @@ func TestWriteClaim_SameRequestDeviceTwoIdx(t *testing.T) { } uid := types.UID("test-uid-shared-0001") - cpus, _ := cpuset.Parse("0") + cpus, _ := libcpu.ParseCpuMask("0") name0 := CDIDeviceName(uid, "myrequest", "punit-0-0", 0) name1 := CDIDeviceName(uid, "myrequest", "punit-0-0", 1) devs := []CDIDevice{ diff --git a/pkg/resmgr/dra/plugin_test.go b/pkg/resmgr/dra/plugin_test.go index 5ad7092f1..7a8bc4522 100644 --- a/pkg/resmgr/dra/plugin_test.go +++ b/pkg/resmgr/dra/plugin_test.go @@ -20,6 +20,7 @@ import ( "context" "errors" "fmt" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "runtime" "strings" "sync" @@ -38,7 +39,6 @@ import ( "github.com/containers/nri-plugins/pkg/log" "github.com/containers/nri-plugins/pkg/resmgr/cpuclass" - "github.com/containers/nri-plugins/pkg/utils/cpuset" "tags.cncf.io/container-device-interface/pkg/parser" ) @@ -59,7 +59,7 @@ func validDeps() Deps { KubeClient: fake.NewClientset(), NodeName: "test-node", ValidateClasses: func() error { return nil }, - ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + ValidateCPUsInPool: func(_ *libcpu.CpuMask) error { return nil }, DeviceLister: &fixedDeviceLister{}, ClaimAllocator: &noopClaimAllocator{}, CDIWriter: &noopCDIWriter{}, @@ -338,13 +338,13 @@ func (e *errorDeviceLister) DRADevices(_ string) ([]resourceapi.Device, error) { // noopClaimAllocator is a ClaimAllocator that succeeds without doing anything. type noopClaimAllocator struct{} -func (*noopClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { - return cpuset.New(), nil +func (*noopClaimAllocator) PickHpCpus(_, _, _ int, _ *libcpu.CpuMask) (*libcpu.CpuMask, error) { + return libcpu.NewCpuMask(), nil } -func (*noopClaimAllocator) ReleaseHpCpus(_, _ int, _ cpuset.CPUSet) {} +func (*noopClaimAllocator) ReleaseHpCpus(_, _ int, _ *libcpu.CpuMask) {} -func (*noopClaimAllocator) AccountHpCpus(_, _ int, _ cpuset.CPUSet) error { return nil } +func (*noopClaimAllocator) AccountHpCpus(_, _ int, _ *libcpu.CpuMask) error { return nil } func (*noopClaimAllocator) IsHPClass(_ string) bool { return false } @@ -540,7 +540,7 @@ func TestPublishResources_Integration(t *testing.T) { RegistrarDir: registrarDir, PluginDataDir: pluginDataDir, ValidateClasses: func() error { return nil }, - ValidateCPUsInPool: func(_ cpuset.CPUSet) error { return nil }, + ValidateCPUsInPool: func(_ *libcpu.CpuMask) error { return nil }, DeviceLister: &fixedDeviceLister{devices: makeTestDevices(5)}, ClaimAllocator: &noopClaimAllocator{}, CDIWriter: &noopCDIWriter{}, @@ -594,28 +594,28 @@ func TestPublishResources_Integration(t *testing.T) { // trackingClaimAllocator tracks PickHpCpus and ReleaseHpCpus calls. type trackingClaimAllocator struct { - pickResult cpuset.CPUSet + pickResult *libcpu.CpuMask pickErr error isHP bool - picks []cpuset.CPUSet // CPUSets returned per PickHpCpus call - releases []cpuset.CPUSet // CPUSets released per ReleaseHpCpus call - accounts []cpuset.CPUSet // CPUSets accounted per AccountHpCpus call + picks []*libcpu.CpuMask // CPUSets returned per PickHpCpus call + releases []*libcpu.CpuMask // CPUSets released per ReleaseHpCpus call + accounts []*libcpu.CpuMask // CPUSets accounted per AccountHpCpus call accountErr error } -func (a *trackingClaimAllocator) PickHpCpus(_, _, _ int, _ cpuset.CPUSet) (cpuset.CPUSet, error) { +func (a *trackingClaimAllocator) PickHpCpus(_, _, _ int, _ *libcpu.CpuMask) (*libcpu.CpuMask, error) { if a.pickErr != nil { - return cpuset.New(), a.pickErr + return libcpu.NewCpuMask(), a.pickErr } a.picks = append(a.picks, a.pickResult) return a.pickResult, nil } -func (a *trackingClaimAllocator) ReleaseHpCpus(_, _ int, cpus cpuset.CPUSet) { +func (a *trackingClaimAllocator) ReleaseHpCpus(_, _ int, cpus *libcpu.CpuMask) { a.releases = append(a.releases, cpus) } -func (a *trackingClaimAllocator) AccountHpCpus(_, _ int, cpus cpuset.CPUSet) error { +func (a *trackingClaimAllocator) AccountHpCpus(_, _ int, cpus *libcpu.CpuMask) error { if a.accountErr != nil { return a.accountErr } @@ -719,7 +719,7 @@ func makeClaim(uid types.UID, driverName, poolName, deviceName, request string, // TestPrepare_SingleHPSuccess verifies that a single HP claim results in a // PrepareResult with one Device and that CPUs are picked and CDI is written. func TestPrepare_SingleHPSuccess(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} cdiW := &trackingCDIWriter{} store := &trackingClaimStore{} deps := validDeps() @@ -768,7 +768,7 @@ func TestPrepare_SingleHPSuccess(t *testing.T) { // same claim with the CDI spec already present returns the same PrepareResult // without re-picking CPUs or re-writing the spec. func TestPrepare_Idempotent_SpecPresent(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} cdiW := &trackingCDIWriter{} store := &trackingClaimStore{} deps := validDeps() @@ -830,7 +830,7 @@ func TestPrepare_Idempotent_SpecPresent(t *testing.T) { // TestPrepare_Idempotent_SpecMissing verifies that a second Prepare call for the // same claim where the CDI spec is missing re-writes the spec without re-picking CPUs. func TestPrepare_Idempotent_SpecMissing(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} cdiW := &trackingCDIWriter{} store := &trackingClaimStore{} deps := validDeps() @@ -1079,7 +1079,7 @@ func TestPrepare_NilAttr(t *testing.T) { // (for packageID or punitID) is handled gracefully: the field defaults to 0 // and Prepare succeeds when all other required attrs are present. func TestPrepare_NilIntAttr(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} deps := validDeps() deps.ClaimAllocator = alloc deps.CDIWriter = &trackingCDIWriter{} @@ -1210,7 +1210,7 @@ func TestPrepare_PickFailure(t *testing.T) { // the picked CPUs and returns a per-claim error. func TestPrepare_CDIWriteFailure(t *testing.T) { writeErr := errors.New("CDI write failed") - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} cdiW := &trackingCDIWriter{writeErr: writeErr} deps := validDeps() deps.ClaimAllocator = alloc @@ -1252,7 +1252,7 @@ func TestPrepare_CDIWriteFailure(t *testing.T) { // state for a claim whose CDI spec/CPUs are already live. func TestPrepare_ClaimStoreSaveFailure(t *testing.T) { saveErr := errors.New("claim store save failed") - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} cdiW := &trackingCDIWriter{} store := &trackingClaimStore{saveErr: saveErr} deps := validDeps() @@ -1296,7 +1296,7 @@ func TestPrepare_ClaimStoreSaveFailure(t *testing.T) { // TestPrepare_MultiResultTwoPunits verifies that a claim spanning punit pools // is rejected before it can be committed. func TestPrepare_MultiResultTwoPunits(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-1"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-1"), isHP: true} cdiW := &trackingCDIWriter{} deps := validDeps() deps.ClaimAllocator = alloc @@ -1356,7 +1356,7 @@ func TestPrepare_MultiResultTwoPunits(t *testing.T) { // topology leaf pool). func TestPrepare_ClaimCPUsOutsideAllocationDomain(t *testing.T) { validateErr := errors.New("spans multiple leaf pools") - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} cdiW := &trackingCDIWriter{} store := &trackingClaimStore{} deps := validDeps() @@ -1364,7 +1364,7 @@ func TestPrepare_ClaimCPUsOutsideAllocationDomain(t *testing.T) { deps.CDIWriter = cdiW deps.ClaimStore = store deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) - deps.ValidateCPUsInPool = func(_ cpuset.CPUSet) error { return validateErr } + deps.ValidateCPUsInPool = func(_ *libcpu.CpuMask) error { return validateErr } p, err := New("test-driver", deps) if err != nil { @@ -1398,7 +1398,7 @@ func TestPrepare_ClaimCPUsOutsideAllocationDomain(t *testing.T) { // TestPrepare_ShareIDNil verifies that a result with a nil ShareID produces a // Device with ShareID == nil. func TestPrepare_ShareIDNil(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} deps := validDeps() deps.ClaimAllocator = alloc deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) @@ -1423,7 +1423,7 @@ func TestPrepare_ShareIDNil(t *testing.T) { // TestPrepare_ShareIDSet verifies that a result with a non-nil ShareID produces a // Device with a matching non-nil ShareID. func TestPrepare_ShareIDSet(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} deps := validDeps() deps.ClaimAllocator = alloc deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) @@ -1465,7 +1465,7 @@ func TestPrepare_ShareIDSet(t *testing.T) { // TestPrepare_AllUIDsInResultMap verifies that every claim UID appears in the // result map even when some claims error. func TestPrepare_AllUIDsInResultMap(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} deps := validDeps() deps.ClaimAllocator = alloc deps.DeviceLister = hpDeviceLister(hpDevice("dev0", "gold", 0, 0)) @@ -1503,7 +1503,7 @@ func TestPrepare_AllUIDsInResultMap(t *testing.T) { // (FirstAvailable subrequest format) produces a valid CDI device name and that // the spec can be written successfully. func TestPrepare_SubrequestSlashInName(t *testing.T) { - alloc := &trackingClaimAllocator{pickResult: cpuset.MustParse("0-3"), isHP: true} + alloc := &trackingClaimAllocator{pickResult: libcpu.MustParseCpuMask("0-3"), isHP: true} cdiW := &trackingCDIWriter{} deps := validDeps() deps.ClaimAllocator = alloc @@ -1901,11 +1901,11 @@ func TestRestoreClaimsLocked_RebuildsAccounting(t *testing.T) { } // Verify the union of all accounted CPU sets matches the expected total. // Map iteration order is non-deterministic so we check the union. - totalAccounted := cpuset.New() + totalAccounted := libcpu.NewCpuMask() for _, cs := range alloc.accounts { totalAccounted = totalAccounted.Union(cs) } - wantTotal := cpuset.MustParse("0-7") + wantTotal := libcpu.MustParseCpuMask("0-7") if !totalAccounted.Equals(wantTotal) { t.Errorf("AccountHpCpus total CPUs = %v, want %v", totalAccounted, wantTotal) } diff --git a/pkg/resmgr/dra/state_test.go b/pkg/resmgr/dra/state_test.go index de7fe5e2a..ffb1fde2d 100644 --- a/pkg/resmgr/dra/state_test.go +++ b/pkg/resmgr/dra/state_test.go @@ -17,6 +17,7 @@ limitations under the License. package dra import ( + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "os" "path/filepath" "testing" @@ -24,7 +25,6 @@ import ( "k8s.io/apimachinery/pkg/types" "github.com/containers/nri-plugins/pkg/resmgr/cache" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) // newTestCache creates a real cache in a temporary directory. The cache rejects @@ -48,7 +48,7 @@ func newTestCache(t *testing.T) cache.Cache { // survives a marshal → unmarshal round-trip without data loss. func TestMarshalUnmarshalClaims_RoundTrip(t *testing.T) { uid := types.UID("test-uid-1234") - originalCPUs := cpuset.New(0, 1, 2, 3) + originalCPUs := libcpu.NewCpuMask(0, 1, 2, 3) claims := map[types.UID]*ClaimState{ uid: { @@ -72,7 +72,7 @@ func TestMarshalUnmarshalClaims_RoundTrip(t *testing.T) { ClassName: "hp", PkgID: 0, PunitID: 1, - CPUs: cpuset.New(4, 5).String(), + CPUs: libcpu.NewCpuMask(4, 5).String(), }, }, }, @@ -100,9 +100,9 @@ func TestMarshalUnmarshalClaims_RoundTrip(t *testing.T) { } // Verify CPUs round-trip via cpuset.Parse. - parsedCPUs, err := cpuset.Parse(cs.Allocs[0].CPUs) + parsedCPUs, err := libcpu.ParseCpuMask(cs.Allocs[0].CPUs) if err != nil { - t.Fatalf("cpuset.Parse(%q) error: %v", cs.Allocs[0].CPUs, err) + t.Fatalf("libcpu.ParseCpuMask(%q) error: %v", cs.Allocs[0].CPUs, err) } if !parsedCPUs.Equals(originalCPUs) { t.Errorf("CPUs round-trip: got %v, want %v", parsedCPUs, originalCPUs) @@ -142,7 +142,7 @@ func TestCacheClaimStore_RoundTrip(t *testing.T) { c := newTestCache(t) uid := types.UID("claim-abc") - originalCPUs := cpuset.New(10, 11, 12, 13) + originalCPUs := libcpu.NewCpuMask(10, 11, 12, 13) claims := map[types.UID]*ClaimState{ uid: { @@ -183,9 +183,9 @@ func TestCacheClaimStore_RoundTrip(t *testing.T) { } // Verify CPUs survive round-trip. - parsedCPUs, err := cpuset.Parse(cs.Allocs[0].CPUs) + parsedCPUs, err := libcpu.ParseCpuMask(cs.Allocs[0].CPUs) if err != nil { - t.Fatalf("cpuset.Parse(%q) error: %v", cs.Allocs[0].CPUs, err) + t.Fatalf("libcpu.ParseCpuMask(%q) error: %v", cs.Allocs[0].CPUs, err) } if !parsedCPUs.Equals(originalCPUs) { t.Errorf("CPU round-trip: got %v, want %v", parsedCPUs, originalCPUs) @@ -225,7 +225,7 @@ func TestCacheClaimStore_LoadReturnsSavedData(t *testing.T) { ClassName: "hp", PkgID: 0, PunitID: 0, - CPUs: cpuset.New(7, 8).String(), + CPUs: libcpu.NewCpuMask(7, 8).String(), }, }, }, @@ -259,10 +259,10 @@ func TestCacheClaimStore_MultiClaim(t *testing.T) { claims := map[types.UID]*ClaimState{ uid1: {UID: string(uid1), Allocs: []ResultAlloc{ - {Request: "r1", Pool: "p", Device: "d1", ClassName: "hp", PkgID: 0, PunitID: 0, CPUs: cpuset.New(0).String()}, + {Request: "r1", Pool: "p", Device: "d1", ClassName: "hp", PkgID: 0, PunitID: 0, CPUs: libcpu.NewCpuMask(0).String()}, }}, uid2: {UID: string(uid2), Allocs: []ResultAlloc{ - {Request: "r2", Pool: "p", Device: "d2", ClassName: "hp", PkgID: 0, PunitID: 1, CPUs: cpuset.New(1).String()}, + {Request: "r2", Pool: "p", Device: "d2", ClassName: "hp", PkgID: 0, PunitID: 1, CPUs: libcpu.NewCpuMask(1).String()}, }}, } diff --git a/pkg/resmgr/policy/policy_test.go b/pkg/resmgr/policy/policy_test.go index 7cc3f2c59..82ec4bffc 100644 --- a/pkg/resmgr/policy/policy_test.go +++ b/pkg/resmgr/policy/policy_test.go @@ -15,9 +15,11 @@ package policy import ( + "github.com/containers/nri-plugins/pkg/lib/hardware" "path/filepath" "sync" "testing" + "testing/fstest" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -113,7 +115,7 @@ var _ Backend = &mockBackend{} func TestPolicyStopForwardsToBackend(t *testing.T) { backend := &mockBackend{} - p, err := NewPolicy(backend, newTestCache(t), &Options{}) + p, err := NewPolicy(backend, newTestCache(t), &Options{Machine: testMachine(t)}) require.NoError(t, err) require.NoError(t, p.Start(nil)) @@ -141,6 +143,7 @@ func TestPolicyStartForwardsKubeClientFnNodeNameAndWithLock(t *testing.T) { } p, err := NewPolicy(backend, newTestCache(t), &Options{ + Machine: testMachine(t), KubeClientFn: func() kubernetes.Interface { return wantClient }, NodeName: "node-under-test", WithLock: withLock, @@ -165,6 +168,7 @@ func TestPolicyStartForwardsKubeClientFnNodeNameAndWithLock(t *testing.T) { func TestPolicyStartKubeClientFnNilWhenNoClient(t *testing.T) { backend := &mockBackend{} p, err := NewPolicy(backend, newTestCache(t), &Options{ + Machine: testMachine(t), KubeClientFn: func() kubernetes.Interface { return nil }, }) require.NoError(t, err) @@ -220,7 +224,7 @@ func TestLockContractWithLockNotReentrant(t *testing.T) { }, } - p, err := NewPolicy(backend, newTestCache(t), &Options{WithLock: stub.run}) + p, err := NewPolicy(backend, newTestCache(t), &Options{Machine: testMachine(t), WithLock: stub.run}) require.NoError(t, err) assert.NotPanics(t, func() { @@ -247,3 +251,27 @@ func TestLockContractReentrantCallPanics(t *testing.T) { type fakeKubeClient struct { kubernetes.Interface } + +// testMachine is the smallest machine NewPolicy will accept. These tests check +// what the policy forwards to its backend, not anything about the hardware, but +// a *hardware.Machine is concrete and cannot be faked. +func testMachine(t *testing.T) *hardware.Machine { + t.Helper() + + file := func(s string) *fstest.MapFile { return &fstest.MapFile{Data: []byte(s)} } + fsys := fstest.MapFS{ + "proc/meminfo": file("MemTotal: 1048576 kB\n"), + "sys/devices/system/cpu/online": file("0\n"), + "sys/devices/system/cpu/present": file("0\n"), + "sys/devices/system/cpu/possible": file("0\n"), + "sys/devices/system/cpu/cpu0/topology/physical_package_id": file("0\n"), + "sys/devices/system/cpu/cpu0/topology/core_id": file("0\n"), + "sys/devices/system/cpu/cpu0/topology/core_cpus_list": file("0\n"), + } + + m, err := hardware.Discover(hardware.WithFS(fsys)) + if err != nil { + t.Fatalf("failed to discover the test machine: %v", err) + } + return m +} From bd0638635019b869a0a4079659950e01bc5359c8 Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 11 Sep 2026 14:58:41 +0300 Subject: [PATCH 38/39] libmem: take CPUs as a libcpu.CpuMask. libmem was the last consumer keeping a converter of its own. A memory node holds the set of CPUs closest to it, and that set is now a mask: WithMachineNodes hands the machine's own set straight to NewNode instead of listing it out and rebuilding it, and CPUSetAffinity takes anything which satisfies the libcpu interface, since all it does is ask what the set intersects. NewNode still takes its own copy of the CPUs, and now seals it. CloseCPUs hands that copy out, and a cpuset.CPUSet, which this used to be, could not be modified by the receiver either, so sealing is what keeps the node's set as immutable as it was. That retires balloons' toCpuSet. The only k8s set left in that policy comes in from the configuration, so toCpuMask stays. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- .../balloons/policy/balloons-policy.go | 12 ++++-------- pkg/resmgr/lib/memory/allocator.go | 10 +++++----- pkg/resmgr/lib/memory/allocator_test.go | 6 +++--- pkg/resmgr/lib/memory/nodes.go | 19 ++++++++++++------- 4 files changed, 24 insertions(+), 23 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 5cf49395e..9d464250f 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -147,13 +147,9 @@ type loadClassVirtDev struct { updateOnEveryCpuAllocation bool } -// toCpuSet and toCpuMask convert between the CPU sets this policy keeps and the -// two interfaces which still take the k8s ones: libmem's CPUSetAffinity, and the -// configuration, which parses an operator's cpuset string. -func toCpuSet(cpus libcpu.CPUSet) cpuset.CPUSet { - return cpuset.New(cpus.List()...) -} - +// toCpuMask converts a CPU set into the kind this policy keeps. It is needed +// for the one interface which still hands out a k8s one: the configuration, +// which parses an operator's cpuset string. func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { return libcpu.NewCpuMask(cpus.List()...) } @@ -2496,7 +2492,7 @@ func memTypeMaskFromStringList(memTypes []string) (libmem.TypeMask, error) { // closestMems returns memory node IDs good for pinning containers // that run on given CPUs func (p *balloons) closestMems(cpus *libcpu.CpuMask) idset.IDSet { - return idset.NewIDSet(p.memAllocator.CPUSetAffinity(toCpuSet(cpus)).Slice()...) + return idset.NewIDSet(p.memAllocator.CPUSetAffinity(cpus).Slice()...) } // resizeCompositeBalloon changes the CPUs allocated for all sub-components diff --git a/pkg/resmgr/lib/memory/allocator.go b/pkg/resmgr/lib/memory/allocator.go index 0e5f66962..40ce684ee 100644 --- a/pkg/resmgr/lib/memory/allocator.go +++ b/pkg/resmgr/lib/memory/allocator.go @@ -21,8 +21,8 @@ import ( "slices" "strings" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" - "github.com/containers/nri-plugins/pkg/utils/cpuset" idset "github.com/intel/goresctrl/pkg/utils" ) @@ -91,7 +91,7 @@ func WithMachineNodes(m *hardware.Machine) AllocatorOption { memType = TypeForKind(node.Kind()) capacity = node.Capacity() isNormal = node.HasNormalMemory() - closeCPUs = cpuset.New(node.CPUs().List()...) + closeCPUs = node.CPUs() distance = node.Distances() ) @@ -136,11 +136,11 @@ func (a *Allocator) Masks() *MaskCache { return a.masks } -// CPUSetAffinity returns the mask of closest nodes for the given cpuset. -func (a *Allocator) CPUSetAffinity(cpus cpuset.CPUSet) NodeMask { +// CPUSetAffinity returns the mask of closest nodes for the given set of CPUs. +func (a *Allocator) CPUSetAffinity(cpus libcpu.CPUSet) NodeMask { nodes := NodeMask(0) a.ForeachNode(a.masks.nodes.all, func(n *Node) bool { - if !cpus.Intersection(n.cpus).IsEmpty() { + if n.cpus.Intersects(cpus) { nodes |= n.Mask() } return ForeachMore diff --git a/pkg/resmgr/lib/memory/allocator_test.go b/pkg/resmgr/lib/memory/allocator_test.go index 89742e7d7..f69a1d798 100644 --- a/pkg/resmgr/lib/memory/allocator_test.go +++ b/pkg/resmgr/lib/memory/allocator_test.go @@ -20,9 +20,9 @@ import ( "github.com/stretchr/testify/require" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "github.com/containers/nri-plugins/pkg/lib/hardware" . "github.com/containers/nri-plugins/pkg/resmgr/lib/memory" - "github.com/containers/nri-plugins/pkg/utils/cpuset" ) func TestNewAllocatorWithMachineNodes(t *testing.T) { @@ -178,7 +178,7 @@ func TestCPUSetAffinity(t *testing.T) { }, } { t.Run(tc.name, func(t *testing.T) { - require.Equal(t, tc.affinity, a.CPUSetAffinity(cpuset.New(tc.cpus...))) + require.Equal(t, tc.affinity, a.CPUSetAffinity(libcpu.NewCpuMask(tc.cpus...))) }) } } @@ -1436,7 +1436,7 @@ func (s *testSetup) nodes(t *testing.T) []*Node { var ( capacity = s.capacities[id] normal = !s.movability[id] - closeCPUs = cpuset.New(s.closeCPUs[id]...) + closeCPUs = libcpu.NewCpuMask(s.closeCPUs[id]...) distance = s.distances[id] ) diff --git a/pkg/resmgr/lib/memory/nodes.go b/pkg/resmgr/lib/memory/nodes.go index 6f27e8afb..de377ef0f 100644 --- a/pkg/resmgr/lib/memory/nodes.go +++ b/pkg/resmgr/lib/memory/nodes.go @@ -23,7 +23,7 @@ import ( "strconv" "strings" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" ) // Node represents a memory node with some amount and type of attached memory. @@ -32,7 +32,7 @@ type Node struct { memType Type capacity int64 normal bool - cpus cpuset.CPUSet + cpus *libcpu.CpuMask distance Distance } @@ -43,8 +43,10 @@ type Distance struct { nodes map[int]NodeMask } -// NewNode creates a new node with the given parameters. -func NewNode(id ID, t Type, capa int64, normal bool, cpus cpuset.CPUSet, d []int) (*Node, error) { +// NewNode creates a new node with the given parameters. The node takes a +// sealed copy of the given set of close CPUs, so the caller stays free to +// modify its own. +func NewNode(id ID, t Type, capa int64, normal bool, cpus *libcpu.CpuMask, d []int) (*Node, error) { if !t.IsValid() { return nil, fmt.Errorf("%w: unknown type %d", ErrInvalidType, t) } @@ -58,12 +60,15 @@ func NewNode(id ID, t Type, capa int64, normal bool, cpus cpuset.CPUSet, d []int return nil, err } + closeCPUs := cpus.Clone() + closeCPUs.Seal() + return &Node{ id: id, memType: t, capacity: capa, normal: normal, - cpus: cpus.Clone(), + cpus: closeCPUs, distance: dist, }, nil } @@ -103,8 +108,8 @@ func (n *Node) HasMemory() bool { return n.capacity > 0 } -// CloseCPUs returns the set of CPUs closest to the node. -func (n *Node) CloseCPUs() cpuset.CPUSet { +// CloseCPUs returns the sealed set of CPUs closest to the node. +func (n *Node) CloseCPUs() *libcpu.CpuMask { return n.cpus } From 7e62ba7bbebac940faf6d52adf27ee8aba8a30dd Mon Sep 17 00:00:00 2001 From: Krisztian Litkey Date: Fri, 11 Sep 2026 14:58:41 +0300 Subject: [PATCH 39/39] config,plugins: parse cpuset amounts into a CpuMask. An Amount is a string, and the cpuset one is parsed on the way in, so nothing about this reaches the wire: the CRD schema for these fields is a plain string either way, and no type in the config API holds a set of CPUs. The return type of the parse method was the last seam, and both policies stitched it with a toCpuMask of their own right at the call. ParseCPUSet keeps its name, since it names what it parses, which is what the field is, and matches the AmountCPUSet and PrefixCPUSet around it. Only what it returns changes. Both toCpuMask helpers go with it. The k8s parser stays in both policies for the NUMA node lists, which are written in cpuset syntax but are not CPUs. The parsers agree on what they accept, empty string included, so only a malformed cpuset reads differently. libcpu names the offending list in its own error, and every caller here already says which amount it was reading, so ParseCPUSet no longer wraps what it gets: the message keeps the two parts it had rather than growing a third. Assisted-by: Claude Opus 5 (1M context) Signed-off-by: Krisztian Litkey --- cmd/plugins/balloons/policy/balloons-policy.go | 18 +++++------------- .../policy/topology-aware-policy.go | 14 +++++++------- cmd/plugins/topology-aware/policy/topology.go | 6 ------ .../config/v1alpha1/resmgr/policy/config.go | 13 ++++++------- 4 files changed, 18 insertions(+), 33 deletions(-) diff --git a/cmd/plugins/balloons/policy/balloons-policy.go b/cmd/plugins/balloons/policy/balloons-policy.go index 9d464250f..1278b7e34 100644 --- a/cmd/plugins/balloons/policy/balloons-policy.go +++ b/cmd/plugins/balloons/policy/balloons-policy.go @@ -147,13 +147,6 @@ type loadClassVirtDev struct { updateOnEveryCpuAllocation bool } -// toCpuMask converts a CPU set into the kind this policy keeps. It is needed -// for the one interface which still hands out a k8s one: the configuration, -// which parses an operator's cpuset string. -func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { - return libcpu.NewCpuMask(cpus.List()...) -} - var log logger.Logger = logger.NewLogger("policy") // String is a stringer for a balloon. @@ -1905,17 +1898,17 @@ func (p *balloons) setConfig(bpoptions *BalloonsOptions) error { amount, kind := bpoptions.AvailableResources.Get(cfgapi.CPU) switch kind { case cfgapi.AmountCPUSet: - cset, err := amount.ParseCPUSet() + cpus, err := amount.ParseCPUSet() if err != nil { return balloonsError("failed to parse available CPU cpuset '%s': %w", amount, err) } - availableCpus = toCpuMask(cset) + availableCpus = cpus case cfgapi.AmountExcludeCPUSet: - cset, err := amount.ParseCPUSet() + cpus, err := amount.ParseCPUSet() if err != nil { return balloonsError("failed to parse available CPU cpuset '%s': %w", amount, err) } - availableCpus = p.machine.PresentCPUs().Difference(toCpuMask(cset)) + availableCpus = p.machine.PresentCPUs().Difference(cpus) case cfgapi.AmountQuantity: return balloonsError("can't handle CPU resources given as resource.Quantity (%v)", amount) @@ -2082,11 +2075,10 @@ func (p *balloons) fillBuiltinBalloonDefs(bpoptions *BalloonsOptions) (*BalloonD // can still allocate CPUs first. If reserved // balloon's MinCpus is undefined, set it to catch all // (or at most MaxCpu) CPUs in the reserved cpuset. - parsed, err := amount.ParseCPUSet() + cset, err := amount.ParseCPUSet() if err != nil { return nil, nil, balloonsError("failed to parse reserved CPU cpuset '%s': %v", amount, err) } - cset := toCpuMask(parsed) if kind == cfgapi.AmountExcludeCPUSet { cset = p.allowed.Difference(cset) } diff --git a/cmd/plugins/topology-aware/policy/topology-aware-policy.go b/cmd/plugins/topology-aware/policy/topology-aware-policy.go index de3ad60e7..93d6029dc 100644 --- a/cmd/plugins/topology-aware/policy/topology-aware-policy.go +++ b/cmd/plugins/topology-aware/policy/topology-aware-policy.go @@ -851,18 +851,18 @@ func (p *policy) checkConstraints() error { amount, kind := p.cfg.AvailableResources.Get(cfgapi.CPU) switch kind { case cfgapi.AmountCPUSet: - cset, err := amount.ParseCPUSet() + cpus, err := amount.ParseCPUSet() if err != nil { return fmt.Errorf("failed to parse available CPU cpuset '%s': %w", amount, err) } - p.allowed = toCpuMask(cset) + p.allowed = cpus case cfgapi.AmountExcludeCPUSet: - cset, err := amount.ParseCPUSet() + cpus, err := amount.ParseCPUSet() if err != nil { return fmt.Errorf("failed to parse available CPU cpuset '%s': %w", amount, err) } - p.allowed = p.machine.PresentCPUs().Difference(toCpuMask(cset)) + p.allowed = p.machine.PresentCPUs().Difference(cpus) case cfgapi.AmountQuantity: return fmt.Errorf("can't handle CPU resources given as resource.Quantity (%v)", amount) @@ -881,14 +881,14 @@ func (p *policy) checkConstraints() error { return policyError("cannot start without CPU reservation") case cfgapi.AmountCPUSet, cfgapi.AmountExcludeCPUSet: - cset, err := amount.ParseCPUSet() + cpus, err := amount.ParseCPUSet() if err != nil { return fmt.Errorf("failed to parse reserved CPU cpuset '%s': %w", amount, err) } if kind == cfgapi.AmountExcludeCPUSet { - p.reserved = p.allowed.Difference(toCpuMask(cset)) + p.reserved = p.allowed.Difference(cpus) } else { - p.reserved = toCpuMask(cset) + p.reserved = cpus } // check that all reserved CPUs are in the allowed set diff --git a/cmd/plugins/topology-aware/policy/topology.go b/cmd/plugins/topology-aware/policy/topology.go index 949877965..4914a4c3c 100644 --- a/cmd/plugins/topology-aware/policy/topology.go +++ b/cmd/plugins/topology-aware/policy/topology.go @@ -23,12 +23,6 @@ import ( idset "github.com/intel/goresctrl/pkg/utils" ) -// toCpuMask converts a set the configuration parsed out of an operator's cpuset -// string into the ones this policy keeps. It is the last of the seam. -func toCpuMask(cpus cpuset.CPUSet) *libcpu.CpuMask { - return libcpu.NewCpuMask(cpus.List()...) -} - // // Packages, dies, clusters and caches // diff --git a/pkg/apis/config/v1alpha1/resmgr/policy/config.go b/pkg/apis/config/v1alpha1/resmgr/policy/config.go index 8e12a88e2..0272b0d81 100644 --- a/pkg/apis/config/v1alpha1/resmgr/policy/config.go +++ b/pkg/apis/config/v1alpha1/resmgr/policy/config.go @@ -18,7 +18,7 @@ import ( "fmt" "strings" - "github.com/containers/nri-plugins/pkg/utils/cpuset" + libcpu "github.com/containers/nri-plugins/pkg/lib/cpu" "k8s.io/apimachinery/pkg/api/resource" nriapi "github.com/containerd/nri/pkg/api" @@ -110,12 +110,11 @@ func (c Constraints) Get(d Domain) (Amount, AmountKind) { } } -func (amount Amount) ParseCPUSet() (cpuset.CPUSet, error) { - cset, err := cpuset.Parse(string(amount)) - if err != nil { - return cset, fmt.Errorf("failed to parse amount '%s' as cpuset: %w", amount, err) - } - return cset, nil +// ParseCPUSet parses the amount as a cpuset. The error is returned as it comes, +// since it names the offending CPU list already, and every caller wraps it with +// which amount it was reading. +func (amount Amount) ParseCPUSet() (*libcpu.CpuMask, error) { + return libcpu.ParseCpuMask(string(amount)) } func (amount Amount) ParseQuantity() (resource.Quantity, error) {