diff --git a/cmd/webhook-manager/app/server.go b/cmd/webhook-manager/app/server.go index 9f2334166fe..e368d487fbd 100644 --- a/cmd/webhook-manager/app/server.go +++ b/cmd/webhook-manager/app/server.go @@ -34,6 +34,7 @@ import ( informers "volcano.sh/apis/pkg/client/informers/externalversions" "volcano.sh/volcano/cmd/webhook-manager/app/options" "volcano.sh/volcano/pkg/kube" + devconfig "volcano.sh/volcano/pkg/scheduler/api/devices/config" "volcano.sh/volcano/pkg/signals" commonutil "volcano.sh/volcano/pkg/util" wkconfig "volcano.sh/volcano/pkg/webhooks/config" @@ -67,7 +68,10 @@ func Run(config *options.Config) error { if admissionConf == nil { klog.Errorf("loadAdmissionConf failed.") } else { - klog.V(2).Infof("loadAdmissionConf:%v", admissionConf.ResGroupsConfig) + klog.V(2).Infof("loadAdmissionConf:%v", admissionConf) + if admissionConf.DeviceMutatorConf.Enable { + devconfig.InitDevicesConfig(admissionConf.DeviceMutatorConf.KnownGeometriesCMName, admissionConf.DeviceMutatorConf.KnownGeometriesCMNamespace) + } } vClient := getVolcanoClient(restConfig) diff --git a/docs/user-guide/how_to_use_vnpu.md b/docs/user-guide/how_to_use_vnpu.md index fd6dabbb91c..b39b5c84090 100644 --- a/docs/user-guide/how_to_use_vnpu.md +++ b/docs/user-guide/how_to_use_vnpu.md @@ -25,7 +25,7 @@ with support for more chip types to come **Description**: -This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature. It supports vNPU feature for both Ascend 310 and Ascend 910. It also supports managing heterogeneous Ascend cluster(Cluster with multiple Ascend types, i.e. 910A,910B2,910B3,310P) +This mode is developed by a third-party community 'HAMi', which is the developer of [volcano-vgpu](./how_to_use_volcano_vgpu.md) feature. It supports vNPU feature for both Ascend 310 and Ascend 910. It also supports managing heterogeneous Ascend cluster(Cluster with multiple Ascend types, i.e. 910A,910B2,910B3,310P). Starting with version 1.16, we support soft device partitioning through [hami-vnpu-core](https://github.com/Project-HAMi/hami-vnpu-core). This feature is known as the `hami-core` mode. **Use case**: @@ -43,7 +43,7 @@ To enable vNPU scheduling, the following components must be set up based on the **Prerequisites**: Kubernetes >= 1.16 -Volcano >= 1.14 +Volcano >= 1.14 (1.16 for `hami-core` mode) [ascend-docker-runtime](https://gitcode.com/Ascend/mind-cluster/tree/master/component/ascend-docker-runtime) (for HAMi Mode) ### Install Volcano: @@ -115,19 +115,21 @@ data: #### HAMi mode -##### Label the Node with `ascend=on` +##### Template vNPU mode + +###### Label the Node with `ascend=on` ``` kubectl label node {ascend-node} ascend=on ``` -##### Deploy `hami-scheduler-device` ConfigMap +###### Deploy `hami-scheduler-device` ConfigMap ``` kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-configmap.yaml ``` -##### Deploy ascend-device-plugin +###### Deploy ascend-device-plugin ``` kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-plugin.yaml @@ -135,7 +137,76 @@ kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-pl For more information, refer to the [ascend-device-plugin documentation](https://github.com/Project-HAMi/ascend-device-plugin). -##### Scheduler Config Update +###### Update Scheduler Config +```yaml +kind: ConfigMap +apiVersion: v1 +metadata: + name: volcano-scheduler-configmap + namespace: volcano-system +data: + volcano-scheduler.conf: | + actions: "enqueue, allocate, backfill" + tiers: + - plugins: + - name: predicates + - name: deviceshare + arguments: + deviceshare.AscendHAMiVNPUEnable: true # enable ascend vnpu + deviceshare.SchedulePolicy: binpack # scheduling policy. binpack / spread + deviceshare.KnownGeometriesCMNamespace: kube-system + deviceshare.KnownGeometriesCMName: hami-scheduler-device +``` + + **Note:** You may notice that, 'volcano-vgpu' has its own GeometriesCMName and GeometriesCMNamespace, which means if you want to use both vNPU and vGPU in a same volcano cluster, you need to merge the configMap from both sides and set it here. + +##### `hami-core` mode + +###### Label the Node with `ascend=on` + +``` +kubectl label node {ascend-node} ascend=on +``` + +###### Deploy `hami-scheduler-device` ConfigMap + +1. Download file +``` +curl -O https://github.com/Project-HAMi/ascend-device-plugin/blob/main/ascend-device-configmap.yaml +``` +2. Set `hamiVnpuCore` to `true` +3. Deploy the yaml + +``` +kubectl apply -f ascend-device-configmap.yaml +``` + +###### Deploy `hami-device-node-config` ConfigMap +1. Download file +``` +curl -O https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/main/ascend-device-node-configmap.yaml +``` +2. Set the `name` field under the `nodes` block to the actual node name. +3. Deploy the yaml +``` +kubectl apply -f ascend-device-node-configmap.yaml +``` + +###### Deploy ascend-device-plugin + +``` +kubectl apply -f https://raw.githubusercontent.com/Project-HAMi/ascend-device-plugin/refs/heads/main/ascend-device-plugin.yaml +``` + +###### Upgrade volcano-adminssion +``` +helm upgrade [RELEASE_NAME] [CHART_PATH] --set custom.device_mutator_enable=true \ + --set custom.enabled_admissions='/jobs/mutate\,/jobs/validate\,/podgroups/validate\,/queues/mutate\,/queues/validate\,/hypernodes/validate\,/cronjobs/validate\,/pods/mutate' +kubectl rollout restart deployment -n [VOLCANO_NAMESPACE] volcano-admission +``` +**Note**: If you deployed Volcano using a Deployment YAML, you need to either generate a new YAML with Helm using the parameters above and deploy it, or uninstall the existing Volcano and redeploy it with Helm. + +###### Update Scheduler Config ```yaml kind: ConfigMap apiVersion: v1 @@ -262,6 +333,8 @@ For detailed information, please consult the official [Ascend MindCluster docume ### HAMi mode +#### Template vNPU mode + ```yaml apiVersion: v1 kind: Pod @@ -289,6 +362,7 @@ The supported Ascend chips and their `ResourceNames` are shown in the following | 910B3 | huawei.com/Ascend910B3 | huawei.com/Ascend910B3-memory | | 910B4 | huawei.com/Ascend910B4 | huawei.com/Ascend910B4-memory | | 910B4-1 | huawei.com/Ascend910B4-1 | huawei.com/Ascend910B4-1-memory | +| 910C | huawei.com/Ascend910C | huawei.com/Ascend910C-memory | | 310P3 | huawei.com/Ascend310P | huawei.com/Ascend310P-memory | #### Hami vNPU scene memory allocation restrictions @@ -299,3 +373,39 @@ The supported Ascend chips and their `ResourceNames` are shown in the following - Pod requests 1 vNPU device with a requested memory of 20480, resulting in an actual allocated memory of 32768 - When a Pod requests multiple vNPU devices, the memory resource request can be left unspecified or filled with the maximum value, The memory allocated to Pod is the actual memory of the entire card + +#### `hami-core` mode + +``` +apiVersion: v1 +kind: Pod +metadata: + name: ascend-pod + annotations: + huawei.com/vnpu-mode: hami-core +spec: + schedulerName: volcano + containers: + - name: ubuntu-container + image: quay.io/ascend/vllm-ascend:v0.18.0-310p + command: ["sleep"] + args: ["100000"] + resources: + limits: + huawei.com/Ascend310P: "1" + huawei.com/Ascend310P-memory: "4096" + huawei.com/Ascend310P-core: "90" +``` +The supported Ascend chips and their `ResourceNames` are shown in the following table: + +| ChipName | ResourceName | ResourceMemoryName | ResourceCoreName +|-------|-------|-------| +| 910A | huawei.com/Ascend910A | huawei.com/Ascend910A-memory | huawei.com/Ascend910A-core | +| 910B2 | huawei.com/Ascend910B2 | huawei.com/Ascend910B2-memory | huawei.com/Ascend910B2-core | +| 910B3 | huawei.com/Ascend910B3 | huawei.com/Ascend910B3-memory | huawei.com/Ascend910B3-core | +| 910B4 | huawei.com/Ascend910B4 | huawei.com/Ascend910B4-memory | huawei.com/Ascend910B4-core | +| 910B4-1 | huawei.com/Ascend910B4-1 | huawei.com/Ascend910B4-1-memory | huawei.com/Ascend910B4-1-core | +| 910C | huawei.com/Ascend910C | huawei.com/Ascend910C-memory | huawei.com/Ascend910C-core | +| 310P3 | huawei.com/Ascend310P | huawei.com/Ascend310P-memory | huawei.com/Ascend310P-core | + +**Note**: If the pod's annotations do not specify `hami-core`, the device will be allocated in the template vNPU mode even if the `hami-core` feature is enabled in the configuration file. \ No newline at end of file diff --git a/installer/helm/chart/volcano/templates/admission.yaml b/installer/helm/chart/volcano/templates/admission.yaml index 72e72cc5d2a..927e253b324 100644 --- a/installer/helm/chart/volcano/templates/admission.yaml +++ b/installer/helm/chart/volcano/templates/admission.yaml @@ -20,6 +20,16 @@ data: {{- .Values.custom.admission_config_override | nindent 4 }} {{- else }} {{- (.Files.Glob .Values.basic.admission_config_file).AsConfig | nindent 2}} + {{- if .Values.custom.device_mutator_enable }} + deviceMutator: + enable: true + {{- if .Values.custom.known_geometries_cm_name }} + knownGeometriesCMName: {{ .Values.custom.known_geometries_cm_name }} + {{- end }} + {{- if .Values.custom.known_geometries_cm_namespace }} + knownGeometriesCMNamespace: {{ .Values.custom.known_geometries_cm_namespace}} + {{- end }} + {{- end }} {{- end }} --- apiVersion: v1 diff --git a/installer/helm/chart/volcano/values.yaml b/installer/helm/chart/volcano/values.yaml index 395548c29d5..b7a2343e1aa 100644 --- a/installer/helm/chart/volcano/values.yaml +++ b/installer/helm/chart/volcano/values.yaml @@ -69,6 +69,10 @@ custom: agent_extend_resource_memory_name: ~ agent_kube_cgroup_root: ~ + device_mutator_enable: false + known_geometries_cm_namespace: "kube-system" + known_geometries_cm_name: "hami-scheduler-device" + # Override the configuration for admission, controller or scheduler. # For example: # diff --git a/pkg/scheduler/api/devices/ascend/hami/constants.go b/pkg/scheduler/api/devices/ascend/hami/constants.go new file mode 100644 index 00000000000..ef510d07e62 --- /dev/null +++ b/pkg/scheduler/api/devices/ascend/hami/constants.go @@ -0,0 +1,23 @@ +/* +Copyright 2026 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hami + +const ( + VNPUModeAnnotation = "huawei.com/vnpu-mode" + VNPUModeHamiCore = "hami-core" + VNPUNodeSelectorAnnotation = "hami-vnpu-core" +) diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info.go b/pkg/scheduler/api/devices/ascend/hami/device_info.go index 1e791e8bfd1..16a1b58a67b 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info.go @@ -60,6 +60,7 @@ type AscendDevice struct { DeviceUsage *devices.DeviceUsage Score float64 PodMap map[string]*devices.DeviceUsage + hamiVnpuCore bool } type AscendDevices struct { @@ -70,8 +71,10 @@ type AscendDevices struct { } type RuntimeInfo struct { - UUID string `json:"UUID,omitempty"` - Temp string `json:"temp,omitempty"` + UUID string `json:"UUID,omitempty"` + Temp string `json:"temp,omitempty"` + Memory int64 `json:"memory,omitempty"` + Core int32 `json:"core,omitempty"` } var ( @@ -102,7 +105,7 @@ func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { klog.V(3).Infof("Node %s does not have allocatable %s resource or value is 0", node.Name, resourceName) continue } - nodeDevices, err := dev.GetNodeDevices(*node) + nodeDevices, nodeSupportHamiCore, err := dev.getNodeDevices(node, curConfig.VNPUs) if err != nil { klog.Warningf("Failed to get node devices. nodeName %s, deviceType %s, error %v", node.Name, dev.CommonWord(), err) continue @@ -124,7 +127,8 @@ func NewAscendDevices(name string, node *v1.Node) map[string]*AscendDevices { Usedmem: 0, Usedcores: 0, }, - PodMap: make(map[string]*devices.DeviceUsage), + PodMap: make(map[string]*devices.DeviceUsage), + hamiVnpuCore: nodeSupportHamiCore, } asDevices.Devices[nd.ID] = cur_dev klog.V(5).Infof("add device. ID %s dev_info %+v", cur_dev.DeviceInfo.ID, cur_dev.DeviceInfo) @@ -191,7 +195,7 @@ func (ads *AscendDevices) SubResource(pod *v1.Pod) { if _, ok := dev.PodMap[string(pod.UID)]; ok { delete(dev.PodMap, string(pod.UID)) ads.SubResourceUsage(dev, cono_dev.Usedcores, cono_dev.Usedmem) - klog.V(5).Infof("sub resource usage for pod %s. device %s usedmem %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem) + klog.V(5).Infof("sub resource usage for pod %s. device %s usedmem %d usedcore %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem, cono_dev.Usedcores) } } } @@ -220,7 +224,7 @@ func (ads *AscendDevices) addResource(annotations map[string]string, pod *v1.Pod Usedmem: cono_dev.Usedmem, } ads.AddResourceUsage(dev, cono_dev.Usedcores, cono_dev.Usedmem) - klog.V(5).Infof("add resource usage for pod %s. device %s usedmem %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem) + klog.V(5).Infof("add resource usage for pod %s. device %s usedmem %d usedcore %d", pod.Name, dev.DeviceInfo.ID, cono_dev.Usedmem, cono_dev.Usedcores) } } } @@ -379,7 +383,7 @@ func (ads *AscendDevices) GetIgnoredDevices() []string { return []string{""} } vnpuConfig := randDev.config - return []string{vnpuConfig.ResourceMemoryName} + return []string{vnpuConfig.ResourceMemoryName, fmt.Sprintf("%s-core", vnpuConfig.ResourceName)} } func (ads *AscendDevices) GetStatus() string { @@ -412,6 +416,7 @@ func (ads *AscendDevices) DeepCopy() interface{} { DeviceUsage: newUsage, Score: dev.Score, PodMap: make(map[string]*devices.DeviceUsage), + hamiVnpuCore: dev.hamiVnpuCore, } for k, v := range dev.PodMap { newDev.PodMap[k] = &devices.DeviceUsage{ @@ -446,10 +451,13 @@ func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (dev usedDevs := make([]*AscendDevice, 0) for _, req := range reqs { klog.V(5).Infof("req %+v", req) - err := verifyReq(req, dupDevs[0]) - if err != nil { - return nil, err + if !isHAMiCoreMode(pod) { + err := verifyReq(req, dupDevs[0]) + if err != nil { + return nil, err + } } + availableDevs := make([]*AscendDevice, 0) for _, dev := range dupDevs { selected := false @@ -467,7 +475,7 @@ func (ads *AscendDevices) selectDevices(pod *v1.Pod, schedulePolicy string) (dev selectedDevs := make([]*AscendDevice, 0) for _, dev := range availableDevs { klog.V(5).Infof("check fit. req %+v dev_info %+v dev_usage %+v", req, dev.DeviceInfo, dev.DeviceUsage) - if !fit(&req, dev) { + if !fit(&req, dev, isHAMiCoreMode(pod)) { klog.V(5).Infof("fit false. dev ID %s", dev.DeviceInfo.ID) continue } @@ -512,10 +520,13 @@ func hasNetworkID(devices []*AscendDevice) bool { return true } -func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice) bool { +func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice, isHAMiCore bool) bool { if req.Type != dev.config.CommonWord { return false } + if isHAMiCore && !dev.hamiVnpuCore { + return false + } deviceUsage := dev.DeviceUsage deviceInfo := dev.DeviceInfo if deviceInfo.Count <= deviceUsage.Used { @@ -524,13 +535,17 @@ func fit(req *devices.ContainerDeviceRequest, dev *AscendDevice) bool { if deviceInfo.Devmem-deviceUsage.Usedmem < req.Memreq { return false } - if deviceInfo.Devcore-deviceUsage.Usedcores < req.Coresreq { + effectiveTotalCore := dev.DeviceInfo.Devcore + if isHAMiCore { + effectiveTotalCore = 100 + } + if effectiveTotalCore-deviceUsage.Usedcores < req.Coresreq { return false } - if deviceInfo.Devcore == 100 && req.Coresreq == 100 && deviceUsage.Used > 0 { + if effectiveTotalCore == 100 && req.Coresreq == 100 && deviceUsage.Used > 0 { return false } - if deviceInfo.Devcore != 0 && deviceUsage.Usedcores == deviceInfo.Devcore && req.Coresreq == 0 { + if effectiveTotalCore != 0 && deviceUsage.Usedcores == effectiveTotalCore && req.Coresreq == 0 { return false } return true @@ -550,7 +565,8 @@ func getDeviceSnapshot(ads *AscendDevices) []*AscendDevice { Usedmem: dev.DeviceUsage.Usedmem, Usedcores: dev.DeviceUsage.Usedcores, }, - PodMap: make(map[string]*devices.DeviceUsage), + PodMap: make(map[string]*devices.DeviceUsage), + hamiVnpuCore: dev.hamiVnpuCore, } for k, v := range dev.PodMap { dupDev.PodMap[k] = &devices.DeviceUsage{ @@ -656,32 +672,42 @@ func (dev *AscendDevice) CommonWord() string { return dev.config.CommonWord } -func (dev *AscendDevice) GetNodeDevices(n v1.Node) ([]*devices.DeviceInfo, error) { +func (dev *AscendDevice) getNodeDevices(n *v1.Node, conf config.VNPUsConfig) ([]*devices.DeviceInfo, bool, error) { + nodeSupportHamiCore := conf.HamiVnpuCore + if val, ok := n.Annotations[VNPUNodeSelectorAnnotation]; ok { + klog.V(5).Infof("hami-vnpu-core is %s in node %s annotation", val, n.Name) + if val == "true" { + nodeSupportHamiCore = true + } else { + nodeSupportHamiCore = false + } + } anno, ok := n.Annotations[dev.nodeRegisterAnno] if !ok { - return []*devices.DeviceInfo{}, fmt.Errorf("annos not found %s", dev.nodeRegisterAnno) + return []*devices.DeviceInfo{}, nodeSupportHamiCore, fmt.Errorf("annos not found %s", dev.nodeRegisterAnno) } nodeDevices, err := devices.UnMarshalNodeDevices(anno) if err != nil { klog.ErrorS(err, "failed to unmarshal node devices", "node", n.Name, "device annotation", anno) - return []*devices.DeviceInfo{}, err + return []*devices.DeviceInfo{}, nodeSupportHamiCore, err } if len(nodeDevices) == 0 { klog.InfoS("no ascend device found", "node", n.Name, "device annotation", anno) - return []*devices.DeviceInfo{}, errors.New("no device found on node") + return []*devices.DeviceInfo{}, nodeSupportHamiCore, errors.New("no device found on node") } - return nodeDevices, nil + return nodeDevices, nodeSupportHamiCore, nil } func (dev *AscendDevice) ResourceReqs(pod *v1.Pod) []devices.ContainerDeviceRequest { - reqs := devices.ExtractResourceRequest(pod, dev.CommonWord(), dev.config.ResourceName, dev.config.ResourceMemoryName, "", "") + resourceCoreName := fmt.Sprintf("%s-core", dev.config.ResourceName) + reqs := devices.ExtractResourceRequest(pod, dev.CommonWord(), dev.config.ResourceName, dev.config.ResourceMemoryName, "", resourceCoreName) for i := range reqs { req := &reqs[i] if req.Memreq == 0 && req.MemPercentagereq != 0 { req.Memreq = int32(dev.DeviceInfo.Devmem * req.MemPercentagereq / 100) klog.V(5).Infof("new memreq %d totalmem %d mempercentage %d", req.Memreq, dev.DeviceInfo.Devmem, req.MemPercentagereq) } - if req.Memreq > 0 { + if req.Memreq > 0 && !isHAMiCoreMode(pod) { m, _ := dev.trimMemory(int64(req.Memreq)) klog.V(5).Infof("raw mem %d, trimed mem %d", req.Memreq, m) req.Memreq = int32(m) @@ -705,11 +731,15 @@ func (ads *AscendDevices) CreateAnnotations(pod *v1.Pod, devList devices.PodSing var rtInfo []RuntimeInfo for _, dp := range devList { for _, val := range dp { - _, temp := dev.trimMemory(int64(val.Usedmem)) - rtInfo = append(rtInfo, RuntimeInfo{ - UUID: val.UUID, - Temp: temp, - }) + info := RuntimeInfo{UUID: val.UUID} + if isHAMiCoreMode(pod) { + info.Memory = int64(val.Usedmem) + info.Core = val.Usedcores + } else { + _, temp := dev.trimMemory(int64(val.Usedmem)) + info.Temp = temp + } + rtInfo = append(rtInfo, info) } } s, err := json.Marshal(rtInfo) @@ -754,3 +784,10 @@ func verifyReq(req devices.ContainerDeviceRequest, dev *AscendDevice) error { } return nil } + +func isHAMiCoreMode(pod *v1.Pod) bool { + if pod == nil { + return false + } + return pod.Annotations[VNPUModeAnnotation] == VNPUModeHamiCore +} diff --git a/pkg/scheduler/api/devices/ascend/hami/device_info_test.go b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go index 16814e2fc6d..4d09c6ddaaa 100644 --- a/pkg/scheduler/api/devices/ascend/hami/device_info_test.go +++ b/pkg/scheduler/api/devices/ascend/hami/device_info_test.go @@ -230,7 +230,7 @@ func Test_fit(t *testing.T) { conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) ascend310PConfig := conf.VNPUConfigs()[len(conf.VNPUConfigs())-1] - device_info := &devices.DeviceInfo{ + deviceInfo := &devices.DeviceInfo{ ID: "68496E64-20E05477-92C31323-6E78030A-BD003019", Index: 0, Count: 7, @@ -238,107 +238,221 @@ func Test_fit(t *testing.T) { Devmem: 21527, } tests := []struct { - name string - req *devices.ContainerDeviceRequest - dev *AscendDevice - result bool + name string + req *devices.ContainerDeviceRequest + dev *AscendDevice + isHAMiCore bool + result bool }{ { - "test1", - &devices.ContainerDeviceRequest{ + name: "test1", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 1024, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 3072, }, }, - true, + isHAMiCore: false, + result: true, }, { - "test2", - &devices.ContainerDeviceRequest{ + name: "test2", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 21527, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 3072, }, }, - false, + isHAMiCore: false, + result: false, }, { - "test3", - &devices.ContainerDeviceRequest{ + name: "test3", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 6144, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 12288, }, }, - true, + isHAMiCore: false, + result: true, }, { - "test4", - &devices.ContainerDeviceRequest{ + name: "test4", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 24576, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 0, Usedmem: 0, }, }, - false, + isHAMiCore: false, + result: false, }, { - "test5_core", - &devices.ContainerDeviceRequest{ + name: "test5_core", + req: &devices.ContainerDeviceRequest{ Nums: 1, Type: "Ascend310P", Memreq: 6144, Coresreq: 4, }, - &AscendDevice{ + dev: &AscendDevice{ config: ascend310PConfig, - DeviceInfo: device_info, + DeviceInfo: deviceInfo, DeviceUsage: &devices.DeviceUsage{ Used: 1, Usedmem: 12288, Usedcores: 6, }, }, - false, + isHAMiCore: false, + result: false, + }, + { + name: "hami_core_mode_should_fail_when_node_not_support_hami_vnpu_core", + req: &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", + Memreq: 1024, + Coresreq: 20, + }, + dev: &AscendDevice{ + config: ascend310PConfig, + DeviceInfo: deviceInfo, + DeviceUsage: &devices.DeviceUsage{}, + hamiVnpuCore: false, + }, + isHAMiCore: true, + result: false, + }, + { + name: "hami_core_mode_should_fail_when_core_is_insufficient", + req: &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", + Memreq: 1024, + Coresreq: 20, + }, + dev: &AscendDevice{ + config: ascend310PConfig, + DeviceInfo: deviceInfo, + DeviceUsage: &devices.DeviceUsage{ + Usedcores: 90, + }, + hamiVnpuCore: true, + }, + isHAMiCore: true, + result: false, + }, + { + name: "hami_core_mode_should_pass_when_node_support_hami_vnpu_core", + req: &devices.ContainerDeviceRequest{ + Nums: 1, + Type: "Ascend310P", + Memreq: 1024, + Coresreq: 20, + }, + dev: &AscendDevice{ + config: ascend310PConfig, + DeviceInfo: deviceInfo, + DeviceUsage: &devices.DeviceUsage{}, + hamiVnpuCore: true, + }, + isHAMiCore: true, + result: true, }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - ret := fit(tt.req, tt.dev) + ret := fit(tt.req, tt.dev, tt.isHAMiCore) assert.Equal(t, tt.result, ret) }) } } +func TestAscendDevice_GetNodeDevices_HAMiVnpuCore(t *testing.T) { + deviceRegisterAnno := fmt.Sprintf("%s/node-register-%s", util.HAMiAnnotationsPrefix, "Ascend910A") + testNodeDevicesJSON := `[{"id":"dev-1","index":0,"count":1,"devmem":32768,"devcore":30}]` + dev := &AscendDevice{ + nodeRegisterAnno: deviceRegisterAnno, + } + var conf config.VNPUsConfig + + tests := []struct { + name string + annotations map[string]string + expectSupport bool + }{ + { + name: "node_support_hami_vnpu_core", + annotations: map[string]string{ + deviceRegisterAnno: testNodeDevicesJSON, + VNPUNodeSelectorAnnotation: "true", + }, + expectSupport: true, + }, + { + name: "node_not_support_hami_vnpu_core", + annotations: map[string]string{ + deviceRegisterAnno: testNodeDevicesJSON, + VNPUNodeSelectorAnnotation: "false", + }, + expectSupport: false, + }, + { + name: "node_without_hami_vnpu_core_annotation", + annotations: map[string]string{ + deviceRegisterAnno: testNodeDevicesJSON, + }, + expectSupport: false, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + node := v1.Node{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-node", + Annotations: tt.annotations, + }, + } + + nodeDevices, nodeSupportHamiCore, err := dev.getNodeDevices(&node, conf) + assert.NoError(t, err) + assert.Len(t, nodeDevices, 1) + assert.Equal(t, tt.expectSupport, nodeSupportHamiCore) + }) + } +} + func Test_verifyReq(t *testing.T) { conf, err := yamlStringToConfig(config_yaml) assert.Nil(t, err) diff --git a/pkg/scheduler/api/devices/config/config.go b/pkg/scheduler/api/devices/config/config.go index 19fa3061094..49f61883616 100644 --- a/pkg/scheduler/api/devices/config/config.go +++ b/pkg/scheduler/api/devices/config/config.go @@ -151,8 +151,8 @@ func InitDevicesConfig(cmName, cmNamespace string) { var err error configs, err = loadConfigFromCM(devices.GetClient(), cmName, cmNamespace) if err != nil { - klog.V(3).InfoS("Volcano device config not found in namespace kube-system, using default config", - "name", cmName) + klog.V(3).Infof("failed to load config from ConfigMap %s in namespace %s: %v. using default config.", + cmName, cmNamespace, err) configs = GetDefaultDevicesConfig() } klog.V(3).InfoS("Initializing volcano device config", "device-configs", configs) diff --git a/pkg/scheduler/api/devices/util.go b/pkg/scheduler/api/devices/util.go index 6e62447ecef..d7434bc5e2b 100644 --- a/pkg/scheduler/api/devices/util.go +++ b/pkg/scheduler/api/devices/util.go @@ -67,19 +67,24 @@ const ( Skip ) -var kubeClient *kubernetes.Clientset +var kubeClient kubernetes.Interface func GetClient() kubernetes.Interface { - var err error if kubeClient == nil { - kubeClient, err = NewClient() + client, err := NewClient() if err != nil { klog.ErrorS(err, "deviceshare initClient failed") + } else { + kubeClient = client } } return kubeClient } +func SetClient(client kubernetes.Interface) { + kubeClient = client +} + // NewClient connects to an API server func NewClient() (*kubernetes.Clientset, error) { config, err := rest.InClusterConfig() diff --git a/pkg/webhooks/admission/pods/mutate/device_mutator.go b/pkg/webhooks/admission/pods/mutate/device_mutator.go new file mode 100644 index 00000000000..292532a20ec --- /dev/null +++ b/pkg/webhooks/admission/pods/mutate/device_mutator.go @@ -0,0 +1,124 @@ +/* +Copyright 2026 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutate + +import ( + "fmt" + + v1 "k8s.io/api/core/v1" + "k8s.io/klog/v2" + + hami "volcano.sh/volcano/pkg/scheduler/api/devices/ascend/hami" + devconfig "volcano.sh/volcano/pkg/scheduler/api/devices/config" +) + +const ( + ascendHAMiMutatorName = "ascend" + jsonPatchOpAdd = "add" +) + +type DeviceMutator interface { + Name() string + MutateAdmission(pod *v1.Pod) []patchOperation +} + +var registeredDeviceMutators = []DeviceMutator{ + newAscendMutator(), +} + +type AscendMutator struct{} + +func newAscendMutator() DeviceMutator { + return &AscendMutator{} +} + +func (m *AscendMutator) Name() string { + return ascendHAMiMutatorName +} + +func (m *AscendMutator) MutateAdmission(pod *v1.Pod) []patchOperation { + if pod == nil || pod.Annotations == nil { + return nil + } + if vnpuMode, ok := pod.Annotations[hami.VNPUModeAnnotation]; !ok || vnpuMode != hami.VNPUModeHamiCore { + return nil + } + klog.V(5).Infof("the hami-core annotation exists in pod %s/%s", pod.Namespace, pod.Name) + config := devconfig.GetConfig() + if config == nil { + klog.V(5).Infof("device configuration is empty.") + return nil + } + var patch []patchOperation + for idx, container := range pod.Spec.Containers { + if container.Lifecycle != nil && container.Lifecycle.PostStart != nil { + klog.V(5).Infof("container %s already has a postStart lifecycle handler, skipping hami-vnpu-core injection", container.Name) + continue + } + hasAscendResource := false + for _, vnpuConfig := range config.VNPUs.Configs { + if checkResource(container.Resources, vnpuConfig.ResourceName) { + resourceCoreName := fmt.Sprintf("%s-core", vnpuConfig.ResourceName) + if checkResource(container.Resources, vnpuConfig.ResourceMemoryName) && checkResource(container.Resources, resourceCoreName) { + hasAscendResource = true + } + break + } + } + if !hasAscendResource { + continue + } + klog.V(3).Infof("Ascend core resource detected, injecting postStart lifecycle for container %s", container.Name) + handler := v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{ + "/bin/sh", + "-c", + "export RUST_LOG=info\n/hami-vnpu-core/limiter > /tmp/limiter_manager.log 2>&1 &", + }, + }, + } + if container.Lifecycle == nil { + patch = append(patch, patchOperation{ + Op: jsonPatchOpAdd, + Path: fmt.Sprintf("/spec/containers/%d/lifecycle", idx), + Value: v1.Lifecycle{ + PostStart: &handler, + }, + }) + continue + } + + patch = append(patch, patchOperation{ + Op: jsonPatchOpAdd, + Path: fmt.Sprintf("/spec/containers/%d/lifecycle/postStart", idx), + Value: handler, + }) + } + + return patch +} + +func checkResource(req v1.ResourceRequirements, resourceName string) bool { + if val, ok := req.Limits[v1.ResourceName(resourceName)]; ok && !val.IsZero() { + return true + } + if val, ok := req.Requests[v1.ResourceName(resourceName)]; ok && !val.IsZero() { + return true + } + return false +} diff --git a/pkg/webhooks/admission/pods/mutate/device_mutator_test.go b/pkg/webhooks/admission/pods/mutate/device_mutator_test.go new file mode 100644 index 00000000000..8ab80dc517b --- /dev/null +++ b/pkg/webhooks/admission/pods/mutate/device_mutator_test.go @@ -0,0 +1,444 @@ +/* +Copyright 2026 The Volcano Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package mutate + +import ( + "testing" + + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes/fake" + devices "volcano.sh/volcano/pkg/scheduler/api/devices" + devconfig "volcano.sh/volcano/pkg/scheduler/api/devices/config" +) + +func TestAscendMutator_Name(t *testing.T) { + mutator := newAscendMutator() + if mutator.Name() != "ascend" { + t.Errorf("expected name 'ascend', got '%s'", mutator.Name()) + } +} + +func TestAscendMutator_MutateAdmission(t *testing.T) { + const ( + knownGeometriesCMName = "test-cm" + knownGeometriesCMNamespace = "test-ns" + deviceConfigDataKey = "device-config.yaml" + ) + + deviceConfigYAML := "vnpus:\n" + + " hamiVnpuCore: true\n" + + " configs:\n" + + " - chipName: cm-310P3\n" + + " commonWord: Ascend310P\n" + + " resourceName: huawei.com/Ascend310P\n" + + " resourceMemoryName: huawei.com/Ascend310P-memory\n" + + " memoryAllocatable: 21527\n" + + " memoryCapacity: 24576\n" + + " aiCore: 8\n" + + " aiCPU: 7\n" + + " templates:\n" + + " - name: vir01\n" + + " memory: 3072\n" + + " aiCore: 1\n" + + " aiCPU: 1\n" + + " - chipName: cm-910A\n" + + " commonWord: Ascend910A\n" + + " resourceName: huawei.com/Ascend910A\n" + + " resourceMemoryName: huawei.com/Ascend910A-memory\n" + + " memoryAllocatable: 32768\n" + + " memoryCapacity: 32768\n" + + " aiCore: 30\n" + + " templates:\n" + + " - name: vir02\n" + + " memory: 2184\n" + + " aiCore: 2\n" + + configMap := &v1.ConfigMap{ + ObjectMeta: metav1.ObjectMeta{ + Name: knownGeometriesCMName, + Namespace: knownGeometriesCMNamespace, + }, + Data: map[string]string{ + deviceConfigDataKey: deviceConfigYAML, + }, + } + + fakeClient := fake.NewSimpleClientset(configMap) + devices.SetClient(fakeClient) + defer devices.SetClient(nil) + devconfig.InitDevicesConfig(knownGeometriesCMName, knownGeometriesCMNamespace) + tests := []struct { + name string + pod *v1.Pod + expectedPatchCount int + expectedPatchPaths []string + expectedPatchOps []string + }{ + { + name: "nil pod returns nil", + pod: nil, + expectedPatchCount: 0, + }, + { + name: "pod without annotations returns nil", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "pod without hami-core annotation returns nil", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "other-mode", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "container without ascend resource returns nil", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "container with lifecycle postStart already exists skip", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + Lifecycle: &v1.Lifecycle{ + PostStart: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{"echo", "hello"}, + }, + }, + }, + }, + }, + }, + }, + expectedPatchCount: 0, + }, + { + name: "successfully add lifecycle to container without lifecycle", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle"}, + expectedPatchOps: []string{"add"}, + }, + { + name: "successfully add postStart to container with lifecycle but no postStart", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + Lifecycle: &v1.Lifecycle{ + PreStop: &v1.LifecycleHandler{ + Exec: &v1.ExecAction{ + Command: []string{"echo", "goodbye"}, + }, + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle/postStart"}, + expectedPatchOps: []string{"add"}, + }, + { + name: "multiple containers with ascend resource", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "container-1", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + { + Name: "container-2", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("2"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + { + Name: "container-3", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "cpu": resource.MustParse("1"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 2, + expectedPatchPaths: []string{ + "/spec/containers/0/lifecycle", + "/spec/containers/1/lifecycle", + }, + expectedPatchOps: []string{"add", "add"}, + }, + { + name: "container with ascend resource in requests", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Requests: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle"}, + expectedPatchOps: []string{"add"}, + }, + { + name: "multiple resource types in config", + pod: &v1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + "huawei.com/vnpu-mode": "hami-core", + }, + }, + Spec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "test-container", + Resources: v1.ResourceRequirements{ + Limits: v1.ResourceList{ + "huawei.com/Ascend310P": resource.MustParse("1"), + "huawei.com/Ascend310P-memory": resource.MustParse("1024"), + "huawei.com/Ascend310P-core": resource.MustParse("10"), + }, + }, + }, + }, + }, + }, + expectedPatchCount: 1, + expectedPatchPaths: []string{"/spec/containers/0/lifecycle"}, + expectedPatchOps: []string{"add"}, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + mutator := newAscendMutator() + patches := mutator.MutateAdmission(tt.pod) + + if len(patches) != tt.expectedPatchCount { + t.Errorf("expected %d patches, got %d", tt.expectedPatchCount, len(patches)) + } + + for i, patch := range patches { + if i < len(tt.expectedPatchPaths) && patch.Path != tt.expectedPatchPaths[i] { + t.Errorf("expected patch path %s, got %s", tt.expectedPatchPaths[i], patch.Path) + } + if i < len(tt.expectedPatchOps) && patch.Op != tt.expectedPatchOps[i] { + t.Errorf("expected patch op %s, got %s", tt.expectedPatchOps[i], patch.Op) + } + if patch.Op == jsonPatchOpAdd { + if patch.Path == "/spec/containers/0/lifecycle" || patch.Path == "/spec/containers/1/lifecycle" { + lifecycle, ok := patch.Value.(v1.Lifecycle) + if !ok { + t.Errorf("patch value is not v1.Lifecycle") + } + if lifecycle.PostStart == nil { + t.Errorf("PostStart handler is nil") + } + if lifecycle.PostStart.Exec == nil { + t.Errorf("Exec action is nil") + } + if len(lifecycle.PostStart.Exec.Command) != 3 { + t.Errorf("expected command length 3, got %d", len(lifecycle.PostStart.Exec.Command)) + } + } + } + } + }) + } + + config := devconfig.GetConfig() + if config == nil { + t.Fatalf("expected devices config loaded from configmap, got nil") + } + if len(config.VNPUs.Configs) != 2 { + t.Fatalf("expected 2 vnpu configs from configmap, got %d", len(config.VNPUs.Configs)) + } + if config.VNPUs.Configs[0].ChipName != "cm-310P3" { + t.Fatalf("expected chipName loaded from configmap to be cm-310P3, got %s", config.VNPUs.Configs[0].ChipName) + } +} + +// TestAscendMutator_RegisteredDevicesMutators verifies that AscendMutator is registered +func TestAscendMutator_RegisteredDevicesMutators(t *testing.T) { + found := false + for _, mutator := range registeredDeviceMutators { + if mutator.Name() == "ascend" { + found = true + break + } + } + if !found { + t.Error("AscendMutator not found in registeredDeviceMutators") + } +} diff --git a/pkg/webhooks/admission/pods/mutate/mutate_pod.go b/pkg/webhooks/admission/pods/mutate/mutate_pod.go index c4d94667b9c..a7e63e7a1f4 100644 --- a/pkg/webhooks/admission/pods/mutate/mutate_pod.go +++ b/pkg/webhooks/admission/pods/mutate/mutate_pod.go @@ -120,6 +120,10 @@ func createPatch(pod *v1.Pod) ([]byte, error) { patch = append(patch, *patchGates) } + if config.ConfigData.DeviceMutatorConf.Enable { + patch = append(patch, patchDeviceMutation(pod)...) + } + for _, resourceGroup := range config.ConfigData.ResGroupsConfig { klog.V(3).Infof("resourceGroup %s", resourceGroup.ResourceGroup) group := GetResGroup(resourceGroup) @@ -260,3 +264,15 @@ func patchSchedulerName(resGroupConfig wkconfig.ResGroupConfig) *patchOperation return &patchOperation{Op: "add", Path: "/spec/schedulerName", Value: resGroupConfig.SchedulerName} } + +func patchDeviceMutation(pod *v1.Pod) []patchOperation { + var patch []patchOperation + for _, mutator := range registeredDeviceMutators { + patches := mutator.MutateAdmission(pod) + if len(patches) == 0 { + continue + } + patch = append(patch, patches...) + } + return patch +} diff --git a/pkg/webhooks/config/config.go b/pkg/webhooks/config/config.go index fe57212792f..2eba5445767 100644 --- a/pkg/webhooks/config/config.go +++ b/pkg/webhooks/config/config.go @@ -45,10 +45,17 @@ type ResGroupConfig struct { Affinity string `yaml:"affinity"` } +type DeviceMutatorConfig struct { + Enable bool `yaml:"enable"` + KnownGeometriesCMName string `yaml:"knownGeometriesCMName"` + KnownGeometriesCMNamespace string `yaml:"knownGeometriesCMNamespace"` +} + // AdmissionConfiguration defines the configuration of admission. type AdmissionConfiguration struct { sync.Mutex - ResGroupsConfig []ResGroupConfig `yaml:"resourceGroups"` + DeviceMutatorConf DeviceMutatorConfig `yaml:"deviceMutator"` + ResGroupsConfig []ResGroupConfig `yaml:"resourceGroups"` } var admissionConf AdmissionConfiguration @@ -73,6 +80,7 @@ func LoadAdmissionConf(confPath string) *AdmissionConfiguration { admissionConf.Lock() admissionConf.ResGroupsConfig = data.ResGroupsConfig + admissionConf.DeviceMutatorConf = data.DeviceMutatorConf admissionConf.Unlock() return &admissionConf }