070-容器云高级特性统一设置
高级特性
一、算力卡参数说明
1. 1 昇腾NPU测试
1.1.1 hami 软切测试YAML
ascend-device-plugin:
ascendContainerToolkit:
enabled: true
ascendDevicePlugin:
enabled: false
ascendDeviceShare:
enabled: true
hard:
enabled: true
nodeSelector:
npu.huawei.sharing: hard
soft:
enabled: true
nodeSelector:
npu.huawei.sharing: soft
enabled: true
global:
registry: '10.30.15.90:5000/topke-system'
gpu-operator:
ccManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
cdi:
enabled: false
dcgm:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
dcgmExporter:
enableGpuAccounting: true
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
serviceMonitor:
enabled: true
devicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
driver:
manager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
enabled: false
gdrcopy:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gds:
enabled: false
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gfd:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
kataManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
migManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
node-feature-discovery:
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
tag: v0.18.2
nodeStatusExporter:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
operator:
initContainer:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
metrics:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
sandboxDevicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
toolkit:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
validator:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vfioManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vgpuDeviceManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
vgpuManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system'
hami:
ascend-device-plugin:
config:
create: false
deviceConfigMapName: hami-scheduler-device
existingDeviceConfigMapName: hami-scheduler-device
enabled: true
hamiVnpuCore:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/projecthami/ascend-device-plugin'
devicePlugin:
enabled: false
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
nodeConfiguration:
config: |-
{
"nodeconfig": [
{
"name": "",
"operatingmode": "hami-core",
"devicememoryscaling": 1,
"devicesplitcount": 10,
"preconfigureddevicememory": 0,
"migstrategy": "none",
"filterdevices": {
"uuid": [],
"index": []
},
"enablegetpreferredallocation": false
}
]
}
runtimeClassName: ''
devices:
ascend:
enabled: true
hamiVnpuCore: true
runtimeClassName: ascend
iluvatar:
enabled: false
enabled: true
scheduler:
extender:
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
kubeScheduler:
image:
registry: '10.30.15.90:5000/registry.k8s.io'
repository: registry.k8s.io/kube-scheduler
leaderElect: false
patch:
imageNew:
registry: '10.30.15.90:5000/topke-system/docker.io'
hygon-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
ix-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
lws:
enabled: false
image:
manager:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/lws/lws'
replicaCount: 2
node-feature-discovery:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
worker:
config:
core:
labelSources:
- custom
volcano:
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override: |
actions: enqueue, allocate, preempt, backfill, reclaim
tiers:
- plugins:
- name: predicates
- name: deviceshare
arguments:
deviceshare.AscendHAMiVNPUEnable: true
deviceshare.SchedulePolicy: binpack
deviceshare.KnownGeometriesCMNamespace: topke-system
deviceshare.KnownGeometriesCMName: hami-scheduler-device
- name: priority
- name: conformance
- name: overcommit
arguments:
overcommit-factor: 2.0
- plugins:
- name: gang
enablePreemptable: false
enableJobStarving: false
- name: drf
enablePreemptable: false
- name: proportion
- name: nodeorder
- name: binpack
enabled: true
volcano-vgpu-device-plugin:
enabled: false
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
monitor:
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
nodeConfig:
configs:
- devicememoryscaling: 1.5
devicesplitcount: 10
filterdevices:
index: []
uuid: []
migstrategy: none
name: ''
operatingmode: hami-core
enabled: true
nodeSelector:
volcano-gpu: 'on'
1.1.2 hami 硬切测试YAML
ascend-device-plugin:
ascendContainerToolkit:
enabled: true
ascendDevicePlugin:
enabled: false
ascendDeviceShare:
enabled: true
hard:
enabled: true
nodeSelector:
npu.huawei.sharing: hard
soft:
enabled: true
nodeSelector:
npu.huawei.sharing: soft
enabled: true
global:
registry: '10.30.15.90:5000/topke-system'
gpu-operator:
ccManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
cdi:
enabled: false
dcgm:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
dcgmExporter:
enableGpuAccounting: true
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
serviceMonitor:
enabled: true
devicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
driver:
manager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
enabled: false
gdrcopy:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gds:
enabled: false
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gfd:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
kataManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
migManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
node-feature-discovery:
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
tag: v0.18.2
nodeStatusExporter:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
operator:
initContainer:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
metrics:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
sandboxDevicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
toolkit:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
validator:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vfioManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vgpuDeviceManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
vgpuManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system'
hami:
ascend-device-plugin:
config:
create: false
deviceConfigMapName: hami-scheduler-device
existingDeviceConfigMapName: hami-scheduler-device
enabled: true
hamiVnpuCore:
enabled: false
image:
repository: '10.30.15.90:5000/topke-system/projecthami/ascend-device-plugin'
devicePlugin:
enabled: false
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
nodeConfiguration:
config: |-
{
"nodeconfig": [
{
"name": "",
"operatingmode": "hami-core",
"devicememoryscaling": 1,
"devicesplitcount": 10,
"preconfigureddevicememory": 0,
"migstrategy": "none",
"filterdevices": {
"uuid": [],
"index": []
},
"enablegetpreferredallocation": false
}
]
}
runtimeClassName: ''
devices:
ascend:
enabled: true
hamiVnpuCore: false
runtimeClassName: ascend
iluvatar:
enabled: false
enabled: true
scheduler:
extender:
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
kubeScheduler:
image:
registry: '10.30.15.90:5000/registry.k8s.io'
repository: registry.k8s.io/kube-scheduler
leaderElect: false
patch:
imageNew:
registry: '10.30.15.90:5000/topke-system/docker.io'
hygon-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
ix-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
lws:
enabled: false
image:
manager:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/lws/lws'
replicaCount: 2
node-feature-discovery:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
worker:
config:
core:
labelSources:
- custom
volcano:
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override: |
actions: enqueue, allocate, preempt, backfill, reclaim
tiers:
- plugins:
- name: predicates
- name: deviceshare
arguments:
deviceshare.AscendHAMiVNPUEnable: true
deviceshare.SchedulePolicy: binpack
deviceshare.KnownGeometriesCMNamespace: topke-system
deviceshare.KnownGeometriesCMName: hami-scheduler-device
- name: priority
- name: conformance
- name: overcommit
arguments:
overcommit-factor: 2.0
- plugins:
- name: gang
enablePreemptable: false
enableJobStarving: false
- name: drf
enablePreemptable: false
- name: proportion
- name: nodeorder
- name: binpack
enabled: true
volcano-vgpu-device-plugin:
enabled: false
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
monitor:
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
nodeConfig:
configs:
- devicememoryscaling: 1.5
devicesplitcount: 10
filterdevices:
index: []
uuid: []
migstrategy: none
name: ''
operatingmode: hami-core
enabled: true
nodeSelector:
volcano-gpu: 'on'
1.1.3 Volcano 软切测试YAML
ascend-device-plugin:
ascendContainerToolkit:
enabled: true
ascendDevicePlugin:
enabled: false
ascendDeviceShare:
enabled: true
hard:
enabled: true
nodeSelector:
npu.huawei.sharing: hard
soft:
enabled: true
nodeSelector:
npu.huawei.sharing: soft
enabled: true
global:
registry: '10.30.15.90:5000/topke-system'
gpu-operator:
ccManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
cdi:
enabled: false
dcgm:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
dcgmExporter:
enableGpuAccounting: true
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
serviceMonitor:
enabled: true
devicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
driver:
manager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
enabled: false
gdrcopy:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gds:
enabled: false
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gfd:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
kataManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
migManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
node-feature-discovery:
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
tag: v0.18.2
nodeStatusExporter:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
operator:
initContainer:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
metrics:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
sandboxDevicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
toolkit:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
validator:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vfioManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vgpuDeviceManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
vgpuManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system'
hami:
ascend-device-plugin:
config:
create: false
deviceConfigMapName: hami-scheduler-device
existingDeviceConfigMapName: hami-scheduler-device
enabled: true
hamiVnpuCore:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/projecthami/ascend-device-plugin'
devicePlugin:
enabled: false
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
nodeConfiguration:
config: |-
{
"nodeconfig": [
{
"name": "",
"operatingmode": "hami-core",
"devicememoryscaling": 1,
"devicesplitcount": 10,
"preconfigureddevicememory": 0,
"migstrategy": "none",
"filterdevices": {
"uuid": [],
"index": []
},
"enablegetpreferredallocation": false
}
]
}
runtimeClassName: ''
devices:
ascend:
enabled: true
hamiVnpuCore: true
runtimeClassName: ascend
iluvatar:
enabled: false
enabled: true
scheduler:
extender:
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
kubeScheduler:
image:
registry: '10.30.15.90:5000/registry.k8s.io'
repository: registry.k8s.io/kube-scheduler
leaderElect: false
patch:
imageNew:
registry: '10.30.15.90:5000/topke-system/docker.io'
hygon-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
ix-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
lws:
enabled: false
image:
manager:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/lws/lws'
replicaCount: 2
node-feature-discovery:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
worker:
config:
core:
labelSources:
- custom
volcano:
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override: |
actions: enqueue, allocate, preempt, backfill, reclaim
tiers:
- plugins:
- name: predicates
- name: deviceshare
arguments:
deviceshare.AscendHAMiVNPUEnable: true
deviceshare.SchedulePolicy: binpack
deviceshare.KnownGeometriesCMNamespace: topke-system
deviceshare.KnownGeometriesCMName: hami-scheduler-device
- name: priority
- name: conformance
- name: overcommit
arguments:
overcommit-factor: 2.0
- plugins:
- name: gang
enablePreemptable: false
enableJobStarving: false
- name: drf
enablePreemptable: false
- name: proportion
- name: nodeorder
- name: binpack
enabled: true
volcano-vgpu-device-plugin:
enabled: false
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
monitor:
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
nodeConfig:
configs:
- devicememoryscaling: 1.5
devicesplitcount: 10
filterdevices:
index: []
uuid: []
migstrategy: none
name: ''
operatingmode: hami-core
enabled: true
nodeSelector:
volcano-gpu: 'on'
1.1.4 Volcano硬切测试YAML
ascend-device-plugin:
ascendContainerToolkit:
enabled: true
ascendDevicePlugin:
enabled: false
ascendDeviceShare:
enabled: true
hard:
enabled: true
nodeSelector:
npu.huawei.sharing: hard
soft:
enabled: true
nodeSelector:
npu.huawei.sharing: soft
enabled: true
global:
registry: '10.30.15.90:5000/topke-system'
gpu-operator:
ccManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
cdi:
enabled: false
dcgm:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
dcgmExporter:
enableGpuAccounting: true
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
serviceMonitor:
enabled: true
devicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
driver:
manager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
enabled: false
gdrcopy:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gds:
enabled: false
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gfd:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
kataManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
migManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
node-feature-discovery:
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
tag: v0.18.2
nodeStatusExporter:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
operator:
initContainer:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
metrics:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
sandboxDevicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
toolkit:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
validator:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vfioManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vgpuDeviceManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
vgpuManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system'
hami:
ascend-device-plugin:
config:
create: false
deviceConfigMapName: hami-scheduler-device
existingDeviceConfigMapName: hami-scheduler-device
enabled: true
hamiVnpuCore:
enabled: false
image:
repository: '10.30.15.90:5000/topke-system/projecthami/ascend-device-plugin'
devicePlugin:
enabled: false
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
nodeConfiguration:
config: |-
{
"nodeconfig": [
{
"name": "",
"operatingmode": "hami-core",
"devicememoryscaling": 1,
"devicesplitcount": 10,
"preconfigureddevicememory": 0,
"migstrategy": "none",
"filterdevices": {
"uuid": [],
"index": []
},
"enablegetpreferredallocation": false
}
]
}
runtimeClassName: ''
devices:
ascend:
enabled: true
hamiVnpuCore: false
runtimeClassName: ascend
iluvatar:
enabled: false
enabled: true
scheduler:
extender:
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
kubeScheduler:
image:
registry: '10.30.15.90:5000/registry.k8s.io'
repository: registry.k8s.io/kube-scheduler
leaderElect: false
patch:
imageNew:
registry: '10.30.15.90:5000/topke-system/docker.io'
hygon-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
ix-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
lws:
enabled: false
image:
manager:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/lws/lws'
replicaCount: 2
node-feature-discovery:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
worker:
config:
core:
labelSources:
- custom
volcano:
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override: |
actions: enqueue, allocate, preempt, backfill, reclaim
tiers:
- plugins:
- name: predicates
- name: deviceshare
arguments:
deviceshare.AscendHAMiVNPUEnable: true
deviceshare.SchedulePolicy: binpack
deviceshare.KnownGeometriesCMNamespace: topke-system
deviceshare.KnownGeometriesCMName: hami-scheduler-device
- name: priority
- name: conformance
- name: overcommit
arguments:
overcommit-factor: 2.0
- plugins:
- name: gang
enablePreemptable: false
enableJobStarving: false
- name: drf
enablePreemptable: false
- name: proportion
- name: nodeorder
- name: binpack
enabled: true
volcano-vgpu-device-plugin:
enabled: false
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
monitor:
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
nodeConfig:
configs:
- devicememoryscaling: 1.5
devicesplitcount: 10
filterdevices:
index: []
uuid: []
migstrategy: none
name: ''
operatingmode: hami-core
enabled: true
nodeSelector:
volcano-gpu: 'on'
1.2 界面参数说明
- x86 英伟达选择
# 整个交互界面参数说明
1. 打开开关显示下面配置
2. 选中算力卡，意思安装该算力卡
2.1 天数GPU：ix-device-plugin.enabled=true
2.2 海光DCU:hygon-device-plugin.enabled=true
2.3 昇腾NPU:ascend-device-plugin.enabled=true
0803新增：选择虚拟化的时候：ascend-device-plugin.ascendDevicePlugin=false
2.4 NVIDIA:gpu-operator.enabled=true
2.5 如果安装NVIDIA时,node-feature-discovery.enabled=false,其他算力卡都要选择true
3. 分布式打开设置：lws.emabled=true,否则就是false
4. 模式选择
4.1 选择整卡模式无插件选择
4.2 仅选中nvidia 才显示MIG模式
4.3 选中虚拟化模式
4.3.1 选中hami, hami.enabled=true
4.3.1.1 打开 numa亲和：vocano.custom.scheduler_config_override 的第二个plugins 覆盖或者增加
- name: numa-aware
arguments:
weight: 10
关闭时去掉
4.3.1.2 打开 优先级抢占调度：vocano.custom.scheduler_config_override，tiers所有的.plugins 里面所有的 enablePreemptable 设置为true
4.3.1.3 打开 公平调度：vocano.custom.scheduler_config_override 修改
- name: drf
enablePreemptable: false // 如果4.3.1.2 开启这里要设置为true
4.3.1.4 打开组调度：vocano.custom.scheduler_config_override tiers[0].plugins 里面查找如果没有就增加，关闭就就删除
- name:
enablePreemptable: false // 如果4.3.1.2 开启这里要设置为true
4.3.1.5 选择volcano调度策略
4.3.1.5.1 选择binpack：
第二plugins 里面增加 - name: binpack 删除 spread
4.3.1.5.2 spread：
第二plugins 里面增加 - name: spread 删除 binpack
4.3.1.6 打开重调度 vocano.custom.descheduler_enable 设置为true
4.3.1.6 打开在离线混部署 vocano.custom.colocation_enable 设置为true
增加labels [{nodename:"10-30-100-155","patch_labels":{"volcano.sh/colocation":"true"}}]
4.3.1.6 打开资源超卖 增加laebs [{nodename:"10-30-100-155","patch_labels":{"volcano.sh/colocation":"true","volcano.sh/oversubscription":"true"}}]
4.3.1.6 设置出口网络带宽 增加laebs [{nodename:"10-30-100-155","patch_labels":{"gpu_core_mode":"whole"，"volcano.sh/colocation":"true","volcano.sh/oversubscription":"true"}},"patch_annotations":{"volcano.sh/network-bandwidth-rate":1000}]
- arm 昇腾选择
2. 选中算力卡，意思安装该算力卡,选中昇腾
4.3 选中虚拟化模式
4.3.1 选中hami, hami.enabled=true
ascend-device-plugin.ascendDevicePlugin=false // 默认是true
hami.devicePlugin.enabled=false,默认是true
hami.devices.ascend.enabled=true
hami.ascend-device-plugin.enabled=true 默认是false
hami.ascend-device-plugin.hamiVnpuCore.enabled=false(默认硬切分) 控制是否软切分，默认就是硬切分，同英伟达的MIG功能
ascend-device-plugin.ascendDeviceShare.enabled=true
4.3.2如果选择了volcano, 配置改成如下:
volcano-scheduler.conf: |
actions: "enqueue, allocate, backfill"
tiers:
- plugins:
- name: predicates
- name: deviceshare
arguments:
deviceshare.AscendHAMiVNPUEnable: true # enable ascend vnpu
deviceshare.SchedulePolicy: binpack # scheduling policy. binpack / spread
deviceshare.KnownGeometriesCMNamespace: kube-system
deviceshare.KnownGeometriesCMName: hami-scheduler-device
标签设置
ascend: "on" # 必传
npu.huawei.sharing=soft # 软切分
npu.huawei.sharing=hard # 硬切分
arm 成功案例
#hami硬切分
ascend-device-plugin:
ascendContainerToolkit:
enabled: true
ascendDevicePlugin:
enabled: false
enabled: true
ascendDeviceShare:
enabled: true
hard:
enabled: true
nodeSelector:
npu.huawei.sharing: hard
soft:
enabled: true
nodeSelector:
npu.huawei.sharing: soft
global:
registry: '10.30.15.90:5000/topke-system'
gpu-operator:
ccManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
cdi:
enabled: false
dcgm:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
dcgmExporter:
enableGpuAccounting: true
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
serviceMonitor:
enabled: true
devicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
driver:
manager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
enabled: false
gdrcopy:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gds:
enabled: false
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gfd:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
kataManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
migManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
node-feature-discovery:
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
tag: v0.18.2
nodeStatusExporter:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
operator:
initContainer:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
metrics:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
sandboxDevicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
toolkit:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
validator:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vfioManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vgpuDeviceManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
vgpuManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system'
hami:
ascend-device-plugin:
config:
create: false
deviceConfigMapName: hami-scheduler-device
existingDeviceConfigMapName: hami-scheduler-device
enabled: true
hamiVnpuCore:
enabled: false
image:
repository: '10.30.15.90:5000/topke-system/projecthami/ascend-device-plugin'
devicePlugin:
enabled: false
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
nodeConfiguration:
config:
nodeconfig:
- name: 10-30-15-89
operatingmode: hami-core
devicememoryscaling: 1
devicesplitcount: 10
preconfigureddevicememory: 0
migstrategy: none
filterdevices:
uuid: []
index: []
enablegetpreferredallocation: false
runtimeClassName: ''
devices:
ascend:
enabled: true
hamiVnpuCore: false
runtimeClassName: ascend
iluvatar:
enabled: false
enabled: true
scheduler:
extender:
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
kubeScheduler:
image:
registry: '10.30.15.90:5000/registry.k8s.io'
repository: registry.k8s.io/kube-scheduler
leaderElect: false
path:
imageNew:
registry: '10.30.15.90:5000/topke-system/docker.io'
hygon-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
ix-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
lws:
enabled: false
image:
manager:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/lws/lws'
replicaCount: 2
node-feature-discovery:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
worker:
config:
core:
labelSources:
- custom
volcano:
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override:
actions: 'enqueue, allocate, backfill'
tiers:
- plugins:
- name: priority
- name: gang
enablePreemptable: false
- name: conformance
- plugins:
- name: overcommit
- name: drf
enablePreemptable: false
- name: predicates
- name: proportion
- name: nodeorder
- name: binpack
enabled: false
volcano-vgpu-device-plugin:
enabled: false
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
monitor:
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
nodeConfig:
configs:
- devicememoryscaling: 1.5
devicesplitcount: 10
filterdevices:
index: []
uuid: []
migstrategy: none
name: ''
operatingmode: hami-core
enabled: true
nodeSelector:
volcano-gpu: 'on'
----
# hami软切分
ascend-device-plugin:
ascendContainerToolkit:
enabled: true
ascendDevicePlugin:
enabled: false
enabled: true
ascendDeviceShare:
enabled: true
hard:
enabled: true
nodeSelector:
npu.huawei.sharing: hard
soft:
enabled: true
nodeSelector:
npu.huawei.sharing: soft
global:
registry: '10.30.15.90:5000/topke-system'
gpu-operator:
ccManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
cdi:
enabled: false
dcgm:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
dcgmExporter:
enableGpuAccounting: true
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
serviceMonitor:
enabled: true
devicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
driver:
manager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
enabled: false
gdrcopy:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gds:
enabled: false
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gfd:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
kataManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
migManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
node-feature-discovery:
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
tag: v0.18.2
nodeStatusExporter:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
operator:
initContainer:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
metrics:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
sandboxDevicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
toolkit:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
validator:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vfioManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vgpuDeviceManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
vgpuManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system'
hami:
ascend-device-plugin:
config:
create: false
deviceConfigMapName: hami-scheduler-device
existingDeviceConfigMapName: hami-scheduler-device
enabled: true
hamiVnpuCore:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/projecthami/ascend-device-plugin'
devicePlugin:
enabled: false
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
nodeConfiguration:
config:
nodeconfig:
- name: 10-30-15-89
operatingmode: hami-core
devicememoryscaling: 1
devicesplitcount: 10
preconfigureddevicememory: 0
migstrategy: none
filterdevices:
uuid: []
index: []
enablegetpreferredallocation: false
runtimeClassName: ''
devices:
ascend:
enabled: true
hamiVnpuCore: true
runtimeClassName: ascend
iluvatar:
enabled: false
enabled: true
scheduler:
extender:
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
kubeScheduler:
image:
registry: '10.30.15.90:5000/registry.k8s.io'
repository: registry.k8s.io/kube-scheduler
leaderElect: false
path:
imageNew:
registry: '10.30.15.90:5000/topke-system/docker.io'
hygon-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
ix-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
lws:
enabled: false
image:
manager:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/lws/lws'
replicaCount: 2
node-feature-discovery:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
worker:
config:
core:
labelSources:
- custom
volcano:
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override:
actions: 'enqueue, allocate, backfill'
tiers:
- plugins:
- name: priority
- name: gang
enablePreemptable: false
- name: conformance
- plugins:
- name: overcommit
- name: drf
enablePreemptable: false
- name: predicates
- name: proportion
- name: nodeorder
- name: binpack
enabled: false
volcano-vgpu-device-plugin:
enabled: false
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
monitor:
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
nodeConfig:
configs:
- devicememoryscaling: 1.5
devicesplitcount: 10
filterdevices:
index: []
uuid: []
migstrategy: none
name: ''
operatingmode: hami-core
enabled: true
nodeSelector:
volcano-gpu: 'on'
---
# volcano 软硬沿用hami
ascend-device-plugin:
ascendContainerToolkit:
enabled: true
ascendDevicePlugin:
enabled: false
enabled: true
ascendDeviceShare:
enabled: true
hard:
enabled: true
nodeSelector:
npu.huawei.sharing: hard
soft:
enabled: true
nodeSelector:
npu.huawei.sharing: soft
global:
registry: '10.30.15.90:5000/topke-system'
gpu-operator:
ccManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
cdi:
enabled: false
dcgm:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
dcgmExporter:
enableGpuAccounting: true
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
serviceMonitor:
enabled: true
devicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
driver:
manager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
enabled: false
gdrcopy:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gds:
enabled: false
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
gfd:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
kataManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
migManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
node-feature-discovery:
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
tag: v0.18.2
nodeStatusExporter:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
operator:
initContainer:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
metrics:
enabled: true
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
sandboxDevicePlugin:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
toolkit:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/k8s'
validator:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vfioManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia'
vgpuDeviceManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
vgpuManager:
driverManager:
repository: '10.30.15.90:5000/topke-system/nvcr.io/nvidia/cloud-native'
repository: '10.30.15.90:5000/topke-system'
hami:
ascend-device-plugin:
config:
create: false
deviceConfigMapName: hami-scheduler-device
existingDeviceConfigMapName: hami-scheduler-device
enabled: true
hamiVnpuCore:
enabled: false
image:
repository: '10.30.15.90:5000/topke-system/projecthami/ascend-device-plugin'
devicePlugin:
enabled: false
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
nodeConfiguration:
config:
nodeconfig:
- name: 10-30-15-89
operatingmode: hami-core
devicememoryscaling: 1
devicesplitcount: 10
preconfigureddevicememory: 0
migstrategy: none
filterdevices:
uuid: []
index: []
enablegetpreferredallocation: false
runtimeClassName: ''
devices:
ascend:
enabled: true
hamiVnpuCore: false
runtimeClassName: ascend
iluvatar:
enabled: false
enabled: true
scheduler:
extender:
image:
registry: '10.30.15.90:5000/topke-system/docker.io'
kubeScheduler:
image:
registry: '10.30.15.90:5000/registry.k8s.io'
repository: registry.k8s.io/kube-scheduler
leaderElect: false
path:
imageNew:
registry: '10.30.15.90:5000/topke-system/docker.io'
hygon-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
ix-device-plugin:
enabled: false
global:
registry: '10.30.15.90:5000/topke-system'
lws:
enabled: false
image:
manager:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/lws/lws'
replicaCount: 2
node-feature-discovery:
enabled: true
image:
repository: '10.30.15.90:5000/topke-system/registry.k8s.io/nfd/node-feature-discovery'
worker:
config:
core:
labelSources:
- custom
volcano:
basic:
image_registry: '10.30.15.90:5000/topke-system/docker.io'
custom:
colocation_enable: false
descheduler_enable: false
scheduler_config_override:
actions: 'enqueue, allocate, backfill'
tiers:
- plugins:
- name: predicates
- name: deviceshare
arguments:
deviceshare.AscendHAMiVNPUEnable: true
deviceshare.SchedulePolicy: binpack
deviceshare.KnownGeometriesCMNamespace: topke-system
deviceshare.KnownGeometriesCMName: hami-scheduler-device
enabled: true
volcano-vgpu-device-plugin:
enabled: false
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
monitor:
image:
repository: >-
10.30.15.90:5000/topke-system/docker.io/projecthami/volcano-vgpu-device-plugin
nodeConfig:
configs:
- devicememoryscaling: 1.5
devicesplitcount: 10
filterdevices:
index: []
uuid: []
migstrategy: none
name: ''
operatingmode: hami-core
enabled: true
nodeSelector:
volcano-gpu: 'on'
----
1.3 模式和插件参数说明
#模式和插件选择参数说明
#模式1：选择整卡模式
gpu_core_mode:whole
hami-dcgm:on
--------------------------------------
#模式2： 选择虚拟化模式
必填标签：
gpu_core_mode:hami-core
选填标签：
如果插件选择：hami-gpu
hami-gpu:on
gpu:on
hami-dcgm:on
volcano-gpu:null
如果插件选择：volcano-gpu
volcano-gpu:on
gpu:null
hami-dcgm:null
hami-gpu:null
----
------hami要修改的配置文件values中----devicePlugin.nodeConfiguration.config 是一个字符串---
{ // 多个节点需要多个对象组成数组并压缩成字符串
"nodeconfig": [
{
"name": "10-30-100-155",
"operatingmode": "hami-core",
"devicememoryscaling": 1.5,
"devicesplitcount": 10,
"migstrategy": "none",
"filterdevices": {
"uuid": [],
"index": []
},
}
]
}
------vocano要修改的配置文件values中----nodeConfig.configs 是一个yaml 对象数组---
nodeConfig:
enabled: true
configs:
- name: "10-30-100-155"
operatingmode: "hami-core"
devicememoryscaling: 1.5
devicesplitcount: 10
migstrategy: "none"
filterdevices:
uuid: []
index: []
----------------------------------------------------------------------------------------------------
#模式3：选择MIG模式
gpu_core_mode:mig
选填标签：
如果插件选择：hami-gpu
hami-gpu:on
gpu:on
hami-dcgm:on
volcano-gpu:null
如果插件选择：volcano-gpu
volcano-gpu:on
gpu:null
hami-dcgm:null
hami-gpu:null
------hami要修改的配置文件values中----devicePlugin.nodeConfiguration.config 是一个字符串---
{ // 多个节点需要多个对象组成数组并压缩成字符串
"nodeconfig": [
{
"name": "10-30-100-155",
"operatingmode": "hami-core",
"devicememoryscaling": 1.5,
"devicesplitcount": 10,
"migstrategy": "none",
"filterdevices": {
"uuid": [],
"index": []
},
}
]
}
------vocano要修改的配置文件values中----nodeConfig.configs 是一个yaml 对象数组---
nodeConfig:
enabled: true
configs:
- name: "10-30-100-155"
operatingmode: "hami-core"
devicememoryscaling: 1.5
devicesplitcount: 10
migstrategy: "none"
filterdevices:
uuid: []
index: []
---------------
device device-config.yaml 配置 volcano-vgpu-device-plugin/hami 下deviceConfig.nvidia.knownMigGeometries //举例
deviceConfig:
nvidia:
knownMigGeometries: // 数据来源于https://10.30.100.155:8443/v1/topke/node/gpu/migInstance
- models: ["A30"] // gpu_type 唯一值，有就修改，没有就新增，选择一个节点就选择一种类型 有就替换，没有就添加
allowedGeometries:
-geometries:
- name: 1g.6gb // Name
memory: 6144 // Memory
count: 4 // InstanceCount
----
选择昇腾
ascend: "on" # 必传
npu.huawei.sharing=soft # 软切分
npu.huawei.sharing=hard # 硬切分
二、volcano高级调度配置
2.1 Hami
Hami（原名 vGPU-Manager，是目前主流的 Kubernetes 异构算力虚拟化/共享开源方案）主要由两个核心组件组成：Hami-Scheduler（调度器）和 Hami-Device-Plugin（设备插件）。
在 Kubernetes 中，它们分工明确：Scheduler 负责在“大脑”层面做决定（把 Pod 调度到哪台显卡最合适的服务器上），而 Device-Plugin 负责在“节点”层面做执行（把显卡切片并真正挂载给容器）。
以下是它们的具体功能和工作原理：
2.1.1. Hami-Scheduler (调度器)
它是扩展 Kubernetes 原生调度器的一个 扩展调度器 (Scheduler Extender) 或 多机调度插件。
🛠️ 核心功能：
- 全局拓扑感知与智能调度：原生的 K8s 调度器只知道节点上有几张显卡，不知道显卡的显存剩余多少。Hami-Scheduler 能够感知集群中每张 GPU 的剩余显存和算力百分比。
- 显存与算力切片调度：当你的 Pod 请求
gpu-mem: 3000MiB（3GB 显存）或gpu-cores: 20%（20% 算力）时，它会扫描集群，精确计算出哪张物理卡的剩余资源满足要求，并将 Pod 指派到该节点。 - Binpack / Spread 策略：支持填充策略。
- Binpack（紧凑布局）：优先把一只显卡塞满，再去用下一张卡，尽量留出完整的空闲卡给大任务。
- Spread（分散布局）：优先把任务均摊到不同显卡上，保障单个任务的性能。
2.1.2 Hami-Device-Plugin (设备插件)
它是运行在集群中每个 GPU 节点上的 DaemonSet（后台守护进程），负责与节点底层的 NVIDIA 驱动、NVML 库以及容器运行时（如 Containerd/Docker）直接打交道。
** 核心功能：**
- 资源上报与虚拟化暴露：它负责向 K8s 节点（Kubelet）注册，告诉 K8s：“我这个节点虽然只有 2 张物理显卡，但我可以提供 200 个虚拟的 GPU 核心（或按显存切片后的虚拟资源）”。
- 容器环境注入（硬件挂载）：当 Hami-Scheduler 决定把 Pod 放到某个节点后，该节点上的 Hami-Device-Plugin 会介入，将指定的物理 GPU 设备的特定切片（通过环境变量或底层 OCI 钩子）注入到容器中。
- 硬隔离与资源限制（核心能力）：
- 显存隔离：确保容器内运行的 AI 模型如果超过了申请的显存（比如申请了 4G 却用了 5G），会直接触发 OOM 或被限制，绝对不会侵占同一张卡上其他容器的显存。
- 算力限制：通过底层技术限制容器的 QPS 或 CUDA Core 使用率，确保多容器共享单卡时的算力公平性。
2.1.3 两者如何协同工作？（一个 Pod 的诞生流程）
- 提交请求：你提交了一个 Pod 申请，声明需要
hami.io/vgpu-mem: 4000（4GB 显存）。 - 调度决策（Scheduler）：Hami-Scheduler 发现 Node-A 的
GPU-0还剩 5GB 显存，于是它做出决策，把这个 Pod 绑定到 Node-A，并在 Pod 的 Annotation（注解）里悄悄写下：“这个 Pod 只能用 Node-A 的 GPU-0 物理卡”。 - 节点落地（Device-Plugin）：Pod 到达 Node-A，Node-A 的 Kubelet 调用本地的 Hami-Device-Plugin。
- 环境准备（Device-Plugin）：Hami-Device-Plugin 读取到刚才的注解，启动容器，通过修改环境变量（如
NVIDIA_VISIBLE_DEVICES）和加载 Hami 自身的libvgpu.so劫持库，把物理GPU-0严格限制在 4GB 显存后，挂载给容器。 - 运行：容器成功启动，安全地在共享显卡上运行。
如果没有 Scheduler，Pod 可能会被盲目分配到显存不足的节点导致不断重启；如果没有 Device-Plugin，物理显卡就无法被切片，容器之间也会因为显存抢占而互相崩溃。
2.2 Volcano
- 核心组件一一对应关系
| 功能角色 | 纯 Hami 方案 | Volcano + vGPU 方案 | 它们在做什么？ |
|---|---|---|---|
| 集群大脑 (调度器) | Hami-Scheduler |
Volcano |
计算和分配：决定哪张 GPU 卡有足够的剩余显存和算力。 |
| 节点手脚 (设备插件) | Hami-Device-Plugin |
volcano-vgpu-device-plugin |
切片与挂载：在节点上上报虚拟资源，并真正拦截和限制容器的显存。 |
- 深入底层的两个关键真相
💡 真相一：volcano-vgpu-device-plugin 内部集成了 Hami-core
你可能已经注意到了，volcano-vgpu-device-plugin 这个开源项目的 GitHub 组织就是 Project-HAMi。
- 技术穿透：当你在集群中部署
volcano-vgpu-device-plugin时，它底层使用的依然是 Hami 研发的libvgpu.so动态链接库。 - 这意味着什么：在单卡显存硬隔离、算力限制等“节点底层控制能力”上，两者的效果是完全相同的。
💡 真相二：为什么会演变出这两个方案？（区别在“调度器”）
两者的主要区别，在于你希望让 谁 来做集群的资源调度：
- 如果你用
Hami-Scheduler：- 它只是一个专注于 GPU 虚拟化 的插件。
- 适用场景：普通的常驻服务、普通的 AI 推理、普通的 Web 应用。它让普通 K8s 调度器具备了切分显卡的能力。
- 如果你用
Volcano：- Volcano 是 CNCF 的顶级开源批处理计算系统（Batch System）。它不仅仅会调度 GPU，还天生自带极其强大的大数据/AI 大模型训练调度策略：作业排队（Queue）、组调度（Gang Scheduling/不齐心不投产）、抢占（Preemption） 和 公平共享（Fair-share）。
- 适用场景：大规模 AI 训练、批量计算、大数据作业。
- 强强联合：为了让在 Volcano 队列里排队的大模型训练任务也能享受到 GPU 切片、共享显存的好处，Hami 社区和 Volcano 社区联合开发了
volcano-vgpu-device-plugin。 [5, 6, 7]
2.3 我该怎么选？
不需要重复安装。根据你的业务特点：
- 选“纯 Hami 方案”（
Hami-Scheduler+Hami-Device-Plugin）：- 如果你们的业务主要是 AI 推理（Inference）、在线 API 服务，Pod 启动了就不会轻易关闭，不需要排队和复杂的批处理调度。
- 选“Volcano vGPU 方案”（
Volcano+volcano-vgpu-device-plugin）：- 如果你们的业务包含 AI 训练（Training）、离线计算、多个研发团队高频提交任务需要抢占和排队。这时候用 Volcano 当大脑，配合 Volcano 的 vgpu 插件，体验是最好的。 [4, 5, 6, 7, 8]
测试参数说明
#旧配置
apiVersion: v1
data:
volcano-scheduler.conf: |
actions: "enqueue, allocate, backfill"
tiers:
- plugins:
- name: priority
- name: gang
enablePreemptable: false
- name: conformance
- plugins:
- name: overcommit
- name: drf
enablePreemptable: false
- name: predicates
- name: proportion
- name: nodeorder
- name: binpack
kind: ConfigMap
metadata:
annotations:
meta.helm.sh/release-name: volcano
meta.helm.sh/release-namespace: topke-system
creationTimestamp: "2026-06-22T08:41:05Z"
labels:
app.kubernetes.io/managed-by: Helm
name: volcano-scheduler-configmap
namespace: topke-system
resourceVersion: "6480926"
uid: b3c2d7c6-cdad-4404-8d62-4b0afc67ccc2
---------------------
#新配置
apiVersion: v1
data:
volcano-scheduler.conf: |
actions: "enqueue, allocate, backfill"
tiers:
- plugins:
- name: priority
- name: gang
enablePreemptable: true
- name: conformance
- plugins:
- name: overcommit
- name: drf
enablePreemptable: true
- name: predicates
- name: proportion
- name: nodeorder
- name: spread
- name: numa-aware
arguments:
weight: 10
kind: ConfigMap
metadata:
annotations:
meta.helm.sh/release-name: volcano
meta.helm.sh/release-namespace: topke-system
topke.cluster.name: xggg
topke.cluster.uuid: 131b128d-5e11-43eb-91e4-8183b514a503
topke.createtime.unix: "1782117665"
topke.tenant.name: ken
topke.tenant.uuid: debe920e-5cc5-4e85-bd8b-a17cfc574602
creationTimestamp: "2026-06-22T08:41:05Z"
labels:
app.kubernetes.io/managed-by: Helm
name: volcano-scheduler-configmap
namespace: topke-system
resourceVersion: "6999765"
uid: b3c2d7c6-cdad-4404-8d62-4b0afc67ccc2
-----------------------------------
apiVersion: v1
data:
volcano-scheduler.conf: |
actions: "enqueue, allocate, backfill"
tiers:
- plugins:
- name: priority
- name: gang
enablePreemptable: true
- name: conformance
- plugins:
- name: overcommit
- name: drf
enablePreemptable: true
- name: predicates
- name: proportion
- name: nodeorder
- name: numa-aware
arguments:
weight: 10
- name: binpack
kind: ConfigMap
metadata:
annotations:
meta.helm.sh/release-name: volcano
meta.helm.sh/release-namespace: topke-system
topke.cluster.name: xggg
topke.cluster.uuid: 131b128d-5e11-43eb-91e4-8183b514a503
topke.createtime.unix: "1782117665"
topke.tenant.name: ken
topke.tenant.uuid: debe920e-5cc5-4e85-bd8b-a17cfc574602
creationTimestamp: "2026-06-22T08:41:05Z"
labels:
app.kubernetes.io/managed-by: Helm
name: volcano-scheduler-configmap
namespace: topke-system
resourceVersion: "7000997"
uid: b3c2d7c6-cdad-4404-8d62-4b0afc67ccc2
自动伸缩
- 镜像列表
cat << EOF > /tmp/autoscapimage.txt
registry.k8s.io/autoscaling/vpa-admission-controller:1.7.0
registry.k8s.io/ingress-nginx/kube-webhook-certgen:v20231011-8b53cabe0
registry.k8s.io/autoscaling/vpa-recommender:1.7.0
registry.k8s.io/autoscaling/vpa-updater:1.7.0
registry.aliyuncs.com/acs/kubernetes-cronhpa-controller:v1.4.3-2f290b2-aliyun
registry.k8s.io/metrics-server/metrics-server:v0.8.1
EOF
while read -r IMAGE; do
MY_IMAGE=$(echo "$IMAGE" | sed 's|^registry\.k8s\.io/|k8s\.flyingtang\.com/|')
FILE_NAME="${IMAGE##*/}"
FILE_NAME="${FILE_NAME/:/_}.tar"
nerdctl pull $MY_IMAGE
nerdctl tag $MY_IMAGE "10.30.13.175:5000/topke-system/$IMAGE"
nerdctl save "10.30.13.175:5000/topke-system/$IMAGE" | gzip > $FILE_NAME
done < /tmp/autoscapimage.txt
Roce-Link
cat << EOF > /tmp/autoscapimage.txt
ghcr.io/k8snetworkplumbingwg/sriov-network-operator:v1.6.0
ghcr.io/k8snetworkplumbingwg/sriov-network-operator-config-daemon:v1.6.0
ghcr.io/k8snetworkplumbingwg/sriov-cni:v2.10.0
ghcr.io/k8snetworkplumbingwg/ib-sriov-cni:v1.3.0
ghcr.io/k8snetworkplumbingwg/ovs-cni-plugin:v0.39.0
ghcr.io/k8snetworkplumbingwg/rdma-cni:v1.6.0
ghcr.io/k8snetworkplumbingwg/sriov-network-device-plugin:v3.11.0
ghcr.io/k8snetworkplumbingwg/network-resources-injector
ghcr.io/k8snetworkplumbingwg/sriov-network-operator-webhook
ghcr.io/k8snetworkplumbingwg/sriov-network-metrics-exporter
registry.k8s.io/kubebuilder/kube-rbac-proxy:v0.15.0
ghcr.io/k8snetworkplumbingwg/multus-cni:snapshot
ghcr.io/k8snetworkplumbingwg/whereabouts:v0.9.3
EOF
while read -r IMAGE; do
MY_IMAGE=$(echo "$IMAGE" | sed 's|^ghcr\.io/|ghcr\.flyingtang\.com/|')
FILE_NAME="${IMAGE##*/}"
FILE_NAME="${FILE_NAME/:/_}.tar"
nerdctl pull $MY_IMAGE
nerdctl tag $MY_IMAGE "10.30.13.175:5000/topke-system/$IMAGE"
nerdctl push "10.30.13.175:5000/topke-system/$IMAGE"
nerdctl save "10.30.13.175:5000/topke-system/$IMAGE" | gzip > $FILE_NAME
done < /tmp/autoscapimage.txt
while read -r IMAGE; do
MY_IMAGE=$(echo "$IMAGE" | sed 's|^registry\.k8s\.io/|k8s\.flyingtang\.com/|')
FILE_NAME="${IMAGE##*/}"
FILE_NAME="${FILE_NAME/:/_}.tar"
nerdctl pull $MY_IMAGE
nerdctl tag $MY_IMAGE "10.30.13.175:5000/topke-system/$IMAGE"
nerdctl push "10.30.13.175:5000/topke-system/$IMAGE"
nerdctl save "10.30.13.175:5000/topke-system/$IMAGE" | gzip > $FILE_NAME
done < /tmp/autoscapimage.txt
volcano-vgpu-device-plugin
docker.io/projecthami/volcano-vgpu-device-plugin:v1.12.0
Hami
docker.io/liangjw/kube-webhook-certgen:v1.1.1
docker.io/projecthami/hami:v2.9.0
三、测试
3.1 整卡测试
四、测试
4.1 x86环境测试
4.1.1 测试脚本
cat << EOF > test_all_gpu.yaml
apiVersion: v1
kind: Pod
metadata:
name: gpu-pod
spec:
containers:
- name: ubuntu-container
image: ubuntu:18.04
command: ["bash", "-c", "sleep 86400"]
resources:
limits:
nvidia.com/gpu: 1
requests:
nvidia.com/gpu: 1
EOF
cat << EOF > testhami.yaml
apiVersion: v1
kind: Pod
metadata:
name: gpu-pod
spec:
# runtimeClassName: nvidia-legacy # 不要用 nvidia
containers:
- name: ubuntu-container
image: ubuntu:18.04
command: ["bash", "-c", "sleep 86400"]
resources:
limits:
nvidia.com/gpu: 1
nvidia.com/gpumem: 1024
requests:
nvidia.com/gpu: 1
nvidia.com/gpumem: 1024
---
apiVersion: v1
kind: Pod
metadata:
name: gpu-pod1
spec:
# runtimeClassName: nvidia-legacy # 不要用 nvidia
containers:
- name: ubuntu-container
image: ubuntu:18.04
command: ["bash", "-c", "sleep 86400"]
resources:
limits:
nvidia.com/gpu: 1
nvidia.com/gpumem: 1025
requests:
nvidia.com/gpu: 1
nvidia.com/gpumem: 1025
EOF
cat << EOF > testvocalno1.yaml
apiVersion: v1
kind: Pod
metadata:
name: gpu-pod
spec:
schedulerName: volcano
containers:
- name: ubuntu-container
image: ubuntu:18.04
command: ["bash", "-c", "sleep 86400"]
resources:
limits:
volcano.sh/vgpu-number: 1
volcano.sh/vgpu-memory: 2024
requests:
volcano.sh/vgpu-number: 1
volcano.sh/vgpu-memory: 2024
EOF
4.1.2 注意事项
- hami和vocalo vGPU切换后会数据残留，创建的资源也会带pending, 报错：0/2 nodes are available: 2 Insufficient volcano.sh/vgpu-cores
- 为啥测试pod的scheduler不会自动注入？
[root@10-30-15-55 compute_test]# kubectl get mutatingwebhookconfiguration | grep -iE 'volcano|hami|scheduler|gpu' volcano-admission-service-jobs-mutate 1 128m volcano-admission-service-queues-mutate 1 128m [root@10-30-15-55 compute_test]# kubectl get pods -A | grep -i volcano-scheduler topke-system compute-card-volcano-scheduler-f64657657-8z9kr 1/1 Running 0 128m [root@10-30-15-55 compute_test]# kubectl get mutatingwebhookconfiguration | grep -iE 'volcano|hami|scheduler|gpu' compute-card-hami-webhook 1 44s volcano-admission-service-jobs-mutate 1 135m volcano-admission-service-queues-mutate 1 135m
4.2 ARM环境测试
source init.sh && make && rm -rf /usr/local/bin/manager && cp bin/manager /usr/local/bin/manager && nerdctl -n system restart manager && sleep 5 && nerdctl -n system exec -it manager -- tail -f /var/log/thci/manager.log
4.2.1 Hami测试脚本
cat << EOF > hami_test.yaml
apiVersion: v1
kind: Pod
metadata:
name: gpu-pod
# 不要加 huawei.com/vnpu-mode: hami-core
spec:
runtimeClassName: ascend
schedulerName: hami-scheduler
containers:
- name: ubuntu-container
image: 10.30.15.90:5000/aarch64/mindie:2.3.1
command: ["bash", "-c", "sleep 86400"]
resources:
limits:
huawei.com/Ascend310P: "1"
huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐
EOF
cat << EOF > hami_soft.yaml
apiVersion: v1
kind: Pod
metadata:
name: gpu-pod
annotations:
huawei.com/vnpu-mode: "hami-core"
spec:
runtimeClassName: ascend
schedulerName: hami-scheduler
containers:
- name: ubuntu-container
image: 10.30.15.90:5000/aarch64/mindie:2.3.1
command: ["bash", "-c", "sleep 86400"]
resources:
limits:
huawei.com/Ascend310P: "1"
huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐
EOF
cat << EOF > volcano_test.yaml
apiVersion: v1
kind: Pod
metadata:
name: gpu-pod
# 不要加 huawei.com/vnpu-mode: hami-core
spec:
runtimeClassName: ascend
schedulerName: volcano # 这里不一样
containers:
- name: ubuntu-container
image: 10.30.15.90:5000/aarch64/mindie:2.3.1
command: ["bash", "-c", "sleep 86400"]
resources:
limits:
huawei.com/Ascend310P: "1"
huawei.com/Ascend310P-memory: "4096" # 会按 vir01/02/04 对齐
EOF
示例输出：
#硬切分
[root@gpu-pod HwHiAiUser]# npu-smi info
+--------------------------------------------------------------------------------------------------------+
| npu-smi 26.0.rc1 Version: 26.0.rc1 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 32800 310Pvir02 | OK | NA 49 0 / 0 |
| 0 0 | 0000:82:00.0 | 0 366 / 10923 |
+===============================+=================+======================================================+
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Chip | Process id | Process name | Process memory(MB) |
+===============================+=================+======================================================+
| No running processes found in NPU 32800 |
+===============================+=================+======================================================+
#软切分
[root@gpu-pod HwHiAiUser]# npu-smi info
+--------------------------------------------------------------------------------------------------------+
| npu-smi 26.0.rc1 Version: 26.0.rc1 |
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Name | Health | Power(W) Temp(C) Hugepages-Usage(page) |
| Chip Device | Bus-Id | AICore(%) Memory-Usage(MB) |
+===============================+=================+======================================================+
| 32800 310P3 | OK | NA 49 0 / 0 |
| 0 0 | 0000:82:00.0 | 0 1530 / 43693 |
+===============================+=================+======================================================+
+-------------------------------+-----------------+------------------------------------------------------+
| NPU Chip | Process id | Process name | Process memory(MB) |
+===============================+=================+======================================================+
| No running processes found in NPU 32800 |
+===============================+=================+======================================================+
4.2.2 Hami注意事项
昇腾应该不用掉这个接口 查询instance接口
huawei.com/Ascend310P-memory 不写入Capacity/Allocatable 属于正常现象
硬切分：
- [root@gpu-pod HwHiAiUser]# npu-smi info
- DrvMngGetConsoleLogLevel failed. (ret=4) dcmi module initialize failed. ret is -8005重启kubelet 然后重建解决
软切分：
软切分会报错：[root@10-30-15-89 hami]# kubectl logs -f gpu-pod bash: symbol lookup error: /hami-vnpu-core/libvnpu.so: undefined symbol: rtStreamGetCaptureInfo （升级环境下解决，原因是容器的cann和驱动不一致）
hami-ascend-device-plugin.hamiVnpuCore.enabled 设置为true
软切分支持多卡
[root@172-16-14-122 ~]# cat test.yaml apiVersion: v1 kind: Pod metadata: name: gpu-pod annotations: huawei.com/vnpu-mode: "hami-core" spec: runtimeClassName: ascend schedulerName: hami-scheduler containers: - name: ubuntu-container image: 172.16.14.123:5000/aarch64/vllm-ascend:0.23.0.0001.910b--huawei-npu-300I command: ["bash", "-c", "sleep 86400"] resources: limits: huawei.com/Ascend910B4: "2" huawei.com/Ascend910B4-memory: "4096" # 会按 vir01/02/04 对齐启用
device-share模式，运行以下命令：
npu-smi set -t device-share -i <id> -d <value>参数 说明 id设备 ID，通过运行 npu-smi info -l命令获取的 NPU ID 即为设备 ID。value容器启用状态： 0（禁用，默认值）或1（启用）。软硬只能选择一种
4.2.3 Volcano脚本测试
Volcano官方文档 或者 hami 中对volcano 的文档
| 插件 | 作用阶段 | 调度对象 | 核心作用 | 详细说明 | 示例 | 适用场景 |
|---|---|---|---|---|---|---|
predicates |
allocate | Node / Pod 基础资源 | 节点过滤 | 类似 Kubernetes Scheduler 的 Filter 阶段，根据 Pod 的资源需求、节点标签、亲和性、污点容忍等条件过滤不可用节点。它决定“哪些节点可以运行”，不负责最终选择。 | Pod 请求nvidia.com/gpu=1，节点 A 没 GPU，则节点 A 被过滤；Pod 请求 20G 显存，节点只有 10G，也会被过滤。 |
所有场景必备，GPU/NPU 调度基础插件 |
deviceshare |
allocate | GPU / NPU / vGPU / vNPU 设备 | 设备共享与切分调度 | HAMI 核心插件，负责感知 GPU/NPU 设备状态，根据显存、算力、设备拓扑等信息，把物理设备切分成多个虚拟资源，并完成具体设备分配。 | 一张 RTX4090（24G 显存）：Pod A 使用 8G，Pod B 使用 12G，deviceshare 负责判断是否能放在同一张 GPU 上。 | HAMI GPU/vGPU、Ascend vNPU、AI 推理平台核心插件 |
deviceshare.SchedulePolicy=binpack |
allocate | GPU/NPU 内部资源 | GPU/NPU 资源集中分配 | deviceshare 内部策略，不是 Volcano 的 binpack 插件。目标是让多个任务尽量复用同一块 GPU/NPU，提高单卡利用率。 | GPU0 已使用 14G，剩余 10G；新任务需要 8G，则继续放 GPU0，而不是启用 GPU1。 | LLM 推理、TTS 推理、多租户 GPU 共享 |
priority |
enqueue / allocate | Pod | 任务优先级排序 | 根据 Kubernetes PriorityClass 决定任务进入调度队列的顺序。高优任务可以优先获得资源。 | 在线推理服务 priority=100，离线训练任务 priority=10；资源不足时先调度推理服务。 | 线上服务、混合业务集群 |
gang |
enqueue | 一组 Pod | 批量调度 / 集合调度 | 要求一组 Pod 同时满足资源条件后才能启动，避免部分启动导致资源浪费。特别适合分布式训练。 | 8 卡训练任务需要 8 个 worker，如果只有 4 张 GPU 空闲，则全部等待；当 8 张 GPU 满足时一起启动。 | AI 训练、MPI、Ray、分布式任务 |
conformance |
enqueue | Pod 配置 | Volcano 配置合法性检查 | 检查 Pod 是否符合 Volcano 调度要求，例如 schedulerName、queue 等配置是否正确。 | Pod 没指定 Volcano scheduler 或 queue 配置错误，可以提前发现。 | 集群治理、生产环境规范检查 |
overcommit |
allocate | CPU / Memory 等资源 | 资源超卖 | 允许资源申请量超过物理资源，提高资源利用率。类似云平台 CPU 超卖机制。 | 节点 CPU=64 核，允许多个低负载服务累计申请 80 核。 | 云平台、资源利用率优先场景 |
drf |
allocate | 多用户 / 多队列资源 | 多租户公平调度 | Dominant Resource Fairness，根据用户占用的主导资源进行公平分配，避免某个租户长期占用大量资源。 | 用户 A 占大量 GPU，用户 B 仍能获得调度机会，而不是一直等待。 | GPU 云平台、多租户 AI 平台 |
proportion |
allocate | Volcano Queue | 队列资源比例控制 | 按 Queue 配置比例分配资源，限制不同团队或业务线资源占比。 | AI 训练队列 70%，开发测试队列 30%；测试任务不会无限占用 GPU。 | 企业内部 AI 平台、多团队共享集群 |
nodeorder |
allocate | Node | 节点评分和选择 | 对 predicates 过滤后的候选节点进行打分，选择最合适节点。通常结合资源利用策略使用。 | 两个节点都满足 GPU 条件，但 node1 剩余资源更多，评分更高，优先选择 node1。 | GPU 集群节点选择优化 |
binpack |
allocate | Node 资源 | 节点资源集中调度 | Volcano 通用 binpack 插件，关注 CPU、Memory、GPU 等节点资源，尽量把 Pod 集中到少量节点。和 deviceshare 的 binpack 不同。 | node1 CPU 使用 50/64，node2 使用 10/64，新 Pod 优先放 node1，让 node2 保持空闲。 | 节省节点、提高整体利用率 |
spread（策略） |
allocate | Node / Device | 资源分散调度 | 与 binpack 相反，让任务分布到不同节点或设备，降低单点压力。 | 两张 GPU：每张使用 40%，而不是一张 80%。 | 高可靠服务、避免 GPU 热点 |
- 两个 binpack 的区别（重点）
| 项目 | deviceshare.SchedulePolicy=binpack |
Volcanobinpack插件 |
|---|---|---|
| 所属模块 | HAMI deviceshare | Volcano scheduler plugin |
| 调度层级 | GPU/NPU 设备内部 | Kubernetes Node 层面 |
| 关注资源 | 显存、算力、GPU slice、NPU slice | CPU、Memory、GPU 数量、Node 资源 |
| 目的 | 提高单卡利用率 | 提高节点利用率 |
| 示例 | 一个 RTX4090 分给多个 Pod | 多个 Pod 尽量放到同一台机器 |
4.2.4 Volcano注意事项
kind: ConfigMap
apiVersion: v1
metadata:
name: volcano-scheduler-configmap
namespace: volcano-system
data:
volcano-scheduler.conf: |
actions: "enqueue, allocate, backfill"
tiers:
- plugins:
- name: predicates
- name: deviceshare
arguments:
deviceshare.AscendHAMiVNPUEnable: true # enable ascend vnpu
deviceshare.SchedulePolicy: binpack # scheduling policy. binpack / spread
deviceshare.KnownGeometriesCMNamespace: kube-system
deviceshare.KnownGeometriesCMName: hami-scheduler-device