Skip to content

Commit ca7b0ea

Browse files
PhilippMatthesPhilippMatthes
authored
Use hv1 effective capacity for weighing + filtering (#583)
In cobaltcore-dev/openstack-hypervisor-operator#257 we introduced a new `effectiveCapacity` field of the hypervisor crd. In cobaltcore-dev/kvm-node-agent#70 we populate this field. Now we can upgrade cortex so it performs scheduling on the overcommitted capacity.
1 parent 13aca98 commit ca7b0ea

7 files changed

Lines changed: 32 additions & 24 deletions

File tree

‎internal/scheduling/nova/integration_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ func newHypervisor(name, cpuCap, cpuAlloc, memCap, memAlloc string) *hv1.Hypervi
4848
Name: name,
4949
},
5050
Status: hv1.HypervisorStatus{
51-
Capacity: map[hv1.ResourceName]resource.Quantity{
51+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
5252
hv1.ResourceCPU: resource.MustParse(cpuCap),
5353
hv1.ResourceMemory: resource.MustParse(memCap),
5454
},

‎internal/scheduling/nova/plugins/filters/filter_has_enough_capacity.go‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -56,8 +56,14 @@ func (s *FilterHasEnoughCapacity) Run(traceLog *slog.Logger, request api.Externa
5656
return nil, err
5757
}
5858
for _, hv := range hvs.Items {
59-
// Start with the total capacity.
60-
freeResourcesByHost[hv.Name] = hv.Status.Capacity
59+
// This case would be caught below, but we want to log this explicitly.
60+
if hv.Status.EffectiveCapacity == nil {
61+
traceLog.Warn("hypervisor with nil effective capacity, skipping", "host", hv.Name)
62+
continue
63+
}
64+
65+
// Start with the total effective capacity which is capacity * overcommit ratio.
66+
freeResourcesByHost[hv.Name] = hv.Status.EffectiveCapacity
6167

6268
// Subtract allocated resources.
6369
for resourceName, allocated := range hv.Status.Allocation {

‎internal/scheduling/nova/plugins/filters/filter_has_enough_capacity_test.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,7 +39,7 @@ func newHypervisor(name, cpuCap, cpuAlloc, memCap, memAlloc string) *hv1.Hypervi
3939
Name: name,
4040
},
4141
Status: hv1.HypervisorStatus{
42-
Capacity: map[hv1.ResourceName]resource.Quantity{
42+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
4343
hv1.ResourceCPU: resource.MustParse(cpuCap),
4444
hv1.ResourceMemory: resource.MustParse(memCap),
4545
},

‎internal/scheduling/nova/plugins/weighers/kvm_binpack.go‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -93,14 +93,15 @@ func (s *KVMBinpackStep) Run(traceLog *slog.Logger, request api.ExternalSchedule
9393
var totalWeightedUtilization, totalWeight float64
9494

9595
for resourceName, weight := range s.Options.ResourceWeights {
96-
capacity, ok := hv.Status.Capacity[resourceName]
96+
// Effective capacity = capacity * overcommit ratio.
97+
capacity, ok := hv.Status.EffectiveCapacity[resourceName]
9798
if !ok {
98-
traceLog.Warn("no capacity in status, skipping",
99+
traceLog.Warn("no effective capacity in status, skipping",
99100
"host", host, "resource", resourceName)
100101
continue
101102
}
102103
if capacity.IsZero() {
103-
traceLog.Warn("capacity is zero, skipping",
104+
traceLog.Warn("effective capacity is zero, skipping",
104105
"host", host, "resource", resourceName)
105106
continue
106107
}

‎internal/scheduling/nova/plugins/weighers/kvm_binpack_test.go‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func newHypervisor(name, capacityCPU, capacityMem, allocationCPU, allocationMem
2222
Name: name,
2323
},
2424
Status: hv1.HypervisorStatus{
25-
Capacity: map[hv1.ResourceName]resource.Quantity{
25+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
2626
hv1.ResourceCPU: resource.MustParse(capacityCPU),
2727
hv1.ResourceMemory: resource.MustParse(capacityMem),
2828
},
@@ -343,7 +343,7 @@ func TestKVMBinpackStep_Run(t *testing.T) {
343343
{
344344
ObjectMeta: metav1.ObjectMeta{Name: "host1"},
345345
Status: hv1.HypervisorStatus{
346-
Capacity: map[hv1.ResourceName]resource.Quantity{
346+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
347347
hv1.ResourceCPU: resource.MustParse("0"),
348348
hv1.ResourceMemory: resource.MustParse("100Gi"),
349349
},
@@ -371,7 +371,7 @@ func TestKVMBinpackStep_Run(t *testing.T) {
371371
{
372372
ObjectMeta: metav1.ObjectMeta{Name: "host1"},
373373
Status: hv1.HypervisorStatus{
374-
Capacity: map[hv1.ResourceName]resource.Quantity{
374+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
375375
hv1.ResourceCPU: resource.MustParse("100"),
376376
},
377377
Allocation: map[hv1.ResourceName]resource.Quantity{
@@ -397,7 +397,7 @@ func TestKVMBinpackStep_Run(t *testing.T) {
397397
{
398398
ObjectMeta: metav1.ObjectMeta{Name: "host1"},
399399
Status: hv1.HypervisorStatus{
400-
Capacity: map[hv1.ResourceName]resource.Quantity{
400+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
401401
// No CPU capacity
402402
},
403403
Allocation: map[hv1.ResourceName]resource.Quantity{

‎internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts.go‎

Lines changed: 8 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -81,9 +81,10 @@ func (s *KVMPreferSmallerHostsStep) Run(traceLog *slog.Logger, request api.Exter
8181
if _, ok := result.Activations[hv.Name]; !ok {
8282
continue
8383
}
84-
capacity, ok := hv.Status.Capacity[resourceName]
84+
// Effective capacity = capacity * overcommit ratio.
85+
capacity, ok := hv.Status.EffectiveCapacity[resourceName]
8586
if !ok {
86-
traceLog.Warn("hypervisor has no capacity for resource, skipping",
87+
traceLog.Warn("hypervisor has no effective capacity for resource, skipping",
8788
"host", hv.Name, "resource", resourceName)
8889
continue
8990
}
@@ -106,9 +107,9 @@ func (s *KVMPreferSmallerHostsStep) Run(traceLog *slog.Logger, request api.Exter
106107
var totalWeightedScore, totalWeight float64
107108

108109
for resourceName, weight := range s.Options.ResourceWeights {
109-
capacity, ok := hv.Status.Capacity[resourceName]
110+
capacity, ok := hv.Status.EffectiveCapacity[resourceName]
110111
if !ok {
111-
traceLog.Warn("hypervisor has no capacity for resource, skipping",
112+
traceLog.Warn("hypervisor has no effective capacity for resource, skipping",
112113
"host", hv.Name, "resource", resourceName)
113114
continue
114115
}
@@ -117,14 +118,14 @@ func (s *KVMPreferSmallerHostsStep) Run(traceLog *slog.Logger, request api.Exter
117118
largestCap := largest[resourceName]
118119

119120
if smallestCap == nil || largestCap == nil {
120-
traceLog.Warn("no capacity range found for resource, skipping",
121+
traceLog.Warn("no effective capacity range found for resource, skipping",
121122
"resource", resourceName)
122123
continue
123124
}
124125

125-
// If all hosts have the same capacity for this resource, skip it
126+
// If all hosts have the same effective capacity for this resource, skip it
126127
if smallestCap.Cmp(*largestCap) == 0 {
127-
traceLog.Info("all hypervisors have the same capacity for resource, skipping",
128+
traceLog.Info("all hypervisors have the same effective capacity for resource, skipping",
128129
"resource", resourceName)
129130
continue
130131
}

‎internal/scheduling/nova/plugins/weighers/kvm_prefer_smaller_hosts_test.go‎

Lines changed: 6 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,7 @@ func newHypervisorWithCapacity(name, capacityCPU, capacityMem string) *hv1.Hyper
2222
Name: name,
2323
},
2424
Status: hv1.HypervisorStatus{
25-
Capacity: map[hv1.ResourceName]resource.Quantity{
25+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
2626
hv1.ResourceCPU: resource.MustParse(capacityCPU),
2727
hv1.ResourceMemory: resource.MustParse(capacityMem),
2828
},
@@ -376,7 +376,7 @@ func TestKVMPreferSmallerHostsStep_Run(t *testing.T) {
376376
{
377377
ObjectMeta: metav1.ObjectMeta{Name: "host3"},
378378
Status: hv1.HypervisorStatus{
379-
Capacity: map[hv1.ResourceName]resource.Quantity{
379+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
380380
hv1.ResourceCPU: resource.MustParse("100"),
381381
// No memory capacity
382382
},
@@ -466,13 +466,13 @@ func TestKVMPreferSmallerHostsStep_Run(t *testing.T) {
466466
{
467467
ObjectMeta: metav1.ObjectMeta{Name: "host1"},
468468
Status: hv1.HypervisorStatus{
469-
Capacity: map[hv1.ResourceName]resource.Quantity{},
469+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{},
470470
},
471471
},
472472
{
473473
ObjectMeta: metav1.ObjectMeta{Name: "host2"},
474474
Status: hv1.HypervisorStatus{
475-
Capacity: map[hv1.ResourceName]resource.Quantity{},
475+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{},
476476
},
477477
},
478478
},
@@ -534,7 +534,7 @@ func TestKVMPreferSmallerHostsStep_Run(t *testing.T) {
534534
{
535535
ObjectMeta: metav1.ObjectMeta{Name: "host1"},
536536
Status: hv1.HypervisorStatus{
537-
Capacity: map[hv1.ResourceName]resource.Quantity{
537+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
538538
hv1.ResourceMemory: resource.MustParse("64Gi"),
539539
// No CPU
540540
},
@@ -543,7 +543,7 @@ func TestKVMPreferSmallerHostsStep_Run(t *testing.T) {
543543
{
544544
ObjectMeta: metav1.ObjectMeta{Name: "host2"},
545545
Status: hv1.HypervisorStatus{
546-
Capacity: map[hv1.ResourceName]resource.Quantity{
546+
EffectiveCapacity: map[hv1.ResourceName]resource.Quantity{
547547
hv1.ResourceMemory: resource.MustParse("128Gi"),
548548
// No CPU
549549
},

0 commit comments

Comments
 (0)