Skip to content

Commit bfdeea0

Browse files
yaroslavborbatyaroslavborbat
committed
improve scheduling for hs and ss hubs
Signed-off-by: Yaroslav Borbat <yaroslav.borbat@flant.com>
1 parent ab0f119 commit bfdeea0

File tree

26 files changed

+364
-152
lines changed

26 files changed

+364
-152
lines changed

‎api/core/v1alpha2/node_device_usb.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -90,6 +90,8 @@ type NodeUSBDeviceAttributes struct {
9090
Major int `json:"major,omitempty"`
9191
// Minor device number.
9292
Minor int `json:"minor,omitempty"`
93+
// Device speed.
94+
Speed int `json:"speed,omitempty"`
9395
// Device name.
9496
Name string `json:"name,omitempty"`
9597
// USB vendor ID in hexadecimal format.

‎crds/doc-ru-nodeusbdevices.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,9 @@ spec:
7575
serial:
7676
description: Серийный номер устройства.
7777
type: string
78+
speed:
79+
description: Скорость устройства.
80+
type: integer
7881
vendorID:
7982
description: USB vendor ID в шестнадцатеричном формате.
8083
type: string

‎crds/doc-ru-usbdevices.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,9 @@ spec:
6666
serial:
6767
description: Серийный номер устройства.
6868
type: string
69+
speed:
70+
description: Скорость устройства.
71+
type: integer
6972
vendorID:
7073
description: USB vendor ID в шестнадцатеричном формате.
7174
type: string

‎crds/nodeusbdevices.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -112,6 +112,9 @@ spec:
112112
serial:
113113
description: Device serial number.
114114
type: string
115+
speed:
116+
description: Device speed.
117+
type: integer
115118
vendorID:
116119
description: USB vendor ID in hexadecimal format.
117120
type: string

‎crds/usbdevices.yaml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,9 @@ spec:
123123
serial:
124124
description: Device serial number.
125125
type: string
126+
speed:
127+
description: Device speed.
128+
type: integer
126129
vendorID:
127130
description: USB vendor ID in hexadecimal format.
128131
type: string

‎images/virtualization-artifact/pkg/common/annotations/annotations.go‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -218,9 +218,11 @@ const (
218218
// AnnUSBDeviceUser is the annotation for device user (owner) in ResourceClaimTemplate.
219219
AnnUSBDeviceUser = "usb.virtualization.deckhouse.io/device-user"
220220

221-
AnnUSBIPTotalPorts = "usb.virtualization.deckhouse.io/usbip-total-ports"
222-
AnnUSBIPUsedPorts = "usb.virtualization.deckhouse.io/usbip-used-ports"
223-
AnnUSBIPAddress = "usb.virtualization.deckhouse.io/usbip-address"
221+
AnnUSBIPTotalPorts = "usb.virtualization.deckhouse.io/usbip-total-ports"
222+
AnnUSBIPUsedPorts = "usb.virtualization.deckhouse.io/usbip-used-ports"
223+
AnnUSBIPHighSpeedHubUsedPorts = "usb.virtualization.deckhouse.io/usbip-high-speed-hub-used-ports"
224+
AnnUSBIPSuperSpeedHubUsedPorts = "usb.virtualization.deckhouse.io/usbip-super-speed-hub-used-ports"
225+
AnnUSBIPAddress = "usb.virtualization.deckhouse.io/usbip-address"
224226

225227
// DefaultUSBDeviceGroup is the default device group ID for USB devices.
226228
DefaultUSBDeviceGroup = "107"

‎images/virtualization-artifact/pkg/controller/resourceslice/internal/handler/device.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,10 @@ func ConvertDeviceToAttributes(device resourcev1.Device, nodeName string) v1alph
8888
if attr.IntValue != nil {
8989
attrs.Minor = int(*attr.IntValue)
9090
}
91+
case "speed":
92+
if attr.IntValue != nil {
93+
attrs.Speed = int(*attr.IntValue)
94+
}
9195
}
9296
}
9397

‎images/virtualization-artifact/pkg/controller/vm/internal/validators/usb_devices_validator.go‎

Lines changed: 104 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,7 @@ import (
3131
"github.com/deckhouse/virtualization-controller/pkg/common/annotations"
3232
"github.com/deckhouse/virtualization-controller/pkg/controller/indexer"
3333
"github.com/deckhouse/virtualization-controller/pkg/featuregates"
34+
"github.com/deckhouse/virtualization-controller/pkg/kubeapi"
3435
"github.com/deckhouse/virtualization/api/core/v1alpha2"
3536
)
3637

@@ -144,14 +145,22 @@ func getUSBDeviceNames(refs []v1alpha2.USBDeviceSpecRef) map[string]struct{} {
144145
}
145146

146147
func (v *USBDevicesValidator) validateAvailableUSBIPPorts(ctx context.Context, vm *v1alpha2.VirtualMachine, oldUSBDevices map[string]struct{}) (admission.Warnings, error) {
148+
if kubeapi.HasDRAPartitionableDevices() {
149+
return v.validateAvailableUSBIPPortsWithPartitionableDevices(ctx, vm, oldUSBDevices)
150+
}
151+
return v.validateAvailableUSBIPPortsDefault(ctx, vm, oldUSBDevices)
152+
}
153+
154+
func (v *USBDevicesValidator) validateAvailableUSBIPPortsWithPartitionableDevices(ctx context.Context, vm *v1alpha2.VirtualMachine, oldUSBDevices map[string]struct{}) (admission.Warnings, error) {
147155
if vm.Status.Node == "" {
148156
return admission.Warnings{}, nil
149157
}
150158
if vm.Spec.USBDevices == nil {
151159
return admission.Warnings{}, nil
152160
}
153161

154-
var usbFromOtherNodes []string
162+
var hsUSBFromOtherNodes []string
163+
var ssUSBFromOtherNodes []string
155164

156165
for _, ref := range vm.Spec.USBDevices {
157166
if _, exists := oldUSBDevices[ref.Name]; exists {
@@ -164,47 +173,126 @@ func (v *USBDevicesValidator) validateAvailableUSBIPPorts(ctx context.Context, v
164173
return admission.Warnings{}, fmt.Errorf("failed to get USB device %s: %w", ref.Name, err)
165174
}
166175

167-
if usbDevice.Status.NodeName != vm.Status.Node {
168-
usbFromOtherNodes = append(usbFromOtherNodes, ref.Name)
176+
if usbDevice.Status.NodeName == vm.Status.Node {
177+
continue
178+
}
179+
180+
isHs, isSS := resolveSpeed(usbDevice.Status.Attributes.Speed)
181+
switch {
182+
case isHs:
183+
hsUSBFromOtherNodes = append(hsUSBFromOtherNodes, ref.Name)
184+
case isSS:
185+
ssUSBFromOtherNodes = append(ssUSBFromOtherNodes, ref.Name)
186+
default:
187+
return admission.Warnings{}, fmt.Errorf("USB device %s has unsupported speed %d", ref.Name, usbDevice.Status.Attributes.Speed)
169188
}
170189
}
171190

172-
if len(usbFromOtherNodes) == 0 {
191+
if len(hsUSBFromOtherNodes) == 0 && len(ssUSBFromOtherNodes) == 0 {
173192
return admission.Warnings{}, nil
174193
}
175194

195+
node, totalPorts, err := v.getNodeTotalPorts(ctx, vm.Status.Node)
196+
if err != nil {
197+
return admission.Warnings{}, err
198+
}
199+
200+
totalPortsPerHub := totalPorts / 2
201+
202+
if len(hsUSBFromOtherNodes) > 0 {
203+
if err = validateUsedPortsByAnnotation(node, annotations.AnnUSBIPHighSpeedHubUsedPorts, hsUSBFromOtherNodes, totalPortsPerHub); err != nil {
204+
return admission.Warnings{}, err
205+
}
206+
}
207+
208+
if len(ssUSBFromOtherNodes) > 0 {
209+
if err = validateUsedPortsByAnnotation(node, annotations.AnnUSBIPSuperSpeedHubUsedPorts, ssUSBFromOtherNodes, totalPortsPerHub); err != nil {
210+
return admission.Warnings{}, err
211+
}
212+
}
213+
214+
return admission.Warnings{}, nil
215+
}
216+
217+
func (v *USBDevicesValidator) getNodeTotalPorts(ctx context.Context, nodeName string) (*corev1.Node, int, error) {
176218
node := &corev1.Node{}
177-
err := v.client.Get(ctx, client.ObjectKey{Name: vm.Status.Node}, node)
219+
err := v.client.Get(ctx, client.ObjectKey{Name: nodeName}, node)
178220
if err != nil {
179-
return admission.Warnings{}, fmt.Errorf("failed to get node %s: %w", vm.Status.Node, err)
221+
return nil, -1, fmt.Errorf("failed to get node %s: %w", nodeName, err)
180222
}
181223

182224
totalPorts, exists := node.Annotations[annotations.AnnUSBIPTotalPorts]
183225
if !exists {
184-
return admission.Warnings{}, fmt.Errorf("node %s does not have %s annotation", vm.Status.Node, annotations.AnnUSBIPTotalPorts)
226+
return nil, -1, fmt.Errorf("node %s does not have %s annotation", nodeName, annotations.AnnUSBIPTotalPorts)
185227
}
186228
totalPortsInt, err := strconv.Atoi(totalPorts)
187229
if err != nil {
188-
return admission.Warnings{}, fmt.Errorf("failed to parse %s annotation: %w", annotations.AnnUSBIPTotalPorts, err)
230+
return nil, -1, fmt.Errorf("failed to parse %s annotation: %w", annotations.AnnUSBIPTotalPorts, err)
189231
}
190232

191-
// total for 2 usb hubs (2.0 and 3.0)
192-
totalPortsInt /= 2
233+
return node, totalPortsInt, nil
234+
}
193235

194-
usedPorts, exists := node.Annotations[annotations.AnnUSBIPUsedPorts]
236+
func validateUsedPortsByAnnotation(node *corev1.Node, anno string, usbFromOtherNodes []string, totalPortsPerHub int) error {
237+
usedPorts, exists := node.Annotations[anno]
195238
if !exists {
196-
return admission.Warnings{}, fmt.Errorf("node %s does not have %s annotation", vm.Status.Node, annotations.AnnUSBIPUsedPorts)
239+
return fmt.Errorf("node %s does not have %s annotation", node.Name, anno)
197240
}
198241
usedPortsInt, err := strconv.Atoi(usedPorts)
199242
if err != nil {
200-
return admission.Warnings{}, fmt.Errorf("failed to parse %s annotation: %w", annotations.AnnUSBIPUsedPorts, err)
243+
return fmt.Errorf("failed to parse %s annotation: %w", anno, err)
201244
}
202245

203246
wantedPorts := usedPortsInt + len(usbFromOtherNodes)
247+
if wantedPorts > totalPortsPerHub {
248+
return fmt.Errorf("node %s not available ports for sharing USB devices %s. total: %d, used: %d, wanted: %d", node.Name, strings.Join(usbFromOtherNodes, ", "), totalPortsPerHub, usedPortsInt, wantedPorts)
249+
}
250+
251+
return nil
252+
}
204253

205-
if wantedPorts > totalPortsInt {
206-
return admission.Warnings{}, fmt.Errorf("node %s not available ports for sharing USB devices %s. total: %d, used: %d, wanted: %d", vm.Status.Node, strings.Join(usbFromOtherNodes, ", "), totalPortsInt, usedPortsInt, wantedPorts)
254+
// https://mjmwired.net/kernel/Documentation/ABI/testing/sysfs-bus-usb#502
255+
func resolveSpeed(speed int) (isHs, isSS bool) {
256+
return speed == 480, speed >= 5000
257+
}
258+
259+
func (v *USBDevicesValidator) validateAvailableUSBIPPortsDefault(ctx context.Context, vm *v1alpha2.VirtualMachine, oldUSBDevices map[string]struct{}) (admission.Warnings, error) {
260+
if vm.Status.Node == "" {
261+
return admission.Warnings{}, nil
262+
}
263+
if vm.Spec.USBDevices == nil {
264+
return admission.Warnings{}, nil
207265
}
208266

209-
return admission.Warnings{}, nil
267+
var usbFromOtherNodes []string
268+
269+
for _, ref := range vm.Spec.USBDevices {
270+
if _, exists := oldUSBDevices[ref.Name]; exists {
271+
continue
272+
}
273+
274+
usbDevice := &v1alpha2.USBDevice{}
275+
err := v.client.Get(ctx, client.ObjectKey{Name: ref.Name, Namespace: vm.Namespace}, usbDevice)
276+
if err != nil {
277+
return admission.Warnings{}, fmt.Errorf("failed to get USB device %s: %w", ref.Name, err)
278+
}
279+
280+
if usbDevice.Status.NodeName != vm.Status.Node {
281+
usbFromOtherNodes = append(usbFromOtherNodes, ref.Name)
282+
}
283+
}
284+
285+
if len(usbFromOtherNodes) == 0 {
286+
return admission.Warnings{}, nil
287+
}
288+
289+
node, totalPorts, err := v.getNodeTotalPorts(ctx, vm.Status.Node)
290+
if err != nil {
291+
return admission.Warnings{}, err
292+
}
293+
294+
// total for 2 usb hubs (2.0 and 3.0)
295+
totalPorts /= 2
296+
297+
return admission.Warnings{}, validateUsedPortsByAnnotation(node, annotations.AnnUSBIPUsedPorts, usbFromOtherNodes, totalPorts)
210298
}

‎images/virtualization-artifact/pkg/kubeapi/kubeapi.go‎

Lines changed: 5 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ package kubeapi
1919
import (
2020
"log/slog"
2121
"os"
22+
"strings"
2223
"sync"
2324

2425
resourcev1 "k8s.io/api/resource/v1"
@@ -78,10 +79,9 @@ func isResourceV1Enabled(clientset kubernetes.Interface) (bool, error) {
7879
}
7980

8081
func HasDRAFeatureGates() bool {
81-
envValue := os.Getenv("HAS_DRA_FEATURE_GATES")
82-
if envValue == "" {
83-
return false
84-
}
82+
return os.Getenv("HAS_DRA_FEATURE_GATES") == "true"
83+
}
8584

86-
return envValue == "true"
85+
func HasDRAPartitionableDevices() bool {
86+
return strings.Contains(os.Getenv("KUBE_APISERVER_FEATURE_GATES"), "DRAPartitionableDevices")
8787
}

‎images/virtualization-dra/cmd/usb/go-usbip/app/app.go‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,8 @@ import (
2323
"github.com/spf13/cobra"
2424
"github.com/spf13/pflag"
2525
"sigs.k8s.io/yaml"
26+
27+
"github.com/deckhouse/virtualization-dra/pkg/logger"
2628
)
2729

2830
const long = `
@@ -37,12 +39,18 @@ const long = `
3739
`
3840

3941
func NewUSBIPCommand() *cobra.Command {
42+
logging := &logger.Options{}
43+
4044
cmd := &cobra.Command{
4145
Use: "usbip",
4246
Short: "USBIP command line tool",
4347
Long: long,
4448
SilenceUsage: true,
4549
SilenceErrors: true,
50+
PersistentPreRunE: func(_ *cobra.Command, _ []string) error {
51+
logger.SetDefaultLogger(logging.Complete())
52+
return nil
53+
},
4654
}
4755

4856
cmd.AddCommand(
@@ -59,6 +67,7 @@ func NewUSBIPCommand() *cobra.Command {
5967
)
6068

6169
printer.AddFlags(cmd.PersistentFlags())
70+
logging.AddFlags(cmd.PersistentFlags())
6271

6372
return cmd
6473
}

0 commit comments

Comments
 (0)