Enable FM in sandbox device plugin when FM shared-nvswitch mode

Signed-off-by: Michail Resvanis <mresvani@redhat.com>

Author: Michail Resvanis

controllers/object_controls.go

@@ -1647,6 +1647,31 @@ func TransformSandboxDevicePlugin(obj *appsv1.DaemonSet, config *gpuv1.ClusterPo
 			setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), env.Name, env.Value)
 		}
 	}
+
+	// Set ENABLE_FABRIC_MANAGER environment variable if shared-nvswitch mode is configured
+	if config.FabricManager.IsSharedNVSwitchMode() {
+		setContainerEnv(&(obj.Spec.Template.Spec.Containers[0]), "ENABLE_FABRIC_MANAGER", "true")
+
+		// Add fabric manager volume mount to the container
+		fabricManagerVolMount := corev1.VolumeMount{
+			Name:      "run-nvidia-fabricmanager",
+			MountPath: "/run/nvidia-fabricmanager",
+		}
+		obj.Spec.Template.Spec.Containers[0].VolumeMounts = append(obj.Spec.Template.Spec.Containers[0].VolumeMounts, fabricManagerVolMount)
+
+		// Add fabric manager volume to the pod spec
+		fabricManagerVol := corev1.Volume{
+			Name: "run-nvidia-fabricmanager",
+			VolumeSource: corev1.VolumeSource{
+				HostPath: &corev1.HostPathVolumeSource{
+					Path: "/run/nvidia-fabricmanager",
+					Type: ptr.To(corev1.HostPathDirectoryOrCreate),
+				},
+			},
+		}
+		obj.Spec.Template.Spec.Volumes = append(obj.Spec.Template.Spec.Volumes, fabricManagerVol)
+	}
+
 	return nil
 }
 

controllers/object_controls_test.go

@@ -882,6 +882,8 @@ func getSandboxDevicePluginTestInput(testCase string) *gpuv1.ClusterPolicy {
 	switch testCase {
 	case "default":
 		// Do nothing
+	case "fabric-manager-shared-nvswitch":
+		cp.Spec.FabricManager.Mode = gpuv1.FabricModeSharedNVSwitch
 	default:
 		return nil
 	}
@@ -897,11 +899,16 @@ func getSandboxDevicePluginTestOutput(testCase string) map[string]interface{} {
 		"numDaemonsets":   1,
 		"image":           "nvcr.io/nvidia/kubevirt-device-plugin:v1.1.0",
 		"imagePullSecret": "ngc-secret",
+		"env":             map[string]string{},
 	}
 
 	switch testCase {
 	case "default":
 		// Do nothing
+	case "fabric-manager-shared-nvswitch":
+		output["env"] = map[string]string{
+			"ENABLE_FABRIC_MANAGER": "true",
+		}
 	default:
 		return nil
 	}
@@ -922,6 +929,11 @@ func TestSandboxDevicePlugin(t *testing.T) {
 			getSandboxDevicePluginTestInput("default"),
 			getSandboxDevicePluginTestOutput("default"),
 		},
+		{
+			"FabricManagerSharedNVSwitch",
+			getSandboxDevicePluginTestInput("fabric-manager-shared-nvswitch"),
+			getSandboxDevicePluginTestOutput("fabric-manager-shared-nvswitch"),
+		},
 	}
 
 	for _, tc := range testCases {
@@ -935,14 +947,26 @@ func TestSandboxDevicePlugin(t *testing.T) {
 			}
 
 			image := ""
+			containerEnv := make(map[string]string)
 			for _, container := range ds.Spec.Template.Spec.Containers {
 				if strings.Contains(container.Name, "nvidia-sandbox-device-plugin-ctr") {
 					image = container.Image
+					for _, env := range container.Env {
+						containerEnv[env.Name] = env.Value
+					}
 					continue
 				}
 			}
 
 			require.Equal(t, tc.output["image"], image, "Unexpected configuration for nvidia-sandbox-device-plugin-ctr image")
+			
+			// Check environment variables
+			expectedEnv := tc.output["env"].(map[string]string)
+			for envName, expectedValue := range expectedEnv {
+				actualValue, found := containerEnv[envName]
+				require.True(t, found, "Expected environment variable %s not found", envName)
+				require.Equal(t, expectedValue, actualValue, "Unexpected value for environment variable %s", envName)
+			}
 
 			// cleanup by deleting all kubernetes objects
 			err = removeState(&clusterPolicyController, clusterPolicyController.idx-1)

controllers/transforms_test.go

@@ -3665,3 +3665,109 @@ func TestTransformDriverVGPUTopologyConfig(t *testing.T) {
 	require.NoError(t, err)
 	require.EqualValues(t, expectedDs, ds)
 }
+
+func TestTransformSandboxDevicePlugin(t *testing.T) {
+	initMockK8sClients()
+	testCases := []struct {
+		description   string
+		ds            Daemonset
+		cpSpec        *gpuv1.ClusterPolicySpec
+		expectedDs    Daemonset
+		errorExpected bool
+	}{
+		{
+			description: "sandbox device plugin with fabric manager shared-nvswitch mode",
+			ds: NewDaemonset().WithContainer(corev1.Container{Name: "nvidia-sandbox-device-plugin-ctr"}).
+				WithInitContainer(corev1.Container{Name: "toolkit-validation"}),
+			cpSpec: &gpuv1.ClusterPolicySpec{
+				SandboxDevicePlugin: gpuv1.SandboxDevicePluginSpec{
+					Repository: "nvcr.io/nvidia",
+					Image:      "kubevirt-device-plugin",
+					Version:    "v1.2.0",
+				},
+				FabricManager: gpuv1.FabricManagerSpec{
+					Mode: gpuv1.FabricModeSharedNVSwitch,
+				},
+				Validator: gpuv1.ValidatorSpec{
+					Repository: "nvcr.io/nvidia/cloud-native",
+					Image:      "gpu-operator-validator",
+					Version:    "v1.0.0",
+				},
+			},
+			expectedDs: NewDaemonset().WithContainer(corev1.Container{
+				Name:            "nvidia-sandbox-device-plugin-ctr",
+				Image:           "nvcr.io/nvidia/kubevirt-device-plugin:v1.2.0",
+				ImagePullPolicy: corev1.PullIfNotPresent,
+				Env: []corev1.EnvVar{
+					{
+						Name:  "ENABLE_FABRIC_MANAGER",
+						Value: "true",
+					},
+				},
+				VolumeMounts: []corev1.VolumeMount{
+					{
+						Name:      "run-nvidia-fabricmanager",
+						MountPath: "/run/nvidia-fabricmanager",
+					},
+				},
+			}).WithInitContainer(corev1.Container{
+				Name:  "toolkit-validation",
+				Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0",
+				SecurityContext: &corev1.SecurityContext{
+					RunAsUser: rootUID,
+				},
+			}).WithVolume(corev1.Volume{
+				Name: "run-nvidia-fabricmanager",
+				VolumeSource: corev1.VolumeSource{
+					HostPath: &corev1.HostPathVolumeSource{
+						Path: "/run/nvidia-fabricmanager",
+						Type: ptr.To(corev1.HostPathDirectoryOrCreate),
+					},
+				},
+			}),
+			errorExpected: false,
+		},
+		{
+			description: "sandbox device plugin without fabric manager shared-nvswitch mode",
+			ds: NewDaemonset().WithContainer(corev1.Container{Name: "nvidia-sandbox-device-plugin-ctr"}).
+				WithInitContainer(corev1.Container{Name: "toolkit-validation"}),
+			cpSpec: &gpuv1.ClusterPolicySpec{
+				SandboxDevicePlugin: gpuv1.SandboxDevicePluginSpec{
+					Repository: "nvcr.io/nvidia",
+					Image:      "kubevirt-device-plugin",
+					Version:    "v1.2.0",
+				},
+				Validator: gpuv1.ValidatorSpec{
+					Repository: "nvcr.io/nvidia/cloud-native",
+					Image:      "gpu-operator-validator",
+					Version:    "v1.0.0",
+				},
+			},
+			expectedDs: NewDaemonset().WithContainer(corev1.Container{
+				Name:            "nvidia-sandbox-device-plugin-ctr",
+				Image:           "nvcr.io/nvidia/kubevirt-device-plugin:v1.2.0",
+				ImagePullPolicy: corev1.PullIfNotPresent,
+			}).WithInitContainer(corev1.Container{
+				Name:  "toolkit-validation",
+				Image: "nvcr.io/nvidia/cloud-native/gpu-operator-validator:v1.0.0",
+				SecurityContext: &corev1.SecurityContext{
+					RunAsUser: rootUID,
+				},
+			}),
+			errorExpected: false,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.description, func(t *testing.T) {
+			err := TransformSandboxDevicePlugin(tc.ds.DaemonSet, tc.cpSpec,
+				ClusterPolicyController{operatorNamespace: "test-ns", logger: ctrl.Log.WithName("test")})
+			if tc.errorExpected {
+				require.Error(t, err)
+				return
+			}
+			require.NoError(t, err)
+			require.EqualValues(t, tc.expectedDs, tc.ds)
+		})
+	}
+}