[CDAP-18568] Support CustomVolumes and CustomVolumeMounts for CDAP System Services

api/v1alpha1/cdapmaster_types.go

@@ -93,6 +93,12 @@ type CDAPMasterSpec struct {
 	Authentication *AuthenticationSpec `json:"authentication,omitempty"`
 	// SecurityContext defines the security context for all pods for all services.
 	SecurityContext *SecurityContext `json:"securityContext,omitempty"`
+	// AdditionalVolumes defines a list of additional volumes for all services.
+	// For information on supported volume types, see https://kubernetes.io/docs/concepts/storage/volumes/.
+	AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
+	// AdditionalVolumeMounts defines a list of additional volume mounts for all services.
+	// For information on suported volume mount types, see https://kubernetes.io/docs/concepts/storage/volumes/.
+	AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty"`
 }
 
 // CDAPServiceSpec defines the base set of specifications applicable to all master services.
@@ -122,6 +128,12 @@ type CDAPServiceSpec struct {
 	// Key is the secret object name. Value is the mount path.
 	// This adds Secret data to the directory specified by the volume mount path.
 	SecretVolumes map[string]string `json:"secretVolumes,omitempty"`
+	// AdditionalVolumes defines a list of additional volumes to mount to the service.
+	// For information on supported volume types, see https://kubernetes.io/docs/concepts/storage/volumes/.
+	AdditionalVolumes []corev1.Volume `json:"additionalVolumes,omitempty"`
+	// AdditionalVolumeMounts defines a list of additional volume mounts for the service.
+	// For information on suported volume mount types, see https://kubernetes.io/docs/concepts/storage/volumes/.
+	AdditionalVolumeMounts []corev1.VolumeMount `json:"additionalVolumeMounts,omitempty"`
 	// SecurityContext overrides the security context for the service pods.
 	SecurityContext *SecurityContext `json:"securityContext,omitempty"`
 }

config/rbac/role.yaml

@@ -1,3 +1,4 @@
+
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole

controllers/deployment.go

@@ -199,6 +199,12 @@ func buildStatefulSets(master *v1alpha1.CDAPMaster, name string, services Servic
 		if _, err := spec.addSecretVolumes(ss.SecretVolumes); err != nil {
 			return nil, err
 		}
+		if _, err := spec.addAdditionalVolumes(ss.AdditionalVolumes); err != nil {
+			return nil, err
+		}
+		if _, err := spec.addAdditionalVolumeMounts(ss.AdditionalVolumeMounts); err != nil {
+			return nil, err
+		}
 	}
 
 	// All services are optional services and are disabled in CR.
@@ -284,6 +290,12 @@ func buildDeployment(master *v1alpha1.CDAPMaster, name string, services ServiceG
 		if _, err := spec.addSecretVolumes(ss.SecretVolumes); err != nil {
 			return nil, err
 		}
+		if _, err := spec.addAdditionalVolumes(ss.AdditionalVolumes); err != nil {
+			return nil, err
+		}
+		if _, err := spec.addAdditionalVolumeMounts(ss.AdditionalVolumeMounts); err != nil {
+			return nil, err
+		}
 	}
 	// All services are optional services and are disabled in CR.
 	// Return nil to indicate no deployment is built.
@@ -327,6 +339,28 @@ func buildStatefulSetsObject(spec *StatefulSpec) (*reconciler.Object, error) {
 	if err != nil {
 		return nil, err
 	}
+	// For custom volumes and custom volume mounts, we directly pass structs from the spec to bypass the YAML templating logic.
+	k8sObj, ok := obj.Obj.(*k8s.Object)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert object to k8s object")
+	}
+	statefulSetObj, ok := k8sObj.Obj.(*appsv1.StatefulSet)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert meta object to statefulset object")
+	}
+	if err := addVolumeToPodSpec(&statefulSetObj.Spec.Template.Spec, spec.Base.AdditionalVolumes); err != nil {
+		return nil, err
+	}
+	for index, _ := range statefulSetObj.Spec.Template.Spec.InitContainers {
+		if err := addVolumeMountToContainer(&statefulSetObj.Spec.Template.Spec.InitContainers[index], spec.Base.AdditionalVolumeMounts); err != nil {
+			return nil, err
+		}
+	}
+	for index, _ := range statefulSetObj.Spec.Template.Spec.Containers {
+		if err := addVolumeMountToContainer(&statefulSetObj.Spec.Template.Spec.Containers[index], spec.Base.AdditionalVolumeMounts); err != nil {
+			return nil, err
+		}
+	}
 	return obj, nil
 }
 
@@ -336,9 +370,55 @@ func buildDeploymentObject(spec *DeploymentSpec) (*reconciler.Object, error) {
 	if err != nil {
 		return nil, err
 	}
+	// For custom volumes and volume mounts, we directly pass structs from the spec to bypass the YAML templating logic.
+	k8sObj, ok := obj.Obj.(*k8s.Object)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert object to k8s object")
+	}
+	deploymentObj, ok := k8sObj.Obj.(*appsv1.Deployment)
+	if !ok {
+		return nil, fmt.Errorf("failed to convert meta object to statefulset object")
+	}
+	if err := addVolumeToPodSpec(&deploymentObj.Spec.Template.Spec, spec.Base.AdditionalVolumes); err != nil {
+		return nil, err
+	}
+	for index, _ := range deploymentObj.Spec.Template.Spec.InitContainers {
+		if err := addVolumeMountToContainer(&deploymentObj.Spec.Template.Spec.InitContainers[index], spec.Base.AdditionalVolumeMounts); err != nil {
+			return nil, err
+		}
+	}
+	for index, _ := range deploymentObj.Spec.Template.Spec.Containers {
+		if err := addVolumeMountToContainer(&deploymentObj.Spec.Template.Spec.Containers[index], spec.Base.AdditionalVolumeMounts); err != nil {
+			return nil, err
+		}
+	}
 	return obj, nil
 }
 
+func addVolumeToPodSpec(podSpec *corev1.PodSpec, volumesToAdd []corev1.Volume) error {
+	for _, volumeToAdd := range volumesToAdd {
+		for _, volume := range podSpec.Volumes {
+			if volume.Name == volumeToAdd.Name {
+				return fmt.Errorf("failed to add custom volume %q to pod spec: already exists", volumeToAdd.Name)
+			}
+		}
+	}
+	podSpec.Volumes = append(podSpec.Volumes, volumesToAdd...)
+	return nil
+}
+
+func addVolumeMountToContainer(container *corev1.Container, volumeMountsToAdd []corev1.VolumeMount) error {
+	for _, volumeMountToAdd := range volumeMountsToAdd {
+		for _, volumeMount := range container.VolumeMounts {
+			if volumeMount.Name == volumeMountToAdd.Name {
+				return fmt.Errorf("failed to mount custom volume %q to container %q at path %q: already mounted", volumeMountToAdd.Name, container.Name, volumeMountToAdd.MountPath)
+			}
+		}
+	}
+	container.VolumeMounts = append(container.VolumeMounts, volumeMountsToAdd...)
+	return nil
+}
+
 // Return a NodePort service to expose the supplied target service
 func buildNetworkService(master *v1alpha1.CDAPMaster, name NetworkServiceName, target ServiceName, labels map[string]string) (*NetworkServiceSpec, error) {
 	s, err := getCDAPExternalServiceSpec(master, target)

controllers/spec.go

@@ -122,22 +122,24 @@ func (s *ContainerSpec) setResources(resources *corev1.ResourceRequirements) *Co
 
 // BaseSpec contains command fields for both StatefulSet and Deployment
 type BaseSpec struct {
-	Name               string                    `json:"name,omitempty"`
-	Namespace          string                    `json:"namespace,omitempty"`
-	Labels             map[string]string         `json:"labels,omitempty"`
-	ServiceAccountName string                    `json:"serviceAccountName,omitempty"`
-	Replicas           int32                     `json:"replicas,omitempty"`
-	NodeSelector       map[string]string         `json:"nodeSelector,omitempty"`
-	RuntimeClassName   string                    `json:"runtimeClassName,omitempty"`
-	PriorityClassName  string                    `json:"priorityClassName,omitempty"`
-	SecuritySecret     string                    `json:"securitySecret,omitempty"`
-	SecuritySecretPath string                    `json:"securitySecretPath,omitempty"`
-	CConf              string                    `json:"cdapConf,omitempty"`
-	HConf              string                    `json:"hadoopConf,omitempty"`
-	SysAppConf         string                    `json:"sysAppConf,omitempty"`
-	ConfigMapVolumes   map[string]string         `json:"configMapVolumes,omitempty"`
-	SecretVolumes      map[string]string         `json:"secretVolumes,omitempty"`
-	SecurityContext    *v1alpha1.SecurityContext `json:"securityContext,omitempty"`
+	Name                   string                    `json:"name,omitempty"`
+	Namespace              string                    `json:"namespace,omitempty"`
+	Labels                 map[string]string         `json:"labels,omitempty"`
+	ServiceAccountName     string                    `json:"serviceAccountName,omitempty"`
+	Replicas               int32                     `json:"replicas,omitempty"`
+	NodeSelector           map[string]string         `json:"nodeSelector,omitempty"`
+	RuntimeClassName       string                    `json:"runtimeClassName,omitempty"`
+	PriorityClassName      string                    `json:"priorityClassName,omitempty"`
+	SecuritySecret         string                    `json:"securitySecret,omitempty"`
+	SecuritySecretPath     string                    `json:"securitySecretPath,omitempty"`
+	CConf                  string                    `json:"cdapConf,omitempty"`
+	HConf                  string                    `json:"hadoopConf,omitempty"`
+	SysAppConf             string                    `json:"sysAppConf,omitempty"`
+	ConfigMapVolumes       map[string]string         `json:"configMapVolumes,omitempty"`
+	SecretVolumes          map[string]string         `json:"secretVolumes,omitempty"`
+	AdditionalVolumes      []corev1.Volume           `json:"additionalVolumes,omitempty"`
+	AdditionalVolumeMounts []corev1.VolumeMount      `json:"additionalVolumeMounts,omitempty"`
+	SecurityContext        *v1alpha1.SecurityContext `json:"securityContext,omitempty"`
 }
 
 func newBaseSpec(master *v1alpha1.CDAPMaster, name string, labels map[string]string, cconf, hconf, sysappconf string) *BaseSpec {
@@ -156,7 +158,8 @@ func newBaseSpec(master *v1alpha1.CDAPMaster, name string, labels map[string]str
 	s.SysAppConf = sysappconf
 	s.ConfigMapVolumes = cloneMap(master.Spec.ConfigMapVolumes)
 	s.SecretVolumes = cloneMap(master.Spec.SecretVolumes)
-
+	s.AdditionalVolumes = master.Spec.AdditionalVolumes
+	s.AdditionalVolumeMounts = master.Spec.AdditionalVolumeMounts
 	return s
 }
 
@@ -218,6 +221,30 @@ func addVolumes(volumes, newVolumes map[string]string, typeName string) error {
 	return nil
 }
 
+func (s *BaseSpec) addAdditionalVolumes(additionalVolumes []corev1.Volume) (*BaseSpec, error) {
+	for _, additionalVolume := range additionalVolumes {
+		for _, specVolume := range s.AdditionalVolumes {
+			if specVolume.Name == additionalVolume.Name {
+				return nil, fmt.Errorf("failed to add custom volume %q due to already mounted as %+v", additionalVolume.Name, specVolume)
+			}
+		}
+		s.AdditionalVolumes = append(s.AdditionalVolumes, additionalVolume)
+	}
+	return s, nil
+}
+
+func (s *BaseSpec) addAdditionalVolumeMounts(additionalVolumeMounts []corev1.VolumeMount) (*BaseSpec, error) {
+	for _, additionalVolumeMount := range additionalVolumeMounts {
+		for _, specVolumeMount := range s.AdditionalVolumeMounts {
+			if specVolumeMount.Name == additionalVolumeMount.Name {
+				return nil, fmt.Errorf("failed to mount custom volume %q at path %q due to already mounted as %+v", additionalVolumeMount.Name, additionalVolumeMount.MountPath, specVolumeMount)
+			}
+		}
+		s.AdditionalVolumeMounts = append(s.AdditionalVolumeMounts, additionalVolumeMount)
+	}
+	return s, nil
+}
+
 func (s *BaseSpec) setSecurityContext(securityContext *v1alpha1.SecurityContext) *BaseSpec {
 	s.SecurityContext = securityContext
 	if securityContext != nil {
@@ -306,6 +333,20 @@ func (s *DeploymentSpec) addSecretVolumes(volumes map[string]string) (*Deploymen
 	return s, nil
 }
 
+func (s *DeploymentSpec) addAdditionalVolumes(additionalVolumes []corev1.Volume) (*DeploymentSpec, error) {
+	if _, err := s.Base.addAdditionalVolumes(additionalVolumes); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *DeploymentSpec) addAdditionalVolumeMounts(additionalVolumeMounts []corev1.VolumeMount) (*DeploymentSpec, error) {
+	if _, err := s.Base.addAdditionalVolumeMounts(additionalVolumeMounts); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
 func (s *DeploymentSpec) setSecurityContext(securityContext *v1alpha1.SecurityContext) *DeploymentSpec {
 	s.Base.setSecurityContext(securityContext)
 	return s
@@ -397,6 +438,20 @@ func (s *StatefulSpec) addSecretVolumes(volumes map[string]string) (*StatefulSpe
 	return s, nil
 }
 
+func (s *StatefulSpec) addAdditionalVolumes(additionalVolumes []corev1.Volume) (*StatefulSpec, error) {
+	if _, err := s.Base.addAdditionalVolumes(additionalVolumes); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
+func (s *StatefulSpec) addAdditionalVolumeMounts(additionalVolumeMounts []corev1.VolumeMount) (*StatefulSpec, error) {
+	if _, err := s.Base.addAdditionalVolumeMounts(additionalVolumeMounts); err != nil {
+		return nil, err
+	}
+	return s, nil
+}
+
 func (s *StatefulSpec) setSecurityContext(securityContext *v1alpha1.SecurityContext) *StatefulSpec {
 	s.Base.setSecurityContext(securityContext)
 	return s

controllers/testdata/appfabric.json

@@ -136,6 +136,15 @@
               {
                 "mountPath": "/my/secret/1",
                 "name": "cdap-se-vol-my-secret-1"
+              },
+              {
+                "mountPath": "/mnt/test",
+                "name": "test-volume",
+                "readOnly": true
+              },
+              {
+                "mountPath": "/mnt/teststorage",
+                "name": "test-persistent-storage"
               }
             ]
           }
@@ -179,16 +188,13 @@
                 "mountPath": "/etc/cdap/security"
               },
               {
-                "name": "cdap-cm-vol-my-config-map-1",
-                "mountPath": "/my/config/map/1"
-              },
-              {
-                "mountPath": "/my/config/map/2",
-                "name": "cdap-cm-vol-my-config-map-2"
+                "mountPath": "/mnt/test",
+                "name": "test-volume",
+                "readOnly": true
               },
               {
-                "mountPath": "/my/secret/1",
-                "name": "cdap-se-vol-my-secret-1"
+                "mountPath": "/mnt/teststorage",
+                "name": "test-persistent-storage"
               }
             ]
           }
@@ -283,6 +289,39 @@
               "defaultMode": 420,
               "secretName": "my-secret-1"
             }
+          },
+          {
+            "name": "test-volume",
+            "projected": {
+              "defaultMode": 420,
+              "sources": [
+                {
+                  "serviceAccountToken": {
+                    "audience": "test-aud",
+                    "expirationSeconds": 3600,
+                    "path": "token"
+                  } 
+                },
+                {
+                  "configMap": {
+                    "name": "test-projected-config",
+                    "items": [
+                      {
+                        "key": "config",
+                        "path": "test-projected-config"
+                      }
+                    ],
+                    "optional": false
+                  }
+                }
+              ]
+            }
+          },
+          {
+            "name": "test-persistent-storage",
+            "persistentVolumeClaim": {
+              "claimName": "test-persistent-volume-claim"
+            }
           }
         ]
       }