diff --git a/api/v1alpha1/cdapmaster_types.go b/api/v1alpha1/cdapmaster_types.go index c48035f4..137b70b2 100644 --- a/api/v1alpha1/cdapmaster_types.go +++ b/api/v1alpha1/cdapmaster_types.go @@ -128,6 +128,14 @@ type CDAPMasterSpec struct { // Mutations can include adding init containers, tolerations and node selectors to pods. To use mutations, // the admission control webhook should be enabled in the cdap operator. MutationConfigs []MutationConfig `json:"mutationConfigs,omitempty"` + // StartupProbe is specification for the startup probe for CDAP system services. + // This is an optional field which ensures that the application is fully initialized + // before it starts receiving traffic. + // To disable the startup probe: either omit or set the field to nil. + // To enable the startup probe: set it to a pointer to a SystemMetricsExporterSpec + // struct (can be an empty struct). CDAPServiceSpec.EnableStartupProbe field also needs + // to be set to true for the deployment services which require startup probe to be enabled. + StartupProbe *StartupProbeSpec `json:"startupProbe,omitempty"` } // CDAPServiceSpec defines the base set of specifications applicable to all master services. @@ -176,6 +184,9 @@ type CDAPServiceSpec struct { Lifecycle *corev1.Lifecycle `json:"lifecycle,omitempty"` // Affinity describes node affinity scheduling rules for the service. Affinity *corev1.Affinity `json:"affinity,omitempty"` + // EnableStartupProbe is an optional field to indicate if StatusProbe should + // be enabled for the container. + EnableStartupProbe *bool `json:"enableStartupProbe,omitempty"` } // CDAPScalableServiceSpec defines the base specification for master services that can have more than one instance. @@ -295,6 +306,24 @@ type SystemMetricExporterSpec struct { CDAPServiceSpec `json:",inline"` } +type StartupProbeSpec struct { + // For Probe config see: + // https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/#configure-probes. + + // Number of seconds after the container has started before probes are initiated. + // Defaults to 0 seconds. Minimum value is 0. + InitialDelaySeconds *int32 `json:"initialDelaySeconds,omitempty"` + // How often (in seconds) to perform the probe. + // Default to 10 seconds. The minimum value is 1. + PeriodSeconds *int32 `json:"periodSeconds,omitempty"` + // Number of seconds after which the probe times out. + // Defaults to 1 second. Minimum value is 1. + TimeoutSeconds *int32 `json:"timeoutSeconds,omitempty"` + // Number of consecutive failures before considering the service not ready. + // Defaults to 3. Minimum value is 1. + FailureThreshold *int32 `json:"failureThreshold,omitempty"` +} + // CDAPMasterStatus defines the observed state of CDAPMaster type CDAPMasterStatus struct { status.Meta `json:",inline"` diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index ce26a1a8..aed95ce7 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -300,6 +300,11 @@ func (in *CDAPMasterSpec) DeepCopyInto(out *CDAPMasterSpec) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + if in.StartupProbe != nil { + in, out := &in.StartupProbe, &out.StartupProbe + *out = new(StartupProbeSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CDAPMasterSpec. @@ -452,6 +457,11 @@ func (in *CDAPServiceSpec) DeepCopyInto(out *CDAPServiceSpec) { *out = new(v1.Affinity) (*in).DeepCopyInto(*out) } + if in.EnableStartupProbe != nil { + in, out := &in.EnableStartupProbe, &out.EnableStartupProbe + *out = new(bool) + **out = **in + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new CDAPServiceSpec. @@ -704,6 +714,41 @@ func (in *SecurityContext) DeepCopy() *SecurityContext { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *StartupProbeSpec) DeepCopyInto(out *StartupProbeSpec) { + *out = *in + if in.InitialDelaySeconds != nil { + in, out := &in.InitialDelaySeconds, &out.InitialDelaySeconds + *out = new(int32) + **out = **in + } + if in.PeriodSeconds != nil { + in, out := &in.PeriodSeconds, &out.PeriodSeconds + *out = new(int32) + **out = **in + } + if in.TimeoutSeconds != nil { + in, out := &in.TimeoutSeconds, &out.TimeoutSeconds + *out = new(int32) + **out = **in + } + if in.FailureThreshold != nil { + in, out := &in.FailureThreshold, &out.FailureThreshold + *out = new(int32) + **out = **in + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StartupProbeSpec. +func (in *StartupProbeSpec) DeepCopy() *StartupProbeSpec { + if in == nil { + return nil + } + out := new(StartupProbeSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *SupportBundleSpec) DeepCopyInto(out *SupportBundleSpec) { *out = *in diff --git a/config/crd/bases/cdap.cdap.io_cdapmasters.yaml b/config/crd/bases/cdap.cdap.io_cdapmasters.yaml index 8afb3193..609c23f3 100644 --- a/config/crd/bases/cdap.cdap.io_cdapmasters.yaml +++ b/config/crd/bases/cdap.cdap.io_cdapmasters.yaml @@ -3995,6 +3995,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -6785,6 +6790,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -9582,6 +9592,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -12379,6 +12394,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -15308,6 +15328,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -18104,6 +18129,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -20897,6 +20927,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -23686,6 +23721,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -27844,6 +27884,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -30644,6 +30689,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -33449,6 +33499,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -33932,6 +33987,41 @@ spec: description: ServiceAccountName is the service account for all the service pods. type: string + startupProbe: + description: |- + StartupProbe is specification for the startup probe for CDAP system services. + This is an optional field which ensures that the application is fully initialized + before it starts receiving traffic. + To disable the startup probe: either omit or set the field to nil. + To enable the startup probe: set it to a pointer to a SystemMetricsExporterSpec + struct (can be an empty struct). CDAPServiceSpec.EnableStartupProbe field also needs + to be set to true for the deployment services which require startup probe to be enabled. + properties: + failureThreshold: + description: |- + Number of consecutive failures before considering the service not ready. + Defaults to 3. Minimum value is 1. + format: int32 + type: integer + initialDelaySeconds: + description: |- + Number of seconds after the container has started before probes are initiated. + Defaults to 0 seconds. Minimum value is 0. + format: int32 + type: integer + periodSeconds: + description: |- + How often (in seconds) to perform the probe. + Default to 10 seconds. The minimum value is 1. + format: int32 + type: integer + timeoutSeconds: + description: |- + Number of seconds after which the probe times out. + Defaults to 1 second. Minimum value is 1. + format: int32 + type: integer + type: object supportBundle: description: |- SupportBundle is specification for the CDAP support-bundle service. @@ -36308,6 +36398,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -39108,6 +39203,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -41905,6 +42005,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter @@ -44705,6 +44810,11 @@ spec: Key is the configmap object name. Value is the mount path. This adds ConfigMap data to the directory specified by the volume mount path. type: object + enableStartupProbe: + description: |- + EnableStartupProbe is an optional field to indicate if StatusProbe should + be enabled for the container. + type: boolean enableSystemMetrics: description: |- EnableSystemMetrics is an optional field that is considered along with CDAPMasterSpec.SystemMetricsExporter diff --git a/controllers/deployment.go b/controllers/deployment.go index 97c4e351..2e932131 100644 --- a/controllers/deployment.go +++ b/controllers/deployment.go @@ -293,6 +293,77 @@ func addSystemMetricsServiceIfEnabled(stsSpec *StatefulSpec, master *v1alpha1.CD return nil } +// addStartupProbeIfEnabled adds a startupProbe to the mainContainer if enabled in service spec. +func addStartupProbeIfEnabled(master *v1alpha1.CDAPMaster, service ServiceName, + mainContainer *ContainerSpec, serviceSpec *v1alpha1.CDAPServiceSpec) error { + if master.Spec.StartupProbe == nil || serviceSpec == nil { + return nil + } + if serviceSpec.EnableStartupProbe == nil { + return nil + } + if *serviceSpec.EnableStartupProbe == false { + return nil + } + + svc := strings.ToLower(service) + endpoint := fmt.Sprintf("https://localhost:$(cat /tmp/%s.port)/v3/system/services/%s/status", svc, svc) + probe, err := startupProbeSpec(master, endpoint) + if err != nil { + return err + } + mainContainer.StartupProbe = probe + return nil +} + +func startupProbeSpec(master *v1alpha1.CDAPMaster, endpoint string) (*corev1.Probe, error) { + if master.Spec.StartupProbe == nil { + return nil, fmt.Errorf("master.Spec.StartupProbe is nil") + } + + id, err := probeConfig("initialDelaySeconds", master.Spec.StartupProbe.InitialDelaySeconds, 0, 0) + if err != nil { + return nil, err + } + ps, err := probeConfig("periodSeconds", master.Spec.StartupProbe.PeriodSeconds, 10, 1) + if err != nil { + return nil, err + } + ts, err := probeConfig("timeoutSeconds", master.Spec.StartupProbe.TimeoutSeconds, 1, 1) + if err != nil { + return nil, err + } + ft, err := probeConfig("failureThreshold", master.Spec.StartupProbe.FailureThreshold, 3, 1) + if err != nil { + return nil, err + } + + return &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + // Use exec to execute a command inside the container + Exec: &corev1.ExecAction{ + // Command to execute + Command: []string{"sh", "-c", "curl -s -f -k " + endpoint}, + }, + }, + // Probe settings + InitialDelaySeconds: id, + PeriodSeconds: ps, + TimeoutSeconds: ts, + FailureThreshold: ft, + }, nil +} + +func probeConfig(key string, config *int32, defaultVal int32, min int32) (int32, error) { + if config == nil { + return defaultVal, nil + } + if *config < min { + return defaultVal, fmt.Errorf("minimum value of %q should be %d", key, min) + } + return *config, nil +} + // Return a single single-/multi- container deployment containing a list of supplied services func buildDeployment(master *v1alpha1.CDAPMaster, name string, services ServiceGroup, labels map[string]string, cconf, hconf, sysappconf, dataDir string) (*DeploymentSpec, error) { objName := getObjName(master, name) @@ -374,6 +445,9 @@ func buildDeployment(master *v1alpha1.CDAPMaster, name string, services ServiceG if _, err := spec.addAdditionalVolumeMounts(ss.AdditionalVolumeMounts); err != nil { return nil, err } + if err := addStartupProbeIfEnabled(master, s, c, ss); err != nil { + return nil, err + } } // All services are optional services and are disabled in CR. // Return nil to indicate no deployment is built. @@ -529,6 +603,7 @@ func buildDeploymentObject(spec *DeploymentSpec) (*reconciler.Object, error) { return nil, err } setLifecycleHookForContainer(&deploymentObj.Spec.Template.Spec.Containers[index], spec.Containers[index].Lifecycle) + setStartupProbeForContainer(&deploymentObj.Spec.Template.Spec.Containers[index], spec.Containers[index].StartupProbe) } return obj, nil } @@ -561,6 +636,10 @@ func setLifecycleHookForContainer(container *corev1.Container, lifecycle *corev1 container.Lifecycle = lifecycle } +func setStartupProbeForContainer(container *corev1.Container, startupProbe *corev1.Probe) { + container.StartupProbe = startupProbe +} + // Return a NodePort service to expose the supplied target service func buildNetworkService(master *v1alpha1.CDAPMaster, name NetworkServiceName, target ServiceName, labels map[string]string) (*NetworkServiceSpec, error) { s, err := getCDAPExternalServiceSpec(master, target) diff --git a/controllers/spec.go b/controllers/spec.go index 49150582..c8107f7a 100644 --- a/controllers/spec.go +++ b/controllers/spec.go @@ -46,6 +46,7 @@ type ContainerSpec struct { ResourceLimits map[string]*resource.Quantity `json:"resourceLimits,omitempty"` DataDir string `json:"dataDir,omitempty"` Lifecycle *corev1.Lifecycle `json:"lifecycle,omitempty"` + StartupProbe *corev1.Probe `json:"startupProbe,omitempty"` } func newContainerSpec(master *v1alpha1.CDAPMaster, name, dataDir string) *ContainerSpec { diff --git a/controllers/testdata/appfabric.json b/controllers/testdata/appfabric.json index 7790ad22..97b20686 100644 --- a/controllers/testdata/appfabric.json +++ b/controllers/testdata/appfabric.json @@ -121,6 +121,18 @@ "privileged": false, "readOnlyRootFilesystem": false }, + "startupProbe": { + "periodSeconds": 2, + "failureThreshold": 60, + "exec": { + "command": [ + "sh", + "-c", + "curl -s -f -k https://localhost:$(cat /tmp/appfabric.port)/v3/system/services/appfabric/status" + ] + }, + "timeoutSeconds": 1 + }, "lifecycle": { "preStop": { "exec": { diff --git a/controllers/testdata/cdap_master_cr.json b/controllers/testdata/cdap_master_cr.json index 960f1258..4ea30661 100644 --- a/controllers/testdata/cdap_master_cr.json +++ b/controllers/testdata/cdap_master_cr.json @@ -100,7 +100,8 @@ } ] } - } + }, + "enableStartupProbe": true }, "appfabricprocessor": { "metadata": { @@ -340,6 +341,11 @@ }, "storageSize": "100Gi" }, + "startupProbe": { + "periodSeconds": 2, + "timeoutSeconds": 1, + "failureThreshold": 60 + }, "systemMetricsExporter": { "metadata": { "creationTimestamp": null