Improve Support for LoadBalancer Service Type

fides-minimal/templates/_helpers.tpl

@@ -18,7 +18,7 @@ If release name contains chart name it will be used as a full name.
       {{- if contains $baseName .Release.Name }}
         {{- $baseName = .Release.Name }}
       {{- else }}
-        {{- printf "%s-%s" .Release.Name $baseName }}
+        {{- $baseName = printf "%s-%s" .Release.Name $baseName }}
       {{- end }}
     {{- end }}
   {{- $baseName | trunc 63 | trimSuffix "-"}}
@@ -150,11 +150,29 @@ Create the name of the config map to store the fides.toml file.
 List of CORS origins, concatenated, deduplicated, and formatted.
 */}}
 {{- define "fides.corsOrigins" -}}
-{{ $cors := list (printf "https://%s" .Values.privacyCenter.publicHostname | quote ) (printf "https://%s" .Values.fides.publicHostname | quote) }}
+{{- $cors := list }}
+
+{{- if .Values.privacyCenter.publicHostname }}
+  {{- $cors = append $cors (printf "https://%s" .Values.privacyCenter.publicHostname | quote) }}
+{{- end }}
+{{- if .Values.fides.publicHostname }}
+  {{- $cors = append $cors (printf "https://%s" .Values.fides.publicHostname | quote) }}
+{{- end }}
+
+{{- if eq .Values.fides.service.type "LoadBalancer" }}
+  {{- if .Values.fides.publicHostname }}
+    {{- $cors = append $cors (printf "http://%s" .Values.fides.publicHostname | quote) }}
+  {{- end }}
+  {{- if and .Values.privacyCenter.enabled .Values.privacyCenter.publicHostname }}
+    {{- $cors = append $cors (printf "http://%s" .Values.privacyCenter.publicHostname | quote) }}
+  {{- end }}
+{{- end }}
+
 {{- range (.Values.fides.configuration.additionalCORSOrigins | compact) }}
-  {{- $cors = . | quote | append $cors }}
+  {{- $cors = append $cors (. | quote) }}
 {{- end }}
-{{ $cors = $cors | uniq }}
+
+{{- $cors = $cors | compact | uniq }}
 {{ printf "[%s]" (join "," $cors) }}
 {{- end }}
 
@@ -167,7 +185,7 @@ The set of environment variables for Fides and workers
 {{- $redisDeployment := .Values.redis }}
 {{- $pgDeployment := .Values.postgresql }}
 {{- with .Values.fides.configuration }}
-{{- .additionalEnvVars | toYaml }}
+{{- include "fides.processedEnvVars" $ }}
 - name: FIDES__DATABASE__SERVER
   valueFrom:
     secretKeyRef:
@@ -210,3 +228,39 @@ The set of environment variables for Fides and workers
       key: REDIS_PASSWORD
 {{- end }}
 {{- end }}
+
+{{/*
+Detect if fidesplus is being used based on the repository name
+*/}}
+{{- define "fides.isFidesplus" -}}
+{{- if contains "fidesplus" (.Values.fides.image.repository | lower) -}}
+true
+{{- else -}}
+false
+{{- end -}}
+{{- end }}
+
+{{/*
+Get processed environment variables with additional settings
+*/}}
+{{- define "fides.processedEnvVars" -}}
+{{- $envVars := .Values.fides.configuration.additionalEnvVars | default list }}
+{{- $hiddenEnvVar := dict "name" "FIDES__EXECUTION__MONITOR_CELERY_TASKS_ENABLED" "value" "true" }}
+{{- $envVars = append $envVars $hiddenEnvVar }}
+{{- $envVars | toYaml }}
+{{- end }}
+
+{{/*
+Validates that all worker types have unique names. Fails if duplicate names are found.
+*/}}
+{{- define "fides.worker.validateUniqueNames" -}}
+{{- $workers := .Values.fides.workerConfiguration.workers | default list }}
+{{- $names := dict }}
+{{- range $workers }}
+{{- if hasKey $names .name }}
+{{- fail (printf "Duplicate worker name found: '%s'. Worker names must be unique" .name) }}
+{{- else }}
+{{- $_ := set $names .name "used" }}
+{{- end }}
+{{- end }}
+{{- end }}

fides-minimal/templates/fides/fides-config.yaml

@@ -14,7 +14,7 @@ data:
     [celery]
     event_queue_prefix = "fides_worker"
     task_default_queue = "fides"
-    task_always_eager = {{ not $.worker }}
+    task_always_eager = false
 
     [security]
     cors_origins = {{ include "fides.corsOrigins" . | trim }}

fides-minimal/templates/fides/worker-config.yaml

@@ -1,3 +1,4 @@
+{{- $_ := set $ "worker" ( ge (len .Values.fides.workerConfiguration.workers) 1) }}
 {{- if $.worker }}
 apiVersion: v1
 kind: ConfigMap
@@ -15,7 +16,7 @@ data:
     [celery]
     event_queue_prefix = "fides_worker"
     task_default_queue = "fides"
-    task_always_eager = true
+    task_always_eager = false
     redis_socket_keepalive = true
 
     [security]

fides-minimal/templates/fides/worker-deployment.yaml

@@ -1,54 +1,110 @@
-{{- $_ := set $ "worker" ( ge (.Values.fides.workers.count | int) 1) }}
+{{- include "fides.worker.validateUniqueNames" . }}
+{{- $userWorkers := .Values.fides.workerConfiguration.workers | default list }}
+{{- $isFidesplus := include "fides.isFidesplus" . }}
+
+{{/* Build default workers list */}}
+{{- $defaultWorkers := list }}
+{{/* Both Fides and Fidesplus get DSR and other workers */}}
+{{- $defaultWorkers = list 
+  (dict "name" "dsr" "count" 1 "queues" (list "fides.dsr") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi")))
+  (dict "name" "other" "count" 1 "excludeQueues" (list "fides.dsr" "fides.privacy_preferences") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi")))
+}}
+{{/* Fidesplus additionally gets classification, helios, and consent workers */}}
+{{- if eq $isFidesplus "true" }}
+{{- $defaultWorkers = concat $defaultWorkers (list
+  (dict "name" "classification" "count" 1 "queues" (list "fidesplus.discovery_monitors_classification") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi")))
+  (dict "name" "helios" "count" 1 "queues" (list "fidesplus.discovery_monitors_promotion" "fidesplus.discovery_monitors_detection") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi")))
+  (dict "name" "consent" "count" 1 "queues" (list "fides.privacy_preferences" "fides.consent_webhooks") "resources" (dict "limits" (dict "memory" "512Mi") "requests" (dict "cpu" "100m" "memory" "256Mi")))
+) }}
+{{- end }}
+
+{{/* Merge user workers with defaults */}}
+{{- $mergedWorkers := list }}
+{{- $userWorkerNames := dict }}
+{{- range $userWorkers }}
+{{- $_ := set $userWorkerNames .name true }}
+{{- $mergedWorkers = append $mergedWorkers . }}
+{{- end }}
+{{- range $defaultWorkers }}
+{{- if not (hasKey $userWorkerNames .name) }}
+{{- $mergedWorkers = append $mergedWorkers . }}
+{{- end }}
+{{- end }}
+
+{{/* Check if we have any active workers */}}
+{{- $hasActiveWorkers := false }}
+{{- range $mergedWorkers }}
+{{- if gt (.count | int) 0 }}
+{{- $hasActiveWorkers = true }}
+{{- end }}
+{{- end }}
+{{- $_ := set $ "worker" $hasActiveWorkers }}
+
 {{- if $.worker }}
+{{- range $mergedWorkers }}
+{{- $workerCount := .count | int }}
+{{- if gt $workerCount 0 }}
 {{- $volume := "config" }}
 {{- $configPath := "/etc/fides/config" }}
 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  name: {{ include "fides.worker.fullname" . }}
+  name: {{ printf "%s-%s" (include "fides.worker.fullname" $) .name}}
   labels:
-    {{- include "fides.labels" . | nindent 4 }}
+    {{- include "fides.labels" $ | nindent 4 }}
+    fid.es/worker: {{ .name }}
 spec:
-  replicas: {{ .Values.fides.workers.count | int }}
+  replicas: {{ $workerCount | int }}
   selector:
     matchLabels:
-      {{- include "fides.worker.selectorLabels" . | nindent 6 }}
+      {{- include "fides.worker.selectorLabels" $ | nindent 6 }}
+      fid.es/worker: {{ .name }}
   strategy:
-    {{- include "fides.deploymentStrategy" . | nindent 4 }}
+    {{- include "fides.deploymentStrategy" $ | nindent 4 }}
   template:
     metadata:
-      {{- with .Values.podAnnotations }}
+      {{- with $.Values.podAnnotations }}
       annotations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
       labels:
-        {{- include "fides.worker.selectorLabels" . | nindent 8 }}
+        {{- include "fides.worker.selectorLabels" $ | nindent 8 }}
+        fid.es/worker: {{ .name }}
     spec:
-      {{- with .Values.imagePullSecrets }}
+      {{- with $.Values.imagePullSecrets }}
       imagePullSecrets:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      serviceAccountName: {{ include "fides.serviceAccountName" . }}
+      serviceAccountName: {{ include "fides.serviceAccountName" $ }}
       securityContext:
-        {{- toYaml .Values.podSecurityContext | nindent 8 }}
+        {{- toYaml $.Values.podSecurityContext | nindent 8 }}
       containers:
         - name: fides
           securityContext:
-            {{- toYaml .Values.securityContext | nindent 12 }}
-          image: {{ printf "%s:%s" .Values.fides.image.repository ( include "fides.dockerTag" .)}}
-          imagePullPolicy: {{ .Values.fides.image.pullPolicy }}
+            {{- toYaml $.Values.securityContext | nindent 12 }}
+          image: {{ printf "%s:%s" $.Values.fides.image.repository (default (include "fides.dockerTag" $) .imageTagOverride) }}
+          imagePullPolicy: {{ $.Values.fides.image.pullPolicy }}
           command: ["fides"]
-          args: ["worker"]
+          {{- if and (hasKey . "queues") (hasKey . "excludeQueues") }}
+            {{- fail (printf "Worker '%s' cannot have both --queues and --exclude-queues passed" .name) }}
+          {{- end }}
+          args:
+            - worker
+            {{- if hasKey . "queues" }}
+            - {{ printf "--queues=%s" (join "," .queues) }}
+            {{- else if hasKey . "excludeQueues" }}
+            - {{ printf "--exclude-queues=%s" (join "," .excludeQueues) }}
+            {{- end }}
           env:
             - name: FIDES__CONFIG_PATH
               value: {{ printf "%s/fides.toml" $configPath }}
-            {{- include "fides.env" . | nindent 12 }}
+            {{- include "fides.env" $ | nindent 12 }}
           envFrom:
             - secretRef: 
-                name: {{ include "fides.fidesSecuritySecretName" . }}
-            {{- if .Values.fides.configuration.additionalEnvVarsSecret }}
+                name: {{ include "fides.fidesSecuritySecretName" $ }}
+            {{- if $.Values.fides.configuration.additionalEnvVarsSecret }}
             - secretRef: 
-                name: {{ .Values.fides.configuration.additionalEnvVarsSecret }}
+                name: {{ $.Values.fides.configuration.additionalEnvVarsSecret }}
             {{- end }}
           livenessProbe:
             exec:
@@ -57,28 +113,31 @@ spec:
                 "-c",
                 "celery --quiet --no-color --app fides.api.tasks inspect ping --destination celery@$HOSTNAME --json"
               ]
-            initialDelaySeconds: {{ .Values.fides.startupTimeSeconds | default 30 }}
+            initialDelaySeconds: {{ $.Values.fides.startupTimeSeconds | default 30 }}
             periodSeconds: 60
-            timeoutSeconds: {{ .Values.fides.healthCheckTimeoutSeconds | default 5 }}
+            timeoutSeconds: {{ $.Values.fides.healthCheckTimeoutSeconds | default 5 }}
           volumeMounts:
           - name: {{ $volume }}
             mountPath: {{ $configPath }}
           resources:
-            {{- toYaml .Values.resources | nindent 12 }}
+            {{- toYaml .resources | nindent 12 }}
       volumes:
         - name: {{ $volume }}
           configMap:
-            name: {{ include "fides.worker.tomlConfigMapName" . }}
-      {{- with .Values.nodeSelector }}
+            name: {{ include "fides.worker.tomlConfigMapName" $ }}
+      {{- with $.Values.nodeSelector }}
       nodeSelector:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.affinity }}
+      {{- with $.Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
       {{- end }}
-      {{- with .Values.tolerations }}
+      {{- with $.Values.tolerations }}
       tolerations:
         {{- toYaml . | nindent 8 }}
       {{- end }}
+---
+{{- end }}
+{{- end }}
 {{- end }}

fides-minimal/values.yaml

@@ -31,15 +31,20 @@ fides:
         value: "false"
       - name: FIDES__REDIS__SSL_CERT_REQS # Accepted values include: none, optional and require.
         value: "none"
+      - name: FIDES__EXECUTION__USE_DSR_3_0
+        value: "true"
       # Additional environment variables may be declared here.
     # fides.configuration.additionalEnvVarsSecret is an optional parameter representing the name of an existing secret containing environment variables to pass into the Fides containers.
     additionalEnvVarsSecret: ""
     # fides.configuration.fidesSecuritySecretName is an optional parameter that respresents the name of a Kubernetes secret containing sensitive Fides configuration elements. If set, this secret must have the following keys:
     # FIDES__SECURITY__APP_ENCRYPTION_KEY, FIDES__SECURITY__OAUTH_ROOT_CLIENT_ID, FIDES__SECURITY__OAUTH_ROOT_CLIENT_SECRET, FIDES__SECURITY__DRP_JWT_SECRET
     fidesSecuritySecretName: ""
     # fides.configuration.additionalCORSOrigins is an optional parameter to configure allowed CORS origins in addition to the Fides and Privacy Center URLs.
+    # Note: When using LoadBalancer service type, the chart automatically includes both HTTP and HTTPS variants of your hostnames
     additionalCORSOrigins: []
   # fides.publicHostname is used to set the allowed CORS origins for Fides, e.g. fides.example.com
+  # For LoadBalancer services: You can leave this empty initially and set it later once you know the LoadBalancer endpoint,
+  # or specify a CNAME/DNS name that points to your LoadBalancer
   publicHostname: ""
   fullnameOverride: ""
   count: 1
@@ -53,10 +58,34 @@ fides:
   startupTimeSeconds: 30
   # fides.healthCheckTimeoutSeconds configures the timeoutSeconds of the liveness and readiness probes.
   healthCheckTimeoutSeconds: 5
-  workers:
-    # fides.workers.count determines how many workers the deployment will use to process DSRs.
-    # To disable workers, set count to 0. This should be set to at least 1 in production environments.
-    count: 0
+  # fides.workerConfiguration configures the Celery workers that process background tasks.
+  # 
+  # DEFAULT WORKER CONFIGURATION:
+  # Both Fides and Fidesplus deployments automatically get:
+  #   - 1 DSR worker
+  #   - 1 other worker  
+  #
+  # Fidesplus deployments additionally get:
+  #   - 1 classification worker
+  #   - 1 helios worker
+  #   - 1 consent worker
+  #
+  # To override defaults, explicitly define workers below. To disable a worker, set count: 0.
+  # For more information, see: https://www.ethyca.com/docs/dev-docs/get-started/advanced#running-workers
+  workerConfiguration:
+    workers: []
+    # Example worker override:
+    # - name: other
+    #   count: 1
+    #   excludeQueues:
+    #   - fides.dsr
+    #   - fides.privacy_preferences
+    #   resources:
+    #     limits:
+    #       memory: 512Mi
+    #     requests:
+    #       cpu: 100m
+    #       memory: 256Mi
 
 # privacyCenter is the end-user facing application where data subjects can submit privacy requests.
 privacyCenter:
@@ -82,6 +111,8 @@ privacyCenter:
     additionalEnvVars: []
   nameOverride: ""
   # privacyCenter.publicHostname is used to set the allowed CORS origins for Fides, e.g. privacy.example.com
+  # For LoadBalancer services: You can leave this empty initially and set it later once you know the LoadBalancer endpoint,
+  # or specify a CNAME/DNS name that points to your LoadBalancer
   publicHostname: ""
   fullnameOverride: ""
   count: 1