From 67cabb599adb1be9f89c0cd40fad2ab3a32df050 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Thu, 28 May 2026 08:15:02 +0200 Subject: [PATCH 01/10] feat: team-specific secret management with sealed secrets --- src/common/sealed-secrets.test.ts | 230 ++++++++++++++++++ src/common/sealed-secrets.ts | 127 +++++++++- .../external-secrets-raw.gotmpl | 5 + values/k8s/k8s-raw-teams.gotmpl | 28 +-- values/team-secrets/team-secrets-raw.gotmpl | 112 --------- 5 files changed, 358 insertions(+), 144 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 69dd2c6e96..e27fa3133c 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -4,6 +4,7 @@ import { applySealedSecretManifests, bootstrapSealedSecrets, buildSecretToNamespaceMap, + buildTeamNamespaceSealedSecretMappings, createSealedSecretManifest, createSealedSecretsKeySecret, createUserSealedSecretManifests, @@ -425,6 +426,7 @@ describe('sealed-secrets', () => { getPemFromCertificate: jest.fn().mockReturnValue('spki-pem'), createSealedSecretsKeySecret: jest.fn(), buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), writeSealedSecretManifests: jest.fn(), createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), @@ -457,6 +459,7 @@ describe('sealed-secrets', () => { getPemFromCertificate: jest.fn().mockReturnValue('existing-spki-pem'), createSealedSecretsKeySecret: jest.fn(), buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), + buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), createSealedSecretManifest: jest.fn().mockResolvedValue({}), writeSealedSecretManifests: jest.fn(), createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), @@ -489,6 +492,7 @@ describe('sealed-secrets', () => { getPemFromCertificate: jest.fn().mockReturnValue('pem'), createSealedSecretsKeySecret: jest.fn(), buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), createSealedSecretManifest: jest.fn(), writeSealedSecretManifests: jest.fn(), createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), @@ -640,4 +644,230 @@ describe('sealed-secrets', () => { expect(manifests).toHaveLength(0) }) }) + + describe('buildTeamNamespaceSealedSecretMappings', () => { + const baseSecrets = { + apps: { + keycloak: { idp: { clientSecret: 'kc-secret' } }, + loki: { adminPassword: 'loki-pass' }, + }, + alerts: { slack: { url: 'https://hooks.slack.com/test' } }, + smtp: { auth_password: 'smtp-pass', auth_secret: 'smtp-secret' }, + } + + const baseValues = { + apps: { keycloak: { idp: { clientID: 'apl' } } }, + } + + it('should return empty array when no teams provided', () => { + const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, baseValues, []) + expect(result).toEqual([]) + }) + + it('should return empty array when grafana and alertmanager are both disabled', () => { + const allValues = { + ...baseValues, + teamConfig: { alpha: { settings: { managedMonitoring: { grafana: false, alertmanager: false } } } }, + } + const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) + expect(result).toEqual([]) + }) + + it('should create grafana secrets when grafana is enabled', () => { + const allSecrets = { ...baseSecrets, teamConfig: { alpha: { settings: { password: 'team-pass' } } } } + const allValues = { + ...baseValues, + teamConfig: { alpha: { settings: { managedMonitoring: { grafana: true } } } }, + } + + const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) + + expect(result).toHaveLength(3) + expect(result.find((m) => m.secretName === 'team-alpha-grafana-admin')).toMatchObject({ + namespace: 'team-alpha', + secretName: 'team-alpha-grafana-admin', + data: { 'admin-user': 'alpha', 'admin-password': 'team-pass' }, + }) + expect(result.find((m) => m.secretName === 'grafana-oidc-secret')).toMatchObject({ + namespace: 'team-alpha', + secretName: 'grafana-oidc-secret', + data: { client_id: 'apl', client_secret: 'kc-secret' }, + }) + expect(result.find((m) => m.secretName === 'grafana-loki-datasource-secret')).toMatchObject({ + namespace: 'team-alpha', + secretName: 'grafana-loki-datasource-secret', + data: { password: 'loki-pass' }, + }) + }) + + it('should skip grafana-admin when team password is missing', () => { + const allSecrets = { ...baseSecrets } // no teamConfig password + const allValues = { + ...baseValues, + teamConfig: { alpha: { settings: { managedMonitoring: { grafana: true } } } }, + } + + const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) + + expect(result.find((m) => m.secretName === 'team-alpha-grafana-admin')).toBeUndefined() + // grafana-oidc and loki-datasource are still created + expect(result.find((m) => m.secretName === 'grafana-oidc-secret')).toBeDefined() + expect(result.find((m) => m.secretName === 'grafana-loki-datasource-secret')).toBeDefined() + }) + + it('should create alertmanager-credentials for slack receiver', () => { + const allValues = { + ...baseValues, + teamConfig: { + alpha: { + settings: { + managedMonitoring: { alertmanager: true }, + alerts: { receivers: ['slack'] }, + }, + }, + }, + } + + const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) + + expect(result).toHaveLength(1) + expect(result[0]).toMatchObject({ + namespace: 'team-alpha', + secretName: 'alertmanager-credentials', + data: { slackUrl: 'https://hooks.slack.com/test' }, + }) + }) + + it('should create alertmanager-credentials for email receiver', () => { + const allValues = { + ...baseValues, + teamConfig: { + alpha: { + settings: { + managedMonitoring: { alertmanager: true }, + alerts: { receivers: ['email'] }, + }, + }, + }, + } + + const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) + + expect(result[0].data).toMatchObject({ + smtpAuthPassword: 'smtp-pass', + smtpAuthSecret: 'smtp-secret', + }) + }) + + it('should skip alertmanager-credentials when receivers is none', () => { + const allValues = { + ...baseValues, + teamConfig: { + alpha: { + settings: { + managedMonitoring: { alertmanager: true }, + alerts: { receivers: ['none'] }, + }, + }, + }, + } + + const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) + expect(result).toHaveLength(0) + }) + + it('should create otomi-pullsecret-global when globalPullSecret is configured', () => { + const allSecrets = { ...baseSecrets, otomi: { globalPullSecret: { password: 'reg-pass' } } } + const allValues = { + ...baseValues, + otomi: { globalPullSecret: { server: 'registry.example.com', username: 'user', email: 'user@example.com' } }, + teamConfig: { alpha: { settings: {} } }, + } + + const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) + + expect(result).toHaveLength(1) + expect(result[0]).toMatchObject({ + namespace: 'team-alpha', + secretName: 'otomi-pullsecret-global', + secretType: 'kubernetes.io/dockerconfigjson', + }) + const dockerConfig = JSON.parse(result[0].data['.dockerconfigjson']) + expect(dockerConfig.auths['registry.example.com']).toMatchObject({ + username: 'user', + password: 'reg-pass', + email: 'user@example.com', + }) + }) + + it('should skip otomi-pullsecret-global when password is missing', () => { + const allValues = { + ...baseValues, + otomi: { globalPullSecret: { server: 'registry.example.com', username: 'user' } }, + teamConfig: { alpha: { settings: {} } }, + } + + const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) + expect(result.find((m) => m.secretName === 'otomi-pullsecret-global')).toBeUndefined() + }) + + it('should use default docker.io server when not specified', () => { + const allSecrets = { ...baseSecrets, otomi: { globalPullSecret: { password: 'reg-pass' } } } + const allValues = { + ...baseValues, + otomi: { globalPullSecret: { username: 'user' } }, // no server + teamConfig: { alpha: { settings: {} } }, + } + + const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) + const dockerConfig = JSON.parse(result[0].data['.dockerconfigjson']) + expect(dockerConfig.auths['docker.io']).toBeDefined() + }) + + it('should create mappings for multiple teams independently', () => { + const allSecrets = { + ...baseSecrets, + teamConfig: { + alpha: { settings: { password: 'alpha-pass' } }, + beta: { settings: { password: 'beta-pass' } }, + }, + } + const allValues = { + ...baseValues, + teamConfig: { + alpha: { settings: { managedMonitoring: { grafana: true } } }, + beta: { settings: { managedMonitoring: { grafana: true } } }, + }, + } + + const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha', 'beta']) + + const alphaAdmin = result.find((m) => m.secretName === 'team-alpha-grafana-admin') + const betaAdmin = result.find((m) => m.secretName === 'team-beta-grafana-admin') + + expect(alphaAdmin?.namespace).toBe('team-alpha') + expect(alphaAdmin?.data['admin-password']).toBe('alpha-pass') + expect(betaAdmin?.namespace).toBe('team-beta') + expect(betaAdmin?.data['admin-password']).toBe('beta-pass') + }) + + it('should use team-level receivers falling back to platform receivers', () => { + const allValues = { + ...baseValues, + alerts: { receivers: ['slack'] }, // platform-level default + teamConfig: { + alpha: { + settings: { + managedMonitoring: { alertmanager: true }, + // No team-level receivers — should use platform default + }, + }, + }, + } + + const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) + + expect(result[0].data).toHaveProperty('slackUrl') + }) + }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index d2618624f2..09be3a7147 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -34,6 +34,8 @@ export interface SecretMapping { namespace: string secretName: string data: Record + /** Optional Kubernetes secret type. Defaults to 'kubernetes.io/opaque'. */ + secretType?: string } export interface SealedSecretManifest { @@ -324,7 +326,7 @@ export const createSealedSecretManifest = async ( template: { immutable: false, metadata: { name: mapping.secretName, namespace: mapping.namespace }, - type: 'kubernetes.io/opaque', + type: mapping.secretType ?? 'kubernetes.io/opaque', }, }, } @@ -571,6 +573,108 @@ export const createUserSealedSecretManifests = async ( return manifests } +/** + * Build SealedSecret mappings for team namespaces. + * These replace ExternalSecrets that previously allowed teams to read platform secrets + * via core-secrets-store ClusterSecretStore — a security hole where any team could + * read any secret in the apl-secrets namespace. + * + * Each mapping is encrypted for the specific team namespace, so platform secrets + * cannot be decrypted in any other namespace. + */ +export const buildTeamNamespaceSealedSecretMappings = ( + allSecrets: Record, + allValues: Record, + teams: string[], +): SecretMapping[] => { + const mappings: SecretMapping[] = [] + + for (const teamId of teams) { + const teamNs = `team-${teamId}` + const settings = get(allValues, `teamConfig.${teamId}.settings`, {}) as Record + const grafanaEnabled = get(settings, 'managedMonitoring.grafana', false) as boolean + const alertmanagerEnabled = get(settings, 'managedMonitoring.alertmanager', false) as boolean + const teamReceivers = get(settings, 'alerts.receivers', get(allValues, 'alerts.receivers', ['slack'])) as string[] + const hasReceivers = !teamReceivers.includes('none') + + if (grafanaEnabled) { + // team--grafana-admin: admin credentials for team's Grafana instance + const teamPassword = get(allSecrets, `teamConfig.${teamId}.settings.password`, '') as string + if (teamPassword) { + mappings.push({ + namespace: teamNs, + secretName: `${teamNs}-grafana-admin`, + data: { 'admin-user': teamId, 'admin-password': teamPassword }, + }) + } + + // grafana-oidc-secret: Keycloak OIDC credentials for Grafana SSO + const clientSecret = get(allSecrets, 'apps.keycloak.idp.clientSecret', '') as string + const clientId = get(allValues, 'apps.keycloak.idp.clientID', '') as string + if (clientSecret) { + mappings.push({ + namespace: teamNs, + secretName: 'grafana-oidc-secret', + data: { client_id: clientId, client_secret: clientSecret }, + }) + } + + // grafana-loki-datasource-secret: Loki admin password for Grafana datasource + const lokiPassword = get(allSecrets, 'apps.loki.adminPassword', '') as string + if (lokiPassword) { + mappings.push({ + namespace: teamNs, + secretName: 'grafana-loki-datasource-secret', + data: { password: lokiPassword }, + }) + } + } + + if (alertmanagerEnabled && hasReceivers) { + // alertmanager-credentials: notification channel credentials for team Alertmanager + const alertData: Record = {} + if (teamReceivers.includes('slack')) { + const slackUrl = get(allSecrets, 'alerts.slack.url', '') as string + if (slackUrl) alertData.slackUrl = slackUrl + } + if (teamReceivers.includes('email')) { + // email receiver uses platform SMTP credentials + const smtpPassword = get(allSecrets, 'smtp.auth_password', '') as string + const smtpSecret = get(allSecrets, 'smtp.auth_secret', '') as string + if (smtpPassword) alertData.smtpAuthPassword = smtpPassword + if (smtpSecret) alertData.smtpAuthSecret = smtpSecret + } + if (teamReceivers.includes('opsgenie')) { + // Legacy receiver — kept for backwards compatibility + const opsgenieKey = get(allSecrets, 'alerts.opsgenie.apiKey', '') as string + if (opsgenieKey) alertData.opsgenieApiKey = opsgenieKey + } + if (Object.keys(alertData).length > 0) { + mappings.push({ namespace: teamNs, secretName: 'alertmanager-credentials', data: alertData }) + } + } + + // otomi-pullsecret-global: docker pull secret for global container registry + // Only created when otomi.globalPullSecret is configured + const pullSecretConfig = get(allValues, 'otomi.globalPullSecret', null) as Record | null + const pullSecretPassword = get(allSecrets, 'otomi.globalPullSecret.password', '') as string + if (pullSecretConfig && pullSecretPassword) { + const server = get(pullSecretConfig, 'server', 'docker.io') as string + const username = get(pullSecretConfig, 'username', '') as string + const email = get(pullSecretConfig, 'email', 'not@val.id') as string + const dockerConfig = JSON.stringify({ auths: { [server]: { username, password: pullSecretPassword, email } } }) + mappings.push({ + namespace: teamNs, + secretName: 'otomi-pullsecret-global', + data: { '.dockerconfigjson': dockerConfig }, + secretType: 'kubernetes.io/dockerconfigjson', + }) + } + } + + return mappings +} + /** * Get the PEM public key from the existing sealed-secrets certificate in the cluster, * or generate a new RSA key pair, store it in the cluster, and return its PEM. @@ -607,6 +711,7 @@ export const bootstrapSealedSecrets = async ( createSealedSecretsKeySecret, getExistingSealedSecretsCert, buildSecretToNamespaceMap, + buildTeamNamespaceSealedSecretMappings, createSealedSecretManifest, writeSealedSecretManifests, createUserSealedSecretManifests, @@ -625,11 +730,11 @@ export const bootstrapSealedSecrets = async ( createSealedSecretsKeySecret: deps.createSealedSecretsKeySecret, }) - // 5. Build secret-to-namespace mapping + // 2. Build secret-to-namespace mapping (platform secrets in apl-secrets namespace) const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) const mappings = await deps.buildSecretToNamespaceMap(secrets, teams, allValues) - // 6. Create SealedSecret manifests + // 3. Create SealedSecret manifests for platform secrets (encrypted for apl-secrets namespace) const manifests: SealedSecretManifest[] = [] for (const mapping of mappings) { const manifest = await deps.createSealedSecretManifest(pem, mapping, { @@ -638,7 +743,19 @@ export const bootstrapSealedSecrets = async ( manifests.push(manifest) } - // 7. Create individual user SealedSecrets in apl-users namespace + // 4. Create team-namespace SealedSecret manifests (replaces ExternalSecrets referencing core-secrets-store) + // These are encrypted for each team's namespace, so platform secrets cannot be decrypted elsewhere. + if (allValues) { + const teamMappings = deps.buildTeamNamespaceSealedSecretMappings(secrets, allValues, teams) + for (const mapping of teamMappings) { + const manifest = await deps.createSealedSecretManifest(pem, mapping, { + encryptSecretItem: deps.encryptSecretItem, + }) + manifests.push(manifest) + } + } + + // 5. Create individual user SealedSecrets in apl-users namespace const { users } = secrets if (Array.isArray(users) && users.length > 0) { const userManifests = await deps.createUserSealedSecretManifests(users, pem, { @@ -648,7 +765,7 @@ export const bootstrapSealedSecrets = async ( manifests.push(...userManifests) } - // 8. Write SealedSecret manifests to disk + // 6. Write all SealedSecret manifests to disk // Note: These manifests are applied later during install, after the sealed-secrets // controller is deployed and the SealedSecret CRD is available. await deps.writeSealedSecretManifests(manifests, envDir) diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl index 91817f92fc..0ccf895b0d 100644 --- a/values/external-secrets/external-secrets-raw.gotmpl +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -31,6 +31,11 @@ resources: metadata: name: core-secrets-store spec: + conditions: + - namespaceSelector: + matchExpressions: + - key: apl.io/team + operator: DoesNotExist provider: kubernetes: remoteNamespace: apl-secrets diff --git a/values/k8s/k8s-raw-teams.gotmpl b/values/k8s/k8s-raw-teams.gotmpl index 11b4af6d50..0f98f5b93b 100644 --- a/values/k8s/k8s-raw-teams.gotmpl +++ b/values/k8s/k8s-raw-teams.gotmpl @@ -10,38 +10,12 @@ resources: labels: name: {{ $ns }} type: team + apl.io/team: "true" {{- if $v.apps.istio.defaultRevision }} istio.io/rev: {{ $v.apps.istio.defaultRevision | quote }} {{- else }} istio-injection: enabled {{- end }} - {{- with $v.otomi | get "globalPullSecret" nil }} - {{- $gpsUsername := . | get "username" "" }} - {{- $gpsServer := . | get "server" "docker.io" }} - {{- $gpsEmail := . | get "email" "not@val.id" }} - - apiVersion: external-secrets.io/v1 - kind: ExternalSecret - metadata: - name: otomi-pullsecret-global - namespace: {{ $ns }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: otomi-pullsecret-global - creationPolicy: Owner - template: - type: kubernetes.io/dockerconfigjson - data: - .dockerconfigjson: '{"auths":{"{{ $gpsServer }}":{"username":"{{ $gpsUsername }}","password":"{{ "{{ .password | toString }}" }}","email":"{{ $gpsEmail }}"}}}' - data: - - secretKey: password - remoteRef: - key: otomi-secrets - property: globalPullSecret_password - {{- end }} # patching service account here as helm does not recognize it as it's own - apiVersion: v1 kind: ServiceAccount diff --git a/values/team-secrets/team-secrets-raw.gotmpl b/values/team-secrets/team-secrets-raw.gotmpl index cc55b341e3..b51a36a4ec 100644 --- a/values/team-secrets/team-secrets-raw.gotmpl +++ b/values/team-secrets/team-secrets-raw.gotmpl @@ -2,85 +2,11 @@ {{- $teamId := .Release.Labels.team }} {{- $tc := $v.teamConfig }} {{- $teamSettings := (index $tc $teamId).settings }} -{{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} -{{- $hasReceivers := not (has "none" $teamReceivers) }} {{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} {{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} {{- $teamAlertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} resources: -{{- if $teamSettings | get "managedMonitoring.grafana" false }} -- apiVersion: external-secrets.io/v1 - kind: ExternalSecret - metadata: - name: team-{{ $teamId }}-grafana-admin - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: team-{{ $teamId }}-grafana-admin - creationPolicy: Owner - template: - type: Opaque - data: - admin-user: {{ $teamId }} - admin-password: '{{ "{{ .password | toString }}" }}' - data: - - secretKey: password - remoteRef: - key: team-{{ $teamId }}-settings-secrets - property: settings_password -{{- end }} -{{- if $teamSettings | get "managedMonitoring.grafana" false }} -- apiVersion: external-secrets.io/v1 - kind: ExternalSecret - metadata: - name: grafana-oidc-secret - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: grafana-oidc-secret - creationPolicy: Owner - template: - type: Opaque - data: - client_id: {{ $v.apps.keycloak.idp.clientID }} - client_secret: '{{ "{{ .clientSecret | toString }}" }}' - data: - - secretKey: clientSecret - remoteRef: - key: keycloak-secrets - property: idp_clientSecret -- apiVersion: external-secrets.io/v1 - kind: ExternalSecret - metadata: - name: grafana-loki-datasource-secret - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: grafana-loki-datasource-secret - creationPolicy: Owner - template: - type: Opaque - data: - password: '{{ "{{ .adminPassword | toString }}" }}' - data: - - secretKey: adminPassword - remoteRef: - key: loki-secrets - property: adminPassword -{{- end }} {{- if $teamSettings | get "managedMonitoring.alertmanager" false }} - apiVersion: v1 kind: Secret @@ -90,42 +16,4 @@ resources: type: Opaque data: alertmanager.yaml: {{ $teamAlertmanagerConfig | b64enc | nindent 6 }} -{{- if $hasReceivers }} -- apiVersion: external-secrets.io/v1 - kind: ExternalSecret - metadata: - name: alertmanager-credentials - namespace: team-{{ $teamId }} - spec: - refreshInterval: 1h - secretStoreRef: - name: core-secrets-store - kind: ClusterSecretStore - target: - name: alertmanager-credentials - creationPolicy: Owner - data: - {{- if has "slack" $teamReceivers }} - - secretKey: slackUrl - remoteRef: - key: alerts-secrets - property: slack_url - {{- end }} - {{- if has "email" $teamReceivers }} - - secretKey: smtpAuthPassword - remoteRef: - key: smtp-secrets - property: auth_password - - secretKey: smtpAuthSecret - remoteRef: - key: smtp-secrets - property: auth_secret - {{- end }} - {{- if has "opsgenie" $teamReceivers }} - - secretKey: opsgenieApiKey - remoteRef: - key: alerts-secrets - property: opsgenie_apiKey - {{- end }} -{{- end }}{{/* if $hasReceivers */}} {{- end }}{{/* if $teamSettings.managedMonitoring.alertmanager */}} From d07874313e6a4d4ea23f5165c4de838d68ae3115 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 00:12:19 +0200 Subject: [PATCH 02/10] feat: improve team secret management --- src/common/sealed-secrets.integration.test.ts | 404 ++++++++++++++++++ src/common/sealed-secrets.ts | 154 ++++++- src/operator/apl-operator.test.ts | 4 + src/operator/apl-operator.ts | 10 + 4 files changed, 568 insertions(+), 4 deletions(-) create mode 100644 src/common/sealed-secrets.integration.test.ts diff --git a/src/common/sealed-secrets.integration.test.ts b/src/common/sealed-secrets.integration.test.ts new file mode 100644 index 0000000000..8df501f02f --- /dev/null +++ b/src/common/sealed-secrets.integration.test.ts @@ -0,0 +1,404 @@ +/** + * Integration tests for sealed-secrets.ts — verify actual YAML files are written to disk + * with the correct structure and content. + * + * These tests complement the unit tests in sealed-secrets.test.ts, which mock the filesystem. + * Here we use a real temporary directory and verify file content after each operation. + */ +import { existsSync } from 'fs' +import { readFile, rm } from 'fs/promises' +import os from 'os' +import path from 'path' +import stubs from 'src/test-stubs' +import { parse as parseYaml } from 'yaml' +import { + bootstrapSealedSecrets, + buildTeamNamespaceSealedSecretMappings, + createSealedSecretManifest, + reconcileTeamSealedSecrets, + SEALED_SECRETS_MANIFESTS_SUBDIR, + writeSealedSecretManifests, +} from './sealed-secrets' + +const { terminal } = stubs + +// Module-level mocks needed so sealed-secrets.ts can load +jest.mock('@linode/kubeseal-encrypt', () => ({ + encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), +})) + +jest.mock('src/common/k8s', () => ({ + getK8sSecret: jest.fn().mockResolvedValue(undefined), + ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), + b64enc: jest.fn((v: string) => Buffer.from(v).toString('base64')), + k8s: { + core: jest.fn().mockReturnValue({ + createNamespacedSecret: jest.fn().mockResolvedValue({}), + }), + app: jest.fn().mockReturnValue({}), + custom: jest.fn().mockReturnValue({}), + }, +})) + +jest.mock('src/common/envalid', () => ({ env: {} })) + +// Deterministic encryption — makes YAML output predictable without a real cluster +const mockEncryptSecretItem = async (_pem: string, _ns: string, value: string) => `encrypted-${value}` + +const FAKE_PEM = '-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\n-----END PUBLIC KEY-----\n' + +// ────────────────────────────────────────────────────────────────────────────── +// Test fixtures +// ────────────────────────────────────────────────────────────────────────────── + +const baseSecrets = { + apps: { + harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret-key' }, + gitea: { adminPassword: 'gitea-pass' }, + keycloak: { idp: { clientSecret: 'kc-secret' } }, + loki: { adminPassword: 'loki-pass' }, + }, + alerts: { slack: { url: 'https://hooks.slack.com/test' } }, + smtp: { auth_password: 'smtp-pass', auth_secret: 'smtp-secret' }, + otomi: { globalPullSecret: { password: 'pull-secret-pass' } }, + teamConfig: { + alpha: { settings: { password: 'alpha-pass' } }, + beta: { settings: { password: 'beta-pass' } }, + }, +} + +const baseValues = { + teamConfig: { + alpha: { + settings: { + managedMonitoring: { grafana: true, alertmanager: true }, + alerts: { receivers: ['slack'] }, + }, + }, + beta: { + settings: { + managedMonitoring: { grafana: false, alertmanager: false }, + alerts: { receivers: ['none'] }, + }, + }, + }, + otomi: { + globalPullSecret: { server: 'registry.example.com', username: 'user', email: 'user@example.com' }, + }, + apps: { keycloak: { idp: { clientID: 'grafana-client' } } }, +} + +// ────────────────────────────────────────────────────────────────────────────── +// Helpers +// ────────────────────────────────────────────────────────────────────────────── + +async function readYamlFile(filePath: string): Promise { + const content = await readFile(filePath, 'utf8') + return parseYaml(content) +} + +function sealedSecretPath(envDir: string, namespace: string, name: string): string { + return path.join(envDir, SEALED_SECRETS_MANIFESTS_SUBDIR, namespace, 'sealedsecrets', `${name}.yaml`) +} + +// ────────────────────────────────────────────────────────────────────────────── +// bootstrapSealedSecrets — file output +// ────────────────────────────────────────────────────────────────────────────── + +describe('bootstrapSealedSecrets — file output', () => { + let tmpDir: string + + beforeEach(() => { + tmpDir = path.join(os.tmpdir(), `apl-test-${Date.now()}-${Math.random().toString(36).slice(2)}`) + }) + + afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }) + }) + + function makeDeps(overrides: Partial[3]> = {}) { + return { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ certificate: 'cert-pem', privateKey: 'key-pem' }), + getPemFromCertificate: jest.fn().mockReturnValue(FAKE_PEM), + createSealedSecretsKeySecret: jest.fn().mockResolvedValue(undefined), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([ + { namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'harbor-pass' } }, + { namespace: 'apl-secrets', secretName: 'gitea-secrets', data: { adminPassword: 'gitea-pass' } }, + ]), + buildTeamNamespaceSealedSecretMappings, + createSealedSecretManifest, + writeSealedSecretManifests, + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), + encryptSecretItem: mockEncryptSecretItem, + ...overrides, + } + } + + it('creates apl-secrets SealedSecret files on disk', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets'))).toBe(true) + expect(existsSync(sealedSecretPath(tmpDir, 'apl-secrets', 'gitea-secrets'))).toBe(true) + }) + + it('writes valid YAML with correct SealedSecret structure', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets')) + + expect(manifest.apiVersion).toBe('bitnami.com/v1alpha1') + expect(manifest.kind).toBe('SealedSecret') + expect(manifest.metadata.name).toBe('harbor-secrets') + expect(manifest.metadata.namespace).toBe('apl-secrets') + expect(manifest.spec.encryptedData).toBeDefined() + expect(manifest.spec.template.type).toBeTruthy() + }) + + it('writes namespace-wide annotation', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets')) + + expect(manifest.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') + }) + + it('encrypts data keys with deterministic mock', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets')) + + expect(manifest.spec.encryptedData.adminPassword).toBe('encrypted-harbor-pass') + }) + + it('creates team namespace files when grafana is enabled', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'team-alpha-grafana-admin'))).toBe(true) + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-loki-datasource-secret'))).toBe(true) + }) + + it('does NOT create grafana files for team with grafana disabled', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-beta', 'grafana-oidc-secret'))).toBe(false) + expect(existsSync(sealedSecretPath(tmpDir, 'team-beta', 'team-beta-grafana-admin'))).toBe(false) + }) + + it('creates alertmanager-credentials when alertmanager enabled with receivers', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'alertmanager-credentials'))).toBe(true) + const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'team-alpha', 'alertmanager-credentials')) + expect(manifest.spec.encryptedData.slackUrl).toBeDefined() + }) + + it('does NOT create alertmanager-credentials when receivers is none', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-beta', 'alertmanager-credentials'))).toBe(false) + }) + + it('creates pull secret with dockerconfigjson type', async () => { + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'otomi-pullsecret-global'))).toBe(true) + const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'team-alpha', 'otomi-pullsecret-global')) + expect(manifest.spec.template.type).toBe('kubernetes.io/dockerconfigjson') + }) + + it('re-running is idempotent (overwrites with same content)', async () => { + const deps = makeDeps() + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, deps) + + const firstContent = await readFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets'), 'utf8') + + await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, deps) + const secondContent = await readFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets'), 'utf8') + + expect(firstContent).toBe(secondContent) + }) +}) + +// ────────────────────────────────────────────────────────────────────────────── +// reconcileTeamSealedSecrets — operator reconcile +// ────────────────────────────────────────────────────────────────────────────── + +describe('reconcileTeamSealedSecrets — operator reconcile', () => { + let tmpDir: string + + beforeEach(async () => { + tmpDir = path.join(os.tmpdir(), `apl-reconcile-test-${Date.now()}-${Math.random().toString(36).slice(2)}`) + }) + + afterEach(async () => { + await rm(tmpDir, { recursive: true, force: true }) + }) + + function makeK8sSecrets(overrides: Record = {}): Record | undefined> { + return { + 'keycloak-secrets': { idp_clientSecret: 'kc-secret' }, + 'loki-secrets': { adminPassword: 'loki-pass' }, + 'alerts-secrets': { slack_url: 'https://hooks.slack.com/test' }, + 'smtp-secrets': { auth_password: 'smtp-pass', auth_secret: 'smtp-secret' }, + 'otomi-secrets': { globalPullSecret_password: 'pull-pass' }, + 'team-alpha-settings-secrets': { settings_password: 'alpha-pass' }, + ...overrides, + } + } + + function makeGetK8sSecret(secrets: Record | undefined>) { + return async (name: string, _ns: string) => secrets[name] + } + + function makeDeps(k8sSecrets: Record | undefined> = makeK8sSecrets()) { + return { + buildAllSecretsFromK8s: async (teams: string[]) => { + // Use real buildAllSecretsFromK8s logic with mocked getK8sSecret + const { buildAllSecretsFromK8s: realFn } = jest.requireActual('./sealed-secrets') as any + return realFn(teams, { getK8sSecret: makeGetK8sSecret(k8sSecrets) }) + }, + buildTeamNamespaceSealedSecretMappings, + createSealedSecretManifest, + writeSealedSecretManifests, + getOrCreateSealedSecretsPem: jest.fn().mockResolvedValue(FAKE_PEM), + encryptSecretItem: mockEncryptSecretItem, + } + } + + const testValues = { + teamConfig: { + alpha: { + settings: { + managedMonitoring: { grafana: true, alertmanager: false }, + alerts: { receivers: ['none'] }, + }, + }, + }, + otomi: { globalPullSecret: null }, + apps: { keycloak: { idp: { clientID: 'grafana-client' } } }, + } + + it('creates team SealedSecrets on first run with hash annotation', async () => { + await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) + + const filePath = sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret') + expect(existsSync(filePath)).toBe(true) + + const manifest = await readYamlFile(filePath) + expect(manifest.metadata.annotations['apl.io/secret-hash']).toBeDefined() + expect(manifest.metadata.annotations['apl.io/secret-hash']).toHaveLength(16) + }) + + it('skips re-encryption when inputs are unchanged (hash match)', async () => { + const deps = makeDeps() + const encryptSpy = jest.fn().mockImplementation(mockEncryptSecretItem) + deps.encryptSecretItem = encryptSpy + + // First run: creates files + await reconcileTeamSealedSecrets(testValues, tmpDir, deps) + const callsAfterFirst = encryptSpy.mock.calls.length + + // Second run with same values: should skip re-encryption + await reconcileTeamSealedSecrets(testValues, tmpDir, deps) + const callsAfterSecond = encryptSpy.mock.calls.length + + expect(callsAfterSecond).toBe(callsAfterFirst) // no new encrypt calls + }) + + it('re-encrypts when secret value changes (hash mismatch)', async () => { + const deps = makeDeps() + await reconcileTeamSealedSecrets(testValues, tmpDir, deps) + + const filePath = sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret') + const firstHash = (await readYamlFile(filePath)).metadata.annotations['apl.io/secret-hash'] + + // Change the secret value + const newDeps = makeDeps(makeK8sSecrets({ 'keycloak-secrets': { idp_clientSecret: 'new-kc-secret' } })) + await reconcileTeamSealedSecrets(testValues, tmpDir, newDeps) + + const secondHash = (await readYamlFile(filePath)).metadata.annotations['apl.io/secret-hash'] + expect(secondHash).not.toBe(firstHash) + }) + + it('creates grafana files when feature is enabled', async () => { + await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'team-alpha-grafana-admin'))).toBe(true) + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-loki-datasource-secret'))).toBe(true) + }) + + it('deletes grafana files when feature is disabled', async () => { + // First run: grafana enabled + await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) + + // Second run: grafana disabled + const disabledValues = { + ...testValues, + teamConfig: { + alpha: { + settings: { + managedMonitoring: { grafana: false, alertmanager: false }, + alerts: { receivers: ['none'] }, + }, + }, + }, + } + await reconcileTeamSealedSecrets(disabledValues, tmpDir, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(false) + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'team-alpha-grafana-admin'))).toBe(false) + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-loki-datasource-secret'))).toBe(false) + }) + + it('skips gracefully when PEM is unavailable (dev env)', async () => { + const deps = makeDeps() + deps.getOrCreateSealedSecretsPem = jest.fn().mockRejectedValue(new Error('not in cluster')) + + // Should not throw + await expect(reconcileTeamSealedSecrets(testValues, tmpDir, deps)).resolves.toBeUndefined() + }) + + it('processes other teams when one team K8s secret read fails', async () => { + const valuesWithTwoTeams = { + ...testValues, + teamConfig: { + ...testValues.teamConfig, + gamma: { + settings: { + managedMonitoring: { grafana: true, alertmanager: false }, + alerts: { receivers: ['none'] }, + }, + }, + }, + } + + // alpha secret read fails, gamma succeeds + const partialSecrets = makeK8sSecrets({ + 'team-alpha-settings-secrets': undefined, // simulates read failure + 'team-gamma-settings-secrets': { settings_password: 'gamma-pass' }, + }) + + await reconcileTeamSealedSecrets(valuesWithTwoTeams, tmpDir, makeDeps(partialSecrets)) + + // gamma should still have oidc secret (no dependency on team settings password) + expect(existsSync(sealedSecretPath(tmpDir, 'team-gamma', 'grafana-oidc-secret'))).toBe(true) + }) + + it('removes files for teams that are deleted from teamConfig', async () => { + // First run with alpha + await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) + + // Second run with no teams + await reconcileTeamSealedSecrets({ teamConfig: {} }, tmpDir, makeDeps()) + + expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(false) + }) +}) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 09be3a7147..eefe6fc0a1 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -1,9 +1,9 @@ import { ApiException, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node' import { encryptSecretItem } from '@linode/kubeseal-encrypt' -import { X509Certificate } from 'crypto' +import { createHash, X509Certificate } from 'crypto' import { existsSync } from 'fs' -import { mkdir, readdir, readFile, writeFile } from 'fs/promises' -import { cloneDeep, get, unset } from 'lodash' +import { mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises' +import { cloneDeep, get, set, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' import { SEALED_SECRETS_NAMESPACE } from 'src/common/constants' @@ -15,7 +15,7 @@ import { objectToYaml } from 'src/common/values' import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' -const SEALED_SECRETS_MANIFESTS_SUBDIR = 'env/manifests/namespaces' +export const SEALED_SECRETS_MANIFESTS_SUBDIR = 'env/manifests/namespaces' /** * Strip ALL x-secret fields from values before writing to disk. @@ -772,3 +772,149 @@ export const bootstrapSealedSecrets = async ( d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) } + +/** + * Read the 5 shared platform secrets + 1 per-team secret from the apl-secrets namespace in K8s. + * Returns an object shaped identically to the `allSecrets` parameter expected by + * `buildTeamNamespaceSealedSecretMappings()` so the operator can drive reconciliation + * from live cluster state rather than values-repo data. + * + * Individual secret reads are caught so one missing secret (e.g. loki not yet deployed) + * does not abort the entire reconcile — the relevant team secrets are simply skipped. + */ +export async function buildAllSecretsFromK8s(teams: string[], deps = { getK8sSecret }): Promise> { + const [keycloak, loki, alerts, smtp, otomi] = await Promise.all([ + deps.getK8sSecret('keycloak-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), + deps.getK8sSecret('loki-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), + deps.getK8sSecret('alerts-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), + deps.getK8sSecret('smtp-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), + deps.getK8sSecret('otomi-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), + ]) + + const teamSecrets: Record = {} + for (const teamId of teams) { + const ts = await deps + .getK8sSecret(`team-${teamId}-settings-secrets`, SEALED_SECRETS_NAMESPACE) + .catch(() => undefined) + set(teamSecrets, `${teamId}.settings.password`, ts?.settings_password ?? '') + } + + return { + teamConfig: teamSecrets, + apps: { + keycloak: { idp: { clientSecret: keycloak?.idp_clientSecret ?? '' } }, + loki: { adminPassword: loki?.adminPassword ?? '' }, + }, + alerts: { + slack: { url: alerts?.slack_url ?? '' }, + opsgenie: { apiKey: alerts?.opsgenie_apiKey ?? '' }, + }, + smtp: { + auth_password: smtp?.auth_password ?? '', + auth_secret: smtp?.auth_secret ?? '', + }, + otomi: { + globalPullSecret: { password: otomi?.globalPullSecret_password ?? '' }, + }, + } +} + +/** + * Reconcile team-namespace SealedSecret manifests on every operator cycle. + * + * Flow: read K8s secrets → build expected mappings → hash-compare plaintext inputs + * → re-encrypt only when changed → write new manifests → delete stale files. + * + * Hash-based idempotency prevents git thrash: SealedSecret encryption is non-deterministic + * (random nonce), so comparing ciphertext always looks "changed". Instead we hash the + * plaintext inputs and store that hash as annotation `apl.io/secret-hash` in the manifest. + */ +export async function reconcileTeamSealedSecrets( + allValues: Record, + envDir: string, + deps = { + buildAllSecretsFromK8s, + buildTeamNamespaceSealedSecretMappings, + createSealedSecretManifest, + writeSealedSecretManifests, + getOrCreateSealedSecretsPem, + encryptSecretItem, + }, +): Promise { + const teams = Object.keys(get(allValues, 'teamConfig', {}) as Record).filter((id) => id !== 'admin') + + const d = terminal(`common:${cmdName}:reconcileTeamSealedSecrets`) + + let pem: string + try { + pem = await deps.getOrCreateSealedSecretsPem() + } catch (e) { + d.warn(`Skipping team SealedSecret reconcile — sealed-secrets PEM unavailable: ${(e as Error).message}`) + return + } + + const allSecrets = await deps.buildAllSecretsFromK8s(teams) + const mappings = deps.buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, teams) + + const expectedPaths = new Set() + const toWrite: SealedSecretManifest[] = [] + + for (const mapping of mappings) { + const manifestPath = join( + envDir, + SEALED_SECRETS_MANIFESTS_SUBDIR, + mapping.namespace, + 'sealedsecrets', + `${mapping.secretName}.yaml`, + ) + expectedPaths.add(manifestPath) + + const inputHash = createHash('sha256') + .update(JSON.stringify({ data: mapping.data, secretType: mapping.secretType ?? '' })) + .digest('hex') + .slice(0, 16) + + let existingHash: string | undefined + try { + const raw = await readFile(manifestPath, 'utf8') + existingHash = (parseYaml(raw) as any)?.metadata?.annotations?.['apl.io/secret-hash'] + } catch { + // File not found — will create + } + + if (existingHash === inputHash) continue + + const manifest = await deps.createSealedSecretManifest(pem, mapping, { encryptSecretItem: deps.encryptSecretItem }) + manifest.metadata.annotations['apl.io/secret-hash'] = inputHash + toWrite.push(manifest) + } + + if (toWrite.length > 0) { + await deps.writeSealedSecretManifests(toWrite, envDir) + } + + // Delete stale files: SealedSecrets for disabled features or removed teams + const baseDir = join(envDir, SEALED_SECRETS_MANIFESTS_SUBDIR) + let teamDirs: string[] = [] + try { + teamDirs = (await readdir(baseDir)).filter((dir) => dir.startsWith('team-')) + } catch { + // Directory doesn't exist yet — nothing to clean up + } + + for (const teamDir of teamDirs) { + const ssDir = join(baseDir, teamDir, 'sealedsecrets') + let files: string[] = [] + try { + files = await readdir(ssDir) + } catch { + continue + } + for (const file of files) { + const filePath = join(ssDir, file) + if (!expectedPaths.has(filePath)) { + await unlink(filePath) + } + } + } +} diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index 55c602bb73..789c65b82f 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -57,6 +57,10 @@ jest.mock('../cmd/commit', () => ({ commit: jest.fn().mockResolvedValue(undefined), })) +jest.mock('../common/sealed-secrets', () => ({ + reconcileTeamSealedSecrets: jest.fn().mockResolvedValue(undefined), +})) + jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), appRevisionMatches: jest.fn().mockResolvedValue(true), diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 70e8f19f85..7e21dd7d31 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -5,6 +5,7 @@ import { env } from '../common/envalid' import { getStoredGitRepoConfig } from '../common/git-config' import { waitTillGitRepoAvailable } from '../common/gitea' import { hfValues } from '../common/hf' +import { reconcileTeamSealedSecrets } from '../common/sealed-secrets' import { ensureManifestDirectories, ensureTeamGitOpsDirectories } from '../common/utils' import { writeValues } from '../common/values' import { HelmArguments } from '../common/yargs' @@ -86,6 +87,15 @@ export class AplOperator { await decrypt() } const values = await hfValues({}, env.ENV_DIR) + + // Reconcile team namespace SealedSecrets — operator is source of truth for these manifests. + // Non-fatal: a missing K8s secret (e.g. loki not yet deployed) retries on next cycle. + try { + await reconcileTeamSealedSecrets(values ?? {}, env.ENV_DIR) + } catch (e) { + this.d.warn(`Team SealedSecret reconcile failed (will retry): ${(e as Error).message}`) + } + await ensureTeamGitOpsDirectories(env.ENV_DIR, values ?? {}) await ensureManifestDirectories() From e346f0d3b000e356339e840a95a6d9aafcf8a2b9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 01:14:58 +0200 Subject: [PATCH 03/10] chore: update sealed secrets tests --- src/common/sealed-secrets.integration.test.ts | 404 ------------------ src/common/sealed-secrets.test.ts | 195 +++++++++ src/common/sealed-secrets.ts | 11 +- tsconfig.json | 3 +- 4 files changed, 204 insertions(+), 409 deletions(-) delete mode 100644 src/common/sealed-secrets.integration.test.ts diff --git a/src/common/sealed-secrets.integration.test.ts b/src/common/sealed-secrets.integration.test.ts deleted file mode 100644 index 8df501f02f..0000000000 --- a/src/common/sealed-secrets.integration.test.ts +++ /dev/null @@ -1,404 +0,0 @@ -/** - * Integration tests for sealed-secrets.ts — verify actual YAML files are written to disk - * with the correct structure and content. - * - * These tests complement the unit tests in sealed-secrets.test.ts, which mock the filesystem. - * Here we use a real temporary directory and verify file content after each operation. - */ -import { existsSync } from 'fs' -import { readFile, rm } from 'fs/promises' -import os from 'os' -import path from 'path' -import stubs from 'src/test-stubs' -import { parse as parseYaml } from 'yaml' -import { - bootstrapSealedSecrets, - buildTeamNamespaceSealedSecretMappings, - createSealedSecretManifest, - reconcileTeamSealedSecrets, - SEALED_SECRETS_MANIFESTS_SUBDIR, - writeSealedSecretManifests, -} from './sealed-secrets' - -const { terminal } = stubs - -// Module-level mocks needed so sealed-secrets.ts can load -jest.mock('@linode/kubeseal-encrypt', () => ({ - encryptSecretItem: jest.fn().mockResolvedValue('encrypted-value'), -})) - -jest.mock('src/common/k8s', () => ({ - getK8sSecret: jest.fn().mockResolvedValue(undefined), - ensureNamespaceExists: jest.fn().mockResolvedValue(undefined), - b64enc: jest.fn((v: string) => Buffer.from(v).toString('base64')), - k8s: { - core: jest.fn().mockReturnValue({ - createNamespacedSecret: jest.fn().mockResolvedValue({}), - }), - app: jest.fn().mockReturnValue({}), - custom: jest.fn().mockReturnValue({}), - }, -})) - -jest.mock('src/common/envalid', () => ({ env: {} })) - -// Deterministic encryption — makes YAML output predictable without a real cluster -const mockEncryptSecretItem = async (_pem: string, _ns: string, value: string) => `encrypted-${value}` - -const FAKE_PEM = '-----BEGIN PUBLIC KEY-----\nMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEA\n-----END PUBLIC KEY-----\n' - -// ────────────────────────────────────────────────────────────────────────────── -// Test fixtures -// ────────────────────────────────────────────────────────────────────────────── - -const baseSecrets = { - apps: { - harbor: { adminPassword: 'harbor-pass', secretKey: 'harbor-secret-key' }, - gitea: { adminPassword: 'gitea-pass' }, - keycloak: { idp: { clientSecret: 'kc-secret' } }, - loki: { adminPassword: 'loki-pass' }, - }, - alerts: { slack: { url: 'https://hooks.slack.com/test' } }, - smtp: { auth_password: 'smtp-pass', auth_secret: 'smtp-secret' }, - otomi: { globalPullSecret: { password: 'pull-secret-pass' } }, - teamConfig: { - alpha: { settings: { password: 'alpha-pass' } }, - beta: { settings: { password: 'beta-pass' } }, - }, -} - -const baseValues = { - teamConfig: { - alpha: { - settings: { - managedMonitoring: { grafana: true, alertmanager: true }, - alerts: { receivers: ['slack'] }, - }, - }, - beta: { - settings: { - managedMonitoring: { grafana: false, alertmanager: false }, - alerts: { receivers: ['none'] }, - }, - }, - }, - otomi: { - globalPullSecret: { server: 'registry.example.com', username: 'user', email: 'user@example.com' }, - }, - apps: { keycloak: { idp: { clientID: 'grafana-client' } } }, -} - -// ────────────────────────────────────────────────────────────────────────────── -// Helpers -// ────────────────────────────────────────────────────────────────────────────── - -async function readYamlFile(filePath: string): Promise { - const content = await readFile(filePath, 'utf8') - return parseYaml(content) -} - -function sealedSecretPath(envDir: string, namespace: string, name: string): string { - return path.join(envDir, SEALED_SECRETS_MANIFESTS_SUBDIR, namespace, 'sealedsecrets', `${name}.yaml`) -} - -// ────────────────────────────────────────────────────────────────────────────── -// bootstrapSealedSecrets — file output -// ────────────────────────────────────────────────────────────────────────────── - -describe('bootstrapSealedSecrets — file output', () => { - let tmpDir: string - - beforeEach(() => { - tmpDir = path.join(os.tmpdir(), `apl-test-${Date.now()}-${Math.random().toString(36).slice(2)}`) - }) - - afterEach(async () => { - await rm(tmpDir, { recursive: true, force: true }) - }) - - function makeDeps(overrides: Partial[3]> = {}) { - return { - terminal, - getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), - generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ certificate: 'cert-pem', privateKey: 'key-pem' }), - getPemFromCertificate: jest.fn().mockReturnValue(FAKE_PEM), - createSealedSecretsKeySecret: jest.fn().mockResolvedValue(undefined), - buildSecretToNamespaceMap: jest.fn().mockResolvedValue([ - { namespace: 'apl-secrets', secretName: 'harbor-secrets', data: { adminPassword: 'harbor-pass' } }, - { namespace: 'apl-secrets', secretName: 'gitea-secrets', data: { adminPassword: 'gitea-pass' } }, - ]), - buildTeamNamespaceSealedSecretMappings, - createSealedSecretManifest, - writeSealedSecretManifests, - createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), - encryptSecretItem: mockEncryptSecretItem, - ...overrides, - } - } - - it('creates apl-secrets SealedSecret files on disk', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets'))).toBe(true) - expect(existsSync(sealedSecretPath(tmpDir, 'apl-secrets', 'gitea-secrets'))).toBe(true) - }) - - it('writes valid YAML with correct SealedSecret structure', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets')) - - expect(manifest.apiVersion).toBe('bitnami.com/v1alpha1') - expect(manifest.kind).toBe('SealedSecret') - expect(manifest.metadata.name).toBe('harbor-secrets') - expect(manifest.metadata.namespace).toBe('apl-secrets') - expect(manifest.spec.encryptedData).toBeDefined() - expect(manifest.spec.template.type).toBeTruthy() - }) - - it('writes namespace-wide annotation', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets')) - - expect(manifest.metadata.annotations['sealedsecrets.bitnami.com/namespace-wide']).toBe('true') - }) - - it('encrypts data keys with deterministic mock', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets')) - - expect(manifest.spec.encryptedData.adminPassword).toBe('encrypted-harbor-pass') - }) - - it('creates team namespace files when grafana is enabled', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'team-alpha-grafana-admin'))).toBe(true) - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-loki-datasource-secret'))).toBe(true) - }) - - it('does NOT create grafana files for team with grafana disabled', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-beta', 'grafana-oidc-secret'))).toBe(false) - expect(existsSync(sealedSecretPath(tmpDir, 'team-beta', 'team-beta-grafana-admin'))).toBe(false) - }) - - it('creates alertmanager-credentials when alertmanager enabled with receivers', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'alertmanager-credentials'))).toBe(true) - const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'team-alpha', 'alertmanager-credentials')) - expect(manifest.spec.encryptedData.slackUrl).toBeDefined() - }) - - it('does NOT create alertmanager-credentials when receivers is none', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-beta', 'alertmanager-credentials'))).toBe(false) - }) - - it('creates pull secret with dockerconfigjson type', async () => { - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'otomi-pullsecret-global'))).toBe(true) - const manifest = await readYamlFile(sealedSecretPath(tmpDir, 'team-alpha', 'otomi-pullsecret-global')) - expect(manifest.spec.template.type).toBe('kubernetes.io/dockerconfigjson') - }) - - it('re-running is idempotent (overwrites with same content)', async () => { - const deps = makeDeps() - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, deps) - - const firstContent = await readFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets'), 'utf8') - - await bootstrapSealedSecrets(baseSecrets, tmpDir, baseValues, deps) - const secondContent = await readFile(sealedSecretPath(tmpDir, 'apl-secrets', 'harbor-secrets'), 'utf8') - - expect(firstContent).toBe(secondContent) - }) -}) - -// ────────────────────────────────────────────────────────────────────────────── -// reconcileTeamSealedSecrets — operator reconcile -// ────────────────────────────────────────────────────────────────────────────── - -describe('reconcileTeamSealedSecrets — operator reconcile', () => { - let tmpDir: string - - beforeEach(async () => { - tmpDir = path.join(os.tmpdir(), `apl-reconcile-test-${Date.now()}-${Math.random().toString(36).slice(2)}`) - }) - - afterEach(async () => { - await rm(tmpDir, { recursive: true, force: true }) - }) - - function makeK8sSecrets(overrides: Record = {}): Record | undefined> { - return { - 'keycloak-secrets': { idp_clientSecret: 'kc-secret' }, - 'loki-secrets': { adminPassword: 'loki-pass' }, - 'alerts-secrets': { slack_url: 'https://hooks.slack.com/test' }, - 'smtp-secrets': { auth_password: 'smtp-pass', auth_secret: 'smtp-secret' }, - 'otomi-secrets': { globalPullSecret_password: 'pull-pass' }, - 'team-alpha-settings-secrets': { settings_password: 'alpha-pass' }, - ...overrides, - } - } - - function makeGetK8sSecret(secrets: Record | undefined>) { - return async (name: string, _ns: string) => secrets[name] - } - - function makeDeps(k8sSecrets: Record | undefined> = makeK8sSecrets()) { - return { - buildAllSecretsFromK8s: async (teams: string[]) => { - // Use real buildAllSecretsFromK8s logic with mocked getK8sSecret - const { buildAllSecretsFromK8s: realFn } = jest.requireActual('./sealed-secrets') as any - return realFn(teams, { getK8sSecret: makeGetK8sSecret(k8sSecrets) }) - }, - buildTeamNamespaceSealedSecretMappings, - createSealedSecretManifest, - writeSealedSecretManifests, - getOrCreateSealedSecretsPem: jest.fn().mockResolvedValue(FAKE_PEM), - encryptSecretItem: mockEncryptSecretItem, - } - } - - const testValues = { - teamConfig: { - alpha: { - settings: { - managedMonitoring: { grafana: true, alertmanager: false }, - alerts: { receivers: ['none'] }, - }, - }, - }, - otomi: { globalPullSecret: null }, - apps: { keycloak: { idp: { clientID: 'grafana-client' } } }, - } - - it('creates team SealedSecrets on first run with hash annotation', async () => { - await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) - - const filePath = sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret') - expect(existsSync(filePath)).toBe(true) - - const manifest = await readYamlFile(filePath) - expect(manifest.metadata.annotations['apl.io/secret-hash']).toBeDefined() - expect(manifest.metadata.annotations['apl.io/secret-hash']).toHaveLength(16) - }) - - it('skips re-encryption when inputs are unchanged (hash match)', async () => { - const deps = makeDeps() - const encryptSpy = jest.fn().mockImplementation(mockEncryptSecretItem) - deps.encryptSecretItem = encryptSpy - - // First run: creates files - await reconcileTeamSealedSecrets(testValues, tmpDir, deps) - const callsAfterFirst = encryptSpy.mock.calls.length - - // Second run with same values: should skip re-encryption - await reconcileTeamSealedSecrets(testValues, tmpDir, deps) - const callsAfterSecond = encryptSpy.mock.calls.length - - expect(callsAfterSecond).toBe(callsAfterFirst) // no new encrypt calls - }) - - it('re-encrypts when secret value changes (hash mismatch)', async () => { - const deps = makeDeps() - await reconcileTeamSealedSecrets(testValues, tmpDir, deps) - - const filePath = sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret') - const firstHash = (await readYamlFile(filePath)).metadata.annotations['apl.io/secret-hash'] - - // Change the secret value - const newDeps = makeDeps(makeK8sSecrets({ 'keycloak-secrets': { idp_clientSecret: 'new-kc-secret' } })) - await reconcileTeamSealedSecrets(testValues, tmpDir, newDeps) - - const secondHash = (await readYamlFile(filePath)).metadata.annotations['apl.io/secret-hash'] - expect(secondHash).not.toBe(firstHash) - }) - - it('creates grafana files when feature is enabled', async () => { - await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'team-alpha-grafana-admin'))).toBe(true) - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-loki-datasource-secret'))).toBe(true) - }) - - it('deletes grafana files when feature is disabled', async () => { - // First run: grafana enabled - await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) - - // Second run: grafana disabled - const disabledValues = { - ...testValues, - teamConfig: { - alpha: { - settings: { - managedMonitoring: { grafana: false, alertmanager: false }, - alerts: { receivers: ['none'] }, - }, - }, - }, - } - await reconcileTeamSealedSecrets(disabledValues, tmpDir, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(false) - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'team-alpha-grafana-admin'))).toBe(false) - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-loki-datasource-secret'))).toBe(false) - }) - - it('skips gracefully when PEM is unavailable (dev env)', async () => { - const deps = makeDeps() - deps.getOrCreateSealedSecretsPem = jest.fn().mockRejectedValue(new Error('not in cluster')) - - // Should not throw - await expect(reconcileTeamSealedSecrets(testValues, tmpDir, deps)).resolves.toBeUndefined() - }) - - it('processes other teams when one team K8s secret read fails', async () => { - const valuesWithTwoTeams = { - ...testValues, - teamConfig: { - ...testValues.teamConfig, - gamma: { - settings: { - managedMonitoring: { grafana: true, alertmanager: false }, - alerts: { receivers: ['none'] }, - }, - }, - }, - } - - // alpha secret read fails, gamma succeeds - const partialSecrets = makeK8sSecrets({ - 'team-alpha-settings-secrets': undefined, // simulates read failure - 'team-gamma-settings-secrets': { settings_password: 'gamma-pass' }, - }) - - await reconcileTeamSealedSecrets(valuesWithTwoTeams, tmpDir, makeDeps(partialSecrets)) - - // gamma should still have oidc secret (no dependency on team settings password) - expect(existsSync(sealedSecretPath(tmpDir, 'team-gamma', 'grafana-oidc-secret'))).toBe(true) - }) - - it('removes files for teams that are deleted from teamConfig', async () => { - // First run with alpha - await reconcileTeamSealedSecrets(testValues, tmpDir, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(true) - - // Second run with no teams - await reconcileTeamSealedSecrets({ teamConfig: {} }, tmpDir, makeDeps()) - - expect(existsSync(sealedSecretPath(tmpDir, 'team-alpha', 'grafana-oidc-secret'))).toBe(false) - }) -}) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index e27fa3133c..109a249ce2 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -1,5 +1,7 @@ +import { createHash } from 'crypto' import { pki } from 'node-forge' import stubs from 'src/test-stubs' +import { stringify as stringifyYaml } from 'yaml' import { applySealedSecretManifests, bootstrapSealedSecrets, @@ -10,6 +12,7 @@ import { createUserSealedSecretManifests, generateSealedSecretsKeyPair, getPemFromCertificate, + reconcileTeamSealedSecrets, restartSealedSecretsController, SealedSecretManifest, stripAllSecrets, @@ -503,6 +506,51 @@ describe('sealed-secrets', () => { expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta'], undefined) }) + + it('should include team namespace manifests in writeSealedSecretManifests when allValues provided', async () => { + const secrets = { + teamConfig: { alpha: { settings: { password: 'alpha-pass' } } }, + apps: { keycloak: { idp: { clientSecret: 'kc-secret' } }, loki: { adminPassword: 'loki-pass' } }, + } + const allValues = { + teamConfig: { alpha: { settings: { managedMonitoring: { grafana: true } } } }, + apps: { keycloak: { idp: { clientID: 'apl' } } }, + } + const mockManifest = (name: string, namespace: string) => ({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name, namespace, annotations: {} }, + spec: { + encryptedData: {}, + template: { immutable: false, metadata: { name, namespace }, type: 'kubernetes.io/opaque' }, + }, + }) + + const deps = { + terminal, + getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), + generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ certificate: 'cert', privateKey: 'key' }), + getPemFromCertificate: jest.fn().mockReturnValue('pem'), + createSealedSecretsKeySecret: jest.fn(), + buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), + buildTeamNamespaceSealedSecretMappings, // real function + createSealedSecretManifest: jest + .fn() + .mockImplementation(async (_pem, mapping) => mockManifest(mapping.secretName, mapping.namespace)), + writeSealedSecretManifests: jest.fn(), + createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), + encryptSecretItem: jest.fn().mockResolvedValue('enc'), + } + + await bootstrapSealedSecrets(secrets, '/test', allValues, deps) + + const written: SealedSecretManifest[] = deps.writeSealedSecretManifests.mock.calls[0][0] + const namespaces = written.map((m) => m.metadata.namespace) + expect(namespaces).toContain('team-alpha') + const names = written.map((m) => m.metadata.name) + expect(names).toContain('grafana-oidc-secret') + expect(names).toContain('team-alpha-grafana-admin') + }) }) describe('secret name derivation', () => { @@ -870,4 +918,151 @@ describe('sealed-secrets', () => { expect(result[0].data).toHaveProperty('slackUrl') }) }) + + describe('reconcileTeamSealedSecrets', () => { + const FAKE_PEM = 'mock-pem' + + const testValues = { + teamConfig: { + alpha: { + settings: { + managedMonitoring: { grafana: true, alertmanager: false }, + alerts: { receivers: ['none'] }, + }, + }, + }, + otomi: { globalPullSecret: null }, + apps: { keycloak: { idp: { clientID: 'grafana-client' } } }, + } + + const testMapping = { + namespace: 'team-alpha', + secretName: 'grafana-oidc-secret', + data: { client_id: 'grafana-client', client_secret: 'kc-secret' }, + } + + const mockManifest = (): SealedSecretManifest => ({ + apiVersion: 'bitnami.com/v1alpha1', + kind: 'SealedSecret', + metadata: { name: testMapping.secretName, namespace: testMapping.namespace, annotations: {} }, + spec: { + encryptedData: { client_secret: 'enc' }, + template: { + immutable: false, + metadata: { name: testMapping.secretName, namespace: testMapping.namespace }, + type: 'kubernetes.io/opaque', + }, + }, + }) + + function makeDeps(overrides: Record = {}) { + return { + buildAllSecretsFromK8s: jest.fn().mockResolvedValue({ + teamConfig: { alpha: { settings: { password: 'alpha-pass' } } }, + apps: { keycloak: { idp: { clientSecret: 'kc-secret' } }, loki: { adminPassword: 'loki-pass' } }, + alerts: { slack: { url: '' }, opsgenie: { apiKey: '' } }, + smtp: { auth_password: '', auth_secret: '' }, + otomi: { globalPullSecret: { password: '' } }, + }), + buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([testMapping]), + createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest()), + writeSealedSecretManifests: jest.fn().mockResolvedValue(undefined), + getOrCreateSealedSecretsPem: jest.fn().mockResolvedValue(FAKE_PEM), + encryptSecretItem: jest.fn().mockResolvedValue('enc'), + readFile: jest.fn().mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })), + readdir: jest.fn().mockResolvedValue([]), + unlink: jest.fn().mockResolvedValue(undefined), + ...overrides, + } + } + + it('calls writeSealedSecretManifests on first run when no existing file', async () => { + const deps = makeDeps() + await reconcileTeamSealedSecrets(testValues, '/test', deps) + expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith( + expect.arrayContaining([ + expect.objectContaining({ metadata: expect.objectContaining({ name: 'grafana-oidc-secret' }) }), + ]), + '/test', + ) + }) + + it('adds apl.io/secret-hash annotation to written manifests', async () => { + const captured: SealedSecretManifest[] = [] + const deps = makeDeps({ + writeSealedSecretManifests: jest.fn().mockImplementation(async (manifests: SealedSecretManifest[]) => { + captured.push(...manifests) + }), + }) + + await reconcileTeamSealedSecrets(testValues, '/test', deps) + + expect(captured).toHaveLength(1) + expect(captured[0].metadata.annotations['apl.io/secret-hash']).toBeDefined() + expect(captured[0].metadata.annotations['apl.io/secret-hash']).toHaveLength(16) + }) + + it('skips re-encryption when hash matches existing file', async () => { + const inputHash = createHash('sha256') + .update(JSON.stringify({ data: testMapping.data, secretType: '' })) + .digest('hex') + .slice(0, 16) + const existingYaml = stringifyYaml({ metadata: { annotations: { 'apl.io/secret-hash': inputHash } } }) + + const deps = makeDeps({ readFile: jest.fn().mockResolvedValue(existingYaml) }) + + await reconcileTeamSealedSecrets(testValues, '/test', deps) + + expect(deps.createSealedSecretManifest).not.toHaveBeenCalled() + expect(deps.writeSealedSecretManifests).not.toHaveBeenCalled() + }) + + it('re-encrypts when hash differs from existing file', async () => { + const existingYaml = stringifyYaml({ metadata: { annotations: { 'apl.io/secret-hash': 'outdatedhash000' } } }) + + const deps = makeDeps({ readFile: jest.fn().mockResolvedValue(existingYaml) }) + + await reconcileTeamSealedSecrets(testValues, '/test', deps) + + expect(deps.createSealedSecretManifest).toHaveBeenCalled() + expect(deps.writeSealedSecretManifests).toHaveBeenCalled() + }) + + it('skips gracefully when PEM is unavailable', async () => { + const deps = makeDeps({ + getOrCreateSealedSecretsPem: jest.fn().mockRejectedValue(new Error('not in cluster')), + }) + + await expect(reconcileTeamSealedSecrets(testValues, '/test', deps)).resolves.toBeUndefined() + expect(deps.writeSealedSecretManifests).not.toHaveBeenCalled() + }) + + it('calls unlink for stale files and not for expected files', async () => { + const expectedFile = `${testMapping.secretName}.yaml` + const staleFile = 'stale-secret.yaml' + + const deps = makeDeps({ + readdir: jest + .fn() + .mockResolvedValueOnce(['team-alpha']) // team dirs + .mockResolvedValueOnce([expectedFile, staleFile]), // files in sealedsecrets/ + }) + + await reconcileTeamSealedSecrets(testValues, '/test', deps) + + const unlinkedPaths: string[] = deps.unlink.mock.calls.map((c: string[]) => c[0]) + expect(unlinkedPaths.some((p) => p.endsWith(staleFile))).toBe(true) + expect(unlinkedPaths.some((p) => p.endsWith(expectedFile))).toBe(false) + }) + + it('does not call writeSealedSecretManifests when teamConfig is empty', async () => { + const deps = makeDeps({ + buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), + }) + + await reconcileTeamSealedSecrets({ teamConfig: {} }, '/test', deps) + + expect(deps.writeSealedSecretManifests).not.toHaveBeenCalled() + }) + }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index eefe6fc0a1..570670ead3 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -839,6 +839,9 @@ export async function reconcileTeamSealedSecrets( writeSealedSecretManifests, getOrCreateSealedSecretsPem, encryptSecretItem, + readFile, + readdir, + unlink, }, ): Promise { const teams = Object.keys(get(allValues, 'teamConfig', {}) as Record).filter((id) => id !== 'admin') @@ -876,7 +879,7 @@ export async function reconcileTeamSealedSecrets( let existingHash: string | undefined try { - const raw = await readFile(manifestPath, 'utf8') + const raw = await deps.readFile(manifestPath, 'utf8') existingHash = (parseYaml(raw) as any)?.metadata?.annotations?.['apl.io/secret-hash'] } catch { // File not found — will create @@ -897,7 +900,7 @@ export async function reconcileTeamSealedSecrets( const baseDir = join(envDir, SEALED_SECRETS_MANIFESTS_SUBDIR) let teamDirs: string[] = [] try { - teamDirs = (await readdir(baseDir)).filter((dir) => dir.startsWith('team-')) + teamDirs = (await deps.readdir(baseDir)).filter((dir) => dir.startsWith('team-')) } catch { // Directory doesn't exist yet — nothing to clean up } @@ -906,14 +909,14 @@ export async function reconcileTeamSealedSecrets( const ssDir = join(baseDir, teamDir, 'sealedsecrets') let files: string[] = [] try { - files = await readdir(ssDir) + files = await deps.readdir(ssDir) } catch { continue } for (const file of files) { const filePath = join(ssDir, file) if (!expectedPaths.has(filePath)) { - await unlink(filePath) + await deps.unlink(filePath) } } } diff --git a/tsconfig.json b/tsconfig.json index 550404d067..77715d0b0c 100644 --- a/tsconfig.json +++ b/tsconfig.json @@ -15,7 +15,8 @@ "sourceMap": true, "strict": false, "strictNullChecks": true, - "target": "esnext" + "target": "esnext", + "types": ["jest", "node"] }, "exclude": ["node_modules", "dist"], "include": ["src", "jest.config.ts"], From 862ac3debbdfa772c2e75d5f4c6737ae11b40d08 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 01:34:24 +0200 Subject: [PATCH 04/10] chore: clean up comments in sealed secrets and operator files for clarity --- src/common/sealed-secrets.ts | 38 +++++------------------------------- src/operator/apl-operator.ts | 2 -- 2 files changed, 5 insertions(+), 35 deletions(-) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 570670ead3..5a025b4a50 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -575,10 +575,6 @@ export const createUserSealedSecretManifests = async ( /** * Build SealedSecret mappings for team namespaces. - * These replace ExternalSecrets that previously allowed teams to read platform secrets - * via core-secrets-store ClusterSecretStore — a security hole where any team could - * read any secret in the apl-secrets namespace. - * * Each mapping is encrypted for the specific team namespace, so platform secrets * cannot be decrypted in any other namespace. */ @@ -598,7 +594,6 @@ export const buildTeamNamespaceSealedSecretMappings = ( const hasReceivers = !teamReceivers.includes('none') if (grafanaEnabled) { - // team--grafana-admin: admin credentials for team's Grafana instance const teamPassword = get(allSecrets, `teamConfig.${teamId}.settings.password`, '') as string if (teamPassword) { mappings.push({ @@ -608,7 +603,6 @@ export const buildTeamNamespaceSealedSecretMappings = ( }) } - // grafana-oidc-secret: Keycloak OIDC credentials for Grafana SSO const clientSecret = get(allSecrets, 'apps.keycloak.idp.clientSecret', '') as string const clientId = get(allValues, 'apps.keycloak.idp.clientID', '') as string if (clientSecret) { @@ -619,7 +613,6 @@ export const buildTeamNamespaceSealedSecretMappings = ( }) } - // grafana-loki-datasource-secret: Loki admin password for Grafana datasource const lokiPassword = get(allSecrets, 'apps.loki.adminPassword', '') as string if (lokiPassword) { mappings.push({ @@ -631,21 +624,18 @@ export const buildTeamNamespaceSealedSecretMappings = ( } if (alertmanagerEnabled && hasReceivers) { - // alertmanager-credentials: notification channel credentials for team Alertmanager const alertData: Record = {} if (teamReceivers.includes('slack')) { const slackUrl = get(allSecrets, 'alerts.slack.url', '') as string if (slackUrl) alertData.slackUrl = slackUrl } if (teamReceivers.includes('email')) { - // email receiver uses platform SMTP credentials const smtpPassword = get(allSecrets, 'smtp.auth_password', '') as string const smtpSecret = get(allSecrets, 'smtp.auth_secret', '') as string if (smtpPassword) alertData.smtpAuthPassword = smtpPassword if (smtpSecret) alertData.smtpAuthSecret = smtpSecret } if (teamReceivers.includes('opsgenie')) { - // Legacy receiver — kept for backwards compatibility const opsgenieKey = get(allSecrets, 'alerts.opsgenie.apiKey', '') as string if (opsgenieKey) alertData.opsgenieApiKey = opsgenieKey } @@ -654,8 +644,6 @@ export const buildTeamNamespaceSealedSecretMappings = ( } } - // otomi-pullsecret-global: docker pull secret for global container registry - // Only created when otomi.globalPullSecret is configured const pullSecretConfig = get(allValues, 'otomi.globalPullSecret', null) as Record | null const pullSecretPassword = get(allSecrets, 'otomi.globalPullSecret.password', '') as string if (pullSecretConfig && pullSecretPassword) { @@ -743,8 +731,7 @@ export const bootstrapSealedSecrets = async ( manifests.push(manifest) } - // 4. Create team-namespace SealedSecret manifests (replaces ExternalSecrets referencing core-secrets-store) - // These are encrypted for each team's namespace, so platform secrets cannot be decrypted elsewhere. + // 4. Create team-namespace SealedSecret manifests (encrypted for each team namespace) if (allValues) { const teamMappings = deps.buildTeamNamespaceSealedSecretMappings(secrets, allValues, teams) for (const mapping of teamMappings) { @@ -773,15 +760,7 @@ export const bootstrapSealedSecrets = async ( d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) } -/** - * Read the 5 shared platform secrets + 1 per-team secret from the apl-secrets namespace in K8s. - * Returns an object shaped identically to the `allSecrets` parameter expected by - * `buildTeamNamespaceSealedSecretMappings()` so the operator can drive reconciliation - * from live cluster state rather than values-repo data. - * - * Individual secret reads are caught so one missing secret (e.g. loki not yet deployed) - * does not abort the entire reconcile — the relevant team secrets are simply skipped. - */ +// Individual reads are caught so one missing secret (e.g. loki not yet deployed) does not abort the entire reconcile. export async function buildAllSecretsFromK8s(teams: string[], deps = { getK8sSecret }): Promise> { const [keycloak, loki, alerts, smtp, otomi] = await Promise.all([ deps.getK8sSecret('keycloak-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), @@ -819,16 +798,9 @@ export async function buildAllSecretsFromK8s(teams: string[], deps = { getK8sSec } } -/** - * Reconcile team-namespace SealedSecret manifests on every operator cycle. - * - * Flow: read K8s secrets → build expected mappings → hash-compare plaintext inputs - * → re-encrypt only when changed → write new manifests → delete stale files. - * - * Hash-based idempotency prevents git thrash: SealedSecret encryption is non-deterministic - * (random nonce), so comparing ciphertext always looks "changed". Instead we hash the - * plaintext inputs and store that hash as annotation `apl.io/secret-hash` in the manifest. - */ +// Hash-based idempotency: SealedSecret encryption is non-deterministic (random nonce), so comparing +// ciphertext always looks "changed". We hash plaintext inputs and store as `apl.io/secret-hash` annotation +// to skip re-encryption when nothing actually changed, preventing a new git commit on every cycle. export async function reconcileTeamSealedSecrets( allValues: Record, envDir: string, diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 7e21dd7d31..5cb4747e70 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -88,8 +88,6 @@ export class AplOperator { } const values = await hfValues({}, env.ENV_DIR) - // Reconcile team namespace SealedSecrets — operator is source of truth for these manifests. - // Non-fatal: a missing K8s secret (e.g. loki not yet deployed) retries on next cycle. try { await reconcileTeamSealedSecrets(values ?? {}, env.ENV_DIR) } catch (e) { From 5973ed918eb4eded875384c8dd49a6fd80b7071d Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 01:39:24 +0200 Subject: [PATCH 05/10] fix: update annotations from 'apl.io' to 'otomi.io' for secret management --- src/common/sealed-secrets.test.ts | 10 +++++----- src/common/sealed-secrets.ts | 6 +++--- values/external-secrets/external-secrets-raw.gotmpl | 2 +- values/k8s/k8s-raw-teams.gotmpl | 2 +- 4 files changed, 10 insertions(+), 10 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 109a249ce2..66e1151200 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -987,7 +987,7 @@ describe('sealed-secrets', () => { ) }) - it('adds apl.io/secret-hash annotation to written manifests', async () => { + it('adds otomi.io/secret-hash annotation to written manifests', async () => { const captured: SealedSecretManifest[] = [] const deps = makeDeps({ writeSealedSecretManifests: jest.fn().mockImplementation(async (manifests: SealedSecretManifest[]) => { @@ -998,8 +998,8 @@ describe('sealed-secrets', () => { await reconcileTeamSealedSecrets(testValues, '/test', deps) expect(captured).toHaveLength(1) - expect(captured[0].metadata.annotations['apl.io/secret-hash']).toBeDefined() - expect(captured[0].metadata.annotations['apl.io/secret-hash']).toHaveLength(16) + expect(captured[0].metadata.annotations['otomi.io/secret-hash']).toBeDefined() + expect(captured[0].metadata.annotations['otomi.io/secret-hash']).toHaveLength(16) }) it('skips re-encryption when hash matches existing file', async () => { @@ -1007,7 +1007,7 @@ describe('sealed-secrets', () => { .update(JSON.stringify({ data: testMapping.data, secretType: '' })) .digest('hex') .slice(0, 16) - const existingYaml = stringifyYaml({ metadata: { annotations: { 'apl.io/secret-hash': inputHash } } }) + const existingYaml = stringifyYaml({ metadata: { annotations: { 'otomi.io/secret-hash': inputHash } } }) const deps = makeDeps({ readFile: jest.fn().mockResolvedValue(existingYaml) }) @@ -1018,7 +1018,7 @@ describe('sealed-secrets', () => { }) it('re-encrypts when hash differs from existing file', async () => { - const existingYaml = stringifyYaml({ metadata: { annotations: { 'apl.io/secret-hash': 'outdatedhash000' } } }) + const existingYaml = stringifyYaml({ metadata: { annotations: { 'otomi.io/secret-hash': 'outdatedhash000' } } }) const deps = makeDeps({ readFile: jest.fn().mockResolvedValue(existingYaml) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 5a025b4a50..4435ed8d80 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -799,7 +799,7 @@ export async function buildAllSecretsFromK8s(teams: string[], deps = { getK8sSec } // Hash-based idempotency: SealedSecret encryption is non-deterministic (random nonce), so comparing -// ciphertext always looks "changed". We hash plaintext inputs and store as `apl.io/secret-hash` annotation +// ciphertext always looks "changed". We hash plaintext inputs and store as `otomi.io/secret-hash` annotation // to skip re-encryption when nothing actually changed, preventing a new git commit on every cycle. export async function reconcileTeamSealedSecrets( allValues: Record, @@ -852,7 +852,7 @@ export async function reconcileTeamSealedSecrets( let existingHash: string | undefined try { const raw = await deps.readFile(manifestPath, 'utf8') - existingHash = (parseYaml(raw) as any)?.metadata?.annotations?.['apl.io/secret-hash'] + existingHash = (parseYaml(raw) as any)?.metadata?.annotations?.['otomi.io/secret-hash'] } catch { // File not found — will create } @@ -860,7 +860,7 @@ export async function reconcileTeamSealedSecrets( if (existingHash === inputHash) continue const manifest = await deps.createSealedSecretManifest(pem, mapping, { encryptSecretItem: deps.encryptSecretItem }) - manifest.metadata.annotations['apl.io/secret-hash'] = inputHash + manifest.metadata.annotations['otomi.io/secret-hash'] = inputHash toWrite.push(manifest) } diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl index 0ccf895b0d..b0dfd2d5d5 100644 --- a/values/external-secrets/external-secrets-raw.gotmpl +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -34,7 +34,7 @@ resources: conditions: - namespaceSelector: matchExpressions: - - key: apl.io/team + - key: otomi.io/team operator: DoesNotExist provider: kubernetes: diff --git a/values/k8s/k8s-raw-teams.gotmpl b/values/k8s/k8s-raw-teams.gotmpl index 0f98f5b93b..433ee885ab 100644 --- a/values/k8s/k8s-raw-teams.gotmpl +++ b/values/k8s/k8s-raw-teams.gotmpl @@ -10,7 +10,7 @@ resources: labels: name: {{ $ns }} type: team - apl.io/team: "true" + otomi.io/team: "true" {{- if $v.apps.istio.defaultRevision }} istio.io/rev: {{ $v.apps.istio.defaultRevision | quote }} {{- else }} From 7fe7f9475ff3138737e6ef302dc59656a849a3d9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 14:33:20 +0200 Subject: [PATCH 06/10] feat: team secret management with ESO push secrets --- src/common/sealed-secrets.test.ts | 425 ------------------ src/common/sealed-secrets.ts | 252 +---------- src/operator/apl-operator.test.ts | 4 - src/operator/apl-operator.ts | 8 - .../external-secrets-raw.gotmpl | 8 + values/k8s/k8s-raw-teams.gotmpl | 13 + values/team-secrets/team-secrets-raw.gotmpl | 275 ++++++++++++ 7 files changed, 304 insertions(+), 681 deletions(-) diff --git a/src/common/sealed-secrets.test.ts b/src/common/sealed-secrets.test.ts index 66e1151200..69dd2c6e96 100644 --- a/src/common/sealed-secrets.test.ts +++ b/src/common/sealed-secrets.test.ts @@ -1,18 +1,14 @@ -import { createHash } from 'crypto' import { pki } from 'node-forge' import stubs from 'src/test-stubs' -import { stringify as stringifyYaml } from 'yaml' import { applySealedSecretManifests, bootstrapSealedSecrets, buildSecretToNamespaceMap, - buildTeamNamespaceSealedSecretMappings, createSealedSecretManifest, createSealedSecretsKeySecret, createUserSealedSecretManifests, generateSealedSecretsKeyPair, getPemFromCertificate, - reconcileTeamSealedSecrets, restartSealedSecretsController, SealedSecretManifest, stripAllSecrets, @@ -429,7 +425,6 @@ describe('sealed-secrets', () => { getPemFromCertificate: jest.fn().mockReturnValue('spki-pem'), createSealedSecretsKeySecret: jest.fn(), buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), - buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest), writeSealedSecretManifests: jest.fn(), createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), @@ -462,7 +457,6 @@ describe('sealed-secrets', () => { getPemFromCertificate: jest.fn().mockReturnValue('existing-spki-pem'), createSealedSecretsKeySecret: jest.fn(), buildSecretToNamespaceMap: jest.fn().mockResolvedValue([mockMapping]), - buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), createSealedSecretManifest: jest.fn().mockResolvedValue({}), writeSealedSecretManifests: jest.fn(), createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), @@ -495,7 +489,6 @@ describe('sealed-secrets', () => { getPemFromCertificate: jest.fn().mockReturnValue('pem'), createSealedSecretsKeySecret: jest.fn(), buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), - buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), createSealedSecretManifest: jest.fn(), writeSealedSecretManifests: jest.fn(), createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), @@ -506,51 +499,6 @@ describe('sealed-secrets', () => { expect(deps.buildSecretToNamespaceMap).toHaveBeenCalledWith(secrets, ['alpha', 'beta'], undefined) }) - - it('should include team namespace manifests in writeSealedSecretManifests when allValues provided', async () => { - const secrets = { - teamConfig: { alpha: { settings: { password: 'alpha-pass' } } }, - apps: { keycloak: { idp: { clientSecret: 'kc-secret' } }, loki: { adminPassword: 'loki-pass' } }, - } - const allValues = { - teamConfig: { alpha: { settings: { managedMonitoring: { grafana: true } } } }, - apps: { keycloak: { idp: { clientID: 'apl' } } }, - } - const mockManifest = (name: string, namespace: string) => ({ - apiVersion: 'bitnami.com/v1alpha1', - kind: 'SealedSecret', - metadata: { name, namespace, annotations: {} }, - spec: { - encryptedData: {}, - template: { immutable: false, metadata: { name, namespace }, type: 'kubernetes.io/opaque' }, - }, - }) - - const deps = { - terminal, - getExistingSealedSecretsCert: jest.fn().mockResolvedValue(undefined), - generateSealedSecretsKeyPair: jest.fn().mockReturnValue({ certificate: 'cert', privateKey: 'key' }), - getPemFromCertificate: jest.fn().mockReturnValue('pem'), - createSealedSecretsKeySecret: jest.fn(), - buildSecretToNamespaceMap: jest.fn().mockResolvedValue([]), - buildTeamNamespaceSealedSecretMappings, // real function - createSealedSecretManifest: jest - .fn() - .mockImplementation(async (_pem, mapping) => mockManifest(mapping.secretName, mapping.namespace)), - writeSealedSecretManifests: jest.fn(), - createUserSealedSecretManifests: jest.fn().mockResolvedValue([]), - encryptSecretItem: jest.fn().mockResolvedValue('enc'), - } - - await bootstrapSealedSecrets(secrets, '/test', allValues, deps) - - const written: SealedSecretManifest[] = deps.writeSealedSecretManifests.mock.calls[0][0] - const namespaces = written.map((m) => m.metadata.namespace) - expect(namespaces).toContain('team-alpha') - const names = written.map((m) => m.metadata.name) - expect(names).toContain('grafana-oidc-secret') - expect(names).toContain('team-alpha-grafana-admin') - }) }) describe('secret name derivation', () => { @@ -692,377 +640,4 @@ describe('sealed-secrets', () => { expect(manifests).toHaveLength(0) }) }) - - describe('buildTeamNamespaceSealedSecretMappings', () => { - const baseSecrets = { - apps: { - keycloak: { idp: { clientSecret: 'kc-secret' } }, - loki: { adminPassword: 'loki-pass' }, - }, - alerts: { slack: { url: 'https://hooks.slack.com/test' } }, - smtp: { auth_password: 'smtp-pass', auth_secret: 'smtp-secret' }, - } - - const baseValues = { - apps: { keycloak: { idp: { clientID: 'apl' } } }, - } - - it('should return empty array when no teams provided', () => { - const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, baseValues, []) - expect(result).toEqual([]) - }) - - it('should return empty array when grafana and alertmanager are both disabled', () => { - const allValues = { - ...baseValues, - teamConfig: { alpha: { settings: { managedMonitoring: { grafana: false, alertmanager: false } } } }, - } - const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) - expect(result).toEqual([]) - }) - - it('should create grafana secrets when grafana is enabled', () => { - const allSecrets = { ...baseSecrets, teamConfig: { alpha: { settings: { password: 'team-pass' } } } } - const allValues = { - ...baseValues, - teamConfig: { alpha: { settings: { managedMonitoring: { grafana: true } } } }, - } - - const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) - - expect(result).toHaveLength(3) - expect(result.find((m) => m.secretName === 'team-alpha-grafana-admin')).toMatchObject({ - namespace: 'team-alpha', - secretName: 'team-alpha-grafana-admin', - data: { 'admin-user': 'alpha', 'admin-password': 'team-pass' }, - }) - expect(result.find((m) => m.secretName === 'grafana-oidc-secret')).toMatchObject({ - namespace: 'team-alpha', - secretName: 'grafana-oidc-secret', - data: { client_id: 'apl', client_secret: 'kc-secret' }, - }) - expect(result.find((m) => m.secretName === 'grafana-loki-datasource-secret')).toMatchObject({ - namespace: 'team-alpha', - secretName: 'grafana-loki-datasource-secret', - data: { password: 'loki-pass' }, - }) - }) - - it('should skip grafana-admin when team password is missing', () => { - const allSecrets = { ...baseSecrets } // no teamConfig password - const allValues = { - ...baseValues, - teamConfig: { alpha: { settings: { managedMonitoring: { grafana: true } } } }, - } - - const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) - - expect(result.find((m) => m.secretName === 'team-alpha-grafana-admin')).toBeUndefined() - // grafana-oidc and loki-datasource are still created - expect(result.find((m) => m.secretName === 'grafana-oidc-secret')).toBeDefined() - expect(result.find((m) => m.secretName === 'grafana-loki-datasource-secret')).toBeDefined() - }) - - it('should create alertmanager-credentials for slack receiver', () => { - const allValues = { - ...baseValues, - teamConfig: { - alpha: { - settings: { - managedMonitoring: { alertmanager: true }, - alerts: { receivers: ['slack'] }, - }, - }, - }, - } - - const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) - - expect(result).toHaveLength(1) - expect(result[0]).toMatchObject({ - namespace: 'team-alpha', - secretName: 'alertmanager-credentials', - data: { slackUrl: 'https://hooks.slack.com/test' }, - }) - }) - - it('should create alertmanager-credentials for email receiver', () => { - const allValues = { - ...baseValues, - teamConfig: { - alpha: { - settings: { - managedMonitoring: { alertmanager: true }, - alerts: { receivers: ['email'] }, - }, - }, - }, - } - - const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) - - expect(result[0].data).toMatchObject({ - smtpAuthPassword: 'smtp-pass', - smtpAuthSecret: 'smtp-secret', - }) - }) - - it('should skip alertmanager-credentials when receivers is none', () => { - const allValues = { - ...baseValues, - teamConfig: { - alpha: { - settings: { - managedMonitoring: { alertmanager: true }, - alerts: { receivers: ['none'] }, - }, - }, - }, - } - - const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) - expect(result).toHaveLength(0) - }) - - it('should create otomi-pullsecret-global when globalPullSecret is configured', () => { - const allSecrets = { ...baseSecrets, otomi: { globalPullSecret: { password: 'reg-pass' } } } - const allValues = { - ...baseValues, - otomi: { globalPullSecret: { server: 'registry.example.com', username: 'user', email: 'user@example.com' } }, - teamConfig: { alpha: { settings: {} } }, - } - - const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) - - expect(result).toHaveLength(1) - expect(result[0]).toMatchObject({ - namespace: 'team-alpha', - secretName: 'otomi-pullsecret-global', - secretType: 'kubernetes.io/dockerconfigjson', - }) - const dockerConfig = JSON.parse(result[0].data['.dockerconfigjson']) - expect(dockerConfig.auths['registry.example.com']).toMatchObject({ - username: 'user', - password: 'reg-pass', - email: 'user@example.com', - }) - }) - - it('should skip otomi-pullsecret-global when password is missing', () => { - const allValues = { - ...baseValues, - otomi: { globalPullSecret: { server: 'registry.example.com', username: 'user' } }, - teamConfig: { alpha: { settings: {} } }, - } - - const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) - expect(result.find((m) => m.secretName === 'otomi-pullsecret-global')).toBeUndefined() - }) - - it('should use default docker.io server when not specified', () => { - const allSecrets = { ...baseSecrets, otomi: { globalPullSecret: { password: 'reg-pass' } } } - const allValues = { - ...baseValues, - otomi: { globalPullSecret: { username: 'user' } }, // no server - teamConfig: { alpha: { settings: {} } }, - } - - const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha']) - const dockerConfig = JSON.parse(result[0].data['.dockerconfigjson']) - expect(dockerConfig.auths['docker.io']).toBeDefined() - }) - - it('should create mappings for multiple teams independently', () => { - const allSecrets = { - ...baseSecrets, - teamConfig: { - alpha: { settings: { password: 'alpha-pass' } }, - beta: { settings: { password: 'beta-pass' } }, - }, - } - const allValues = { - ...baseValues, - teamConfig: { - alpha: { settings: { managedMonitoring: { grafana: true } } }, - beta: { settings: { managedMonitoring: { grafana: true } } }, - }, - } - - const result = buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, ['alpha', 'beta']) - - const alphaAdmin = result.find((m) => m.secretName === 'team-alpha-grafana-admin') - const betaAdmin = result.find((m) => m.secretName === 'team-beta-grafana-admin') - - expect(alphaAdmin?.namespace).toBe('team-alpha') - expect(alphaAdmin?.data['admin-password']).toBe('alpha-pass') - expect(betaAdmin?.namespace).toBe('team-beta') - expect(betaAdmin?.data['admin-password']).toBe('beta-pass') - }) - - it('should use team-level receivers falling back to platform receivers', () => { - const allValues = { - ...baseValues, - alerts: { receivers: ['slack'] }, // platform-level default - teamConfig: { - alpha: { - settings: { - managedMonitoring: { alertmanager: true }, - // No team-level receivers — should use platform default - }, - }, - }, - } - - const result = buildTeamNamespaceSealedSecretMappings(baseSecrets, allValues, ['alpha']) - - expect(result[0].data).toHaveProperty('slackUrl') - }) - }) - - describe('reconcileTeamSealedSecrets', () => { - const FAKE_PEM = 'mock-pem' - - const testValues = { - teamConfig: { - alpha: { - settings: { - managedMonitoring: { grafana: true, alertmanager: false }, - alerts: { receivers: ['none'] }, - }, - }, - }, - otomi: { globalPullSecret: null }, - apps: { keycloak: { idp: { clientID: 'grafana-client' } } }, - } - - const testMapping = { - namespace: 'team-alpha', - secretName: 'grafana-oidc-secret', - data: { client_id: 'grafana-client', client_secret: 'kc-secret' }, - } - - const mockManifest = (): SealedSecretManifest => ({ - apiVersion: 'bitnami.com/v1alpha1', - kind: 'SealedSecret', - metadata: { name: testMapping.secretName, namespace: testMapping.namespace, annotations: {} }, - spec: { - encryptedData: { client_secret: 'enc' }, - template: { - immutable: false, - metadata: { name: testMapping.secretName, namespace: testMapping.namespace }, - type: 'kubernetes.io/opaque', - }, - }, - }) - - function makeDeps(overrides: Record = {}) { - return { - buildAllSecretsFromK8s: jest.fn().mockResolvedValue({ - teamConfig: { alpha: { settings: { password: 'alpha-pass' } } }, - apps: { keycloak: { idp: { clientSecret: 'kc-secret' } }, loki: { adminPassword: 'loki-pass' } }, - alerts: { slack: { url: '' }, opsgenie: { apiKey: '' } }, - smtp: { auth_password: '', auth_secret: '' }, - otomi: { globalPullSecret: { password: '' } }, - }), - buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([testMapping]), - createSealedSecretManifest: jest.fn().mockResolvedValue(mockManifest()), - writeSealedSecretManifests: jest.fn().mockResolvedValue(undefined), - getOrCreateSealedSecretsPem: jest.fn().mockResolvedValue(FAKE_PEM), - encryptSecretItem: jest.fn().mockResolvedValue('enc'), - readFile: jest.fn().mockRejectedValue(Object.assign(new Error('ENOENT'), { code: 'ENOENT' })), - readdir: jest.fn().mockResolvedValue([]), - unlink: jest.fn().mockResolvedValue(undefined), - ...overrides, - } - } - - it('calls writeSealedSecretManifests on first run when no existing file', async () => { - const deps = makeDeps() - await reconcileTeamSealedSecrets(testValues, '/test', deps) - expect(deps.writeSealedSecretManifests).toHaveBeenCalledWith( - expect.arrayContaining([ - expect.objectContaining({ metadata: expect.objectContaining({ name: 'grafana-oidc-secret' }) }), - ]), - '/test', - ) - }) - - it('adds otomi.io/secret-hash annotation to written manifests', async () => { - const captured: SealedSecretManifest[] = [] - const deps = makeDeps({ - writeSealedSecretManifests: jest.fn().mockImplementation(async (manifests: SealedSecretManifest[]) => { - captured.push(...manifests) - }), - }) - - await reconcileTeamSealedSecrets(testValues, '/test', deps) - - expect(captured).toHaveLength(1) - expect(captured[0].metadata.annotations['otomi.io/secret-hash']).toBeDefined() - expect(captured[0].metadata.annotations['otomi.io/secret-hash']).toHaveLength(16) - }) - - it('skips re-encryption when hash matches existing file', async () => { - const inputHash = createHash('sha256') - .update(JSON.stringify({ data: testMapping.data, secretType: '' })) - .digest('hex') - .slice(0, 16) - const existingYaml = stringifyYaml({ metadata: { annotations: { 'otomi.io/secret-hash': inputHash } } }) - - const deps = makeDeps({ readFile: jest.fn().mockResolvedValue(existingYaml) }) - - await reconcileTeamSealedSecrets(testValues, '/test', deps) - - expect(deps.createSealedSecretManifest).not.toHaveBeenCalled() - expect(deps.writeSealedSecretManifests).not.toHaveBeenCalled() - }) - - it('re-encrypts when hash differs from existing file', async () => { - const existingYaml = stringifyYaml({ metadata: { annotations: { 'otomi.io/secret-hash': 'outdatedhash000' } } }) - - const deps = makeDeps({ readFile: jest.fn().mockResolvedValue(existingYaml) }) - - await reconcileTeamSealedSecrets(testValues, '/test', deps) - - expect(deps.createSealedSecretManifest).toHaveBeenCalled() - expect(deps.writeSealedSecretManifests).toHaveBeenCalled() - }) - - it('skips gracefully when PEM is unavailable', async () => { - const deps = makeDeps({ - getOrCreateSealedSecretsPem: jest.fn().mockRejectedValue(new Error('not in cluster')), - }) - - await expect(reconcileTeamSealedSecrets(testValues, '/test', deps)).resolves.toBeUndefined() - expect(deps.writeSealedSecretManifests).not.toHaveBeenCalled() - }) - - it('calls unlink for stale files and not for expected files', async () => { - const expectedFile = `${testMapping.secretName}.yaml` - const staleFile = 'stale-secret.yaml' - - const deps = makeDeps({ - readdir: jest - .fn() - .mockResolvedValueOnce(['team-alpha']) // team dirs - .mockResolvedValueOnce([expectedFile, staleFile]), // files in sealedsecrets/ - }) - - await reconcileTeamSealedSecrets(testValues, '/test', deps) - - const unlinkedPaths: string[] = deps.unlink.mock.calls.map((c: string[]) => c[0]) - expect(unlinkedPaths.some((p) => p.endsWith(staleFile))).toBe(true) - expect(unlinkedPaths.some((p) => p.endsWith(expectedFile))).toBe(false) - }) - - it('does not call writeSealedSecretManifests when teamConfig is empty', async () => { - const deps = makeDeps({ - buildTeamNamespaceSealedSecretMappings: jest.fn().mockReturnValue([]), - }) - - await reconcileTeamSealedSecrets({ teamConfig: {} }, '/test', deps) - - expect(deps.writeSealedSecretManifests).not.toHaveBeenCalled() - }) - }) }) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index 4435ed8d80..e84a3c326a 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -1,9 +1,9 @@ import { ApiException, PatchStrategy, setHeaderOptions } from '@kubernetes/client-node' import { encryptSecretItem } from '@linode/kubeseal-encrypt' -import { createHash, X509Certificate } from 'crypto' +import { X509Certificate } from 'crypto' import { existsSync } from 'fs' -import { mkdir, readdir, readFile, unlink, writeFile } from 'fs/promises' -import { cloneDeep, get, set, unset } from 'lodash' +import { mkdir, readdir, readFile, writeFile } from 'fs/promises' +import { cloneDeep, get, unset } from 'lodash' import { pki } from 'node-forge' import { join } from 'path' import { SEALED_SECRETS_NAMESPACE } from 'src/common/constants' @@ -15,7 +15,7 @@ import { objectToYaml } from 'src/common/values' import { parse as parseYaml } from 'yaml' const cmdName = 'sealed-secrets' -export const SEALED_SECRETS_MANIFESTS_SUBDIR = 'env/manifests/namespaces' +const SEALED_SECRETS_MANIFESTS_SUBDIR = 'env/manifests/namespaces' /** * Strip ALL x-secret fields from values before writing to disk. @@ -573,96 +573,6 @@ export const createUserSealedSecretManifests = async ( return manifests } -/** - * Build SealedSecret mappings for team namespaces. - * Each mapping is encrypted for the specific team namespace, so platform secrets - * cannot be decrypted in any other namespace. - */ -export const buildTeamNamespaceSealedSecretMappings = ( - allSecrets: Record, - allValues: Record, - teams: string[], -): SecretMapping[] => { - const mappings: SecretMapping[] = [] - - for (const teamId of teams) { - const teamNs = `team-${teamId}` - const settings = get(allValues, `teamConfig.${teamId}.settings`, {}) as Record - const grafanaEnabled = get(settings, 'managedMonitoring.grafana', false) as boolean - const alertmanagerEnabled = get(settings, 'managedMonitoring.alertmanager', false) as boolean - const teamReceivers = get(settings, 'alerts.receivers', get(allValues, 'alerts.receivers', ['slack'])) as string[] - const hasReceivers = !teamReceivers.includes('none') - - if (grafanaEnabled) { - const teamPassword = get(allSecrets, `teamConfig.${teamId}.settings.password`, '') as string - if (teamPassword) { - mappings.push({ - namespace: teamNs, - secretName: `${teamNs}-grafana-admin`, - data: { 'admin-user': teamId, 'admin-password': teamPassword }, - }) - } - - const clientSecret = get(allSecrets, 'apps.keycloak.idp.clientSecret', '') as string - const clientId = get(allValues, 'apps.keycloak.idp.clientID', '') as string - if (clientSecret) { - mappings.push({ - namespace: teamNs, - secretName: 'grafana-oidc-secret', - data: { client_id: clientId, client_secret: clientSecret }, - }) - } - - const lokiPassword = get(allSecrets, 'apps.loki.adminPassword', '') as string - if (lokiPassword) { - mappings.push({ - namespace: teamNs, - secretName: 'grafana-loki-datasource-secret', - data: { password: lokiPassword }, - }) - } - } - - if (alertmanagerEnabled && hasReceivers) { - const alertData: Record = {} - if (teamReceivers.includes('slack')) { - const slackUrl = get(allSecrets, 'alerts.slack.url', '') as string - if (slackUrl) alertData.slackUrl = slackUrl - } - if (teamReceivers.includes('email')) { - const smtpPassword = get(allSecrets, 'smtp.auth_password', '') as string - const smtpSecret = get(allSecrets, 'smtp.auth_secret', '') as string - if (smtpPassword) alertData.smtpAuthPassword = smtpPassword - if (smtpSecret) alertData.smtpAuthSecret = smtpSecret - } - if (teamReceivers.includes('opsgenie')) { - const opsgenieKey = get(allSecrets, 'alerts.opsgenie.apiKey', '') as string - if (opsgenieKey) alertData.opsgenieApiKey = opsgenieKey - } - if (Object.keys(alertData).length > 0) { - mappings.push({ namespace: teamNs, secretName: 'alertmanager-credentials', data: alertData }) - } - } - - const pullSecretConfig = get(allValues, 'otomi.globalPullSecret', null) as Record | null - const pullSecretPassword = get(allSecrets, 'otomi.globalPullSecret.password', '') as string - if (pullSecretConfig && pullSecretPassword) { - const server = get(pullSecretConfig, 'server', 'docker.io') as string - const username = get(pullSecretConfig, 'username', '') as string - const email = get(pullSecretConfig, 'email', 'not@val.id') as string - const dockerConfig = JSON.stringify({ auths: { [server]: { username, password: pullSecretPassword, email } } }) - mappings.push({ - namespace: teamNs, - secretName: 'otomi-pullsecret-global', - data: { '.dockerconfigjson': dockerConfig }, - secretType: 'kubernetes.io/dockerconfigjson', - }) - } - } - - return mappings -} - /** * Get the PEM public key from the existing sealed-secrets certificate in the cluster, * or generate a new RSA key pair, store it in the cluster, and return its PEM. @@ -699,7 +609,6 @@ export const bootstrapSealedSecrets = async ( createSealedSecretsKeySecret, getExistingSealedSecretsCert, buildSecretToNamespaceMap, - buildTeamNamespaceSealedSecretMappings, createSealedSecretManifest, writeSealedSecretManifests, createUserSealedSecretManifests, @@ -718,11 +627,11 @@ export const bootstrapSealedSecrets = async ( createSealedSecretsKeySecret: deps.createSealedSecretsKeySecret, }) - // 2. Build secret-to-namespace mapping (platform secrets in apl-secrets namespace) + // 5. Build secret-to-namespace mapping const teams = Object.keys(get(secrets, 'teamConfig', {}) as Record) const mappings = await deps.buildSecretToNamespaceMap(secrets, teams, allValues) - // 3. Create SealedSecret manifests for platform secrets (encrypted for apl-secrets namespace) + // 6. Create SealedSecret manifests const manifests: SealedSecretManifest[] = [] for (const mapping of mappings) { const manifest = await deps.createSealedSecretManifest(pem, mapping, { @@ -731,18 +640,7 @@ export const bootstrapSealedSecrets = async ( manifests.push(manifest) } - // 4. Create team-namespace SealedSecret manifests (encrypted for each team namespace) - if (allValues) { - const teamMappings = deps.buildTeamNamespaceSealedSecretMappings(secrets, allValues, teams) - for (const mapping of teamMappings) { - const manifest = await deps.createSealedSecretManifest(pem, mapping, { - encryptSecretItem: deps.encryptSecretItem, - }) - manifests.push(manifest) - } - } - - // 5. Create individual user SealedSecrets in apl-users namespace + // 7. Create individual user SealedSecrets in apl-users namespace const { users } = secrets if (Array.isArray(users) && users.length > 0) { const userManifests = await deps.createUserSealedSecretManifests(users, pem, { @@ -752,144 +650,10 @@ export const bootstrapSealedSecrets = async ( manifests.push(...userManifests) } - // 6. Write all SealedSecret manifests to disk + // 8. Write SealedSecret manifests to disk // Note: These manifests are applied later during install, after the sealed-secrets // controller is deployed and the SealedSecret CRD is available. await deps.writeSealedSecretManifests(manifests, envDir) d.info(`Bootstrapped ${manifests.length} sealed secret manifests`) } - -// Individual reads are caught so one missing secret (e.g. loki not yet deployed) does not abort the entire reconcile. -export async function buildAllSecretsFromK8s(teams: string[], deps = { getK8sSecret }): Promise> { - const [keycloak, loki, alerts, smtp, otomi] = await Promise.all([ - deps.getK8sSecret('keycloak-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), - deps.getK8sSecret('loki-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), - deps.getK8sSecret('alerts-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), - deps.getK8sSecret('smtp-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), - deps.getK8sSecret('otomi-secrets', SEALED_SECRETS_NAMESPACE).catch(() => undefined), - ]) - - const teamSecrets: Record = {} - for (const teamId of teams) { - const ts = await deps - .getK8sSecret(`team-${teamId}-settings-secrets`, SEALED_SECRETS_NAMESPACE) - .catch(() => undefined) - set(teamSecrets, `${teamId}.settings.password`, ts?.settings_password ?? '') - } - - return { - teamConfig: teamSecrets, - apps: { - keycloak: { idp: { clientSecret: keycloak?.idp_clientSecret ?? '' } }, - loki: { adminPassword: loki?.adminPassword ?? '' }, - }, - alerts: { - slack: { url: alerts?.slack_url ?? '' }, - opsgenie: { apiKey: alerts?.opsgenie_apiKey ?? '' }, - }, - smtp: { - auth_password: smtp?.auth_password ?? '', - auth_secret: smtp?.auth_secret ?? '', - }, - otomi: { - globalPullSecret: { password: otomi?.globalPullSecret_password ?? '' }, - }, - } -} - -// Hash-based idempotency: SealedSecret encryption is non-deterministic (random nonce), so comparing -// ciphertext always looks "changed". We hash plaintext inputs and store as `otomi.io/secret-hash` annotation -// to skip re-encryption when nothing actually changed, preventing a new git commit on every cycle. -export async function reconcileTeamSealedSecrets( - allValues: Record, - envDir: string, - deps = { - buildAllSecretsFromK8s, - buildTeamNamespaceSealedSecretMappings, - createSealedSecretManifest, - writeSealedSecretManifests, - getOrCreateSealedSecretsPem, - encryptSecretItem, - readFile, - readdir, - unlink, - }, -): Promise { - const teams = Object.keys(get(allValues, 'teamConfig', {}) as Record).filter((id) => id !== 'admin') - - const d = terminal(`common:${cmdName}:reconcileTeamSealedSecrets`) - - let pem: string - try { - pem = await deps.getOrCreateSealedSecretsPem() - } catch (e) { - d.warn(`Skipping team SealedSecret reconcile — sealed-secrets PEM unavailable: ${(e as Error).message}`) - return - } - - const allSecrets = await deps.buildAllSecretsFromK8s(teams) - const mappings = deps.buildTeamNamespaceSealedSecretMappings(allSecrets, allValues, teams) - - const expectedPaths = new Set() - const toWrite: SealedSecretManifest[] = [] - - for (const mapping of mappings) { - const manifestPath = join( - envDir, - SEALED_SECRETS_MANIFESTS_SUBDIR, - mapping.namespace, - 'sealedsecrets', - `${mapping.secretName}.yaml`, - ) - expectedPaths.add(manifestPath) - - const inputHash = createHash('sha256') - .update(JSON.stringify({ data: mapping.data, secretType: mapping.secretType ?? '' })) - .digest('hex') - .slice(0, 16) - - let existingHash: string | undefined - try { - const raw = await deps.readFile(manifestPath, 'utf8') - existingHash = (parseYaml(raw) as any)?.metadata?.annotations?.['otomi.io/secret-hash'] - } catch { - // File not found — will create - } - - if (existingHash === inputHash) continue - - const manifest = await deps.createSealedSecretManifest(pem, mapping, { encryptSecretItem: deps.encryptSecretItem }) - manifest.metadata.annotations['otomi.io/secret-hash'] = inputHash - toWrite.push(manifest) - } - - if (toWrite.length > 0) { - await deps.writeSealedSecretManifests(toWrite, envDir) - } - - // Delete stale files: SealedSecrets for disabled features or removed teams - const baseDir = join(envDir, SEALED_SECRETS_MANIFESTS_SUBDIR) - let teamDirs: string[] = [] - try { - teamDirs = (await deps.readdir(baseDir)).filter((dir) => dir.startsWith('team-')) - } catch { - // Directory doesn't exist yet — nothing to clean up - } - - for (const teamDir of teamDirs) { - const ssDir = join(baseDir, teamDir, 'sealedsecrets') - let files: string[] = [] - try { - files = await deps.readdir(ssDir) - } catch { - continue - } - for (const file of files) { - const filePath = join(ssDir, file) - if (!expectedPaths.has(filePath)) { - await deps.unlink(filePath) - } - } - } -} diff --git a/src/operator/apl-operator.test.ts b/src/operator/apl-operator.test.ts index 789c65b82f..55c602bb73 100644 --- a/src/operator/apl-operator.test.ts +++ b/src/operator/apl-operator.test.ts @@ -57,10 +57,6 @@ jest.mock('../cmd/commit', () => ({ commit: jest.fn().mockResolvedValue(undefined), })) -jest.mock('../common/sealed-secrets', () => ({ - reconcileTeamSealedSecrets: jest.fn().mockResolvedValue(undefined), -})) - jest.mock('./k8s', () => ({ updateApplyState: jest.fn().mockResolvedValue(undefined), appRevisionMatches: jest.fn().mockResolvedValue(true), diff --git a/src/operator/apl-operator.ts b/src/operator/apl-operator.ts index 5cb4747e70..70e8f19f85 100644 --- a/src/operator/apl-operator.ts +++ b/src/operator/apl-operator.ts @@ -5,7 +5,6 @@ import { env } from '../common/envalid' import { getStoredGitRepoConfig } from '../common/git-config' import { waitTillGitRepoAvailable } from '../common/gitea' import { hfValues } from '../common/hf' -import { reconcileTeamSealedSecrets } from '../common/sealed-secrets' import { ensureManifestDirectories, ensureTeamGitOpsDirectories } from '../common/utils' import { writeValues } from '../common/values' import { HelmArguments } from '../common/yargs' @@ -87,13 +86,6 @@ export class AplOperator { await decrypt() } const values = await hfValues({}, env.ENV_DIR) - - try { - await reconcileTeamSealedSecrets(values ?? {}, env.ENV_DIR) - } catch (e) { - this.d.warn(`Team SealedSecret reconcile failed (will retry): ${(e as Error).message}`) - } - await ensureTeamGitOpsDirectories(env.ENV_DIR, values ?? {}) await ensureManifestDirectories() diff --git a/values/external-secrets/external-secrets-raw.gotmpl b/values/external-secrets/external-secrets-raw.gotmpl index b0dfd2d5d5..9b768e7230 100644 --- a/values/external-secrets/external-secrets-raw.gotmpl +++ b/values/external-secrets/external-secrets-raw.gotmpl @@ -14,6 +14,14 @@ resources: - apiGroups: [""] resources: ["secrets"] verbs: ["get", "list", "watch"] + - apiVersion: rbac.authorization.k8s.io/v1 + kind: ClusterRole + metadata: + name: eso-push-secret-writer + rules: + - apiGroups: [""] + resources: ["secrets"] + verbs: ["get", "create", "update", "patch", "delete"] - apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding metadata: diff --git a/values/k8s/k8s-raw-teams.gotmpl b/values/k8s/k8s-raw-teams.gotmpl index 433ee885ab..a8cf12e47f 100644 --- a/values/k8s/k8s-raw-teams.gotmpl +++ b/values/k8s/k8s-raw-teams.gotmpl @@ -31,4 +31,17 @@ resources: - name: harbor-pullsecret {{- end }} {{- end }} + - apiVersion: rbac.authorization.k8s.io/v1 + kind: RoleBinding + metadata: + name: eso-push-secret-writer + namespace: {{ $ns }} + subjects: + - kind: ServiceAccount + name: eso-store-sa + namespace: external-secrets + roleRef: + kind: ClusterRole + name: eso-push-secret-writer + apiGroup: rbac.authorization.k8s.io {{- end }} diff --git a/values/team-secrets/team-secrets-raw.gotmpl b/values/team-secrets/team-secrets-raw.gotmpl index b51a36a4ec..9f1db729c0 100644 --- a/values/team-secrets/team-secrets-raw.gotmpl +++ b/values/team-secrets/team-secrets-raw.gotmpl @@ -2,11 +2,286 @@ {{- $teamId := .Release.Labels.team }} {{- $tc := $v.teamConfig }} {{- $teamSettings := (index $tc $teamId).settings }} +{{- $teamNs := printf "team-%s" $teamId }} +{{- $teamReceivers := $teamSettings | get "alerts.receivers" ($v | get "alerts.receivers" (list "slack")) }} +{{- $hasReceivers := not (has "none" $teamReceivers) }} {{- $slackTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/slack.gotmpl") $v | toString }} {{- $opsgenieTpl := tpl (readFile "../../helmfile.d/snippets/alertmanager/opsgenie.gotmpl") $v | toString }} {{- $teamAlertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} resources: +# SecretStore in apl-secrets namespace — ESO uses this to push secrets to the team namespace +- apiVersion: external-secrets.io/v1 + kind: SecretStore + metadata: + name: {{ $teamNs }}-push-store + namespace: apl-secrets + spec: + provider: + kubernetes: + remoteNamespace: {{ $teamNs }} + server: + url: "https://kubernetes.default.svc" + caProvider: + type: ConfigMap + name: kube-root-ca.crt + namespace: external-secrets + key: ca.crt + auth: + serviceAccount: + name: eso-store-sa + namespace: external-secrets + +{{- if $teamSettings | get "managedMonitoring.grafana" false }} +# Grafana admin secret: composite in apl-secrets (literal admin-user + secret admin-password) +- apiVersion: external-secrets.io/v1 + kind: ExternalSecret + metadata: + name: {{ $teamNs }}-grafana-admin-composite + namespace: apl-secrets + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: {{ $teamNs }}-grafana-admin-composite + creationPolicy: Owner + template: + data: + admin-user: {{ $teamId }} + admin-password: '{{ "{{ .settings_password | toString }}" }}' + data: + - secretKey: settings_password + remoteRef: + key: team-{{ $teamId }}-settings-secrets + property: settings_password +- apiVersion: external-secrets.io/v1alpha1 + kind: PushSecret + metadata: + name: {{ $teamNs }}-grafana-admin-push + namespace: apl-secrets + spec: + refreshInterval: 1h + deletionPolicy: Delete + secretStoreRefs: + - name: {{ $teamNs }}-push-store + kind: SecretStore + selector: + secret: + name: {{ $teamNs }}-grafana-admin-composite + data: + - match: + secretKey: admin-user + remoteRef: + remoteKey: {{ $teamNs }}-grafana-admin + property: admin-user + - match: + secretKey: admin-password + remoteRef: + remoteKey: {{ $teamNs }}-grafana-admin + property: admin-password +# Grafana OIDC secret: composite in apl-secrets (literal client_id from values + secret client_secret) +- apiVersion: external-secrets.io/v1 + kind: ExternalSecret + metadata: + name: {{ $teamNs }}-grafana-oidc-composite + namespace: apl-secrets + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: {{ $teamNs }}-grafana-oidc-composite + creationPolicy: Owner + template: + data: + client_id: {{ $v.apps.keycloak.idp.clientID }} + client_secret: '{{ "{{ .idp_clientSecret | toString }}" }}' + data: + - secretKey: idp_clientSecret + remoteRef: + key: keycloak-secrets + property: idp_clientSecret +- apiVersion: external-secrets.io/v1alpha1 + kind: PushSecret + metadata: + name: {{ $teamNs }}-grafana-oidc-push + namespace: apl-secrets + spec: + refreshInterval: 1h + deletionPolicy: Delete + secretStoreRefs: + - name: {{ $teamNs }}-push-store + kind: SecretStore + selector: + secret: + name: {{ $teamNs }}-grafana-oidc-composite + data: + - match: + secretKey: client_id + remoteRef: + remoteKey: grafana-oidc-secret + property: client_id + - match: + secretKey: client_secret + remoteRef: + remoteKey: grafana-oidc-secret + property: client_secret +# Grafana Loki datasource: direct push from loki-secrets (no composite needed) +- apiVersion: external-secrets.io/v1alpha1 + kind: PushSecret + metadata: + name: {{ $teamNs }}-grafana-loki-push + namespace: apl-secrets + spec: + refreshInterval: 1h + deletionPolicy: Delete + secretStoreRefs: + - name: {{ $teamNs }}-push-store + kind: SecretStore + selector: + secret: + name: loki-secrets + data: + - match: + secretKey: adminPassword + remoteRef: + remoteKey: grafana-loki-datasource-secret + property: password +{{- end }}{{/* managedMonitoring.grafana */}} + +{{- if and ($teamSettings | get "managedMonitoring.alertmanager" false) $hasReceivers }} +# Alertmanager credentials: composite aggregates keys from multiple source secrets +- apiVersion: external-secrets.io/v1 + kind: ExternalSecret + metadata: + name: {{ $teamNs }}-alertmanager-composite + namespace: apl-secrets + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: {{ $teamNs }}-alertmanager-composite + creationPolicy: Owner + data: + {{- if has "slack" $teamReceivers }} + - secretKey: slackUrl + remoteRef: + key: alerts-secrets + property: slack_url + {{- end }} + {{- if has "email" $teamReceivers }} + - secretKey: smtpAuthPassword + remoteRef: + key: smtp-secrets + property: auth_password + - secretKey: smtpAuthSecret + remoteRef: + key: smtp-secrets + property: auth_secret + {{- end }} + {{- if has "opsgenie" $teamReceivers }} + - secretKey: opsgenieApiKey + remoteRef: + key: alerts-secrets + property: opsgenie_apiKey + {{- end }} +- apiVersion: external-secrets.io/v1alpha1 + kind: PushSecret + metadata: + name: {{ $teamNs }}-alertmanager-push + namespace: apl-secrets + spec: + refreshInterval: 1h + deletionPolicy: Delete + secretStoreRefs: + - name: {{ $teamNs }}-push-store + kind: SecretStore + selector: + secret: + name: {{ $teamNs }}-alertmanager-composite + data: + {{- if has "slack" $teamReceivers }} + - match: + secretKey: slackUrl + remoteRef: + remoteKey: alertmanager-credentials + property: slackUrl + {{- end }} + {{- if has "email" $teamReceivers }} + - match: + secretKey: smtpAuthPassword + remoteRef: + remoteKey: alertmanager-credentials + property: smtpAuthPassword + - match: + secretKey: smtpAuthSecret + remoteRef: + remoteKey: alertmanager-credentials + property: smtpAuthSecret + {{- end }} + {{- if has "opsgenie" $teamReceivers }} + - match: + secretKey: opsgenieApiKey + remoteRef: + remoteKey: alertmanager-credentials + property: opsgenieApiKey + {{- end }} +{{- end }}{{/* alertmanager + hasReceivers */}} + +{{- with $v.otomi | get "globalPullSecret" nil }} +{{- $gpsUsername := . | get "username" "" }} +{{- $gpsServer := . | get "server" "docker.io" }} +{{- $gpsEmail := . | get "email" "not@val.id" }} +# Global pull secret: composite with JSON-constructed dockerconfigjson +- apiVersion: external-secrets.io/v1 + kind: ExternalSecret + metadata: + name: {{ $teamNs }}-pullsecret-composite + namespace: apl-secrets + spec: + refreshInterval: 1h + secretStoreRef: + name: core-secrets-store + kind: ClusterSecretStore + target: + name: {{ $teamNs }}-pullsecret-composite + creationPolicy: Owner + template: + type: kubernetes.io/dockerconfigjson + data: + .dockerconfigjson: '{"auths":{"{{ $gpsServer }}":{"username":"{{ $gpsUsername }}","password":"{{ "{{ .globalPullSecret_password | toString }}" }}","email":"{{ $gpsEmail }}"}}}' + data: + - secretKey: globalPullSecret_password + remoteRef: + key: otomi-secrets + property: globalPullSecret_password +- apiVersion: external-secrets.io/v1alpha1 + kind: PushSecret + metadata: + name: {{ $teamNs }}-pullsecret-push + namespace: apl-secrets + spec: + refreshInterval: 1h + deletionPolicy: Delete + secretStoreRefs: + - name: {{ $teamNs }}-push-store + kind: SecretStore + selector: + secret: + name: {{ $teamNs }}-pullsecret-composite + data: + - match: + secretKey: .dockerconfigjson + remoteRef: + remoteKey: otomi-pullsecret-global + property: .dockerconfigjson +{{- end }}{{/* globalPullSecret */}} + {{- if $teamSettings | get "managedMonitoring.alertmanager" false }} - apiVersion: v1 kind: Secret From 2b7caf9f9c93a6fe701c51a807efd855584cfaca Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 15:11:36 +0200 Subject: [PATCH 07/10] feat: add PushSecret CustomResourceDefinition for external secrets management --- charts/external-secrets/crds/pushsecret.yaml | 638 +++++++++++++++++++ 1 file changed, 638 insertions(+) create mode 100644 charts/external-secrets/crds/pushsecret.yaml diff --git a/charts/external-secrets/crds/pushsecret.yaml b/charts/external-secrets/crds/pushsecret.yaml new file mode 100644 index 0000000000..1a0edfb786 --- /dev/null +++ b/charts/external-secrets/crds/pushsecret.yaml @@ -0,0 +1,638 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + annotations: + controller-gen.kubebuilder.io/version: v0.19.0 + labels: + external-secrets.io/component: controller + name: pushsecrets.external-secrets.io +spec: + group: external-secrets.io + names: + categories: + - external-secrets + kind: PushSecret + listKind: PushSecretList + plural: pushsecrets + shortNames: + - ps + singular: pushsecret + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .metadata.creationTimestamp + name: AGE + type: date + - jsonPath: .status.conditions[?(@.type=="Ready")].reason + name: Status + type: string + - jsonPath: .status.refreshTime + name: Last Sync + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: PushSecret is the Schema for the PushSecrets API that enables pushing Kubernetes secrets to external secret providers. + properties: + apiVersion: + description: |- + APIVersion defines the versioned schema of this representation of an object. + Servers should convert recognized schemas to the latest internal value, and + may reject unrecognized values. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources + type: string + kind: + description: |- + Kind is a string value representing the REST resource this object represents. + Servers may infer this from the endpoint the client submits requests to. + Cannot be updated. + In CamelCase. + More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds + type: string + metadata: + type: object + spec: + description: PushSecretSpec configures the behavior of the PushSecret. + properties: + data: + description: Secret Data that should be pushed to providers + items: + description: PushSecretData defines data to be pushed to the provider and associated metadata. + properties: + conversionStrategy: + default: None + description: Used to define a conversion Strategy for the secret keys + enum: + - None + - ReverseUnicode + type: string + match: + description: Match a given Secret Key to be pushed to the provider. + properties: + remoteRef: + description: Remote Refs to push to providers. + properties: + property: + description: Name of the property in the resulting secret + type: string + remoteKey: + description: Name of the resulting provider secret. + type: string + required: + - remoteKey + type: object + secretKey: + description: Secret Key to be pushed + type: string + required: + - remoteRef + type: object + metadata: + description: |- + Metadata is metadata attached to the secret. + The structure of metadata is provider specific, please look it up in the provider documentation. + x-kubernetes-preserve-unknown-fields: true + required: + - match + type: object + type: array + dataTo: + description: DataTo defines bulk push rules that expand source Secret keys into provider entries. + items: + description: PushSecretDataTo defines how to bulk-push secrets to providers without explicit per-key mappings. + properties: + conversionStrategy: + default: None + description: Used to define a conversion Strategy for the secret keys + enum: + - None + - ReverseUnicode + type: string + match: + description: |- + Match pattern for selecting keys from the source Secret. + If not specified, all keys are selected. + properties: + regexp: + description: |- + Regexp matches keys by regular expression. + If not specified, all keys are matched. + type: string + type: object + metadata: + description: |- + Metadata is metadata attached to the secret. + The structure of metadata is provider specific, please look it up in the provider documentation. + x-kubernetes-preserve-unknown-fields: true + remoteKey: + description: |- + RemoteKey is the name of the single provider secret that will receive ALL + matched keys bundled as a JSON object (e.g. {"DB_HOST":"...","DB_USER":"..."}). + When set, per-key expansion is skipped and a single push is performed. + The provider's store prefix (if any) is still prepended to this value. + When not set, each matched key is pushed as its own individual provider secret. + type: string + rewrite: + description: |- + Rewrite operations to transform keys before pushing to the provider. + Operations are applied sequentially. + items: + description: PushSecretRewrite defines how to transform secret keys before pushing. + properties: + regexp: + description: Used to rewrite with regular expressions. + properties: + source: + description: Used to define the regular expression of a re.Compiler. + type: string + target: + description: Used to define the target pattern of a ReplaceAll operation. + type: string + required: + - source + - target + type: object + transform: + description: Used to apply string transformation on the secrets. + properties: + template: + description: |- + Used to define the template to apply on the secret name. + `.value ` will specify the secret name in the template. + type: string + required: + - template + type: object + type: object + x-kubernetes-validations: + - message: exactly one of regexp or transform must be set + rule: (has(self.regexp) && !has(self.transform)) || (!has(self.regexp) && has(self.transform)) + type: array + storeRef: + description: StoreRef specifies which SecretStore to push to. Required. + properties: + kind: + default: SecretStore + description: Kind of the SecretStore resource (SecretStore or ClusterSecretStore) + enum: + - SecretStore + - ClusterSecretStore + type: string + labelSelector: + description: Optionally, sync to secret stores with label selector + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: Optionally, sync to the SecretStore of the given name + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + type: object + x-kubernetes-validations: + - message: storeRef must specify either name or labelSelector + rule: has(self.storeRef) && (has(self.storeRef.name) || has(self.storeRef.labelSelector)) + - message: 'remoteKey and rewrite are mutually exclusive: rewrite is only supported in per-key mode (without remoteKey)' + rule: '!has(self.remoteKey) || !has(self.rewrite) || size(self.rewrite) == 0' + type: array + deletionPolicy: + default: None + description: Deletion Policy to handle Secrets in the provider. + enum: + - Delete + - None + type: string + refreshInterval: + default: 1h0m0s + description: The Interval to which External Secrets will try to push a secret definition + type: string + secretStoreRefs: + items: + description: PushSecretStoreRef contains a reference on how to sync to a SecretStore. + properties: + kind: + default: SecretStore + description: Kind of the SecretStore resource (SecretStore or ClusterSecretStore) + enum: + - SecretStore + - ClusterSecretStore + type: string + labelSelector: + description: Optionally, sync to secret stores with label selector + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + name: + description: Optionally, sync to the SecretStore of the given name + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + type: object + type: array + selector: + description: The Secret Selector (k8s source) for the Push Secret + maxProperties: 1 + minProperties: 1 + properties: + generatorRef: + description: Point to a generator to create a Secret. + properties: + apiVersion: + default: generators.external-secrets.io/v1alpha1 + description: Specify the apiVersion of the generator resource + type: string + kind: + description: Specify the Kind of the generator resource + enum: + - ACRAccessToken + - ClusterGenerator + - CloudsmithAccessToken + - ECRAuthorizationToken + - Fake + - GCRAccessToken + - GithubAccessToken + - QuayAccessToken + - Password + - SSHKey + - STSSessionToken + - UUID + - VaultDynamicSecret + - Webhook + - Grafana + - MFA + type: string + name: + description: Specify the name of the generator resource + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - kind + - name + type: object + secret: + description: Select a Secret to Push. + properties: + name: + description: |- + Name of the Secret. + The Secret must exist in the same namespace as the PushSecret manifest. + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + selector: + description: Selector chooses secrets using a labelSelector. + properties: + matchExpressions: + description: matchExpressions is a list of label selector requirements. The requirements are ANDed. + items: + description: |- + A label selector requirement is a selector that contains values, a key, and an operator that + relates the key and values. + properties: + key: + description: key is the label key that the selector applies to. + type: string + operator: + description: |- + operator represents a key's relationship to a set of values. + Valid operators are In, NotIn, Exists and DoesNotExist. + type: string + values: + description: |- + values is an array of string values. If the operator is In or NotIn, + the values array must be non-empty. If the operator is Exists or DoesNotExist, + the values array must be empty. This array is replaced during a strategic + merge patch. + items: + type: string + type: array + x-kubernetes-list-type: atomic + required: + - key + - operator + type: object + type: array + x-kubernetes-list-type: atomic + matchLabels: + additionalProperties: + type: string + description: |- + matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, whose key field is "key", the + operator is "In", and the values array contains only "value". The requirements are ANDed. + type: object + type: object + x-kubernetes-map-type: atomic + type: object + type: object + template: + description: Template defines a blueprint for the created Secret resource. + properties: + data: + additionalProperties: + type: string + type: object + engineVersion: + default: v2 + description: |- + EngineVersion specifies the template engine version + that should be used to compile/execute the + template specified in .data and .templateFrom[]. + enum: + - v2 + type: string + mergePolicy: + default: Replace + description: TemplateMergePolicy defines how the rendered template should be merged with the existing Secret data. + enum: + - Replace + - Merge + type: string + metadata: + description: ExternalSecretTemplateMetadata defines metadata fields for the Secret blueprint. + properties: + annotations: + additionalProperties: + type: string + type: object + finalizers: + items: + type: string + type: array + labels: + additionalProperties: + type: string + type: object + type: object + templateFrom: + items: + description: |- + TemplateFrom specifies a source for templates. + Each item in the list can either reference a ConfigMap or a Secret resource. + properties: + configMap: + description: TemplateRef specifies a reference to either a ConfigMap or a Secret resource. + properties: + items: + description: A list of keys in the ConfigMap/Secret to use as templates for Secret data + items: + description: TemplateRefItem specifies a key in the ConfigMap/Secret to use as a template for Secret data. + properties: + key: + description: A key in the ConfigMap/Secret + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + templateAs: + default: Values + description: TemplateScope specifies how the template keys should be interpreted. + enum: + - Values + - KeysAndValues + type: string + required: + - key + type: object + type: array + name: + description: The name of the ConfigMap/Secret resource + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - items + - name + type: object + literal: + type: string + secret: + description: TemplateRef specifies a reference to either a ConfigMap or a Secret resource. + properties: + items: + description: A list of keys in the ConfigMap/Secret to use as templates for Secret data + items: + description: TemplateRefItem specifies a key in the ConfigMap/Secret to use as a template for Secret data. + properties: + key: + description: A key in the ConfigMap/Secret + maxLength: 253 + minLength: 1 + pattern: ^[-._a-zA-Z0-9]+$ + type: string + templateAs: + default: Values + description: TemplateScope specifies how the template keys should be interpreted. + enum: + - Values + - KeysAndValues + type: string + required: + - key + type: object + type: array + name: + description: The name of the ConfigMap/Secret resource + maxLength: 253 + minLength: 1 + pattern: ^[a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*$ + type: string + required: + - items + - name + type: object + target: + default: Data + description: |- + Target specifies where to place the template result. + For Secret resources, common values are: "Data", "Annotations", "Labels". + For custom resources (when spec.target.manifest is set), this supports + nested paths like "spec.database.config" or "data". + type: string + type: object + type: array + type: + type: string + type: object + updatePolicy: + default: Replace + description: UpdatePolicy to handle Secrets in the provider. + enum: + - Replace + - IfNotExists + type: string + required: + - secretStoreRefs + - selector + type: object + status: + description: PushSecretStatus indicates the history of the status of PushSecret. + properties: + conditions: + items: + description: PushSecretStatusCondition indicates the status of the PushSecret. + properties: + lastTransitionTime: + format: date-time + type: string + message: + type: string + reason: + type: string + status: + type: string + type: + description: PushSecretConditionType indicates the condition of the PushSecret. + type: string + required: + - status + - type + type: object + type: array + refreshTime: + description: |- + refreshTime is the time and date the external secret was fetched and + the target secret updated + format: date-time + nullable: true + type: string + syncedPushSecrets: + additionalProperties: + additionalProperties: + description: PushSecretData defines data to be pushed to the provider and associated metadata. + properties: + conversionStrategy: + default: None + description: Used to define a conversion Strategy for the secret keys + enum: + - None + - ReverseUnicode + type: string + match: + description: Match a given Secret Key to be pushed to the provider. + properties: + remoteRef: + description: Remote Refs to push to providers. + properties: + property: + description: Name of the property in the resulting secret + type: string + remoteKey: + description: Name of the resulting provider secret. + type: string + required: + - remoteKey + type: object + secretKey: + description: Secret Key to be pushed + type: string + required: + - remoteRef + type: object + metadata: + description: |- + Metadata is metadata attached to the secret. + The structure of metadata is provider specific, please look it up in the provider documentation. + x-kubernetes-preserve-unknown-fields: true + required: + - match + type: object + type: object + description: |- + Synced PushSecrets, including secrets that already exist in provider. + Matches secret stores to PushSecretData that was stored to that secret store. + type: object + syncedResourceVersion: + description: SyncedResourceVersion keeps track of the last synced version. + type: string + type: object + type: object + served: true + storage: true + subresources: + status: {} From 12c56656e256f33e86fd383209ec510819b9bacc Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 16:02:12 +0200 Subject: [PATCH 08/10] feat: remove optional secretType and namespace field --- src/common/sealed-secrets.ts | 4 +--- values/team-secrets/team-secrets-raw.gotmpl | 1 - 2 files changed, 1 insertion(+), 4 deletions(-) diff --git a/src/common/sealed-secrets.ts b/src/common/sealed-secrets.ts index e84a3c326a..d2618624f2 100644 --- a/src/common/sealed-secrets.ts +++ b/src/common/sealed-secrets.ts @@ -34,8 +34,6 @@ export interface SecretMapping { namespace: string secretName: string data: Record - /** Optional Kubernetes secret type. Defaults to 'kubernetes.io/opaque'. */ - secretType?: string } export interface SealedSecretManifest { @@ -326,7 +324,7 @@ export const createSealedSecretManifest = async ( template: { immutable: false, metadata: { name: mapping.secretName, namespace: mapping.namespace }, - type: mapping.secretType ?? 'kubernetes.io/opaque', + type: 'kubernetes.io/opaque', }, }, } diff --git a/values/team-secrets/team-secrets-raw.gotmpl b/values/team-secrets/team-secrets-raw.gotmpl index 9f1db729c0..9483029fe0 100644 --- a/values/team-secrets/team-secrets-raw.gotmpl +++ b/values/team-secrets/team-secrets-raw.gotmpl @@ -25,7 +25,6 @@ resources: caProvider: type: ConfigMap name: kube-root-ca.crt - namespace: external-secrets key: ca.crt auth: serviceAccount: From 87c182fe76a631309f7987907fd4d50c4fe109f9 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 16:19:49 +0200 Subject: [PATCH 09/10] feat: update SecretStore to ClusterSecretStore for team secret management --- values/team-secrets/team-secrets-raw.gotmpl | 21 +++++++++++++-------- 1 file changed, 13 insertions(+), 8 deletions(-) diff --git a/values/team-secrets/team-secrets-raw.gotmpl b/values/team-secrets/team-secrets-raw.gotmpl index 9483029fe0..d984a63b3c 100644 --- a/values/team-secrets/team-secrets-raw.gotmpl +++ b/values/team-secrets/team-secrets-raw.gotmpl @@ -10,13 +10,17 @@ {{- $teamAlertmanagerConfig := tpl (readFile "../../helmfile.d/snippets/alertmanager-teams.gotmpl") (dict "instance" $teamSettings "root" $v "slackTpl" $slackTpl "opsgenieTpl" $opsgenieTpl) | toString }} resources: -# SecretStore in apl-secrets namespace — ESO uses this to push secrets to the team namespace +# ClusterSecretStore per team — ESO uses this to push secrets to the team namespace. +# Restricted to apl-secrets namespace via conditions to prevent team namespaces from using it. - apiVersion: external-secrets.io/v1 - kind: SecretStore + kind: ClusterSecretStore metadata: name: {{ $teamNs }}-push-store - namespace: apl-secrets spec: + conditions: + - namespaceSelector: + matchLabels: + kubernetes.io/metadata.name: apl-secrets provider: kubernetes: remoteNamespace: {{ $teamNs }} @@ -25,6 +29,7 @@ resources: caProvider: type: ConfigMap name: kube-root-ca.crt + namespace: external-secrets key: ca.crt auth: serviceAccount: @@ -65,7 +70,7 @@ resources: deletionPolicy: Delete secretStoreRefs: - name: {{ $teamNs }}-push-store - kind: SecretStore + kind: ClusterSecretStore selector: secret: name: {{ $teamNs }}-grafana-admin-composite @@ -113,7 +118,7 @@ resources: deletionPolicy: Delete secretStoreRefs: - name: {{ $teamNs }}-push-store - kind: SecretStore + kind: ClusterSecretStore selector: secret: name: {{ $teamNs }}-grafana-oidc-composite @@ -139,7 +144,7 @@ resources: deletionPolicy: Delete secretStoreRefs: - name: {{ $teamNs }}-push-store - kind: SecretStore + kind: ClusterSecretStore selector: secret: name: loki-secrets @@ -199,7 +204,7 @@ resources: deletionPolicy: Delete secretStoreRefs: - name: {{ $teamNs }}-push-store - kind: SecretStore + kind: ClusterSecretStore selector: secret: name: {{ $teamNs }}-alertmanager-composite @@ -269,7 +274,7 @@ resources: deletionPolicy: Delete secretStoreRefs: - name: {{ $teamNs }}-push-store - kind: SecretStore + kind: ClusterSecretStore selector: secret: name: {{ $teamNs }}-pullsecret-composite From 8717c2d79279fa1aab4db2cd6859ed558ccdffc8 Mon Sep 17 00:00:00 2001 From: Ferruh Cihan <63190600+ferruhcihan@users.noreply.github.com> Date: Fri, 29 May 2026 16:29:43 +0200 Subject: [PATCH 10/10] feat: enable processPushSecret for external secrets management --- values/external-secrets/external-secrets.gotmpl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/values/external-secrets/external-secrets.gotmpl b/values/external-secrets/external-secrets.gotmpl index 33dee161c2..ebbfb3c650 100644 --- a/values/external-secrets/external-secrets.gotmpl +++ b/values/external-secrets/external-secrets.gotmpl @@ -21,4 +21,4 @@ installCRDs: false processClusterExternalSecret: false processClusterPushSecret: false processClusterGenerator: false -processPushSecret: false +processPushSecret: true