From 7a26a7cc8aebd4d528d63d4dd2a2bbb3483568e7 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Wed, 20 May 2026 06:59:49 -0400 Subject: [PATCH 1/3] kms: support deploying the vault mock plugin --- .../workload/sync_openshift_oauth_apiserver.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/pkg/operator/workload/sync_openshift_oauth_apiserver.go b/pkg/operator/workload/sync_openshift_oauth_apiserver.go index bc7213022..571c83900 100644 --- a/pkg/operator/workload/sync_openshift_oauth_apiserver.go +++ b/pkg/operator/workload/sync_openshift_oauth_apiserver.go @@ -24,7 +24,7 @@ import ( "github.com/openshift/library-go/pkg/controller/factory" libgoetcd "github.com/openshift/library-go/pkg/operator/configobserver/etcd" "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" - encryptionkms "github.com/openshift/library-go/pkg/operator/encryption/kms" + kmspluginlifecycle "github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle" "github.com/openshift/library-go/pkg/operator/events" "github.com/openshift/library-go/pkg/operator/resource/resourceapply" "github.com/openshift/library-go/pkg/operator/resource/resourcehash" @@ -315,8 +315,14 @@ func (c *OAuthAPIServerWorkload) syncStandardDeployment(ctx context.Context, ope } required.Spec.Replicas = masterNodeCount - if err := encryptionkms.AddKMSPluginVolumeAndMountToPodSpec(&required.Spec.Template.Spec, "oauth-apiserver", c.featureGateAccessor); err != nil { - return nil, fmt.Errorf("failed to add KMS encryption volumes: %w", err) + if err := kmspluginlifecycle.AddKMSPluginSidecarToPodSpec( + ctx, &required.Spec.Template.Spec, + required.Spec.Template.Name, + c.targetNamespace, + fmt.Sprintf("encryption-config-%d", operatorStatus.LatestAvailableRevision), + c.kubeClient.CoreV1(), + c.featureGateAccessor); err != nil { + return nil, fmt.Errorf("failed to add KMS plugin to pod spec: %w", err) } deployment, _, err := resourceapply.ApplyDeployment(ctx, c.kubeClient.AppsV1(), eventRecorder, required, resourcemerge.ExpectedDeploymentGeneration(required, operatorStatus.Generations)) From e3bc61e423d796de62c126a3d419c4b0d1d45747 Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Wed, 20 May 2026 11:41:49 -0400 Subject: [PATCH 2/3] bump(openshift/library-go): to get KMS plugin lifecycle changes --- .../encryption/kms/pluginlifecycle/sidecar.go | 184 ++++++++++++++++++ .../encryption/kms/pluginlifecycle/vault.go | 77 ++++++++ vendor/modules.txt | 1 + 3 files changed, 262 insertions(+) create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go create mode 100644 vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go new file mode 100644 index 000000000..1925cec3e --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/sidecar.go @@ -0,0 +1,184 @@ +package pluginlifecycle + +import ( + "context" + "fmt" + + configv1 "github.com/openshift/api/config/v1" + "github.com/openshift/api/features" + "github.com/openshift/library-go/pkg/operator/configobserver/featuregates" + "github.com/openshift/library-go/pkg/operator/encryption/encryptiondata" + corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + corev1client "k8s.io/client-go/kubernetes/typed/core/v1" + "k8s.io/klog/v2" +) + +// sidecarProvider abstracts the construction of a KMS plugin sidecar container for a specific provider (e.g. Vault). +type sidecarProvider interface { + // Name returns the identifier used to name the sidecar container and locate its volume mounts. + Name() string + // BuildSidecarContainer returns a fully configured sidecar container ready to be injected into the API server pod + BuildSidecarContainer() (corev1.Container, error) +} + +// newSidecarProvider creates a provider-specific SidecarProvider for the given keyID, UDS endpoint, and plugin configuration. +func newSidecarProvider(keyID string, udsPath string, pluginConfig configv1.KMSPluginConfig) (sidecarProvider, error) { + switch pluginConfig.Type { + case configv1.VaultKMSProvider: + return newVaultSidecarProvider("vault-kms-plugin", keyID, udsPath, pluginConfig) + default: + return nil, fmt.Errorf("unsupported KMS plugin configuration") + } +} + +// AddKMSPluginSidecarToPodSpec discovers KMS plugins from the encryption-config secret and injects a sidecar container for each one into the pod spec. +// It is a no-op when the KMSEncryption feature gate is not enabled or the encryption-config secret does not exist. +// It uses an uncached client to avoid injecting sidecars based on a stale encryption configuration. +func AddKMSPluginSidecarToPodSpec(ctx context.Context, podSpec *corev1.PodSpec, containerName string, encryptionConfigNamespace string, encryptionConfigSecretName string, secretClient corev1client.SecretsGetter, featureGateAccessor featuregates.FeatureGateAccess) error { + if podSpec == nil { + return fmt.Errorf("pod spec cannot be nil") + } + + if containerName == "" { + return fmt.Errorf("container name cannot be empty") + } + + if !featureGateAccessor.AreInitialFeatureGatesObserved() { + return nil + } + + featureGates, err := featureGateAccessor.CurrentFeatureGates() + if err != nil { + return fmt.Errorf("failed to get feature gates: %w", err) + } + + if !featureGates.Enabled(features.FeatureGateKMSEncryption) { + return nil + } + + encryptionConfigurationSecret, err := secretClient.Secrets(encryptionConfigNamespace).Get(ctx, encryptionConfigSecretName, metav1.GetOptions{}) + if apierrors.IsNotFound(err) { + klog.V(4).Infof("skipping KMS sidecar injection: %s/%s secret not found", encryptionConfigNamespace, encryptionConfigSecretName) + return nil + } + if err != nil { + return fmt.Errorf("failed to get %s/%s secret: %w", encryptionConfigNamespace, encryptionConfigSecretName, err) + } + + encryptionConfig, err := encryptiondata.FromSecret(encryptionConfigurationSecret) + if err != nil { + return fmt.Errorf("failed to extract encryption config from %s/%s secret: %w", encryptionConfigNamespace, encryptionConfigSecretName, err) + } + + kmsConfigurations, err := encryptiondata.ExtractUniqueAndSortedKMSConfigurations(encryptionConfig) + if err != nil { + return fmt.Errorf("failed to get KMS configurations: %w", err) + } + if len(kmsConfigurations) == 0 { + klog.V(4).Infof("skipping KMS sidecar injection: no KMS plugins found in EncryptionConfiguration") + return nil + } + + klog.V(4).Infof("injecting %d KMS sidecar(s)", len(kmsConfigurations)) + + for _, kmsConfiguration := range kmsConfigurations { + // ExtractUniqueAndSortedKMSConfigurations function rewrites the .Name field to include only the key ID + keyID := kmsConfiguration.Name + udsPath := kmsConfiguration.Endpoint + + pluginConfig, ok := encryptionConfig.KMSPlugins[keyID] + if !ok { + return fmt.Errorf("missing plugin config for keyID %s", keyID) + } + + sidecarProvider, err := newSidecarProvider(keyID, udsPath, pluginConfig) + if err != nil { + return fmt.Errorf("failed to create a sidecar provider for keyID %s: %w", keyID, err) + } + + if err := ensureSidecarContainer(podSpec, sidecarProvider); err != nil { + return err + } + + if err := ensureSocketVolumeMountInContainer(podSpec.InitContainers, sidecarProvider.Name()); err != nil { + return err + } + } + + if err := ensureSocketVolumeMountInContainer(podSpec.Containers, containerName); err != nil { + return err + } + + // The volume mount in the kube-apiserver and KMS plugin containers requires a volume in the podSpec + ensureSocketVolume(podSpec) + + return nil +} + +func ensureSidecarContainer(podSpec *corev1.PodSpec, provider sidecarProvider) error { + sidecar, err := provider.BuildSidecarContainer() + if err != nil { + return fmt.Errorf("failed to build sidecar container: %w", err) + } + + for i, container := range podSpec.InitContainers { + if container.Name == sidecar.Name { + podSpec.InitContainers[i] = sidecar + return nil + } + } + + podSpec.InitContainers = append(podSpec.InitContainers, sidecar) + return nil +} + +func ensureSocketVolumeMountInContainer(containers []corev1.Container, containerName string) error { + containerIndex := -1 + for i, container := range containers { + if container.Name == containerName { + containerIndex = i + break + } + } + + if containerIndex < 0 { + return fmt.Errorf("container %s not found", containerName) + } + + foundMount := false + container := &containers[containerIndex] + for _, m := range container.VolumeMounts { + if m.Name == "kms-plugin-socket" { + foundMount = true + break + } + } + if !foundMount { + container.VolumeMounts = append(container.VolumeMounts, + corev1.VolumeMount{ + Name: "kms-plugin-socket", + MountPath: "/var/run/kmsplugin", + }, + ) + } + return nil +} + +func ensureSocketVolume(podSpec *corev1.PodSpec) { + for _, volume := range podSpec.Volumes { + if volume.Name == "kms-plugin-socket" { + return + } + } + + podSpec.Volumes = append(podSpec.Volumes, + corev1.Volume{ + Name: "kms-plugin-socket", + VolumeSource: corev1.VolumeSource{ + EmptyDir: &corev1.EmptyDirVolumeSource{}, + }, + }, + ) +} diff --git a/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go new file mode 100644 index 000000000..937bdfd74 --- /dev/null +++ b/vendor/github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle/vault.go @@ -0,0 +1,77 @@ +package pluginlifecycle + +import ( + "fmt" + + configv1 "github.com/openshift/api/config/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/resource" + "k8s.io/utils/ptr" +) + +// newVaultSidecarProvider creates a Vault sidecar provider from the given KMS plugin configuration. +func newVaultSidecarProvider(name, keyID, udsPath string, pluginConfig configv1.KMSPluginConfig) (*vault, error) { + return &vault{ + name: name, + keyID: keyID, + udsPath: udsPath, + config: &pluginConfig.Vault, + }, nil +} + +// vault implements SidecarProvider for HashiCorp Vault KMS. +type vault struct { + name string + keyID string + udsPath string + config *configv1.VaultKMSPluginConfig +} + +// Name returns the sidecar name appended by the key id. +func (v *vault) Name() string { + return fmt.Sprintf("%s-%s", v.name, v.keyID) +} + +// BuildSidecarContainer returns a container spec for the Vault KMS plugin sidecar +// configured with the Vault address, namespace, transit mount, and transit key. +func (v *vault) BuildSidecarContainer() (corev1.Container, error) { + // Required API fields: always set. + args := []string{ + fmt.Sprintf("-listen-address=%s", v.udsPath), + fmt.Sprintf("-vault-address=%s", v.config.VaultAddress), + fmt.Sprintf("-transit-key=%s", v.config.TransitKey), + // TODO(bertinatto): dummy value for the Vault mock plugin; will come from the encryption-config secret. + fmt.Sprintf("-approle-role-id=dummy-role-id-%s", v.keyID), + // TODO(bertinatto): placeholder path for the Vault mock plugin; will differ per operator (KASO vs. aggregated apiserver operators). + fmt.Sprintf("-approle-secret-id-path=/var/run/secrets/vault-kms/secret-id-%s", v.keyID), + } + + // Optional fields: only pass non-empty values. + if v.config.VaultNamespace != "" { + args = append(args, fmt.Sprintf("-vault-namespace=%s", v.config.VaultNamespace)) + } + + if v.config.TransitMount != "" { + args = append(args, fmt.Sprintf("-transit-mount=%s", v.config.TransitMount)) + } + + return corev1.Container{ + Name: v.Name(), + Image: v.config.KMSPluginImage, + Args: args, + ImagePullPolicy: corev1.PullIfNotPresent, + // We place the container in InitContainers with RestartPolicyAlways so the kubelet starts it before + // regular containers and keeps it running for the pod's lifetime. + RestartPolicy: ptr.To(corev1.ContainerRestartPolicyAlways), + TerminationMessagePolicy: corev1.TerminationMessageFallbackToLogsOnError, + // TODO(bertinatto): the plugin sidecar needs to be measure under heavy load to figure out good defaults. + // For now follow what most sidecars in the kube-apiserver pod do. xref: + // https://github.com/openshift/cluster-kube-apiserver-operator/commit/e15a19cd2474c8b60ce17ac16dd8f422c729847a + Resources: corev1.ResourceRequirements{ + Requests: corev1.ResourceList{ + corev1.ResourceMemory: resource.MustParse("50Mi"), + corev1.ResourceCPU: resource.MustParse("5m"), + }, + }, + }, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 11670cbdf..58aa8d7d6 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -429,6 +429,7 @@ github.com/openshift/library-go/pkg/operator/encryption/deployer github.com/openshift/library-go/pkg/operator/encryption/encoding github.com/openshift/library-go/pkg/operator/encryption/encryptiondata github.com/openshift/library-go/pkg/operator/encryption/kms +github.com/openshift/library-go/pkg/operator/encryption/kms/pluginlifecycle github.com/openshift/library-go/pkg/operator/encryption/observer github.com/openshift/library-go/pkg/operator/encryption/secrets github.com/openshift/library-go/pkg/operator/encryption/state From 362db9b90ceae13306d7cb5da29d6081ce7fb7ba Mon Sep 17 00:00:00 2001 From: Fabio Bertinatto Date: Thu, 21 May 2026 05:02:22 -0400 Subject: [PATCH 3/3] kms: use the right container name --- pkg/operator/workload/sync_openshift_oauth_apiserver.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pkg/operator/workload/sync_openshift_oauth_apiserver.go b/pkg/operator/workload/sync_openshift_oauth_apiserver.go index 571c83900..ad22d5eca 100644 --- a/pkg/operator/workload/sync_openshift_oauth_apiserver.go +++ b/pkg/operator/workload/sync_openshift_oauth_apiserver.go @@ -316,8 +316,9 @@ func (c *OAuthAPIServerWorkload) syncStandardDeployment(ctx context.Context, ope required.Spec.Replicas = masterNodeCount if err := kmspluginlifecycle.AddKMSPluginSidecarToPodSpec( - ctx, &required.Spec.Template.Spec, - required.Spec.Template.Name, + ctx, + &required.Spec.Template.Spec, + "oauth-apiserver", c.targetNamespace, fmt.Sprintf("encryption-config-%d", operatorStatus.LatestAvailableRevision), c.kubeClient.CoreV1(),