enable HA hub agent

Wei Weng · Wei Weng · commit c7cb9b4a2198 · 2025-12-15T15:59:59.000Z
Signed-off-by: Wei Weng &lt;Wei.Weng@microsoft.com&gt;
diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml
@@ -14,6 +14,7 @@ on:
 
 env:
   GO_VERSION: '1.24.9'
+  CERT_MANAGER_VERSION: 'v1.16.2'
 
 jobs:
   detect-noop:
@@ -125,6 +126,7 @@ jobs:
           PROPERTY_PROVIDER: 'azure'
           RESOURCE_SNAPSHOT_CREATION_MINIMUM_INTERVAL: ${{ matrix.resource-snapshot-creation-minimum-interval }}
           RESOURCE_CHANGES_COLLECTION_DURATION: ${{ matrix.resource-changes-collection-duration }}
+          CERT_MANAGER_VERSION: ${{ env.CERT_MANAGER_VERSION }}
       
       - name: Collect logs
         if: always()
diff --git a/charts/hub-agent/README.md b/charts/hub-agent/README.md
@@ -2,11 +2,33 @@
 
 ## Install Chart
 
+### Default Installation (Self-Signed Certificates)
+
 ```console
 # Helm install with fleet-system namespace already created
 helm install hub-agent ./charts/hub-agent/
 ```
 
+### Installation with cert-manager
+
+When using cert-manager for certificate management, install cert-manager as a prerequisite first:
+
+```console
+# Install cert-manager
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+helm install cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --create-namespace \
+  --version v1.16.2 \
+  --set crds.enabled=true
+
+# Then install hub-agent with cert-manager enabled
+helm install hub-agent ./charts/hub-agent --set useCertManager=true
+```
+
+This configures cert-manager to manage webhook certificates.
+
 ## Upgrade Chart
 
 ```console
@@ -32,6 +54,11 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen
 | `affinity`                                | Node affinity for hub-agent pods                                                           | `{}`                                             |
 | `tolerations`                             | Tolerations for hub-agent pods                                                             | `[]`                                             |
 | `logVerbosity`                            | Log level (klog V logs)                                                                    | `5`                                              |
+| `enableWebhook`                           | Enable webhook server                                                                      | `true`                                           |
+| `webhookServiceName`                      | Webhook service name                                                                       | `fleetwebhook`                                   |
+| `enableGuardRail`                         | Enable guard rail webhook configurations                                                   | `true`                                           |
+| `webhookClientConnectionType`             | Connection type for webhook client (service or url)                                        | `service`                                        |
+| `useCertManager`                          | Use cert-manager for webhook certificate management                                        | `false`                                          |
 | `enableV1Beta1APIs`                       | Watch for v1beta1 APIs                                                                     | `true`                                           |
 | `hubAPIQPS`                               | QPS for fleet-apiserver (not including events/node heartbeat)                              | `250`                                            |
 | `hubAPIBurst`                             | Burst for fleet-apiserver (not including events/node heartbeat)                            | `1000`                                           |
@@ -41,4 +68,38 @@ _See [helm install](https://helm.sh/docs/helm/helm_install/) for command documen
 | `MaxFleetSizeSupported`                   | Max number of member clusters supported                                                    | `100`                                            |
 | `resourceSnapshotCreationMinimumInterval` | The minimum interval at which resource snapshots could be created.                         | `30s`                                            |
 | `resourceChangesCollectionDuration`       | The duration for collecting resource changes into one snapshot.                            | `15s`                                            |
-| `enableWorkload`                          | Enable kubernetes builtin workload to run in hub cluster.                           | `false`                                          |
+| `enableWorkload`                          | Enable kubernetes builtin workload to run in hub cluster.                                  | `false`                                          |
+
+## Certificate Management
+
+The hub-agent supports two modes for webhook certificate management:
+
+### Automatic Certificate Generation (Default)
+
+By default, the hub-agent generates certificates automatically at startup. This mode:
+- Requires no external dependencies
+- Works out of the box
+- Certificates are valid for 10 years
+
+### cert-manager (Optional)
+
+When `useCertManager=true`, certificates are managed by cert-manager. This mode:
+- Requires cert-manager to be installed as a prerequisite
+- Handles certificate rotation automatically (90-day certificates)
+- Follows industry-standard certificate management practices
+- Suitable for production environments
+
+To switch to cert-manager mode:
+```console
+# Install cert-manager first
+helm repo add jetstack https://charts.jetstack.io
+helm repo update
+helm install cert-manager jetstack/cert-manager \
+  --namespace cert-manager \
+  --create-namespace \
+  --version v1.16.2 \
+  --set crds.enabled=true
+
+# Then install hub-agent with cert-manager enabled
+helm install hub-agent ./charts/hub-agent --set useCertManager=true
+```
diff --git a/charts/hub-agent/templates/certificate.yaml b/charts/hub-agent/templates/certificate.yaml
@@ -0,0 +1,62 @@
+{{- if and .Values.enableWebhook .Values.useCertManager }}
+---
+apiVersion: cert-manager.io/v1
+kind: Certificate
+metadata:
+  name: fleet-webhook-server-cert
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- include "hub-agent.labels" . | nindent 4 }}
+spec:
+  # Secret name where cert-manager will store the certificate
+  secretName: fleet-webhook-server-cert
+  
+  # Certificate duration (90 days is cert-manager's default and recommended)
+  duration: 2160h # 90 days
+  
+  # Renew certificate 30 days before expiry
+  renewBefore: 720h # 30 days
+  
+  # Subject configuration
+  subject:
+    organizations:
+      - KubeFleet
+  
+  # Common name
+  commonName: fleet-webhook.{{ .Values.namespace }}.svc
+  
+  # DNS names for the certificate
+  dnsNames:
+    - {{ .Values.webhookServiceName }}
+    - {{ .Values.webhookServiceName }}.{{ .Values.namespace }}
+    - {{ .Values.webhookServiceName }}.{{ .Values.namespace }}.svc
+    - {{ .Values.webhookServiceName }}.{{ .Values.namespace }}.svc.cluster.local
+  
+  # Issuer reference - using self-signed issuer
+  issuerRef:
+    name: fleet-selfsigned-issuer
+    kind: Issuer
+    group: cert-manager.io
+  
+  # Private key configuration
+  privateKey:
+    algorithm: ECDSA
+    size: 256
+  
+  # Key usages
+  usages:
+    - digital signature
+    - key encipherment
+    - server auth
+---
+# Self-signed issuer for generating the certificate
+apiVersion: cert-manager.io/v1
+kind: Issuer
+metadata:
+  name: fleet-selfsigned-issuer
+  namespace: {{ .Values.namespace }}
+  labels:
+    {{- include "hub-agent.labels" . | nindent 4 }}
+spec:
+  selfSigned: {}
+{{- end }}
diff --git a/charts/hub-agent/templates/deployment.yaml b/charts/hub-agent/templates/deployment.yaml
@@ -6,6 +6,7 @@ metadata:
   labels:
     {{- include "hub-agent.labels" . | nindent 4 }}
 spec:
+  replicas: {{ .Values.replicaCount }}
   selector:
     matchLabels:
       {{- include "hub-agent.selectorLabels" . | nindent 6 }}
@@ -25,6 +26,7 @@ spec:
             - --webhook-service-name={{ .Values.webhookServiceName }}
             - --enable-guard-rail={{ .Values.enableGuardRail }}
             - --enable-workload={{ .Values.enableWorkload }}
+            - --use-cert-manager={{ .Values.useCertManager }}
             - --whitelisted-users=system:serviceaccount:fleet-system:hub-agent-sa
             - --webhook-client-connection-type={{.Values.webhookClientConnectionType}}
             - --v={{ .Values.logVerbosity }}
@@ -73,6 +75,19 @@ spec:
                 fieldPath: metadata.namespace
           resources:
             {{- toYaml .Values.resources | nindent 12 }}
+          {{- if .Values.useCertManager }}
+          volumeMounts:
+          - name: webhook-cert
+            mountPath: /tmp/k8s-webhook-server/serving-certs
+            readOnly: true
+          {{- end }}
+      {{- if .Values.useCertManager }}
+      volumes:
+      - name: webhook-cert
+        secret:
+          secretName: fleet-webhook-server-cert
+          defaultMode: 0644
+      {{- end }}
       {{- with .Values.affinity }}
       affinity:
         {{- toYaml . | nindent 8 }}
diff --git a/charts/hub-agent/values.yaml b/charts/hub-agent/values.yaml
@@ -17,13 +17,17 @@ webhookServiceName: fleetwebhook
 enableGuardRail: true
 webhookClientConnectionType: service
 enableWorkload: false
+# useCertManager enables cert-manager for webhook certificate management
+# When enabled, cert-manager will be installed as a dependency
+# and a Certificate resource will be created
+useCertManager: false
+
 forceDeleteWaitTime: 15m0s
 clusterUnhealthyThreshold: 3m0s
 resourceSnapshotCreationMinimumInterval: 30s
 resourceChangesCollectionDuration: 15s
 
-namespace:
-  fleet-system
+namespace: fleet-system
 
 resources:
   limits:
diff --git a/cmd/hubagent/main.go b/cmd/hubagent/main.go
@@ -159,7 +159,7 @@ func main() {
 	if opts.EnableWebhook {
 		whiteListedUsers := strings.Split(opts.WhiteListedUsers, ",")
 		if err := SetupWebhook(mgr, options.WebhookClientConnectionType(opts.WebhookClientConnectionType), opts.WebhookServiceName, whiteListedUsers,
-			opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels, opts.EnableWorkload, opts.NetworkingAgentsEnabled); err != nil {
+			opts.EnableGuardRail, opts.EnableV1Beta1APIs, opts.DenyModifyMemberClusterLabels, opts.EnableWorkload, opts.NetworkingAgentsEnabled, opts.UseCertManager); err != nil {
 			klog.ErrorS(err, "unable to set up webhook")
 			exitWithErrorFunc()
 		}
@@ -202,9 +202,9 @@ func main() {
 
 // SetupWebhook generates the webhook cert and then set up the webhook configurator.
 func SetupWebhook(mgr manager.Manager, webhookClientConnectionType options.WebhookClientConnectionType, webhookServiceName string,
-	whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool, enableWorkload bool, networkingAgentsEnabled bool) error {
+	whiteListedUsers []string, enableGuardRail, isFleetV1Beta1API bool, denyModifyMemberClusterLabels bool, enableWorkload bool, networkingAgentsEnabled bool, useCertManager bool) error {
 	// Generate self-signed key and crt files in FleetWebhookCertDir for the webhook server to start.
-	w, err := webhook.NewWebhookConfig(mgr, webhookServiceName, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir, enableGuardRail, denyModifyMemberClusterLabels, enableWorkload)
+	w, err := webhook.NewWebhookConfig(mgr, webhookServiceName, FleetWebhookPort, &webhookClientConnectionType, FleetWebhookCertDir, enableGuardRail, denyModifyMemberClusterLabels, enableWorkload, useCertManager)
 	if err != nil {
 		klog.ErrorS(err, "fail to generate WebhookConfig")
 		return err
diff --git a/cmd/hubagent/options/options.go b/cmd/hubagent/options/options.go
@@ -110,6 +110,9 @@ type Options struct {
 	// EnableWorkload enables workload resources (pods and replicasets) to be created in the hub cluster.
 	// When set to true, the pod and replicaset validating webhooks are disabled.
 	EnableWorkload bool
+	// UseCertManager indicates whether to use cert-manager for webhook certificate management.
+	// When enabled, webhook certificates are managed by cert-manager instead of self-signed generation.
+	UseCertManager bool
 	// ResourceSnapshotCreationMinimumInterval is the minimum interval at which resource snapshots could be created.
 	// Whether the resource snapshot is created or not depends on the both ResourceSnapshotCreationMinimumInterval and ResourceChangesCollectionDuration.
 	ResourceSnapshotCreationMinimumInterval time.Duration
@@ -185,6 +188,7 @@ func (o *Options) AddFlags(flags *flag.FlagSet) {
 	flags.IntVar(&o.PprofPort, "pprof-port", 6065, "The port for pprof profiling.")
 	flags.BoolVar(&o.DenyModifyMemberClusterLabels, "deny-modify-member-cluster-labels", false, "If set, users not in the system:masters cannot modify member cluster labels.")
 	flags.BoolVar(&o.EnableWorkload, "enable-workload", false, "If set, workloads (pods and replicasets) can be created in the hub cluster. This disables the pod and replicaset validating webhooks.")
+	flags.BoolVar(&o.UseCertManager, "use-cert-manager", false, "If set, cert-manager will be used for webhook certificate management instead of self-signed certificates.")
 	flags.DurationVar(&o.ResourceSnapshotCreationMinimumInterval, "resource-snapshot-creation-minimum-interval", 30*time.Second, "The minimum interval at which resource snapshots could be created.")
 	flags.DurationVar(&o.ResourceChangesCollectionDuration, "resource-changes-collection-duration", 15*time.Second,
 		"The duration for collecting resource changes into one snapshot. The default is 15 seconds, which means that the controller will collect resource changes for 15 seconds before creating a resource snapshot.")
diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go
@@ -70,6 +70,7 @@ import (
 const (
 	fleetWebhookCertFileName      = "tls.crt"
 	fleetWebhookKeyFileName       = "tls.key"
+	fleetWebhookCertSecretName    = "fleet-webhook-server-cert" //nolint:gosec // This is a Secret name, not a credential
 	fleetValidatingWebhookCfgName = "fleet-validating-webhook-configuration"
 	fleetGuardRailWebhookCfgName  = "fleet-guard-rail-webhook-configuration"
 	fleetMutatingWebhookCfgName   = "fleet-mutating-webhook-configuration"
@@ -162,9 +163,11 @@ type Config struct {
 
 	denyModifyMemberClusterLabels bool
 	enableWorkload                bool
+	// useCertManager indicates whether cert-manager is used for certificate management
+	useCertManager bool
 }
 
-func NewWebhookConfig(mgr manager.Manager, webhookServiceName string, port int32, clientConnectionType *options.WebhookClientConnectionType, certDir string, enableGuardRail bool, denyModifyMemberClusterLabels bool, enableWorkload bool) (*Config, error) {
+func NewWebhookConfig(mgr manager.Manager, webhookServiceName string, port int32, clientConnectionType *options.WebhookClientConnectionType, certDir string, enableGuardRail bool, denyModifyMemberClusterLabels bool, enableWorkload bool, useCertManager bool) (*Config, error) {
 	// We assume the Pod namespace should be passed to env through downward API in the Pod spec.
 	namespace := os.Getenv("POD_NAMESPACE")
 	if namespace == "" {
@@ -180,13 +183,29 @@ func NewWebhookConfig(mgr manager.Manager, webhookServiceName string, port int32
 		enableGuardRail:               enableGuardRail,
 		denyModifyMemberClusterLabels: denyModifyMemberClusterLabels,
 		enableWorkload:                enableWorkload,
+		useCertManager:                useCertManager,
 	}
-	caPEM, err := w.genCertificate(certDir)
-	if err != nil {
-		return nil, err
+
+	var caPEM []byte
+	var err error
+
+	if useCertManager {
+		// When using cert-manager, certificates are mounted as files by Kubernetes
+		// cert-manager creates tls.crt and tls.key, but we need ca.crt for the webhook config
+		caPEM, err = w.loadCertManagerCA(certDir)
+		if err != nil {
+			return nil, fmt.Errorf("failed to load cert-manager CA certificate: %w", err)
+		}
+	} else {
+		// Use self-signed certificate generation (original flow)
+		caPEM, err = w.genCertificate(certDir)
+		if err != nil {
+			return nil, err
+		}
 	}
+
 	w.caPEM = caPEM
-	return &w, err
+	return &w, nil
 }
 
 func (w *Config) Start(ctx context.Context) error {
@@ -216,12 +235,19 @@ func (w *Config) createFleetWebhookConfiguration(ctx context.Context) error {
 
 // createMutatingWebhookConfiguration creates the MutatingWebhookConfiguration object for the webhook.
 func (w *Config) createMutatingWebhookConfiguration(ctx context.Context, webhooks []admv1.MutatingWebhook, configName string) error {
+	annotations := map[string]string{}
+	if w.useCertManager {
+		// Tell cert-manager's CA injector to automatically inject the CA bundle
+		annotations["cert-manager.io/inject-ca-from"] = fmt.Sprintf("%s/%s", w.serviceNamespace, fleetWebhookCertSecretName)
+	}
+
 	mutatingWebhookConfig := admv1.MutatingWebhookConfiguration{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: configName,
 			Labels: map[string]string{
 				"admissions.enforcer/disabled": "true",
 			},
+			Annotations: annotations,
 		},
 		Webhooks: webhooks,
 	}
@@ -269,12 +295,19 @@ func (w *Config) buildFleetMutatingWebhooks() []admv1.MutatingWebhook {
 }
 
 func (w *Config) createValidatingWebhookConfiguration(ctx context.Context, webhooks []admv1.ValidatingWebhook, configName string) error {
+	annotations := map[string]string{}
+	if w.useCertManager {
+		// Tell cert-manager's CA injector to automatically inject the CA bundle
+		annotations["cert-manager.io/inject-ca-from"] = fmt.Sprintf("%s/%s", w.serviceNamespace, fleetWebhookCertSecretName)
+	}
+
 	validatingWebhookConfig := admv1.ValidatingWebhookConfiguration{
 		ObjectMeta: metav1.ObjectMeta{
 			Name: configName,
 			Labels: map[string]string{
 				"admissions.enforcer/disabled": "true",
 			},
+			Annotations: annotations,
 		},
 		Webhooks: webhooks,
 	}
@@ -659,6 +692,26 @@ func (w *Config) genCertificate(certDir string) ([]byte, error) {
 	return caPEM, nil
 }
 
+// loadCertManagerCA loads the CA certificate from the mounted cert-manager Secret.
+// When using cert-manager, Kubernetes mounts the Secret as files in the certDir.
+// cert-manager creates: ca.crt, tls.crt, and tls.key
+// The tls.crt and tls.key are automatically used by the webhook server.
+// We only need to read ca.crt for the webhook configuration's CABundle.
+func (w *Config) loadCertManagerCA(certDir string) ([]byte, error) {
+	caPath := filepath.Join(certDir, "ca.crt")
+	caCert, err := os.ReadFile(caPath)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read ca.crt from %s: %w", caPath, err)
+	}
+
+	if len(caCert) == 0 {
+		return nil, fmt.Errorf("ca.crt is empty at %s", caPath)
+	}
+
+	klog.V(2).InfoS("Successfully loaded CA certificate from cert-manager mounted Secret", "path", caPath)
+	return caCert, nil
+}
+
 // genSelfSignedCert generates the self signed Certificate/Key pair
 func (w *Config) genSelfSignedCert() (caPEMByte, certPEMByte, keyPEMByte []byte, err error) {
 	// CA config
diff --git a/pkg/webhook/webhook_test.go b/pkg/webhook/webhook_test.go
diff --git a/test/e2e/setup.sh b/test/e2e/setup.sh
