Move InstanceHa Enabling/Disabling to own controller

The instance ha controller will eventually move out to the instance-ha service
itself. But for now, we keep it as a stand in to reflect the spec value
over the https api.

Author: fwiesel

api/v1/eviction_types.go

@@ -52,6 +52,9 @@ const (
 	// ConditionTypeHypervisorDisabled is the type of condition for hypervisor disabled status
 	ConditionTypeHypervisorDisabled = "HypervisorDisabled"
 
+	// ConditionTypeHaEnabled is the type of condition for signalling if HA is enabled / disabled for the hypervisor
+	ConditionTypeHaEnabled = "HaEnabled"
+
 	// ConditionTypeEvicting is the type of condition for eviction status
 	ConditionTypeEvicting = "Evicting"
 )

api/v1/hypervisor_types.go

@@ -72,6 +72,10 @@ const (
 	ConditionReasonTestAggregates     = "TestAggregates"
 	ConditionReasonTerminating        = "Terminating"
 	ConditionReasonEvictionInProgress = "EvictionInProgress"
+
+	// ConditionTypeHaEnabled reasons
+	ConditionReasonHaEvicted    = "Evicted"    // HA disabled due to eviction
+	ConditionReasonHaOnboarding = "Onboarding" // HA disabled during onboarding
 )
 
 // HypervisorSpec defines the desired state of Hypervisor

cmd/main.go

@@ -252,6 +252,13 @@ func main() {
 		setupLog.Error(err, "unable to create controller", "controller", "Eviction")
 		os.Exit(1)
 	}
+	if err = (&controller.HypervisorInstanceHaController{
+		Client: mgr.GetClient(),
+		Scheme: mgr.GetScheme(),
+	}).SetupWithManager(mgr); err != nil {
+		setupLog.Error(err, "unable to create controller", "controller", "HypervisorInstanceHa")
+		os.Exit(1)
+	}
 
 	if err = (&controller.NodeDecommissionReconciler{
 		Client: mgr.GetClient(),

internal/controller/gardener_node_lifecycle_controller.go

@@ -93,9 +93,9 @@ func (r *GardenerNodeLifecycleController) Reconcile(ctx context.Context, req ctr
 		minAvailable = 0
 
 		if onboardingCompleted && isTerminating(node) {
-			// Onboarded & terminating & eviction complete -> disable HA
-			if err := disableInstanceHA(hv); err != nil {
-				return ctrl.Result{}, err
+			// Wait for HypervisorInstanceHa controller to disable HA
+			if !meta.IsStatusConditionFalse(hv.Status.Conditions, kvmv1.ConditionTypeHaEnabled) {
+				return ctrl.Result{}, nil // Will be reconciled again when condition changes
 			}
 		}
 	}

internal/controller/hypervisor_instance_ha_controller.go

@@ -0,0 +1,156 @@
+/*
+SPDX-FileCopyrightText: Copyright 2024 SAP SE or an SAP affiliate company and cobaltcore-dev contributors
+SPDX-License-Identifier: Apache-2.0
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+	http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package controller
+
+import (
+	"context"
+	"errors"
+
+	"k8s.io/apimachinery/pkg/api/equality"
+	"k8s.io/apimachinery/pkg/api/meta"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	ctrl "sigs.k8s.io/controller-runtime"
+	k8sclient "sigs.k8s.io/controller-runtime/pkg/client"
+	logger "sigs.k8s.io/controller-runtime/pkg/log"
+
+	kvmv1 "github.com/cobaltcore-dev/openstack-hypervisor-operator/api/v1"
+)
+
+const (
+	HypervisorInstanceHaControllerName = "HypervisorInstanceHa"
+)
+
+type HypervisorInstanceHaController struct {
+	k8sclient.Client
+	Scheme *runtime.Scheme
+}
+
+// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors,verbs=get;list;watch;update;patch
+// +kubebuilder:rbac:groups=kvm.cloud.sap,resources=hypervisors/status,verbs=get;list;watch;update;patch
+func (r *HypervisorInstanceHaController) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) {
+	log := logger.FromContext(ctx).WithName(req.Name)
+	ctx = logger.IntoContext(ctx, log)
+
+	hv := &kvmv1.Hypervisor{}
+	if err := r.Get(ctx, req.NamespacedName, hv); err != nil {
+		// ignore not found errors, could be deleted
+		return ctrl.Result{}, k8sclient.IgnoreNotFound(err)
+	}
+
+	onboardingCondition := meta.FindStatusCondition(hv.Status.Conditions, kvmv1.ConditionTypeOnboarding)
+
+	if onboardingCondition == nil {
+		return ctrl.Result{}, nil // Onboarding not started, so no hypervisor and nothing to do
+	}
+
+	old := hv.DeepCopy()
+
+	// Determine if HA should be disabled and the reason
+	evicted := meta.IsStatusConditionFalse(hv.Status.Conditions, kvmv1.ConditionTypeEvicting)
+	testing := onboardingCondition.Status == metav1.ConditionTrue &&
+		onboardingCondition.Reason != kvmv1.ConditionReasonHandover // Onboarding still testing (not yet in Handover phase)
+	aborted := onboardingCondition.Status == metav1.ConditionFalse &&
+		onboardingCondition.Reason == kvmv1.ConditionReasonAborted // Onboarding was aborted
+	shouldDisable := !hv.Spec.HighAvailability || // HA not requested
+		evicted || // HA not needed as it is empty
+		testing || // HA not needed for test VMs
+		aborted // HA not needed when onboarding aborted
+
+	if shouldDisable {
+		// Determine the reason based on why HA is being disabled
+		var reason string
+		var message string
+		switch {
+		case evicted:
+			reason = kvmv1.ConditionReasonHaEvicted
+			message = "HA disabled due to eviction"
+		case testing, aborted:
+			reason = kvmv1.ConditionReasonHaOnboarding
+			message = "HA disabled before onboarding"
+		default:
+			reason = kvmv1.ConditionReasonSucceeded
+			message = "HA disabled per spec"
+		}
+
+		if !meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeHaEnabled,
+			Status:  metav1.ConditionFalse,
+			Message: message,
+			Reason:  reason,
+		}) {
+			// Desired state already achieved
+			return ctrl.Result{}, nil
+		}
+
+		if err := disableInstanceHA(hv); err != nil {
+			meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+				Type:    kvmv1.ConditionTypeHaEnabled,
+				Status:  metav1.ConditionUnknown,
+				Message: err.Error(),
+				Reason:  kvmv1.ConditionReasonFailed,
+			})
+
+			if patchErr := r.Status().Patch(ctx, hv, k8sclient.MergeFromWithOptions(old, k8sclient.MergeFromWithOptimisticLock{}), k8sclient.FieldOwner(HypervisorInstanceHaControllerName)); patchErr != nil {
+				return ctrl.Result{}, errors.Join(err, patchErr)
+			}
+			return ctrl.Result{}, err
+		}
+	} else {
+		if !meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+			Type:    kvmv1.ConditionTypeHaEnabled,
+			Status:  metav1.ConditionTrue,
+			Message: "HA is enabled",
+			Reason:  kvmv1.ConditionReasonSucceeded,
+		}) {
+			// Desired state already achieved
+			return ctrl.Result{}, nil
+		}
+
+		if err := enableInstanceHA(hv); err != nil {
+			meta.SetStatusCondition(&hv.Status.Conditions, metav1.Condition{
+				Type:    kvmv1.ConditionTypeHaEnabled,
+				Status:  metav1.ConditionUnknown,
+				Message: err.Error(),
+				Reason:  kvmv1.ConditionReasonFailed,
+			})
+
+			if patchErr := r.Status().Patch(ctx, hv, k8sclient.MergeFromWithOptions(old, k8sclient.MergeFromWithOptimisticLock{}), k8sclient.FieldOwner(HypervisorInstanceHaControllerName)); patchErr != nil {
+				return ctrl.Result{}, errors.Join(err, patchErr)
+			}
+			return ctrl.Result{}, err
+		}
+	}
+
+	if equality.Semantic.DeepEqual(hv, old) {
+		return ctrl.Result{}, nil
+	}
+
+	return ctrl.Result{}, r.Status().Patch(ctx, hv, k8sclient.MergeFromWithOptions(old, k8sclient.MergeFromWithOptimisticLock{}), k8sclient.FieldOwner(HypervisorInstanceHaControllerName))
+}
+
+// SetupWithManager sets up the controller with the Manager.
+func (r *HypervisorInstanceHaController) SetupWithManager(mgr ctrl.Manager) error {
+	ctx := context.Background()
+	_ = logger.FromContext(ctx)
+
+	return ctrl.NewControllerManagedBy(mgr).
+		Named(HypervisorInstanceHaControllerName).
+		For(&kvmv1.Hypervisor{}). // trigger the r.Reconcile whenever a hypervisor is created/updated/deleted.
+		Complete(r)
+}