Update featuregates to read from featuregateaccess

Author: JoelSpeed

cmd/cluster-cloud-controller-manager-operator/main.go

@@ -17,6 +17,7 @@ limitations under the License.
 package main
 
 import (
+	"errors"
 	"flag"
 	"os"
 	"time"
@@ -25,6 +26,7 @@ import (
 
 	// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
 	// to ensure that exec-entrypoint and run can make use of them.
+	"k8s.io/client-go/kubernetes"
 	_ "k8s.io/client-go/plugin/pkg/client/auth"
 
 	"k8s.io/apimachinery/pkg/runtime"
@@ -39,6 +41,10 @@ import (
 
 	configv1 "github.com/openshift/api/config/v1"
 	operatorv1 "github.com/openshift/api/operator/v1"
+	configv1client "github.com/openshift/client-go/config/clientset/versioned"
+	configinformers "github.com/openshift/client-go/config/informers/externalversions"
+	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
+	"github.com/openshift/library-go/pkg/operator/events"
 
 	"github.com/openshift/cluster-cloud-controller-manager-operator/pkg/controllers"
 	"github.com/openshift/cluster-cloud-controller-manager-operator/pkg/restmapper"
@@ -111,6 +117,8 @@ func main() {
 		LeaseDuration: leaderElectionConfig.LeaseDuration,
 	})
 
+	ctx := ctrl.SetupSignalHandler()
+
 	syncPeriod := 10 * time.Minute
 	mgr, err := ctrl.NewManager(restConfig, ctrl.Options{
 		Namespace:          *managedNamespace,
@@ -137,15 +145,62 @@ func main() {
 		os.Exit(1)
 	}
 
+	// Setup for the feature gate accessor. This reads and monitors feature gates
+	// from the FeatureGate object status for the given version.
+	desiredVersion := controllers.GetReleaseVersion()
+	missingVersion := "0.0.1-snapshot"
+
+	configClient, err := configv1client.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		setupLog.Error(err, "unable to create config client")
+		os.Exit(1)
+	}
+	configInformers := configinformers.NewSharedInformerFactory(configClient, 10*time.Minute)
+
+	kubeClient, err := kubernetes.NewForConfig(mgr.GetConfig())
+	if err != nil {
+		setupLog.Error(err, "unable to create kube client")
+		os.Exit(1)
+	}
+
+	controllerRef, err := events.GetControllerReferenceForCurrentPod(ctx, kubeClient, *managedNamespace, nil)
+	if err != nil {
+		klog.Warningf("unable to get owner reference (falling back to namespace): %v", err)
+	}
+
+	recorder := events.NewKubeRecorder(kubeClient.CoreV1().Events(*managedNamespace), "cloud-controller-manager-operator", controllerRef)
+	featureGateAccessor := featuregates.NewFeatureGateAccess(
+		desiredVersion, missingVersion,
+		configInformers.Config().V1().ClusterVersions(), configInformers.Config().V1().FeatureGates(),
+		recorder,
+	)
+
+	featureGateAccessor.SetChangeHandler(func(featureChange featuregates.FeatureChange) {
+		// Do nothing here. The controller watches feature gate changes and will react to them.
+		klog.InfoS("FeatureGates changed", "enabled", featureChange.New.Enabled, "disabled", featureChange.New.Disabled)
+	})
+
+	go featureGateAccessor.Run(ctx)
+	go configInformers.Start(ctx.Done())
+
+	select {
+	case <-featureGateAccessor.InitialFeatureGatesObserved():
+		enabled, disabled, _ := featureGateAccessor.CurrentFeatureGates()
+		setupLog.Info("FeatureGates initialized", "enabled", enabled, "disabled", disabled)
+	case <-time.After(1 * time.Minute):
+		setupLog.Error(errors.New("timed out waiting for FeatureGate detection"), "unable to start manager")
+	}
+
 	if err = (&controllers.CloudOperatorReconciler{
 		ClusterOperatorStatusClient: controllers.ClusterOperatorStatusClient{
 			Client:           mgr.GetClient(),
 			Recorder:         mgr.GetEventRecorderFor("cloud-controller-manager-operator"),
 			ReleaseVersion:   controllers.GetReleaseVersion(),
 			ManagedNamespace: *managedNamespace,
 		},
-		Scheme:     mgr.GetScheme(),
-		ImagesFile: *imagesFile,
+		Scheme:            mgr.GetScheme(),
+		ImagesFile:        *imagesFile,
+		FeatureGateAccess: featureGateAccessor,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "ClusterOperator")
 		os.Exit(1)
@@ -162,7 +217,7 @@ func main() {
 	}
 
 	setupLog.Info("starting manager")
-	if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil {
+	if err := mgr.Start(ctx); err != nil {
 		setupLog.Error(err, "problem running manager")
 		os.Exit(1)
 	}

pkg/controllers/clusteroperator_controller.go

@@ -23,6 +23,7 @@ import (
 	configv1 "github.com/openshift/api/config/v1"
 	operatorv1 "github.com/openshift/api/operator/v1"
 	"github.com/openshift/library-go/pkg/cloudprovider"
+	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -49,9 +50,10 @@ const (
 // CloudOperatorReconciler reconciles a ClusterOperator object
 type CloudOperatorReconciler struct {
 	ClusterOperatorStatusClient
-	Scheme     *runtime.Scheme
-	watcher    ObjectWatcher
-	ImagesFile string
+	Scheme            *runtime.Scheme
+	watcher           ObjectWatcher
+	ImagesFile        string
+	FeatureGateAccess featuregates.FeatureGateAccess
 }
 
 // +kubebuilder:rbac:groups=config.openshift.io,resources=clusteroperators,verbs=get;list;watch;create;update;patch;delete
@@ -234,27 +236,8 @@ func (r *CloudOperatorReconciler) provisioningAllowed(ctx context.Context, infra
 		return true, nil
 	}
 
-	featureGate := &configv1.FeatureGate{}
-	if err := r.Get(ctx, client.ObjectKey{Name: externalFeatureGateName}, featureGate); errors.IsNotFound(err) {
-		klog.Infof("FeatureGate cluster does not exist. Skipping...")
-
-		if err := r.setStatusAvailable(ctx); err != nil {
-			klog.Errorf("Unable to sync cluster operator status: %s", err)
-			return false, err
-		}
-		return false, nil
-	} else if err != nil {
-		klog.Errorf("Unable to retrive FeatureGate object: %v", err)
-
-		if err := r.setStatusDegraded(ctx, err); err != nil {
-			klog.Errorf("Error syncing ClusterOperatorStatus: %v", err)
-			return false, fmt.Errorf("error syncing ClusterOperatorStatus: %v", err)
-		}
-		return false, err
-	}
-
 	// Verify FeatureGate ExternalCloudProvider is enabled for operator to work in TP phase
-	external, err := cloudprovider.IsCloudProviderExternal(infra.Status.PlatformStatus, featureGate)
+	external, err := cloudprovider.IsCloudProviderExternal(infra.Status.PlatformStatus, r.FeatureGateAccess)
 	if err != nil {
 		klog.Errorf("Could not determine external cloud provider state: %v", err)
 

pkg/controllers/clusteroperator_controller_test.go

@@ -11,8 +11,8 @@ import (
 	. "github.com/onsi/gomega"
 	configv1 "github.com/openshift/api/config/v1"
 	operatorv1 "github.com/openshift/api/operator/v1"
-	"github.com/openshift/library-go/pkg/cloudprovider"
 	"github.com/openshift/library-go/pkg/config/clusteroperator/v1helpers"
+	"github.com/openshift/library-go/pkg/operator/configobserver/featuregates"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -221,21 +221,16 @@ func constructKeyForWatchedObject(object client.Object, scheme *runtime.Scheme)
 
 var _ = Describe("Component sync controller", func() {
 	var infra *configv1.Infrastructure
-	var fg *configv1.FeatureGate
 	var kcm *operatorv1.KubeControllerManager
 	var co *configv1.ClusterOperator
 	var operatorController *CloudOperatorReconciler
 	var operands []client.Object
 	var watcher mockedWatcher
 
-	externalFeatureGateSpec := &configv1.FeatureGateSpec{
-		FeatureGateSelection: configv1.FeatureGateSelection{
-			FeatureSet: configv1.CustomNoUpgrade,
-			CustomNoUpgrade: &configv1.CustomFeatureGates{
-				Enabled: []configv1.FeatureGateName{cloudprovider.ExternalCloudProviderFeature},
-			},
-		},
-	}
+	externalFeatureGateAccessor := featuregates.NewHardcodedFeatureGateAccess(
+		[]configv1.FeatureGateName{configv1.FeatureGateExternalCloudProvider},
+		nil,
+	)
 
 	kcmStatus := &operatorv1.KubeControllerManagerStatus{
 		StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{
@@ -299,9 +294,6 @@ var _ = Describe("Component sync controller", func() {
 		infra = &configv1.Infrastructure{}
 		infra.SetName(infrastructureResourceName)
 
-		fg = &configv1.FeatureGate{}
-		fg.SetName(externalFeatureGateName)
-
 		kcm = &operatorv1.KubeControllerManager{
 			ObjectMeta: metav1.ObjectMeta{
 				Name: kcmResourceName,
@@ -326,29 +318,27 @@ var _ = Describe("Component sync controller", func() {
 				ManagedNamespace: testManagedNamespace,
 				Recorder:         record.NewFakeRecorder(32),
 			},
-			Scheme:     scheme.Scheme,
-			watcher:    w,
-			ImagesFile: testImagesFilePath,
+			Scheme:            scheme.Scheme,
+			watcher:           w,
+			ImagesFile:        testImagesFilePath,
+			FeatureGateAccess: featuregates.NewHardcodedFeatureGateAccess(nil, nil),
 		}
 		originalWatcher, _ := w.(*objectWatcher)
 		watcher = mockedWatcher{watcher: originalWatcher}
 
 		Expect(cl.Create(context.Background(), infra.DeepCopy())).To(Succeed())
-		Expect(cl.Create(context.Background(), fg.DeepCopy())).To(Succeed())
 		Expect(cl.Create(context.Background(), kcm.DeepCopy())).To(Succeed())
 		Expect(cl.Create(context.Background(), co.DeepCopy())).To(Succeed())
 	})
 
 	AfterEach(func() {
 		Expect(cl.Delete(context.Background(), infra.DeepCopy())).To(Succeed())
-		Expect(cl.Delete(context.Background(), fg.DeepCopy())).To(Succeed())
 		Expect(cl.Delete(context.Background(), kcm.DeepCopy())).To(Succeed())
 		Expect(cl.Delete(context.Background(), co.DeepCopy())).To(Succeed())
 
 		Eventually(func() bool {
 			return apierrors.IsNotFound(cl.Get(context.Background(), client.ObjectKeyFromObject(infra), infra.DeepCopy())) &&
-				apierrors.IsNotFound(cl.Get(context.Background(), client.ObjectKeyFromObject(fg), fg.DeepCopy())) &&
-				apierrors.IsNotFound(cl.Get(context.Background(), client.ObjectKeyFromObject(co), fg.DeepCopy())) &&
+				apierrors.IsNotFound(cl.Get(context.Background(), client.ObjectKeyFromObject(co), co.DeepCopy())) &&
 				apierrors.IsNotFound(cl.Get(context.Background(), client.ObjectKeyFromObject(kcm), kcm.DeepCopy()))
 		}, timeout).Should(BeTrue())
 
@@ -363,7 +353,7 @@ var _ = Describe("Component sync controller", func() {
 
 	type testCase struct {
 		status            *configv1.InfrastructureStatus
-		featureGateSpec   *configv1.FeatureGateSpec
+		featureGate       featuregates.FeatureGateAccess
 		kcmStatus         *operatorv1.KubeControllerManagerStatus
 		coStatus          *configv1.ClusterOperatorStatus
 		expectProvisioned bool
@@ -376,10 +366,8 @@ var _ = Describe("Component sync controller", func() {
 			infra.Status = *tc.status
 			Expect(cl.Status().Update(context.Background(), infra.DeepCopy())).To(Succeed())
 
-			if tc.featureGateSpec != nil {
-				Expect(cl.Get(context.Background(), client.ObjectKeyFromObject(fg), fg)).To(Succeed())
-				fg.Spec = *tc.featureGateSpec
-				Expect(cl.Update(context.Background(), fg.DeepCopy())).To(Succeed())
+			if tc.featureGate != nil {
+				operatorController.FeatureGateAccess = tc.featureGate
 			}
 
 			if tc.kcmStatus != nil {
@@ -454,7 +442,7 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.AWSPlatformType,
 				},
 			},
-			featureGateSpec:   externalFeatureGateSpec,
+			featureGate:       externalFeatureGateAccessor,
 			kcmStatus:         kcmStatus,
 			coStatus:          coStatus,
 			expectProvisioned: true,
@@ -468,7 +456,7 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.OpenStackPlatformType,
 				},
 			},
-			featureGateSpec:   externalFeatureGateSpec,
+			featureGate:       externalFeatureGateAccessor,
 			kcmStatus:         kcmStatus,
 			coStatus:          coStatus,
 			expectProvisioned: true,
@@ -482,7 +470,7 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.AWSPlatformType,
 				},
 			},
-			featureGateSpec:   externalFeatureGateSpec,
+			featureGate:       externalFeatureGateAccessor,
 			kcmStatus:         &operatorv1.KubeControllerManagerStatus{},
 			coStatus:          coStatus,
 			expectProvisioned: true,
@@ -496,7 +484,7 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.KubevirtPlatformType,
 				},
 			},
-			featureGateSpec:   externalFeatureGateSpec,
+			featureGate:       externalFeatureGateAccessor,
 			kcmStatus:         kcmStatus,
 			coStatus:          coStatus,
 			expectProvisioned: false,
@@ -534,7 +522,7 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.AWSPlatformType,
 				},
 			},
-			featureGateSpec: externalFeatureGateSpec,
+			featureGate: externalFeatureGateAccessor,
 			kcmStatus: &operatorv1.KubeControllerManagerStatus{
 				StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{
 					OperatorStatus: operatorv1.OperatorStatus{
@@ -560,7 +548,7 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.AWSPlatformType,
 				},
 			},
-			featureGateSpec: externalFeatureGateSpec,
+			featureGate: externalFeatureGateAccessor,
 			kcmStatus: &operatorv1.KubeControllerManagerStatus{
 				StaticPodOperatorStatus: operatorv1.StaticPodOperatorStatus{
 					OperatorStatus: operatorv1.OperatorStatus{
@@ -586,8 +574,8 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.AWSPlatformType,
 				},
 			},
-			featureGateSpec: externalFeatureGateSpec,
-			kcmStatus:       kcmStatus,
+			featureGate: externalFeatureGateAccessor,
+			kcmStatus:   kcmStatus,
 			coStatus: &configv1.ClusterOperatorStatus{
 				Conditions: []configv1.ClusterOperatorStatusCondition{
 					{
@@ -624,8 +612,8 @@ var _ = Describe("Component sync controller", func() {
 					Type: configv1.AWSPlatformType,
 				},
 			},
-			featureGateSpec: externalFeatureGateSpec,
-			kcmStatus:       kcmStatus,
+			featureGate: externalFeatureGateAccessor,
+			kcmStatus:   kcmStatus,
 			coStatus: &configv1.ClusterOperatorStatus{
 				Conditions: []configv1.ClusterOperatorStatusCondition{
 					{