From c341417dafd845645799333db1a7bcb0fa1de086 Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Tue, 16 Dec 2025 22:34:52 +0000 Subject: [PATCH 1/3] decouple informer cache population and event handling Signed-off-by: Wei Weng --- cmd/hubagent/workload/setup.go | 16 + pkg/resourcewatcher/change_dector.go | 44 +- pkg/resourcewatcher/change_detector_test.go | 200 ++++++++ pkg/resourcewatcher/informer_populator.go | 102 ++++ .../informer_populator_test.go | 437 ++++++++++++++++++ pkg/resourcewatcher/resource_collector.go | 48 +- .../resource_collector_test.go | 221 +++++++++ pkg/utils/informer/informermanager.go | 110 ++--- pkg/utils/informer/informermanager_test.go | 217 +++++++++ pkg/utils/informer/readiness/readiness.go | 2 +- test/utils/handler/handler.go | 47 ++ test/utils/informer/manager.go | 7 + 12 files changed, 1354 insertions(+), 97 deletions(-) create mode 100644 pkg/resourcewatcher/change_detector_test.go create mode 100644 pkg/resourcewatcher/informer_populator.go create mode 100644 pkg/resourcewatcher/informer_populator_test.go create mode 100644 pkg/resourcewatcher/resource_collector_test.go create mode 100644 test/utils/handler/handler.go diff --git a/cmd/hubagent/workload/setup.go b/cmd/hubagent/workload/setup.go index dbb47de4f..1397a2f1a 100644 --- a/cmd/hubagent/workload/setup.go +++ b/cmd/hubagent/workload/setup.go @@ -496,7 +496,23 @@ func SetupControllers(ctx context.Context, wg *sync.WaitGroup, mgr ctrl.Manager, } resourceChangeController := controller.NewController(resourceChangeControllerName, controller.ClusterWideKeyFunc, rcr.Reconcile, rateLimiter) + // Set up the InformerPopulator that runs on ALL pods (leader and followers) + // This ensures all pods have synced informer caches for webhook validation + klog.Info("Setting up informer populator") + informerPopulator := &resourcewatcher.InformerPopulator{ + DiscoveryClient: discoverClient, + RESTMapper: mgr.GetRESTMapper(), + InformerManager: dynamicInformerManager, + ResourceConfig: resourceConfig, + } + + if err := mgr.Add(informerPopulator); err != nil { + klog.ErrorS(err, "Failed to setup informer populator") + return err + } + // Set up a runner that starts all the custom controllers we created above + // This runs ONLY on the leader and adds event handlers to the informers created by InformerPopulator resourceChangeDetector := &resourcewatcher.ChangeDetector{ DiscoveryClient: discoverClient, RESTMapper: mgr.GetRESTMapper(), diff --git a/pkg/resourcewatcher/change_dector.go b/pkg/resourcewatcher/change_dector.go index 9a4011628..0dcfc41e6 100644 --- a/pkg/resourcewatcher/change_dector.go +++ b/pkg/resourcewatcher/change_dector.go @@ -23,7 +23,6 @@ import ( "golang.org/x/sync/errgroup" "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" - "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/discovery" "k8s.io/client-go/tools/cache" @@ -45,7 +44,7 @@ var ( // ChangeDetector is a resource watcher which watches all types of resources in the cluster and reconcile the events. type ChangeDetector struct { // DiscoveryClient is used to do resource discovery. - DiscoveryClient *discovery.DiscoveryClient + DiscoveryClient discovery.DiscoveryInterface // RESTMapper is used to convert between GVK and GVR RESTMapper meta.RESTMapper @@ -137,43 +136,20 @@ func (d *ChangeDetector) discoverAPIResourcesLoop(ctx context.Context, period ti }, period) } -// discoverResources goes through all the api resources in the cluster and create informers on selected types +// discoverResources goes through all the api resources in the cluster and adds event handlers to informers func (d *ChangeDetector) discoverResources(dynamicResourceEventHandler cache.ResourceEventHandler) { - newResources, err := d.getWatchableResources() - var dynamicResources []informer.APIResourceMeta - if err != nil { - klog.ErrorS(err, "Failed to get all the api resources from the cluster") - } - for _, res := range newResources { - // all the static resources are disabled by default - if d.shouldWatchResource(res.GroupVersionResource) { - dynamicResources = append(dynamicResources, res) - } + resourcesToWatch := discoverWatchableResources(d.DiscoveryClient, d.RESTMapper, d.ResourceConfig) + + // On the leader, add event handlers to informers that were already created by InformerPopulator + // The informers exist on all pods, but only the leader adds handlers and processes events + for _, res := range resourcesToWatch { + d.InformerManager.AddEventHandlerToInformer(res.GroupVersionResource, dynamicResourceEventHandler) } - d.InformerManager.AddDynamicResources(dynamicResources, dynamicResourceEventHandler, err == nil) + // this will start the newly added informers if there is any d.InformerManager.Start() -} - -// gvrDisabled returns whether GroupVersionResource is disabled. -func (d *ChangeDetector) shouldWatchResource(gvr schema.GroupVersionResource) bool { - // By default, all of the APIs are allowed. - if d.ResourceConfig == nil { - return true - } - gvks, err := d.RESTMapper.KindsFor(gvr) - if err != nil { - klog.ErrorS(err, "gvr transform failed", "gvr", gvr.String()) - return false - } - for _, gvk := range gvks { - if d.ResourceConfig.IsResourceDisabled(gvk) { - klog.V(4).InfoS("Skip watch resource", "group version kind", gvk.String()) - return false - } - } - return true + klog.V(2).InfoS("Change detector: discovered resources", "count", len(resourcesToWatch)) } // dynamicResourceFilter filters out resources that we don't want to watch diff --git a/pkg/resourcewatcher/change_detector_test.go b/pkg/resourcewatcher/change_detector_test.go new file mode 100644 index 000000000..e0f83e3bf --- /dev/null +++ b/pkg/resourcewatcher/change_detector_test.go @@ -0,0 +1,200 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcewatcher + +import ( + "testing" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/restmapper" + "k8s.io/client-go/tools/cache" + + "github.com/kubefleet-dev/kubefleet/pkg/utils" + testinformer "github.com/kubefleet-dev/kubefleet/test/utils/informer" +) + +func TestChangeDetector_discoverResources(t *testing.T) { + tests := []struct { + name string + discoveryResources []*metav1.APIResourceList + resourceConfig *utils.ResourceConfig + }{ + { + name: "discovers and adds handlers for watchable resources", + discoveryResources: []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + { + Name: "secrets", + Kind: "Secret", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + resourceConfig: nil, // Allow all resources + }, + { + name: "skips resources without list/watch verbs", + discoveryResources: []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"get", "delete"}, // Missing list/watch + }, + }, + }, + }, + resourceConfig: nil, + }, + { + name: "respects resource config filtering", + discoveryResources: []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + { + Name: "secrets", + Kind: "Secret", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + resourceConfig: func() *utils.ResourceConfig { + rc := utils.NewResourceConfig(false) // Skip mode + _ = rc.Parse("v1/Secret") // Skip secrets + return rc + }(), + }, + { + name: "discovers apps group resources", + discoveryResources: []*metav1.APIResourceList{ + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + { + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + resourceConfig: nil, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create fake discovery client + fakeClient := fake.NewSimpleClientset() + fakeDiscovery, ok := fakeClient.Discovery().(*fakediscovery.FakeDiscovery) + if !ok { + t.Fatal("Failed to cast to FakeDiscovery") + } + fakeDiscovery.Resources = tt.discoveryResources + + // Create REST mapper + groupResources := []*restmapper.APIGroupResources{} + for _, resourceList := range tt.discoveryResources { + gv, err := schema.ParseGroupVersion(resourceList.GroupVersion) + if err != nil { + t.Fatalf("Failed to parse group version: %v", err) + } + + groupResources = append(groupResources, &restmapper.APIGroupResources{ + Group: metav1.APIGroup{ + Name: gv.Group, + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: resourceList.GroupVersion, Version: gv.Version}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: resourceList.GroupVersion, + Version: gv.Version, + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + gv.Version: resourceList.APIResources, + }, + }) + } + restMapper := restmapper.NewDiscoveryRESTMapper(groupResources) + + // Create fake informer manager + fakeInformerManager := &testinformer.FakeManager{ + APIResources: make(map[schema.GroupVersionKind]bool), + } + + // Track handler additions + testHandler := cache.ResourceEventHandlerFuncs{ + AddFunc: func(obj interface{}) {}, + } + + // Create ChangeDetector with the interface type + detector := &ChangeDetector{ + DiscoveryClient: fakeDiscovery, + RESTMapper: restMapper, + InformerManager: fakeInformerManager, + ResourceConfig: tt.resourceConfig, + } + + // Test discoverResources which discovers resources and adds handlers + detector.discoverResources(testHandler) + + // The main goal is to verify no panics occur during discovery and handler addition + }) + } +} + +func TestChangeDetector_NeedLeaderElection(t *testing.T) { + detector := &ChangeDetector{} + + // ChangeDetector SHOULD need leader election so only the leader processes events + if !detector.NeedLeaderElection() { + t.Error("ChangeDetector should need leader election") + } +} diff --git a/pkg/resourcewatcher/informer_populator.go b/pkg/resourcewatcher/informer_populator.go new file mode 100644 index 000000000..8162f606a --- /dev/null +++ b/pkg/resourcewatcher/informer_populator.go @@ -0,0 +1,102 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcewatcher + +import ( + "context" + "time" + + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/util/wait" + "k8s.io/client-go/discovery" + "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/manager" + + "github.com/kubefleet-dev/kubefleet/pkg/utils" + "github.com/kubefleet-dev/kubefleet/pkg/utils/informer" +) + +const ( + // informerPopulatorDiscoveryPeriod is how often the InformerPopulator rediscovers API resources + informerPopulatorDiscoveryPeriod = 30 * time.Second +) + +// make sure that our InformerPopulator implements controller runtime interfaces +var ( + _ manager.Runnable = &InformerPopulator{} + _ manager.LeaderElectionRunnable = &InformerPopulator{} +) + +// InformerPopulator discovers API resources and creates informers for them WITHOUT adding event handlers. +// This allows follower pods to have synced informer caches for webhook validation while the leader's +// ChangeDetector adds event handlers and runs controllers. +type InformerPopulator struct { + // DiscoveryClient is used to do resource discovery. + DiscoveryClient discovery.DiscoveryInterface + + // RESTMapper is used to convert between GVK and GVR + RESTMapper meta.RESTMapper + + // InformerManager manages all the dynamic informers created by the discovery client + InformerManager informer.Manager + + // ResourceConfig contains all the API resources that we won't select based on the allowed or skipped propagating APIs option. + ResourceConfig *utils.ResourceConfig +} + +// Start runs the informer populator, discovering resources and creating informers. +// This runs on ALL pods (leader and followers) to ensure all have synced caches. +func (p *InformerPopulator) Start(ctx context.Context) error { + klog.InfoS("Starting the informer populator") + defer klog.InfoS("The informer populator is stopped") + + // Run initial discovery to create informers + p.discoverAndCreateInformers() + + // Wait for initial cache sync + p.InformerManager.WaitForCacheSync() + klog.InfoS("Informer populator: initial cache sync complete") + + // Continue discovering resources periodically to handle CRD installations + wait.UntilWithContext(ctx, func(ctx context.Context) { + p.discoverAndCreateInformers() + }, informerPopulatorDiscoveryPeriod) + + return nil +} + +// discoverAndCreateInformers discovers API resources and creates informers WITHOUT adding event handlers +func (p *InformerPopulator) discoverAndCreateInformers() { + resourcesToWatch := discoverWatchableResources(p.DiscoveryClient, p.RESTMapper, p.ResourceConfig) + + // Create informers directly without adding event handlers. + // This avoids adding any event handlers on follower pods + for _, res := range resourcesToWatch { + p.InformerManager.CreateInformerForResource(res) + } + + // Start any newly created informers + p.InformerManager.Start() + + klog.V(2).InfoS("Informer populator: discovered resources", "count", len(resourcesToWatch)) +} + +// NeedLeaderElection implements LeaderElectionRunnable interface. +// Returns false so this runs on ALL pods (leader and followers). +func (p *InformerPopulator) NeedLeaderElection() bool { + return false +} diff --git a/pkg/resourcewatcher/informer_populator_test.go b/pkg/resourcewatcher/informer_populator_test.go new file mode 100644 index 000000000..592be39b2 --- /dev/null +++ b/pkg/resourcewatcher/informer_populator_test.go @@ -0,0 +1,437 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcewatcher + +import ( + "context" + "testing" + "time" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + fakediscovery "k8s.io/client-go/discovery/fake" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/restmapper" + + "github.com/kubefleet-dev/kubefleet/pkg/utils" + testinformer "github.com/kubefleet-dev/kubefleet/test/utils/informer" +) + +const ( + // testTimeout is the timeout for test operations + testTimeout = 200 * time.Millisecond + // testSleep is how long to sleep to allow periodic operations + testSleep = 150 * time.Millisecond +) + +func TestInformerPopulator_NeedLeaderElection(t *testing.T) { + populator := &InformerPopulator{} + + // InformerPopulator should NOT need leader election so it runs on all pods + if populator.NeedLeaderElection() { + t.Error("InformerPopulator should not need leader election") + } +} + +func TestInformerPopulator_discoverAndCreateInformers(t *testing.T) { + tests := []struct { + name string + discoveryResources []*metav1.APIResourceList + resourceConfig *utils.ResourceConfig + expectedInformerCreated bool + expectedResourceCount int + }{ + { + name: "creates informers for watchable resources", + discoveryResources: []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + resourceConfig: nil, // Allow all resources + expectedInformerCreated: true, + expectedResourceCount: 1, + }, + { + name: "skips resources without list/watch verbs", + discoveryResources: []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"get", "delete"}, // Missing list/watch + }, + }, + }, + }, + resourceConfig: nil, + expectedInformerCreated: false, + expectedResourceCount: 0, + }, + { + name: "respects resource config filtering", + discoveryResources: []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "secrets", + Kind: "Secret", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + resourceConfig: func() *utils.ResourceConfig { + rc := utils.NewResourceConfig(false) // Skip mode + _ = rc.Parse("v1/Secret") // Skip secrets + return rc + }(), + expectedInformerCreated: false, + expectedResourceCount: 0, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + // Create fake discovery client + fakeClient := fake.NewSimpleClientset() + fakeDiscovery, ok := fakeClient.Discovery().(*fakediscovery.FakeDiscovery) + if !ok { + t.Fatal("Failed to cast to FakeDiscovery") + } + fakeDiscovery.Resources = tt.discoveryResources + + // Create REST mapper + groupResources := []*restmapper.APIGroupResources{} + for _, resourceList := range tt.discoveryResources { + gv, err := schema.ParseGroupVersion(resourceList.GroupVersion) + if err != nil { + t.Fatalf("Failed to parse group version: %v", err) + } + + apiResources := []metav1.APIResource{} + apiResources = append(apiResources, resourceList.APIResources...) + + groupResources = append(groupResources, &restmapper.APIGroupResources{ + Group: metav1.APIGroup{ + Name: gv.Group, + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: resourceList.GroupVersion, Version: gv.Version}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: resourceList.GroupVersion, + Version: gv.Version, + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + gv.Version: apiResources, + }, + }) + } + restMapper := restmapper.NewDiscoveryRESTMapper(groupResources) + + // Create fake informer manager + fakeInformerManager := &testinformer.FakeManager{ + APIResources: make(map[schema.GroupVersionKind]bool), + } + + // Track calls to CreateInformerForResource + populator := &InformerPopulator{ + DiscoveryClient: fakeDiscovery, + RESTMapper: restMapper, + InformerManager: fakeInformerManager, + ResourceConfig: tt.resourceConfig, + } + + // Run discovery + populator.discoverAndCreateInformers() + + // Note: FakeManager doesn't track calls, so we verify no panics occurred + }) + } +} + +func TestInformerPopulator_Start(t *testing.T) { + // Create fake discovery client with some resources + fakeClient := fake.NewSimpleClientset() + fakeDiscovery, ok := fakeClient.Discovery().(*fakediscovery.FakeDiscovery) + if !ok { + t.Fatal("Failed to cast to FakeDiscovery") + } + + fakeDiscovery.Resources = []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + } + + // Create REST mapper + gv := schema.GroupVersion{Group: "", Version: "v1"} + groupResources := []*restmapper.APIGroupResources{ + { + Group: metav1.APIGroup{ + Name: "", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": { + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + } + restMapper := restmapper.NewDiscoveryRESTMapper(groupResources) + + // Create fake informer manager + fakeInformerManager := &testinformer.FakeManager{ + APIResources: map[schema.GroupVersionKind]bool{ + gv.WithKind("ConfigMap"): true, + }, + IsClusterScopedResource: false, + } + + populator := &InformerPopulator{ + DiscoveryClient: fakeDiscovery, + RESTMapper: restMapper, + InformerManager: fakeInformerManager, + ResourceConfig: nil, + } + + // Create a context that will cancel after a short time + // Use half of testTimeout to ensure we have time to verify after cancellation + ctx, cancel := context.WithTimeout(context.Background(), testTimeout/2) + defer cancel() + + // Start the populator in a goroutine + done := make(chan error, 1) + go func() { + done <- populator.Start(ctx) + }() + + // Wait for context to cancel or error + select { + case err := <-done: + // Should return nil when context is canceled + if err != nil { + t.Errorf("Start should not return error on context cancellation: %v", err) + } + case <-time.After(testTimeout): + t.Fatal("Start did not exit after context cancellation") + } +} + +func TestInformerPopulator_Integration(t *testing.T) { + // This test verifies the integration between InformerPopulator and the informer manager + + // Create fake discovery with multiple resource types + fakeClient := fake.NewSimpleClientset() + fakeDiscovery, ok := fakeClient.Discovery().(*fakediscovery.FakeDiscovery) + if !ok { + t.Fatal("Failed to cast to FakeDiscovery") + } + + fakeDiscovery.Resources = []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + { + Name: "secrets", + Kind: "Secret", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + { + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + } + + // Create REST mapper + groupResources := []*restmapper.APIGroupResources{ + { + Group: metav1.APIGroup{ + Name: "", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": fakeDiscovery.Resources[0].APIResources, + }, + }, + { + Group: metav1.APIGroup{ + Name: "apps", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "apps/v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "apps/v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": fakeDiscovery.Resources[1].APIResources, + }, + }, + } + restMapper := restmapper.NewDiscoveryRESTMapper(groupResources) + + // Create resource config that skips secrets + resourceConfig := utils.NewResourceConfig(false) + err := resourceConfig.Parse("v1/Secret") + if err != nil { + t.Fatalf("Failed to parse resource config: %v", err) + } + + fakeInformerManager := &testinformer.FakeManager{ + APIResources: make(map[schema.GroupVersionKind]bool), + IsClusterScopedResource: false, + } + + populator := &InformerPopulator{ + DiscoveryClient: fakeDiscovery, + RESTMapper: restMapper, + InformerManager: fakeInformerManager, + ResourceConfig: resourceConfig, + } + + // Run discovery + populator.discoverAndCreateInformers() + + // Note: FakeManager doesn't track calls, so we just verify no panics +} + +func TestInformerPopulator_PeriodicDiscovery(t *testing.T) { + // This test verifies that the populator continues to discover resources periodically + + fakeClient := fake.NewSimpleClientset() + fakeDiscovery, ok := fakeClient.Discovery().(*fakediscovery.FakeDiscovery) + if !ok { + t.Fatal("Failed to cast to FakeDiscovery") + } + + fakeDiscovery.Resources = []*metav1.APIResourceList{ + { + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + } + + groupResources := []*restmapper.APIGroupResources{ + { + Group: metav1.APIGroup{ + Name: "", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": fakeDiscovery.Resources[0].APIResources, + }, + }, + } + restMapper := restmapper.NewDiscoveryRESTMapper(groupResources) + + fakeInformerManager := &testinformer.FakeManager{ + APIResources: make(map[schema.GroupVersionKind]bool), + IsClusterScopedResource: false, + } + + populator := &InformerPopulator{ + DiscoveryClient: fakeDiscovery, + RESTMapper: restMapper, + InformerManager: fakeInformerManager, + ResourceConfig: nil, + } + + // Override the discovery period for testing + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + defer cancel() + + // Start the populator + go func() { + _ = populator.Start(ctx) + }() + + // Wait a bit to allow multiple discovery cycles + time.Sleep(testSleep) + + // Note: FakeManager doesn't track calls, so we just verify successful execution +} diff --git a/pkg/resourcewatcher/resource_collector.go b/pkg/resourcewatcher/resource_collector.go index 86e2cf235..b740bfc8c 100644 --- a/pkg/resourcewatcher/resource_collector.go +++ b/pkg/resourcewatcher/resource_collector.go @@ -17,12 +17,14 @@ limitations under the License. package resourcewatcher import ( + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/discovery" "k8s.io/klog/v2" metricsV1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" + "github.com/kubefleet-dev/kubefleet/pkg/utils" "github.com/kubefleet-dev/kubefleet/pkg/utils/informer" ) @@ -30,10 +32,11 @@ import ( // More specifically, all api resources which support the 'list', and 'watch' verbs. // All discovery errors are considered temporary. Upon encountering any error, // getWatchableResources will log and return any discovered resources it was able to process (which may be none). -func (d *ChangeDetector) getWatchableResources() ([]informer.APIResourceMeta, error) { +// This is a standalone function that can be used by both ChangeDetector and InformerPopulator. +func getWatchableResources(discoveryClient discovery.ServerResourcesInterface) ([]informer.APIResourceMeta, error) { // Get all the resources this cluster has. We only need to care about the preferred version as the informers watch // the preferred version will get watch event for resources on the other versions since there is only one version in etcd. - allResources, discoverError := d.DiscoveryClient.ServerPreferredResources() + allResources, discoverError := discoveryClient.ServerPreferredResources() allErr := make([]error, 0) if discoverError != nil { if discovery.IsGroupDiscoveryFailedError(discoverError) { @@ -82,3 +85,44 @@ func (d *ChangeDetector) getWatchableResources() ([]informer.APIResourceMeta, er return watchableGroupVersionResources, errors.NewAggregate(allErr) } + +// shouldWatchResource returns whether a GroupVersionResource should be watched. +// This is a standalone function that can be used by both ChangeDetector and InformerPopulator. +func shouldWatchResource(gvr schema.GroupVersionResource, restMapper meta.RESTMapper, resourceConfig *utils.ResourceConfig) bool { + // By default, all of the APIs are allowed. + if resourceConfig == nil { + return true + } + + gvks, err := restMapper.KindsFor(gvr) + if err != nil { + klog.ErrorS(err, "gvr transform failed", "gvr", gvr.String()) + return false + } + for _, gvk := range gvks { + if resourceConfig.IsResourceDisabled(gvk) { + klog.V(4).InfoS("Skip watch resource", "group version kind", gvk.String()) + return false + } + } + return true +} + +// discoverWatchableResources discovers all API resources in the cluster and filters them +// based on the resource configuration. This is a shared helper used by both InformerPopulator +// and ChangeDetector to ensure consistent resource discovery logic. +func discoverWatchableResources(discoveryClient discovery.DiscoveryInterface, restMapper meta.RESTMapper, resourceConfig *utils.ResourceConfig) []informer.APIResourceMeta { + newResources, err := getWatchableResources(discoveryClient) + if err != nil { + klog.ErrorS(err, "Failed to get all the api resources from the cluster") + } + + var resourcesToWatch []informer.APIResourceMeta + for _, res := range newResources { + if shouldWatchResource(res.GroupVersionResource, restMapper, resourceConfig) { + resourcesToWatch = append(resourcesToWatch, res) + } + } + + return resourcesToWatch +} diff --git a/pkg/resourcewatcher/resource_collector_test.go b/pkg/resourcewatcher/resource_collector_test.go new file mode 100644 index 000000000..33a5d1d1f --- /dev/null +++ b/pkg/resourcewatcher/resource_collector_test.go @@ -0,0 +1,221 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resourcewatcher + +import ( + "testing" + + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/restmapper" + + "github.com/kubefleet-dev/kubefleet/pkg/utils" +) + +func TestShouldWatchResource(t *testing.T) { + tests := []struct { + name string + gvr schema.GroupVersionResource + resourceConfig *utils.ResourceConfig + setupMapper func() meta.RESTMapper + expected bool + }{ + { + name: "returns true when resourceConfig is nil", + gvr: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + }, + resourceConfig: nil, + setupMapper: func() meta.RESTMapper { + groupResources := []*restmapper.APIGroupResources{ + { + Group: metav1.APIGroup{ + Name: "", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": { + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + } + return restmapper.NewDiscoveryRESTMapper(groupResources) + }, + expected: true, + }, + { + name: "returns true when resource is not disabled", + gvr: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + }, + resourceConfig: func() *utils.ResourceConfig { + rc := utils.NewResourceConfig(false) + // Disable secrets, but not configmaps + _ = rc.Parse("v1/Secret") + return rc + }(), + setupMapper: func() meta.RESTMapper { + groupResources := []*restmapper.APIGroupResources{ + { + Group: metav1.APIGroup{ + Name: "", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": { + { + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + } + return restmapper.NewDiscoveryRESTMapper(groupResources) + }, + expected: true, + }, + { + name: "returns false when resource is disabled", + gvr: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "secrets", + }, + resourceConfig: func() *utils.ResourceConfig { + rc := utils.NewResourceConfig(false) + _ = rc.Parse("v1/Secret") + return rc + }(), + setupMapper: func() meta.RESTMapper { + groupResources := []*restmapper.APIGroupResources{ + { + Group: metav1.APIGroup{ + Name: "", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": { + { + Name: "secrets", + Kind: "Secret", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + } + return restmapper.NewDiscoveryRESTMapper(groupResources) + }, + expected: false, + }, + { + name: "returns false when GVR mapping fails", + gvr: schema.GroupVersionResource{ + Group: "invalid.group", + Version: "v1", + Resource: "nonexistent", + }, + resourceConfig: utils.NewResourceConfig(false), + setupMapper: func() meta.RESTMapper { + // Empty mapper - will fail to map the GVR + groupResources := []*restmapper.APIGroupResources{} + return restmapper.NewDiscoveryRESTMapper(groupResources) + }, + expected: false, + }, + { + name: "handles apps group resources correctly", + gvr: schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + resourceConfig: nil, + setupMapper: func() meta.RESTMapper { + groupResources := []*restmapper.APIGroupResources{ + { + Group: metav1.APIGroup{ + Name: "apps", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "apps/v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "apps/v1", + Version: "v1", + }, + }, + VersionedResources: map[string][]metav1.APIResource{ + "v1": { + { + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + Verbs: []string{"list", "watch", "get"}, + }, + }, + }, + }, + } + return restmapper.NewDiscoveryRESTMapper(groupResources) + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + restMapper := tt.setupMapper() + result := shouldWatchResource(tt.gvr, restMapper, tt.resourceConfig) + if result != tt.expected { + t.Errorf("shouldWatchResource() = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/utils/informer/informermanager.go b/pkg/utils/informer/informermanager.go index 53da3343a..984a6d392 100644 --- a/pkg/utils/informer/informermanager.go +++ b/pkg/utils/informer/informermanager.go @@ -33,11 +33,6 @@ import ( // InformerManager manages dynamic shared informer for all resources, include Kubernetes resource and // custom resources defined by CustomResourceDefinition. type Manager interface { - // AddDynamicResources builds a dynamicInformer for each resource in the resources list with the event handler. - // A resource is dynamic if its definition can be created/deleted/updated during runtime. - // Normally, it is a custom resource that is installed by users. The handler should not be nil. - AddDynamicResources(resources []APIResourceMeta, handler cache.ResourceEventHandler, listComplete bool) - // AddStaticResource creates a dynamicInformer for the static 'resource' and set its event handler. // A resource is static if its definition is pre-determined and immutable during runtime. // Normally, it is a resource that is pre-installed by the system. @@ -72,6 +67,16 @@ type Manager interface { // GetClient returns the dynamic dynamicClient. GetClient() dynamic.Interface + + // AddEventHandlerToInformer adds an event handler to an existing informer for the given resource. + // If the informer doesn't exist, it will be created. This is used by the leader's ChangeDetector + // to add event handlers to informers that were created by the InformerPopulator. + AddEventHandlerToInformer(resource schema.GroupVersionResource, handler cache.ResourceEventHandler) + + // CreateInformerForResource creates an informer for the given resource without adding any event handlers. + // This is used by InformerPopulator to create informers on all pods (leader and followers) so they have + // synced caches for webhook validation. The leader's ChangeDetector will add event handlers later. + CreateInformerForResource(resource APIResourceMeta) } // NewInformerManager constructs a new instance of informerManagerImpl. @@ -124,61 +129,6 @@ type informerManagerImpl struct { resourcesLock sync.RWMutex } -func (s *informerManagerImpl) AddDynamicResources(dynResources []APIResourceMeta, handler cache.ResourceEventHandler, listComplete bool) { - newGVKs := make(map[schema.GroupVersionKind]bool, len(dynResources)) - - addInformerFunc := func(newRes APIResourceMeta) { - dynRes, exist := s.apiResources[newRes.GroupVersionKind] - if !exist { - newRes.isPresent = true - s.apiResources[newRes.GroupVersionKind] = &newRes - // TODO (rzhang): remember the ResourceEventHandlerRegistration and remove it when the resource is deleted - // TODO: handle error which only happens if the informer is stopped - informer := s.informerFactory.ForResource(newRes.GroupVersionResource).Informer() - // Strip away the ManagedFields info from objects to save memory. - // - // TO-DO (chenyu1): evaluate if there are other fields, e.g., owner refs, status, that can also be stripped - // away to save memory. - if err := informer.SetTransform(ctrlcache.TransformStripManagedFields()); err != nil { - // The SetTransform func would only fail if the informer has already started. In this case, - // no further action is needed. - klog.ErrorS(err, "Failed to set transform func for informer", "gvr", newRes.GroupVersionResource) - } - _, _ = informer.AddEventHandler(handler) - klog.InfoS("Added an informer for a new resource", "res", newRes) - } else if !dynRes.isPresent { - // we just mark it as enabled as we should not add another eventhandler to the informer as it's still - // in the informerFactory - // TODO: add the Event handler back - dynRes.isPresent = true - klog.InfoS("Reactivated an informer for a reappeared resource", "res", dynRes) - } - } - - s.resourcesLock.Lock() - defer s.resourcesLock.Unlock() - - // Add the new dynResources that do not exist yet while build a map to speed up lookup - for _, newRes := range dynResources { - newGVKs[newRes.GroupVersionKind] = true - addInformerFunc(newRes) - } - - if !listComplete { - // do not disable any informer if we know the resource list is not complete - return - } - - // mark the disappeared dynResources from the handler map - for gvk, dynRes := range s.apiResources { - if !newGVKs[gvk] && !dynRes.isStaticResource && dynRes.isPresent { - // TODO: Remove the Event handler from the informer using the resourceEventHandlerRegistration during creat - dynRes.isPresent = false - klog.InfoS("Disabled an informer for a disappeared resource", "res", dynRes) - } - } -} - func (s *informerManagerImpl) AddStaticResource(resource APIResourceMeta, handler cache.ResourceEventHandler) { s.resourcesLock.Lock() defer s.resourcesLock.Unlock() @@ -255,6 +205,46 @@ func (s *informerManagerImpl) Stop() { s.cancel() } +func (s *informerManagerImpl) AddEventHandlerToInformer(resource schema.GroupVersionResource, handler cache.ResourceEventHandler) { + // Get or create the informer (this is idempotent - if it exists, we get the same instance) + // The idempotent behavior is important because it is called by both the change detector and informer populator, + // which run concurrently on the leader hub agent. + informer := s.informerFactory.ForResource(resource).Informer() + + // Add the event handler to the informer + _, _ = informer.AddEventHandler(handler) + + klog.V(2).InfoS("Added event handler to informer", "gvr", resource) +} + +func (s *informerManagerImpl) CreateInformerForResource(resource APIResourceMeta) { + s.resourcesLock.Lock() + defer s.resourcesLock.Unlock() + + dynRes, exist := s.apiResources[resource.GroupVersionKind] + if !exist { + // Register this resource in our tracking map + resource.isPresent = true + s.apiResources[resource.GroupVersionKind] = &resource + + // Create the informer without adding any event handler + informer := s.informerFactory.ForResource(resource.GroupVersionResource).Informer() + + // Strip away the ManagedFields info from objects to save memory + if err := informer.SetTransform(ctrlcache.TransformStripManagedFields()); err != nil { + // The SetTransform func would only fail if the informer has already started. + // In this case, no further action is needed. + klog.ErrorS(err, "Failed to set transform func for informer", "gvr", resource.GroupVersionResource) + } + + klog.V(3).InfoS("Created informer without handler", "res", resource) + } else if !dynRes.isPresent { + // Mark it as present again (resource reappeared) + dynRes.isPresent = true + klog.V(3).InfoS("Reactivated informer for reappeared resource", "res", dynRes) + } +} + // ContextForChannel derives a child context from a parent channel. // // The derived context's Done channel is closed when the returned cancel function diff --git a/pkg/utils/informer/informermanager_test.go b/pkg/utils/informer/informermanager_test.go index 53f2ce74a..5c8b1c764 100644 --- a/pkg/utils/informer/informermanager_test.go +++ b/pkg/utils/informer/informermanager_test.go @@ -22,6 +22,8 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/dynamic/fake" "k8s.io/client-go/kubernetes/scheme" + + testhandler "github.com/kubefleet-dev/kubefleet/test/utils/handler" ) func TestGetAllResources(t *testing.T) { @@ -287,3 +289,218 @@ func TestGetAllResources_NotPresent(t *testing.T) { t.Errorf("GetAllResources()[0] = %v, want %v", got, presentRes.GroupVersionResource) } } + +func TestAddEventHandlerToInformer(t *testing.T) { + tests := []struct { + name string + gvr schema.GroupVersionResource + addTwice bool // Test adding handler twice to same informer + }{ + { + name: "add handler to new informer", + gvr: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + }, + addTwice: false, + }, + { + name: "add multiple handlers to same informer", + gvr: schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + addTwice: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleDynamicClient(scheme.Scheme) + stopCh := make(chan struct{}) + defer close(stopCh) + + mgr := NewInformerManager(fakeClient, 0, stopCh) + + // Track handler calls + callCount := 0 + handler := &testhandler.TestHandler{ + OnAddFunc: func() { callCount++ }, + } + + // Add the handler + mgr.AddEventHandlerToInformer(tt.gvr, handler) + + // Verify informer was created + implMgr := mgr.(*informerManagerImpl) + informer := implMgr.informerFactory.ForResource(tt.gvr).Informer() + if informer == nil { + t.Fatal("Expected informer to be created") + } + + if tt.addTwice { + // Add another handler to the same informer + handler2 := &testhandler.TestHandler{ + OnAddFunc: func() { callCount++ }, + } + mgr.AddEventHandlerToInformer(tt.gvr, handler2) + } + + // Test is successful if no panic occurred and informer exists + }) + } +} + +func TestCreateInformerForResource(t *testing.T) { + tests := []struct { + name string + resource APIResourceMeta + createTwice bool + markNotPresent bool // Mark resource as not present before second create + }{ + { + name: "create new informer", + resource: APIResourceMeta{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "ConfigMap", + }, + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "configmaps", + }, + IsClusterScoped: false, + }, + createTwice: false, + }, + { + name: "create informer twice (idempotent)", + resource: APIResourceMeta{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "apps", + Version: "v1", + Kind: "Deployment", + }, + GroupVersionResource: schema.GroupVersionResource{ + Group: "apps", + Version: "v1", + Resource: "deployments", + }, + IsClusterScoped: false, + }, + createTwice: true, + }, + { + name: "recreate informer for reappeared resource", + resource: APIResourceMeta{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Secret", + }, + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "secrets", + }, + IsClusterScoped: false, + }, + createTwice: true, + markNotPresent: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + fakeClient := fake.NewSimpleDynamicClient(scheme.Scheme) + stopCh := make(chan struct{}) + defer close(stopCh) + + mgr := NewInformerManager(fakeClient, 0, stopCh) + implMgr := mgr.(*informerManagerImpl) + + // Create the informer + mgr.CreateInformerForResource(tt.resource) + + // Verify resource is tracked + resMeta, exists := implMgr.apiResources[tt.resource.GroupVersionKind] + if !exists { + t.Fatal("Expected resource to be tracked in apiResources map") + } + if !resMeta.isPresent { + t.Error("Expected resource to be marked as present") + } + if resMeta.IsClusterScoped != tt.resource.IsClusterScoped { + t.Errorf("IsClusterScoped = %v, want %v", resMeta.IsClusterScoped, tt.resource.IsClusterScoped) + } + + // Verify informer was created + informer := implMgr.informerFactory.ForResource(tt.resource.GroupVersionResource).Informer() + if informer == nil { + t.Fatal("Expected informer to be created") + } + + if tt.createTwice { + if tt.markNotPresent { + // Mark as not present (simulating resource deletion) + resMeta.isPresent = false + } + + // Create again + mgr.CreateInformerForResource(tt.resource) + + // Verify it's marked as present again + if !resMeta.isPresent { + t.Error("Expected resource to be marked as present after recreation") + } + } + }) + } +} + +func TestCreateInformerForResource_IsIdempotent(t *testing.T) { + // Test that creating the same informer multiple times doesn't cause issues + fakeClient := fake.NewSimpleDynamicClient(scheme.Scheme) + stopCh := make(chan struct{}) + defer close(stopCh) + + mgr := NewInformerManager(fakeClient, 0, stopCh) + implMgr := mgr.(*informerManagerImpl) + + resource := APIResourceMeta{ + GroupVersionKind: schema.GroupVersionKind{ + Group: "", + Version: "v1", + Kind: "Pod", + }, + GroupVersionResource: schema.GroupVersionResource{ + Group: "", + Version: "v1", + Resource: "pods", + }, + IsClusterScoped: false, + } + + // Create multiple times + for i := 0; i < 3; i++ { + mgr.CreateInformerForResource(resource) + } + + // Should only have one entry in apiResources + if len(implMgr.apiResources) != 1 { + t.Errorf("Expected 1 resource in apiResources, got %d", len(implMgr.apiResources)) + } + + // Verify resource is still tracked correctly + resMeta, exists := implMgr.apiResources[resource.GroupVersionKind] + if !exists { + t.Fatal("Expected resource to be tracked") + } + if !resMeta.isPresent { + t.Error("Expected resource to be marked as present") + } +} diff --git a/pkg/utils/informer/readiness/readiness.go b/pkg/utils/informer/readiness/readiness.go index 23e12cf86..749be5592 100644 --- a/pkg/utils/informer/readiness/readiness.go +++ b/pkg/utils/informer/readiness/readiness.go @@ -37,7 +37,7 @@ func InformerReadinessChecker(resourceInformer informer.Manager) func(*http.Requ // Require ALL informer caches to be synced before marking ready allResources := resourceInformer.GetAllResources() if len(allResources) == 0 { - // This can happen during startup when the ResourceInformer is created but the ChangeDetector + // This can happen during startup when the ResourceInformer is created but the InformerPopulator // hasn't discovered and registered any resources yet via AddDynamicResources(). return fmt.Errorf("resource informer not ready: no resources registered") } diff --git a/test/utils/handler/handler.go b/test/utils/handler/handler.go new file mode 100644 index 000000000..67d382f48 --- /dev/null +++ b/test/utils/handler/handler.go @@ -0,0 +1,47 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +// Package handler provides test utilities for Kubernetes event handlers. +package handler + +// TestHandler is a simple implementation of cache.ResourceEventHandler for testing. +// It allows tests to track when specific event handler methods are called. +type TestHandler struct { + OnAddFunc func() + OnUpdateFunc func() + OnDeleteFunc func() +} + +// OnAdd is called when an object is added. +func (h *TestHandler) OnAdd(obj interface{}, isInInitialList bool) { + if h.OnAddFunc != nil { + h.OnAddFunc() + } +} + +// OnUpdate is called when an object is updated. +func (h *TestHandler) OnUpdate(oldObj, newObj interface{}) { + if h.OnUpdateFunc != nil { + h.OnUpdateFunc() + } +} + +// OnDelete is called when an object is deleted. +func (h *TestHandler) OnDelete(obj interface{}) { + if h.OnDeleteFunc != nil { + h.OnDeleteFunc() + } +} diff --git a/test/utils/informer/manager.go b/test/utils/informer/manager.go index fde757292..004ef9364 100644 --- a/test/utils/informer/manager.go +++ b/test/utils/informer/manager.go @@ -185,3 +185,10 @@ func (m *FakeManager) WaitForCacheSync() { func (m *FakeManager) GetClient() dynamic.Interface { return nil } +func (m *FakeManager) AddEventHandlerToInformer(_ schema.GroupVersionResource, _ cache.ResourceEventHandler) { + // No-op for testing +} + +func (m *FakeManager) CreateInformerForResource(_ informer.APIResourceMeta) { + // No-op for testing +} From 088ea477fd3755f32c47946e1d3253604a2db363 Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Wed, 17 Dec 2025 00:56:44 +0000 Subject: [PATCH 2/3] resolve AI comments Signed-off-by: Wei Weng --- pkg/utils/informer/informermanager.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/pkg/utils/informer/informermanager.go b/pkg/utils/informer/informermanager.go index 984a6d392..3f9616790 100644 --- a/pkg/utils/informer/informermanager.go +++ b/pkg/utils/informer/informermanager.go @@ -206,9 +206,12 @@ func (s *informerManagerImpl) Stop() { } func (s *informerManagerImpl) AddEventHandlerToInformer(resource schema.GroupVersionResource, handler cache.ResourceEventHandler) { + s.resourcesLock.Lock() + defer s.resourcesLock.Unlock() + // Get or create the informer (this is idempotent - if it exists, we get the same instance) - // The idempotent behavior is important because it is called by both the change detector and informer populator, - // which run concurrently on the leader hub agent. + // The idempotent behavior is important because this method may be called multiple times, + // potentially concurrently, and relies on the shared informer instance from the factory. informer := s.informerFactory.ForResource(resource).Informer() // Add the event handler to the informer @@ -225,6 +228,7 @@ func (s *informerManagerImpl) CreateInformerForResource(resource APIResourceMeta if !exist { // Register this resource in our tracking map resource.isPresent = true + resource.isStaticResource = false s.apiResources[resource.GroupVersionKind] = &resource // Create the informer without adding any event handler From 55e8022a0d75cd40d2d0889b721564429cbe1afb Mon Sep 17 00:00:00 2001 From: Wei Weng Date: Wed, 17 Dec 2025 02:14:55 +0000 Subject: [PATCH 3/3] refactor tests Signed-off-by: Wei Weng --- .../placement/resource_selector.go | 22 +- pkg/resourcewatcher/change_detector_test.go | 50 +-- .../informer_populator_test.go | 113 +----- pkg/resourcewatcher/resource_collector.go | 24 +- .../resource_collector_test.go | 221 ------------ pkg/utils/apiresources.go | 25 ++ pkg/utils/apiresources_test.go | 150 ++++++++ pkg/utils/informer/informermanager_test.go | 200 +++-------- test/utils/resource/apiresource.go | 322 ++++++++++++++++++ 9 files changed, 571 insertions(+), 556 deletions(-) delete mode 100644 pkg/resourcewatcher/resource_collector_test.go create mode 100644 test/utils/resource/apiresource.go diff --git a/pkg/controllers/placement/resource_selector.go b/pkg/controllers/placement/resource_selector.go index 0805889b7..0eb15cda1 100644 --- a/pkg/controllers/placement/resource_selector.go +++ b/pkg/controllers/placement/resource_selector.go @@ -380,7 +380,7 @@ func (r *Reconciler) fetchAllResourcesInOneNamespace(namespaceName string, place trackedResource := r.InformerManager.GetNameSpaceScopedResources() for _, gvr := range trackedResource { - if !r.shouldSelectResource(gvr) { + if !utils.ShouldProcessResource(gvr, r.RestMapper, r.ResourceConfig) { continue } if !r.InformerManager.IsInformerSynced(gvr) { @@ -406,26 +406,6 @@ func (r *Reconciler) fetchAllResourcesInOneNamespace(namespaceName string, place return resources, nil } -// shouldSelectResource returns whether a resource should be selected for propagation. -func (r *Reconciler) shouldSelectResource(gvr schema.GroupVersionResource) bool { - // By default, all of the APIs are allowed. - if r.ResourceConfig == nil { - return true - } - gvks, err := r.RestMapper.KindsFor(gvr) - if err != nil { - klog.ErrorS(err, "gvr(%s) transform failed: %v", gvr.String(), err) - return false - } - for _, gvk := range gvks { - if r.ResourceConfig.IsResourceDisabled(gvk) { - klog.V(2).InfoS("Skip watch resource", "group version kind", gvk.String()) - return false - } - } - return true -} - // generateRawContent strips all the unnecessary fields to prepare the objects for dispatch. func generateRawContent(object *unstructured.Unstructured) ([]byte, error) { // Make a deep copy of the object as we are modifying it. diff --git a/pkg/resourcewatcher/change_detector_test.go b/pkg/resourcewatcher/change_detector_test.go index e0f83e3bf..da521efa9 100644 --- a/pkg/resourcewatcher/change_detector_test.go +++ b/pkg/resourcewatcher/change_detector_test.go @@ -28,6 +28,7 @@ import ( "github.com/kubefleet-dev/kubefleet/pkg/utils" testinformer "github.com/kubefleet-dev/kubefleet/test/utils/informer" + testresource "github.com/kubefleet-dev/kubefleet/test/utils/resource" ) func TestChangeDetector_discoverResources(t *testing.T) { @@ -42,18 +43,8 @@ func TestChangeDetector_discoverResources(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - { - Name: "secrets", - Kind: "Secret", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceConfigMap(), + testresource.APIResourceSecret(), }, }, }, @@ -65,12 +56,7 @@ func TestChangeDetector_discoverResources(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"get", "delete"}, // Missing list/watch - }, + testresource.APIResourceWithVerbs("configmaps", "ConfigMap", true, []string{"get", "delete"}), // Missing list/watch }, }, }, @@ -82,18 +68,8 @@ func TestChangeDetector_discoverResources(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - { - Name: "secrets", - Kind: "Secret", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceConfigMap(), + testresource.APIResourceSecret(), }, }, }, @@ -109,18 +85,8 @@ func TestChangeDetector_discoverResources(t *testing.T) { { GroupVersion: "apps/v1", APIResources: []metav1.APIResource{ - { - Name: "deployments", - Kind: "Deployment", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - { - Name: "statefulsets", - Kind: "StatefulSet", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceDeployment(), + testresource.APIResourceStatefulSet(), }, }, }, diff --git a/pkg/resourcewatcher/informer_populator_test.go b/pkg/resourcewatcher/informer_populator_test.go index 592be39b2..a20065434 100644 --- a/pkg/resourcewatcher/informer_populator_test.go +++ b/pkg/resourcewatcher/informer_populator_test.go @@ -29,6 +29,7 @@ import ( "github.com/kubefleet-dev/kubefleet/pkg/utils" testinformer "github.com/kubefleet-dev/kubefleet/test/utils/informer" + testresource "github.com/kubefleet-dev/kubefleet/test/utils/resource" ) const ( @@ -61,12 +62,7 @@ func TestInformerPopulator_discoverAndCreateInformers(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceConfigMap(), }, }, }, @@ -80,12 +76,7 @@ func TestInformerPopulator_discoverAndCreateInformers(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"get", "delete"}, // Missing list/watch - }, + testresource.APIResourceWithVerbs("configmaps", "ConfigMap", true, []string{"get", "delete"}), // Missing list/watch }, }, }, @@ -99,12 +90,7 @@ func TestInformerPopulator_discoverAndCreateInformers(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "secrets", - Kind: "Secret", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceSecret(), }, }, }, @@ -190,12 +176,7 @@ func TestInformerPopulator_Start(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceConfigMap(), }, }, } @@ -203,28 +184,7 @@ func TestInformerPopulator_Start(t *testing.T) { // Create REST mapper gv := schema.GroupVersion{Group: "", Version: "v1"} groupResources := []*restmapper.APIGroupResources{ - { - Group: metav1.APIGroup{ - Name: "", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "v1", - Version: "v1", - }, - }, - VersionedResources: map[string][]metav1.APIResource{ - "v1": { - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - }, - }, - }, + testresource.APIGroupResourcesV1(testresource.APIResourceConfigMap()), } restMapper := restmapper.NewDiscoveryRESTMapper(groupResources) @@ -280,29 +240,14 @@ func TestInformerPopulator_Integration(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - { - Name: "secrets", - Kind: "Secret", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceConfigMap(), + testresource.APIResourceSecret(), }, }, { GroupVersion: "apps/v1", APIResources: []metav1.APIResource{ - { - Name: "deployments", - Kind: "Deployment", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceDeployment(), }, }, } @@ -310,31 +255,13 @@ func TestInformerPopulator_Integration(t *testing.T) { // Create REST mapper groupResources := []*restmapper.APIGroupResources{ { - Group: metav1.APIGroup{ - Name: "", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "v1", - Version: "v1", - }, - }, + Group: testresource.APIGroupV1(), VersionedResources: map[string][]metav1.APIResource{ "v1": fakeDiscovery.Resources[0].APIResources, }, }, { - Group: metav1.APIGroup{ - Name: "apps", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "apps/v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "apps/v1", - Version: "v1", - }, - }, + Group: testresource.APIGroupAppsV1(), VersionedResources: map[string][]metav1.APIResource{ "v1": fakeDiscovery.Resources[1].APIResources, }, @@ -380,28 +307,14 @@ func TestInformerPopulator_PeriodicDiscovery(t *testing.T) { { GroupVersion: "v1", APIResources: []metav1.APIResource{ - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, + testresource.APIResourceConfigMap(), }, }, } groupResources := []*restmapper.APIGroupResources{ { - Group: metav1.APIGroup{ - Name: "", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "v1", - Version: "v1", - }, - }, + Group: testresource.APIGroupV1(), VersionedResources: map[string][]metav1.APIResource{ "v1": fakeDiscovery.Resources[0].APIResources, }, diff --git a/pkg/resourcewatcher/resource_collector.go b/pkg/resourcewatcher/resource_collector.go index b740bfc8c..1347a64b8 100644 --- a/pkg/resourcewatcher/resource_collector.go +++ b/pkg/resourcewatcher/resource_collector.go @@ -86,28 +86,6 @@ func getWatchableResources(discoveryClient discovery.ServerResourcesInterface) ( return watchableGroupVersionResources, errors.NewAggregate(allErr) } -// shouldWatchResource returns whether a GroupVersionResource should be watched. -// This is a standalone function that can be used by both ChangeDetector and InformerPopulator. -func shouldWatchResource(gvr schema.GroupVersionResource, restMapper meta.RESTMapper, resourceConfig *utils.ResourceConfig) bool { - // By default, all of the APIs are allowed. - if resourceConfig == nil { - return true - } - - gvks, err := restMapper.KindsFor(gvr) - if err != nil { - klog.ErrorS(err, "gvr transform failed", "gvr", gvr.String()) - return false - } - for _, gvk := range gvks { - if resourceConfig.IsResourceDisabled(gvk) { - klog.V(4).InfoS("Skip watch resource", "group version kind", gvk.String()) - return false - } - } - return true -} - // discoverWatchableResources discovers all API resources in the cluster and filters them // based on the resource configuration. This is a shared helper used by both InformerPopulator // and ChangeDetector to ensure consistent resource discovery logic. @@ -119,7 +97,7 @@ func discoverWatchableResources(discoveryClient discovery.DiscoveryInterface, re var resourcesToWatch []informer.APIResourceMeta for _, res := range newResources { - if shouldWatchResource(res.GroupVersionResource, restMapper, resourceConfig) { + if utils.ShouldProcessResource(res.GroupVersionResource, restMapper, resourceConfig) { resourcesToWatch = append(resourcesToWatch, res) } } diff --git a/pkg/resourcewatcher/resource_collector_test.go b/pkg/resourcewatcher/resource_collector_test.go deleted file mode 100644 index 33a5d1d1f..000000000 --- a/pkg/resourcewatcher/resource_collector_test.go +++ /dev/null @@ -1,221 +0,0 @@ -/* -Copyright 2025 The KubeFleet Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package resourcewatcher - -import ( - "testing" - - "k8s.io/apimachinery/pkg/api/meta" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime/schema" - "k8s.io/client-go/restmapper" - - "github.com/kubefleet-dev/kubefleet/pkg/utils" -) - -func TestShouldWatchResource(t *testing.T) { - tests := []struct { - name string - gvr schema.GroupVersionResource - resourceConfig *utils.ResourceConfig - setupMapper func() meta.RESTMapper - expected bool - }{ - { - name: "returns true when resourceConfig is nil", - gvr: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "configmaps", - }, - resourceConfig: nil, - setupMapper: func() meta.RESTMapper { - groupResources := []*restmapper.APIGroupResources{ - { - Group: metav1.APIGroup{ - Name: "", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "v1", - Version: "v1", - }, - }, - VersionedResources: map[string][]metav1.APIResource{ - "v1": { - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - }, - }, - }, - } - return restmapper.NewDiscoveryRESTMapper(groupResources) - }, - expected: true, - }, - { - name: "returns true when resource is not disabled", - gvr: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "configmaps", - }, - resourceConfig: func() *utils.ResourceConfig { - rc := utils.NewResourceConfig(false) - // Disable secrets, but not configmaps - _ = rc.Parse("v1/Secret") - return rc - }(), - setupMapper: func() meta.RESTMapper { - groupResources := []*restmapper.APIGroupResources{ - { - Group: metav1.APIGroup{ - Name: "", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "v1", - Version: "v1", - }, - }, - VersionedResources: map[string][]metav1.APIResource{ - "v1": { - { - Name: "configmaps", - Kind: "ConfigMap", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - }, - }, - }, - } - return restmapper.NewDiscoveryRESTMapper(groupResources) - }, - expected: true, - }, - { - name: "returns false when resource is disabled", - gvr: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "secrets", - }, - resourceConfig: func() *utils.ResourceConfig { - rc := utils.NewResourceConfig(false) - _ = rc.Parse("v1/Secret") - return rc - }(), - setupMapper: func() meta.RESTMapper { - groupResources := []*restmapper.APIGroupResources{ - { - Group: metav1.APIGroup{ - Name: "", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "v1", - Version: "v1", - }, - }, - VersionedResources: map[string][]metav1.APIResource{ - "v1": { - { - Name: "secrets", - Kind: "Secret", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - }, - }, - }, - } - return restmapper.NewDiscoveryRESTMapper(groupResources) - }, - expected: false, - }, - { - name: "returns false when GVR mapping fails", - gvr: schema.GroupVersionResource{ - Group: "invalid.group", - Version: "v1", - Resource: "nonexistent", - }, - resourceConfig: utils.NewResourceConfig(false), - setupMapper: func() meta.RESTMapper { - // Empty mapper - will fail to map the GVR - groupResources := []*restmapper.APIGroupResources{} - return restmapper.NewDiscoveryRESTMapper(groupResources) - }, - expected: false, - }, - { - name: "handles apps group resources correctly", - gvr: schema.GroupVersionResource{ - Group: "apps", - Version: "v1", - Resource: "deployments", - }, - resourceConfig: nil, - setupMapper: func() meta.RESTMapper { - groupResources := []*restmapper.APIGroupResources{ - { - Group: metav1.APIGroup{ - Name: "apps", - Versions: []metav1.GroupVersionForDiscovery{ - {GroupVersion: "apps/v1", Version: "v1"}, - }, - PreferredVersion: metav1.GroupVersionForDiscovery{ - GroupVersion: "apps/v1", - Version: "v1", - }, - }, - VersionedResources: map[string][]metav1.APIResource{ - "v1": { - { - Name: "deployments", - Kind: "Deployment", - Namespaced: true, - Verbs: []string{"list", "watch", "get"}, - }, - }, - }, - }, - } - return restmapper.NewDiscoveryRESTMapper(groupResources) - }, - expected: true, - }, - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - restMapper := tt.setupMapper() - result := shouldWatchResource(tt.gvr, restMapper, tt.resourceConfig) - if result != tt.expected { - t.Errorf("shouldWatchResource() = %v, want %v", result, tt.expected) - } - }) - } -} diff --git a/pkg/utils/apiresources.go b/pkg/utils/apiresources.go index 7c00bafe8..a767eed72 100644 --- a/pkg/utils/apiresources.go +++ b/pkg/utils/apiresources.go @@ -23,7 +23,9 @@ import ( coordv1 "k8s.io/api/coordination/v1" corev1 "k8s.io/api/core/v1" eventsv1 "k8s.io/api/events/v1" + "k8s.io/apimachinery/pkg/api/meta" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/klog/v2" metricsV1beta1 "k8s.io/metrics/pkg/apis/metrics/v1beta1" clusterv1beta1 "github.com/kubefleet-dev/kubefleet/apis/cluster/v1beta1" @@ -362,3 +364,26 @@ func (r *ResourceConfig) AddGroupKind(gk schema.GroupKind) { func (r *ResourceConfig) AddGroupVersionKind(gvk schema.GroupVersionKind) { r.groupVersionKinds[gvk] = struct{}{} } + +// ShouldProcessResource returns whether a GroupVersionResource should be processed (watched or selected). +// It checks if the resource is enabled based on the ResourceConfig settings. +// Returns true if resourceConfig is nil (all APIs allowed by default) or if the resource is not disabled. +func ShouldProcessResource(gvr schema.GroupVersionResource, restMapper meta.RESTMapper, resourceConfig *ResourceConfig) bool { + // By default, all of the APIs are allowed. + if resourceConfig == nil { + return true + } + + gvks, err := restMapper.KindsFor(gvr) + if err != nil { + klog.ErrorS(err, "gvr transform failed", "gvr", gvr.String()) + return false + } + for _, gvk := range gvks { + if resourceConfig.IsResourceDisabled(gvk) { + klog.V(4).InfoS("Skip processing resource", "group version kind", gvk.String()) + return false + } + } + return true +} diff --git a/pkg/utils/apiresources_test.go b/pkg/utils/apiresources_test.go index b01929ce4..e8ed30648 100644 --- a/pkg/utils/apiresources_test.go +++ b/pkg/utils/apiresources_test.go @@ -19,7 +19,11 @@ package utils import ( "testing" + "github.com/kubefleet-dev/kubefleet/test/utils/resource" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/restmapper" ) func TestResourceConfigGVKParse(t *testing.T) { @@ -609,3 +613,149 @@ func checkIfResourcesAreEnabledInConfig(t *testing.T, r *ResourceConfig, resourc } } } + +// testResource represents a simplified API resource for testing +type testResource struct { + Group string + Version string + Resource string + Kind string +} + +// newTestRESTMapper creates a RESTMapper with the specified resources for testing. +// Each resource is configured with standard settings (namespaced, standard verbs). +// Assumes input resources are valid and well-formed. +func newTestRESTMapper(resources ...testResource) meta.RESTMapper { + groupMap := make(map[string]*restmapper.APIGroupResources) + + for _, res := range resources { + groupVersion := res.Version + if res.Group != "" { + groupVersion = res.Group + "/" + res.Version + } + + // Initialize group if not exists + if groupMap[res.Group] == nil { + groupMap[res.Group] = &restmapper.APIGroupResources{ + Group: metav1.APIGroup{ + Name: res.Group, + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: groupVersion, Version: res.Version}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: groupVersion, + Version: res.Version, + }, + }, + VersionedResources: make(map[string][]metav1.APIResource), + } + } + + // Add resource to the version + groupMap[res.Group].VersionedResources[res.Version] = append( + groupMap[res.Group].VersionedResources[res.Version], + metav1.APIResource{ + Name: res.Resource, + Kind: res.Kind, + Namespaced: true, + Verbs: resource.VerbsAll, + }, + ) + } + + // Convert map to slice + groupResources := make([]*restmapper.APIGroupResources, 0, len(groupMap)) + for _, group := range groupMap { + groupResources = append(groupResources, group) + } + + return restmapper.NewDiscoveryRESTMapper(groupResources) +} + +func TestShouldProcessResource(t *testing.T) { + tests := []struct { + name string + gvr schema.GroupVersionResource + resourceConfig *ResourceConfig + setupMapper func() meta.RESTMapper + expected bool + }{ + { + name: "returns true when resourceConfig is nil", + gvr: schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, + resourceConfig: nil, + setupMapper: func() meta.RESTMapper { + return newTestRESTMapper( + testResource{Group: "", Version: "v1", Resource: "configmaps", Kind: "ConfigMap"}, + ) + }, + expected: true, + }, + { + name: "returns true when resource is not disabled", + gvr: schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"}, + resourceConfig: func() *ResourceConfig { + rc := NewResourceConfig(false) + // Disable secrets, but not configmaps + _ = rc.Parse("v1/Secret") + return rc + }(), + setupMapper: func() meta.RESTMapper { + return newTestRESTMapper( + testResource{Group: "", Version: "v1", Resource: "configmaps", Kind: "ConfigMap"}, + ) + }, + expected: true, + }, + { + name: "returns false when resource is disabled", + gvr: schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"}, + resourceConfig: func() *ResourceConfig { + rc := NewResourceConfig(false) + _ = rc.Parse("v1/Secret") + return rc + }(), + setupMapper: func() meta.RESTMapper { + return newTestRESTMapper( + testResource{Group: "", Version: "v1", Resource: "secrets", Kind: "Secret"}, + ) + }, + expected: false, + }, + { + name: "returns false when GVR mapping fails", + gvr: schema.GroupVersionResource{ + Group: "invalid.group", + Version: "v1", + Resource: "nonexistent", + }, + resourceConfig: NewResourceConfig(false), + setupMapper: func() meta.RESTMapper { + // Empty mapper - will fail to map the GVR + return newTestRESTMapper() + }, + expected: false, + }, + { + name: "handles apps group resources correctly", + gvr: schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"}, + resourceConfig: nil, + setupMapper: func() meta.RESTMapper { + return newTestRESTMapper( + testResource{Group: "apps", Version: "v1", Resource: "deployments", Kind: "Deployment"}, + ) + }, + expected: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + restMapper := tt.setupMapper() + result := ShouldProcessResource(tt.gvr, restMapper, tt.resourceConfig) + if result != tt.expected { + t.Errorf("ShouldProcessResource() = %v, want %v", result, tt.expected) + } + }) + } +} diff --git a/pkg/utils/informer/informermanager_test.go b/pkg/utils/informer/informermanager_test.go index 5c8b1c764..eae4425c4 100644 --- a/pkg/utils/informer/informermanager_test.go +++ b/pkg/utils/informer/informermanager_test.go @@ -24,6 +24,7 @@ import ( "k8s.io/client-go/kubernetes/scheme" testhandler "github.com/kubefleet-dev/kubefleet/test/utils/handler" + testresource "github.com/kubefleet-dev/kubefleet/test/utils/resource" ) func TestGetAllResources(t *testing.T) { @@ -39,61 +40,29 @@ func TestGetAllResources(t *testing.T) { name: "mixed cluster and namespace scoped resources", namespaceScopedResources: []APIResourceMeta{ { - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "ConfigMap", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "configmaps", - }, - IsClusterScoped: false, + GroupVersionKind: testresource.GVKConfigMap(), + GroupVersionResource: testresource.GVRConfigMap(), + IsClusterScoped: false, }, { - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Secret", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "secrets", - }, - IsClusterScoped: false, + GroupVersionKind: testresource.GVKSecret(), + GroupVersionResource: testresource.GVRSecret(), + IsClusterScoped: false, }, }, clusterScopedResources: []APIResourceMeta{ { - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Namespace", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "namespaces", - }, - IsClusterScoped: true, + GroupVersionKind: testresource.GVKNamespace(), + GroupVersionResource: testresource.GVRNamespace(), + IsClusterScoped: true, }, }, staticResources: []APIResourceMeta{ { - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Node", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "nodes", - }, - IsClusterScoped: true, - isStaticResource: true, + GroupVersionKind: testresource.GVKNode(), + GroupVersionResource: testresource.GVRNode(), + IsClusterScoped: true, + isStaticResource: true, }, }, expectedResourceCount: 4, // All resources including static @@ -108,17 +77,9 @@ func TestGetAllResources(t *testing.T) { name: "only namespace scoped resources", namespaceScopedResources: []APIResourceMeta{ { - GroupVersionKind: schema.GroupVersionKind{ - Group: "apps", - Version: "v1", - Kind: "Deployment", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "apps", - Version: "v1", - Resource: "deployments", - }, - IsClusterScoped: false, + GroupVersionKind: testresource.GVKDeployment(), + GroupVersionResource: testresource.GVRDeployment(), + IsClusterScoped: false, }, }, expectedResourceCount: 1, @@ -128,17 +89,9 @@ func TestGetAllResources(t *testing.T) { name: "only cluster scoped resources", clusterScopedResources: []APIResourceMeta{ { - GroupVersionKind: schema.GroupVersionKind{ - Group: "rbac.authorization.k8s.io", - Version: "v1", - Kind: "ClusterRole", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "rbac.authorization.k8s.io", - Version: "v1", - Resource: "clusterroles", - }, - IsClusterScoped: true, + GroupVersionKind: testresource.GVKClusterRole(), + GroupVersionResource: testresource.GVRClusterRole(), + IsClusterScoped: true, }, }, expectedResourceCount: 1, @@ -249,35 +202,19 @@ func TestGetAllResources_NotPresent(t *testing.T) { // Add a resource that is present presentRes := APIResourceMeta{ - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "ConfigMap", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "configmaps", - }, - IsClusterScoped: false, - isPresent: true, + GroupVersionKind: testresource.GVKConfigMap(), + GroupVersionResource: testresource.GVRConfigMap(), + IsClusterScoped: false, + isPresent: true, } implMgr.apiResources[presentRes.GroupVersionKind] = &presentRes // Add a resource that is NOT present (deleted) notPresentRes := APIResourceMeta{ - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Secret", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "secrets", - }, - IsClusterScoped: false, - isPresent: false, + GroupVersionKind: testresource.GVKSecret(), + GroupVersionResource: testresource.GVRSecret(), + IsClusterScoped: false, + isPresent: false, } implMgr.apiResources[notPresentRes.GroupVersionKind] = ¬PresentRes @@ -297,21 +234,13 @@ func TestAddEventHandlerToInformer(t *testing.T) { addTwice bool // Test adding handler twice to same informer }{ { - name: "add handler to new informer", - gvr: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "configmaps", - }, + name: "add handler to new informer", + gvr: testresource.GVRConfigMap(), addTwice: false, }, { - name: "add multiple handlers to same informer", - gvr: schema.GroupVersionResource{ - Group: "apps", - Version: "v1", - Resource: "deployments", - }, + name: "add multiple handlers to same informer", + gvr: testresource.GVRDeployment(), addTwice: true, }, } @@ -363,51 +292,27 @@ func TestCreateInformerForResource(t *testing.T) { { name: "create new informer", resource: APIResourceMeta{ - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "ConfigMap", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "configmaps", - }, - IsClusterScoped: false, + GroupVersionKind: testresource.GVKConfigMap(), + GroupVersionResource: testresource.GVRConfigMap(), + IsClusterScoped: false, }, createTwice: false, }, { name: "create informer twice (idempotent)", resource: APIResourceMeta{ - GroupVersionKind: schema.GroupVersionKind{ - Group: "apps", - Version: "v1", - Kind: "Deployment", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "apps", - Version: "v1", - Resource: "deployments", - }, - IsClusterScoped: false, + GroupVersionKind: testresource.GVKDeployment(), + GroupVersionResource: testresource.GVRDeployment(), + IsClusterScoped: false, }, createTwice: true, }, { name: "recreate informer for reappeared resource", resource: APIResourceMeta{ - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Secret", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "secrets", - }, - IsClusterScoped: false, + GroupVersionKind: testresource.GVKSecret(), + GroupVersionResource: testresource.GVRSecret(), + IsClusterScoped: false, }, createTwice: true, markNotPresent: true, @@ -463,6 +368,10 @@ func TestCreateInformerForResource(t *testing.T) { } func TestCreateInformerForResource_IsIdempotent(t *testing.T) { + // Use 3 attempts to verify idempotency works consistently across multiple calls, + // not just a single retry scenario + const createAttempts = 3 + // Test that creating the same informer multiple times doesn't cause issues fakeClient := fake.NewSimpleDynamicClient(scheme.Scheme) stopCh := make(chan struct{}) @@ -472,25 +381,18 @@ func TestCreateInformerForResource_IsIdempotent(t *testing.T) { implMgr := mgr.(*informerManagerImpl) resource := APIResourceMeta{ - GroupVersionKind: schema.GroupVersionKind{ - Group: "", - Version: "v1", - Kind: "Pod", - }, - GroupVersionResource: schema.GroupVersionResource{ - Group: "", - Version: "v1", - Resource: "pods", - }, - IsClusterScoped: false, + GroupVersionKind: testresource.GVKPod(), + GroupVersionResource: testresource.GVRPod(), + IsClusterScoped: false, } // Create multiple times - for i := 0; i < 3; i++ { + for i := 0; i < createAttempts; i++ { mgr.CreateInformerForResource(resource) } - // Should only have one entry in apiResources + // Should only have one entry in apiResources after + // we create the same informer multiple times if len(implMgr.apiResources) != 1 { t.Errorf("Expected 1 resource in apiResources, got %d", len(implMgr.apiResources)) } diff --git a/test/utils/resource/apiresource.go b/test/utils/resource/apiresource.go new file mode 100644 index 000000000..863a7d26c --- /dev/null +++ b/test/utils/resource/apiresource.go @@ -0,0 +1,322 @@ +/* +Copyright 2025 The KubeFleet Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package resource + +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/client-go/restmapper" +) + +// Common verbs for API resources +var ( + // VerbsAll includes all standard Kubernetes verbs + VerbsAll = []string{"list", "watch", "get", "create", "update", "patch", "delete"} + // VerbsReadOnly includes verbs for read-only access + VerbsReadOnly = []string{"list", "watch", "get"} + // VerbsNoWatch includes verbs without watch capability + VerbsNoWatch = []string{"get", "create", "update", "patch", "delete"} +) + +// APIGroupV1 returns a standard core v1 API group for testing +func APIGroupV1() metav1.APIGroup { + return metav1.APIGroup{ + Name: "", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "v1", + Version: "v1", + }, + } +} + +// APIGroupAppsV1 returns a standard apps/v1 API group for testing +func APIGroupAppsV1() metav1.APIGroup { + return metav1.APIGroup{ + Name: "apps", + Versions: []metav1.GroupVersionForDiscovery{ + {GroupVersion: "apps/v1", Version: "v1"}, + }, + PreferredVersion: metav1.GroupVersionForDiscovery{ + GroupVersion: "apps/v1", + Version: "v1", + }, + } +} + +// APIGroupResourcesV1 returns APIGroupResources for core v1 with the provided resources +func APIGroupResourcesV1(resources ...metav1.APIResource) *restmapper.APIGroupResources { + return &restmapper.APIGroupResources{ + Group: APIGroupV1(), + VersionedResources: map[string][]metav1.APIResource{ + "v1": resources, + }, + } +} + +// APIGroupResourcesAppsV1 returns APIGroupResources for apps/v1 with the provided resources +func APIGroupResourcesAppsV1(resources ...metav1.APIResource) *restmapper.APIGroupResources { + return &restmapper.APIGroupResources{ + Group: APIGroupAppsV1(), + VersionedResources: map[string][]metav1.APIResource{ + "v1": resources, + }, + } +} + +// APIResourceConfigMap returns a standard ConfigMap APIResource for testing +func APIResourceConfigMap() metav1.APIResource { + return metav1.APIResource{ + Name: "configmaps", + Kind: "ConfigMap", + Namespaced: true, + Verbs: VerbsReadOnly, + } +} + +// APIResourceSecret returns a standard Secret APIResource for testing +func APIResourceSecret() metav1.APIResource { + return metav1.APIResource{ + Name: "secrets", + Kind: "Secret", + Namespaced: true, + Verbs: VerbsReadOnly, + } +} + +// APIResourcePod returns a standard Pod APIResource for testing +func APIResourcePod() metav1.APIResource { + return metav1.APIResource{ + Name: "pods", + Kind: "Pod", + Namespaced: true, + Verbs: VerbsReadOnly, + } +} + +// APIResourceService returns a standard Service APIResource for testing +func APIResourceService() metav1.APIResource { + return metav1.APIResource{ + Name: "services", + Kind: "Service", + Namespaced: true, + Verbs: VerbsReadOnly, + } +} + +// APIResourceNamespace returns a standard Namespace APIResource for testing +func APIResourceNamespace() metav1.APIResource { + return metav1.APIResource{ + Name: "namespaces", + Kind: "Namespace", + Namespaced: false, + Verbs: VerbsReadOnly, + } +} + +// APIResourceNode returns a standard Node APIResource for testing +func APIResourceNode() metav1.APIResource { + return metav1.APIResource{ + Name: "nodes", + Kind: "Node", + Namespaced: false, + Verbs: VerbsReadOnly, + } +} + +// APIResourceDeployment returns a standard Deployment APIResource for testing +func APIResourceDeployment() metav1.APIResource { + return metav1.APIResource{ + Name: "deployments", + Kind: "Deployment", + Namespaced: true, + Verbs: VerbsReadOnly, + } +} + +// APIResourceStatefulSet returns a standard StatefulSet APIResource for testing +func APIResourceStatefulSet() metav1.APIResource { + return metav1.APIResource{ + Name: "statefulsets", + Kind: "StatefulSet", + Namespaced: true, + Verbs: VerbsReadOnly, + } +} + +// APIResourceDaemonSet returns a standard DaemonSet APIResource for testing +func APIResourceDaemonSet() metav1.APIResource { + return metav1.APIResource{ + Name: "daemonsets", + Kind: "DaemonSet", + Namespaced: true, + Verbs: VerbsReadOnly, + } +} + +// APIResourceClusterRole returns a standard ClusterRole APIResource for testing +func APIResourceClusterRole() metav1.APIResource { + return metav1.APIResource{ + Name: "clusterroles", + Kind: "ClusterRole", + Namespaced: false, + Verbs: VerbsReadOnly, + } +} + +// APIResourceListV1 returns a standard v1 APIResourceList for testing with common core resources +func APIResourceListV1() *metav1.APIResourceList { + return &metav1.APIResourceList{ + GroupVersion: "v1", + APIResources: []metav1.APIResource{ + APIResourceConfigMap(), + APIResourceSecret(), + APIResourcePod(), + APIResourceService(), + APIResourceNamespace(), + APIResourceNode(), + }, + } +} + +// APIResourceListAppsV1 returns a standard apps/v1 APIResourceList for testing +func APIResourceListAppsV1() *metav1.APIResourceList { + return &metav1.APIResourceList{ + GroupVersion: "apps/v1", + APIResources: []metav1.APIResource{ + APIResourceDeployment(), + APIResourceStatefulSet(), + APIResourceDaemonSet(), + }, + } +} + +// APIResourceWithVerbs creates a custom APIResource with specified verbs for testing +func APIResourceWithVerbs(name, kind string, namespaced bool, verbs []string) metav1.APIResource { + return metav1.APIResource{ + Name: name, + Kind: kind, + Namespaced: namespaced, + Verbs: verbs, + } +} + +// GVK helpers - GroupVersionKind for common resources + +// GVKConfigMap returns the GroupVersionKind for ConfigMap +func GVKConfigMap() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "", Version: "v1", Kind: "ConfigMap"} +} + +// GVKSecret returns the GroupVersionKind for Secret +func GVKSecret() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Secret"} +} + +// GVKPod returns the GroupVersionKind for Pod +func GVKPod() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Pod"} +} + +// GVKService returns the GroupVersionKind for Service +func GVKService() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Service"} +} + +// GVKNamespace returns the GroupVersionKind for Namespace +func GVKNamespace() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Namespace"} +} + +// GVKNode returns the GroupVersionKind for Node +func GVKNode() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "", Version: "v1", Kind: "Node"} +} + +// GVKDeployment returns the GroupVersionKind for Deployment +func GVKDeployment() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "Deployment"} +} + +// GVKStatefulSet returns the GroupVersionKind for StatefulSet +func GVKStatefulSet() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "StatefulSet"} +} + +// GVKDaemonSet returns the GroupVersionKind for DaemonSet +func GVKDaemonSet() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "apps", Version: "v1", Kind: "DaemonSet"} +} + +// GVKClusterRole returns the GroupVersionKind for ClusterRole +func GVKClusterRole() schema.GroupVersionKind { + return schema.GroupVersionKind{Group: "rbac.authorization.k8s.io", Version: "v1", Kind: "ClusterRole"} +} + +// GVR helpers - GroupVersionResource for common resources + +// GVRConfigMap returns the GroupVersionResource for configmaps +func GVRConfigMap() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "", Version: "v1", Resource: "configmaps"} +} + +// GVRSecret returns the GroupVersionResource for secrets +func GVRSecret() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "", Version: "v1", Resource: "secrets"} +} + +// GVRPod returns the GroupVersionResource for pods +func GVRPod() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "", Version: "v1", Resource: "pods"} +} + +// GVRService returns the GroupVersionResource for services +func GVRService() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "", Version: "v1", Resource: "services"} +} + +// GVRNamespace returns the GroupVersionResource for namespaces +func GVRNamespace() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "", Version: "v1", Resource: "namespaces"} +} + +// GVRNode returns the GroupVersionResource for nodes +func GVRNode() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "", Version: "v1", Resource: "nodes"} +} + +// GVRDeployment returns the GroupVersionResource for deployments +func GVRDeployment() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "deployments"} +} + +// GVRStatefulSet returns the GroupVersionResource for statefulsets +func GVRStatefulSet() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "statefulsets"} +} + +// GVRDaemonSet returns the GroupVersionResource for daemonsets +func GVRDaemonSet() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "apps", Version: "v1", Resource: "daemonsets"} +} + +// GVRClusterRole returns the GroupVersionResource for clusterroles +func GVRClusterRole() schema.GroupVersionResource { + return schema.GroupVersionResource{Group: "rbac.authorization.k8s.io", Version: "v1", Resource: "clusterroles"} +}