diff --git a/cloud/linode/client/client.go b/cloud/linode/client/client.go index b184c602..f0ac40de 100644 --- a/cloud/linode/client/client.go +++ b/cloud/linode/client/client.go @@ -23,6 +23,8 @@ const ( DefaultLinodeAPIURL = "https://api.linode.com" ) +type TokenProvider func(context.Context) (string, error) + type Client interface { GetInstance(context.Context, int) (*linodego.Instance, error) ListInstances(context.Context, *linodego.ListOptions) ([]linodego.Instance, error) @@ -75,21 +77,48 @@ type Client interface { // linodego.Client implements Client var _ Client = (*linodego.Client)(nil) -// New creates a new linode client with a given token and default timeout -func New(token string, timeout time.Duration) (*linodego.Client, error) { +type tokenTransport struct { + base http.RoundTripper + tokenProvider TokenProvider +} + +func (t *tokenTransport) RoundTrip(req *http.Request) (*http.Response, error) { + token, err := t.tokenProvider(req.Context()) + if err != nil { + return nil, err + } + + clone := req.Clone(req.Context()) + clone.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + return t.base.RoundTrip(clone) +} + +// New creates a new linode client with a given token and default timeout. +func New(token string, timeout time.Duration, tokenProvider TokenProvider) (*linodego.Client, error) { userAgent := fmt.Sprintf("linode-cloud-controller-manager %s", linodego.DefaultUserAgent) apiURL := os.Getenv("LINODE_URL") if apiURL == "" { apiURL = DefaultLinodeAPIURL } + if tokenProvider == nil { + tokenProvider = func(context.Context) (string, error) { + return token, nil + } + } + + httpClient := &http.Client{Timeout: timeout} + httpClient.Transport = &tokenTransport{ + base: http.DefaultTransport, + tokenProvider: tokenProvider, + } - linodeClient := linodego.NewClient(&http.Client{Timeout: timeout}) + linodeClient := linodego.NewClient(httpClient) client, err := linodeClient.UseURL(apiURL) if err != nil { return nil, err } client.SetUserAgent(userAgent) - client.SetToken(token) klog.V(3).Infof("Linode client created with default timeout of %v", timeout) return client, nil diff --git a/cloud/linode/cloud.go b/cloud/linode/cloud.go index 01a5b003..2362e587 100644 --- a/cloud/linode/cloud.go +++ b/cloud/linode/cloud.go @@ -7,10 +7,15 @@ import ( "os" "regexp" "strconv" + "sync" "time" "golang.org/x/exp/slices" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes" + "k8s.io/client-go/rest" + "k8s.io/client-go/tools/clientcmd" cloudprovider "k8s.io/cloud-provider" "k8s.io/klog/v2" @@ -21,12 +26,16 @@ import ( const ( // The name of this cloudprovider - ProviderName = "linode" - accessTokenEnv = "LINODE_API_TOKEN" - regionEnv = "LINODE_REGION" - ciliumLBType = "cilium-bgp" - nodeBalancerLBType = "nodebalancer" - tokenHealthCheckPeriod = 5 * time.Minute + ProviderName = "linode" + regionEnv = "LINODE_REGION" + defaultTokenSecretName = "ccm-linode" + defaultTokenSecretKey = "apiToken" + defaultTokenSecretNamespace = "kube-system" + tokenSecretCacheTTLEnv = "LINODE_API_TOKEN_CACHE_TTL_SECONDS" + defaultTokenSecretCacheTTL = time.Minute + ciliumLBType = "cilium-bgp" + nodeBalancerLBType = "nodebalancer" + tokenHealthCheckPeriod = 5 * time.Minute ) var supportedLoadBalancerTypes = []string{ciliumLBType, nodeBalancerLBType} @@ -43,8 +52,69 @@ var ( instanceCache *services.Instances ipHolderCharLimit int = 23 NodeBalancerPrefixCharLimit int = 19 + + newKubernetesClient = defaultKubernetesClient ) +type tokenSecretProvider struct { + kubeClient kubernetes.Interface + namespace string + name string + key string + now func() time.Time + cacheTTL time.Duration + + mu sync.RWMutex + cachedToken string + expiresAt time.Time +} + +func (t *tokenSecretProvider) String() string { + return fmt.Sprintf("%s/%s[%s]", t.namespace, t.name, t.key) +} + +func (t *tokenSecretProvider) nowTime() time.Time { + if t.now != nil { + return t.now() + } + + return time.Now() +} + +func (t *tokenSecretProvider) GetToken(ctx context.Context) (string, error) { + now := t.nowTime() + + t.mu.RLock() + if t.cachedToken != "" && now.Before(t.expiresAt) { + token := t.cachedToken + t.mu.RUnlock() + return token, nil + } + t.mu.RUnlock() + + secret, err := t.kubeClient.CoreV1().Secrets(t.namespace).Get(ctx, t.name, metav1.GetOptions{}) + if err != nil { + return "", fmt.Errorf("failed to get secret %s: %w", t.String(), err) + } + + rawToken, found := secret.Data[t.key] + if !found { + return "", fmt.Errorf("secret %s does not contain key %q", t.String(), t.key) + } + + token := string(rawToken) + if token == "" { + return "", fmt.Errorf("secret %s key %q is empty", t.String(), t.key) + } + + t.mu.Lock() + t.cachedToken = token + t.expiresAt = t.nowTime().Add(t.cacheTTL) + t.mu.Unlock() + + return token, nil +} + func init() { registerMetrics() cloudprovider.RegisterCloudProvider( @@ -56,8 +126,8 @@ func init() { // newLinodeClientWithPrometheus creates a new client kept in its own local // scope and returns an instrumented one that should be used and passed around -func newLinodeClientWithPrometheus(apiToken string, timeout time.Duration) (client.Client, error) { - linodeClient, err := client.New(apiToken, timeout) +func newLinodeClientWithPrometheus(apiToken string, timeout time.Duration, tokenProvider client.TokenProvider) (client.Client, error) { + linodeClient, err := client.New(apiToken, timeout, tokenProvider) if err != nil { return nil, fmt.Errorf("client was not created successfully: %w", err) } @@ -69,16 +139,70 @@ func newLinodeClientWithPrometheus(apiToken string, timeout time.Duration) (clie return client.NewClientWithPrometheus(linodeClient), nil } +func defaultKubernetesClient() (kubernetes.Interface, error) { + var ( + kubeConfig *rest.Config + err error + ) + + kubeconfigFlag := options.Options.KubeconfigFlag + if kubeconfigFlag == nil || kubeconfigFlag.Value.String() == "" { + kubeConfig, err = rest.InClusterConfig() + } else { + kubeConfig, err = clientcmd.BuildConfigFromFlags("", kubeconfigFlag.Value.String()) + } + if err != nil { + return nil, err + } + + return kubernetes.NewForConfig(kubeConfig) +} + +func tokenSecretCacheTTLFromEnv() time.Duration { + tokenCacheTTL := defaultTokenSecretCacheTTL + if raw, ok := os.LookupEnv(tokenSecretCacheTTLEnv); ok { + if ttlSeconds, err := strconv.Atoi(raw); err == nil && ttlSeconds > 0 { + tokenCacheTTL = time.Duration(ttlSeconds) * time.Second + } + } + + return tokenCacheTTL +} + func newCloud() (cloudprovider.Interface, error) { region := os.Getenv(regionEnv) if region == "" { return nil, fmt.Errorf("%s must be set in the environment (use a k8s secret)", regionEnv) } - // Read environment variables (from secrets) - apiToken := os.Getenv(accessTokenEnv) - if apiToken == "" { - return nil, fmt.Errorf("%s must be set in the environment (use a k8s secret)", accessTokenEnv) + secretName := options.Options.LinodeAPITokenSecretName + if secretName == "" { + secretName = defaultTokenSecretName + } + secretKey := options.Options.LinodeAPITokenSecretKey + if secretKey == "" { + secretKey = defaultTokenSecretKey + } + secretNamespace := options.Options.LinodeAPITokenSecretNamespace + if secretNamespace == "" { + secretNamespace = defaultTokenSecretNamespace + } + + kubeClient, err := newKubernetesClient() + if err != nil { + return nil, fmt.Errorf("failed to create kubernetes client for token secret retrieval: %w", err) + } + + tokenProvider := tokenSecretProvider{ + kubeClient: kubeClient, + namespace: secretNamespace, + name: secretName, + key: secretKey, + } + + apiToken, err := tokenProvider.GetToken(context.Background()) + if err != nil { + return nil, err } // set timeout used by linodeclient for API calls @@ -89,7 +213,9 @@ func newCloud() (cloudprovider.Interface, error) { } } - linodeClient, err := newLinodeClientWithPrometheus(apiToken, timeout) + tokenProvider.cacheTTL = tokenSecretCacheTTLFromEnv() + + linodeClient, err := newLinodeClientWithPrometheus(apiToken, timeout, tokenProvider.GetToken) if err != nil { return nil, err } @@ -104,7 +230,7 @@ func newCloud() (cloudprovider.Interface, error) { } if !authenticated { - return nil, fmt.Errorf("linode api token %q is invalid", accessTokenEnv) + return nil, fmt.Errorf("linode api token from secret %s is invalid", tokenProvider.String()) } healthChecker = newHealthChecker(linodeClient, tokenHealthCheckPeriod, options.Options.GlobalStopChannel) diff --git a/cloud/linode/cloud_test.go b/cloud/linode/cloud_test.go index 4aed65e2..e9a52e20 100644 --- a/cloud/linode/cloud_test.go +++ b/cloud/linode/cloud_test.go @@ -4,10 +4,15 @@ import ( "reflect" "strings" "testing" + "time" "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/kubernetes" + k8sfake "k8s.io/client-go/kubernetes/fake" cloudprovider "k8s.io/cloud-provider" "github.com/linode/linode-cloud-controller-manager/cloud/linode/client/mocks" @@ -15,13 +20,108 @@ import ( "github.com/linode/linode-cloud-controller-manager/cloud/linode/services" ) +func configureKubernetesClientWithTokenSecret(t *testing.T, namespace string, tokenData map[string]string) { + t.Helper() + + fakeClient := k8sfake.NewSimpleClientset() + secretData := make(map[string][]byte, len(tokenData)) + for key, value := range tokenData { + secretData[key] = []byte(value) + } + + _, err := fakeClient.CoreV1().Secrets(namespace).Create(t.Context(), &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: "ccm-linode", + Namespace: namespace, + }, + Data: secretData, + }, metav1.CreateOptions{}) + require.NoError(t, err) + + original := newKubernetesClient + t.Cleanup(func() { + newKubernetesClient = original + }) + + newKubernetesClient = func() (kubernetes.Interface, error) { return fakeClient, nil } +} + +func TestTokenSecretProviderCache(t *testing.T) { + namespace := "kube-system" + secretName := "ccm-linode" + secretKey := "apiToken" + + client := k8sfake.NewSimpleClientset(&v1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Name: secretName, + Namespace: namespace, + }, + Data: map[string][]byte{ + secretKey: []byte("token-v1"), + }, + }) + + now := time.Now() + provider := &tokenSecretProvider{ + kubeClient: client, + namespace: namespace, + name: secretName, + key: secretKey, + cacheTTL: defaultTokenSecretCacheTTL, + now: func() time.Time { + return now + }, + } + + firstToken, err := provider.GetToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-v1", firstToken) + + secret, err := client.CoreV1().Secrets(namespace).Get(t.Context(), secretName, metav1.GetOptions{}) + require.NoError(t, err) + secret.Data[secretKey] = []byte("token-v2") + _, err = client.CoreV1().Secrets(namespace).Update(t.Context(), secret, metav1.UpdateOptions{}) + require.NoError(t, err) + + cachedToken, err := provider.GetToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-v1", cachedToken) + + now = now.Add(defaultTokenSecretCacheTTL + time.Second) + refreshedToken, err := provider.GetToken(t.Context()) + require.NoError(t, err) + assert.Equal(t, "token-v2", refreshedToken) +} + +func TestTokenSecretCacheTTLFromEnv(t *testing.T) { + t.Run("uses default ttl", func(t *testing.T) { + t.Setenv(tokenSecretCacheTTLEnv, "") + assert.Equal(t, defaultTokenSecretCacheTTL, tokenSecretCacheTTLFromEnv()) + }) + + t.Run("uses configured ttl when valid", func(t *testing.T) { + t.Setenv(tokenSecretCacheTTLEnv, "7") + assert.Equal(t, 7*time.Second, tokenSecretCacheTTLFromEnv()) + }) + + t.Run("falls back to default ttl when invalid", func(t *testing.T) { + t.Setenv(tokenSecretCacheTTLEnv, "invalid") + assert.Equal(t, defaultTokenSecretCacheTTL, tokenSecretCacheTTLFromEnv()) + }) + + t.Run("falls back to default ttl when non-positive", func(t *testing.T) { + t.Setenv(tokenSecretCacheTTLEnv, "0") + assert.Equal(t, defaultTokenSecretCacheTTL, tokenSecretCacheTTLFromEnv()) + }) +} + func TestNewCloudRouteControllerDisabled(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - t.Setenv("LINODE_API_TOKEN", "dummyapitoken") t.Setenv("LINODE_REGION", "us-east") t.Setenv("LINODE_REQUEST_TIMEOUT_SECONDS", "10") + configureKubernetesClientWithTokenSecret(t, "kube-system", map[string]string{"apiToken": "dummyapitoken"}) options.Options.NodeBalancerPrefix = "ccm" t.Run("should not fail if vpc is empty and routecontroller is disabled", func(t *testing.T) { @@ -43,16 +143,16 @@ func TestNewCloud(t *testing.T) { ctrl := gomock.NewController(t) defer ctrl.Finish() - t.Setenv("LINODE_API_TOKEN", "dummyapitoken") t.Setenv("LINODE_REGION", "us-east") t.Setenv("LINODE_REQUEST_TIMEOUT_SECONDS", "10") t.Setenv("LINODE_ROUTES_CACHE_TTL_SECONDS", "60") t.Setenv("LINODE_URL", "https://api.linode.com/v4") + configureKubernetesClientWithTokenSecret(t, "kube-system", map[string]string{"apiToken": "dummyapitoken"}) options.Options.LinodeGoDebug = true options.Options.NodeBalancerPrefix = "ccm" t.Run("should fail if api token is empty", func(t *testing.T) { - t.Setenv("LINODE_API_TOKEN", "") + configureKubernetesClientWithTokenSecret(t, "kube-system", map[string]string{"apiToken": ""}) _, err := newCloud() assert.Error(t, err, "expected error when api token is empty") }) diff --git a/cloud/linode/options/options.go b/cloud/linode/options/options.go index aac562f9..2271de30 100644 --- a/cloud/linode/options/options.go +++ b/cloud/linode/options/options.go @@ -37,4 +37,7 @@ var Options struct { NodeCIDRMaskSizeIPv4 int NodeCIDRMaskSizeIPv6 int NodeBalancerPrefix string + LinodeAPITokenSecretName string + LinodeAPITokenSecretKey string + LinodeAPITokenSecretNamespace string } diff --git a/deploy/ccm-linode-template.yaml b/deploy/ccm-linode-template.yaml index 829da922..604b68ab 100644 --- a/deploy/ccm-linode-template.yaml +++ b/deploy/ccm-linode-template.yaml @@ -115,15 +115,13 @@ spec: - --v=3 - --secure-port=10253 - --webhook-secure-port=0 + - --linode-api-token-secret-name=ccm-linode + - --linode-api-token-secret-key=apiToken + - --linode-api-token-secret-namespace=kube-system volumeMounts: - mountPath: /etc/kubernetes name: k8s env: - - name: LINODE_API_TOKEN - valueFrom: - secretKeyRef: - name: ccm-linode - key: apiToken - name: LINODE_REGION valueFrom: secretKeyRef: diff --git a/deploy/chart/templates/daemonset.yaml b/deploy/chart/templates/daemonset.yaml index f1f37e4e..cce6fdc6 100644 --- a/deploy/chart/templates/daemonset.yaml +++ b/deploy/chart/templates/daemonset.yaml @@ -160,6 +160,9 @@ spec: {{- with .Values.tokenHealthChecker }} - --enable-token-health-checker={{ . }} {{- end }} + - --linode-api-token-secret-name={{ if .Values.secretRef }}{{ .Values.secretRef.name | default "ccm-linode" }}{{ else }}ccm-linode{{ end }} + - --linode-api-token-secret-key={{ if .Values.secretRef }}{{ .Values.secretRef.apiTokenRef | default "apiToken" }}{{ else }}apiToken{{ end }} + - --linode-api-token-secret-namespace={{ if .Values.secretRef }}{{ .Values.secretRef.namespace | default .Values.namespace }}{{ else }}{{ .Values.namespace }}{{ end }} {{- with .Values.nodeBalancerTags }} - --nodebalancer-tags={{ join " " . }} {{- end }} @@ -215,11 +218,6 @@ spec: - name: KUBERNETES_SERVICE_PORT value: {{ .Values.k8sServicePort | quote }} {{- end }} - - name: LINODE_API_TOKEN - valueFrom: - secretKeyRef: - name: {{ if .Values.secretRef }}{{ .Values.secretRef.name | default "ccm-linode" }}{{ else }}"ccm-linode"{{ end }} - key: {{ if .Values.secretRef }}{{ .Values.secretRef.apiTokenRef | default "apiToken" }}{{ else }}"apiToken"{{ end }} - name: LINODE_REGION valueFrom: secretKeyRef: diff --git a/deploy/chart/values.yaml b/deploy/chart/values.yaml index 17c9d4ed..2173d1c3 100644 --- a/deploy/chart/values.yaml +++ b/deploy/chart/values.yaml @@ -7,6 +7,7 @@ region: "" # Set these values if your APIToken and region are already present in a k8s secret. # secretRef: # name: "linode-ccm" +# namespace: "kube-system" # apiTokenRef: "apiToken" # regionRef: "region" diff --git a/main.go b/main.go index afe5941c..0b58dadb 100644 --- a/main.go +++ b/main.go @@ -103,6 +103,9 @@ func main() { command.Flags().BoolVar(&ccmOptions.Options.DisableNodeBalancerVPCBackends, "disable-nodebalancer-vpc-backends", false, "disables nodebalancer backends in VPCs (when enabled, nodebalancers will only have private IPs as backends for backward compatibility)") command.Flags().StringVar(&ccmOptions.Options.NodeBalancerPrefix, "nodebalancer-prefix", "ccm", fmt.Sprintf("Name prefix for NoadBalancers. (max. %v char.)", linode.NodeBalancerPrefixCharLimit)) command.Flags().BoolVar(&ccmOptions.Options.DisableIPv6NodeCIDRAllocation, "disable-ipv6-node-cidr-allocation", false, "disables IPv6 node cidr allocation by ipam controller (when enabled, IPv6 cidr ranges will be allocated to nodes)") + command.Flags().StringVar(&ccmOptions.Options.LinodeAPITokenSecretName, "linode-api-token-secret-name", "", "name of kubernetes secret containing Linode API token") + command.Flags().StringVar(&ccmOptions.Options.LinodeAPITokenSecretKey, "linode-api-token-secret-key", "", "key in kubernetes secret containing Linode API token") + command.Flags().StringVar(&ccmOptions.Options.LinodeAPITokenSecretNamespace, "linode-api-token-secret-namespace", "", "namespace of kubernetes secret containing Linode API token") // Set static flags command.Flags().VisitAll(func(fl *pflag.Flag) {