fix(auth/m2m): remove double-caching to enable proactive token refresh

NEXT_CHANGELOG.md

@@ -15,12 +15,14 @@
 
  * Fix double-caching of OAuth tokens in Azure client secret credentials ([#1549](https://github.com/databricks/databricks-sdk-go/issues/1549)).
  * Disable async token refresh for GCP credential providers to avoid wasted refresh attempts caused by double-caching with Google's internal `oauth2.ReuseTokenSource` ([#1549](https://github.com/databricks/databricks-sdk-go/issues/1549)).
+ * Fixed double-caching in M2M OAuth that prevented the proactive async token refresh from reaching the HTTP endpoint until ~10s before expiry, causing bursts of 401 errors at token rotation boundaries ([#1549](https://github.com/databricks/databricks-sdk-go/issues/1549)).
 
 ### Documentation
 
 ### Internal Changes
 
  * Normalize internal token sources on `auth.TokenSource` for proper context propagation ([#1577](https://github.com/databricks/databricks-sdk-go/pull/1577)).
+ * Fix `TestAzureGithubOIDCCredentials` hang caused by missing `HTTPTransport` stub: `EnsureResolved` now calls `resolveHostMetadata`, which makes a real network request when no transport is set ([#1550](https://github.com/databricks/databricks-sdk-go/pull/1550)).
  * Bump golang.org/x/crypto from 0.21.0 to 0.45.0 in /examples/slog ([#1566](https://github.com/databricks/databricks-sdk-go/pull/1566)).
  * Bump golang.org/x/net from 0.23.0 to 0.33.0 in /examples/slog ([#1127](https://github.com/databricks/databricks-sdk-go/pull/1127)).
  * Bump golang.org/x/oauth2 from 0.20.0 to 0.27.0 ([#1563](https://github.com/databricks/databricks-sdk-go/pull/1563)).

config/auth_azure_github_oidc_test.go

@@ -49,6 +49,7 @@ func TestAzureGithubOIDCCredentials(t *testing.T) {
 				AzureTenantID:              "test-tenant-id",
 				ActionsIDTokenRequestURL:   "http://endpoint.com/test?version=1",
 				ActionsIDTokenRequestToken: "token-1337",
+				HTTPTransport:              fixtures.MappingTransport{},
 			},
 			wantErrPrefix: errPrefix("github-oidc-azure auth: not configured"),
 		},
@@ -79,6 +80,7 @@ func TestAzureGithubOIDCCredentials(t *testing.T) {
 				AzureClientID:              "test-client-id",
 				ActionsIDTokenRequestURL:   "http://endpoint.com/test?version=1",
 				ActionsIDTokenRequestToken: "token-1337",
+				HTTPTransport:              fixtures.MappingTransport{},
 			},
 			wantErrPrefix: errPrefix("github-oidc-azure auth: not configured"),
 		},
@@ -94,6 +96,7 @@ func TestAzureGithubOIDCCredentials(t *testing.T) {
 				AzureTenantID:            "test-tenant-id",
 				Host:                     "http://host.com/test",
 				ActionsIDTokenRequestURL: "http://endpoint.com/test?version=1",
+				HTTPTransport:            fixtures.MappingTransport{},
 			},
 			wantErrPrefix: errPrefix("github-oidc-azure auth: not configured"),
 		},
@@ -109,6 +112,7 @@ func TestAzureGithubOIDCCredentials(t *testing.T) {
 				AzureTenantID:              "test-tenant-id",
 				Host:                       "http://host.com/test",
 				ActionsIDTokenRequestToken: "token-1337",
+				HTTPTransport:              fixtures.MappingTransport{},
 			},
 			wantErrPrefix: errPrefix("github-oidc-azure auth: not configured"),
 		},

config/auth_m2m.go

@@ -9,7 +9,6 @@ import (
 
 	"github.com/databricks/databricks-sdk-go/config/credentials"
 	"github.com/databricks/databricks-sdk-go/config/experimental/auth"
-	"github.com/databricks/databricks-sdk-go/config/experimental/auth/authconv"
 	"github.com/databricks/databricks-sdk-go/logger"
 )
 
@@ -28,16 +27,29 @@ func (c M2mCredentials) Configure(ctx context.Context, cfg *Config) (credentials
 		return nil, fmt.Errorf("oidc: %w", err)
 	}
 	logger.Debugf(ctx, "Generating Databricks OAuth token for Service Principal (%s)", cfg.ClientID)
-	ts := (&clientcredentials.Config{
+	ccfg := &clientcredentials.Config{
 		ClientID:     cfg.ClientID,
 		ClientSecret: cfg.ClientSecret,
 		AuthStyle:    oauth2.AuthStyleInHeader,
 		TokenURL:     endpoints.TokenEndpoint,
 		Scopes:       cfg.GetScopes(),
-	}).TokenSource(ctx)
+	}
 
-	authTs := authconv.AuthTokenSource(ts)
+	// Use a direct (non-caching) token source so that cachedTokenSource is the
+	// single cache layer. clientcredentials.Config.TokenSource returns an
+	// oauth2.ReuseTokenSource which adds a second cache with a 10 s expiryDelta.
+	// With double-caching the async refresh in cachedTokenSource calls through to
+	// ReuseTokenSource, which returns its own cached token without making an HTTP
+	// request until only ~10 s remain — defeating the proactive 20-min refresh
+	// window and causing a burst of 401s at token expiry.
+	// See https://github.com/databricks/databricks-sdk-go/issues/1549.
+	//
+	// ctx is captured from Configure; the oauth2 HTTP client is bound to it via
+	// InContextForOAuth2 and must be used for all token requests.
+	directTS := auth.TokenSourceFn(func(_ context.Context) (*oauth2.Token, error) {
+		return ccfg.Token(ctx)
+	})
 	return credentials.NewOAuthCredentialsProviderFromTokenSource(
-		auth.NewCachedTokenSource(authTs, cacheOptions(cfg)...),
+		auth.NewCachedTokenSource(directTS, cacheOptions(cfg)...),
 	), nil
 }

config/auth_m2m_test.go

@@ -1,11 +1,14 @@
 package config
 
 import (
+	"context"
 	"errors"
 	"net/http"
 	"net/url"
+	"sync/atomic"
 	"testing"
 
+	"github.com/databricks/databricks-sdk-go/config/credentials"
 	"github.com/databricks/databricks-sdk-go/credentials/u2m"
 	"github.com/databricks/databricks-sdk-go/httpclient/fixtures"
 	"golang.org/x/oauth2"
@@ -96,6 +99,83 @@ func TestM2mNotSupported(t *testing.T) {
 	}
 }
 
+// TestM2mCredentials_DirectTokenSource verifies that M2mCredentials.Configure
+// plumbs a direct token source (ccfg.Token) through cachedTokenSource rather
+// than wrapping clientcredentials.Config.TokenSource, which returns an
+// oauth2.ReuseTokenSource that adds a second cache layer. With double-caching,
+// the proactive async refresh in cachedTokenSource is silently suppressed until
+// ~10 s before expiry, causing bursts of 401 errors at token rotation boundaries.
+// See https://github.com/databricks/databricks-sdk-go/issues/1549.
+func TestM2mCredentials_DirectTokenSource(t *testing.T) {
+	var tokenCalls int32
+	transport := &postCountingTransport{
+		calls: &tokenCalls,
+		inner: fixtures.MappingTransport{
+			"GET /oidc/.well-known/oauth-authorization-server": {
+				Response: u2m.OAuthAuthorizationServer{
+					TokenEndpoint: "https://localhost/token",
+				},
+			},
+			"POST /token": {
+				Response: oauth2.Token{
+					TokenType:   "Bearer",
+					AccessToken: "test-token",
+				},
+			},
+		},
+	}
+
+	cfg := &Config{
+		Host:          "a",
+		ClientID:      "b",
+		ClientSecret:  "c",
+		AuthType:      "oauth-m2m",
+		ConfigFile:    "/dev/null",
+		HTTPTransport: transport,
+	}
+
+	err := cfg.EnsureResolved()
+	if err != nil {
+		t.Fatalf("EnsureResolved(): %v", err)
+	}
+
+	ctx := cfg.refreshClient.InContextForOAuth2(cfg.refreshCtx)
+	provider, err := M2mCredentials{}.Configure(ctx, cfg)
+	if err != nil {
+		t.Fatalf("Configure(): %v", err)
+	}
+
+	oauthProvider := provider.(credentials.OAuthCredentialsProvider)
+
+	// Token() goes through cachedTokenSource, which fetches once and caches.
+	// Verify the endpoint is reached (not short-circuited by an inner cache).
+	tok, err := oauthProvider.Token(context.Background())
+	if err != nil {
+		t.Fatalf("Token(): %v", err)
+	}
+	if tok.AccessToken == "" {
+		t.Fatalf("Token(): empty access token")
+	}
+
+	if got := int(atomic.LoadInt32(&tokenCalls)); got != 1 {
+		t.Errorf("token endpoint calls = %d, want 1", got)
+	}
+}
+
+type postCountingTransport struct {
+	calls *int32
+	inner http.RoundTripper
+}
+
+func (t *postCountingTransport) RoundTrip(req *http.Request) (*http.Response, error) {
+	if req.Method == "POST" {
+		atomic.AddInt32(t.calls, 1)
+	}
+	return t.inner.RoundTrip(req)
+}
+
+func (t *postCountingTransport) SkipRetryOnIO() bool { return true }
+
 func TestM2M_Scopes(t *testing.T) {
 	tests := []struct {
 		name   string