feat: metrics for error codes

cmd/serve_cmd.go

@@ -197,7 +197,7 @@ func serve(ctx context.Context) {
 		log.WithError(err).Fatal("http server listen failed")
 	}
 	err = httpSrv.Serve(listener)
-	if err == http.ErrServerClosed {
+	if errors.Is(err, http.ErrServerClosed) {
 		log.Info("http server closed")
 	} else if err != nil {
 		log.WithError(err).Fatal("http server serve failed")

internal/api/apierrors/errorcode.go

@@ -2,6 +2,7 @@ package apierrors
 
 type ErrorCode = string
 
+// All error codes the auth server returns are defined here.
 const (
 	// ErrorCodeUnknown should not be used directly, it only indicates a failure in the error handling system in such a way that an error code was not assigned properly.
 	ErrorCodeUnknown ErrorCode = "unknown"

internal/api/apierrors/errorcode_gen.go

@@ -0,0 +1,107 @@
+package apierrors
+
+//go:generate go test -run TestGenerate -args -generate
+//go:generate go fmt
+
+var errorCodesMap = map[string]string{
+	"anonymous_provider_disabled":                "ErrorCodeAnonymousProviderDisabled",
+	"bad_code_verifier":                          "ErrorCodeBadCodeVerifier",
+	"bad_json":                                   "ErrorCodeBadJSON",
+	"bad_jwt":                                    "ErrorCodeBadJWT",
+	"bad_oauth_callback":                         "ErrorCodeBadOAuthCallback",
+	"bad_oauth_state":                            "ErrorCodeBadOAuthState",
+	"captcha_failed":                             "ErrorCodeCaptchaFailed",
+	"conflict":                                   "ErrorCodeConflict",
+	"current_password_invalid":                   "ErrorCodeCurrentPasswordMismatch",
+	"current_password_required":                  "ErrorCodeCurrentPasswordRequired",
+	"custom_provider_not_found":                  "ErrorCodeCustomProviderNotFound",
+	"email_address_invalid":                      "ErrorCodeEmailAddressInvalid",
+	"email_address_not_authorized":               "ErrorCodeEmailAddressNotAuthorized",
+	"email_address_not_provided":                 "ErrorCodeEmailAddressNotProvided",
+	"email_conflict_identity_not_deletable":      "ErrorCodeEmailConflictIdentityNotDeletable",
+	"email_exists":                               "ErrorCodeEmailExists",
+	"email_not_confirmed":                        "ErrorCodeEmailNotConfirmed",
+	"email_provider_disabled":                    "ErrorCodeEmailProviderDisabled",
+	"feature_disabled":                           "ErrorCodeFeatureDisabled",
+	"flow_state_expired":                         "ErrorCodeFlowStateExpired",
+	"flow_state_not_found":                       "ErrorCodeFlowStateNotFound",
+	"hook_payload_invalid_content_type":          "ErrorCodeHookPayloadInvalidContentType",
+	"hook_payload_over_size_limit":               "ErrorCodeHookPayloadOverSizeLimit",
+	"hook_timeout":                               "ErrorCodeHookTimeout",
+	"hook_timeout_after_retry":                   "ErrorCodeHookTimeoutAfterRetry",
+	"identity_already_exists":                    "ErrorCodeIdentityAlreadyExists",
+	"identity_not_found":                         "ErrorCodeIdentityNotFound",
+	"insufficient_aal":                           "ErrorCodeInsufficientAAL",
+	"invalid_credentials":                        "ErrorCodeInvalidCredentials",
+	"invite_not_found":                           "ErrorCodeInviteNotFound",
+	"manual_linking_disabled":                    "ErrorCodeManualLinkingDisabled",
+	"mfa_challenge_expired":                      "ErrorCodeMFAChallengeExpired",
+	"mfa_factor_name_conflict":                   "ErrorCodeMFAFactorNameConflict",
+	"mfa_factor_not_found":                       "ErrorCodeMFAFactorNotFound",
+	"mfa_ip_address_mismatch":                    "ErrorCodeMFAIPAddressMismatch",
+	"mfa_phone_enroll_not_enabled":               "ErrorCodeMFAPhoneEnrollDisabled",
+	"mfa_phone_verify_not_enabled":               "ErrorCodeMFAPhoneVerifyDisabled",
+	"mfa_totp_enroll_not_enabled":                "ErrorCodeMFATOTPEnrollDisabled",
+	"mfa_totp_verify_not_enabled":                "ErrorCodeMFATOTPVerifyDisabled",
+	"mfa_verification_failed":                    "ErrorCodeMFAVerificationFailed",
+	"mfa_verification_rejected":                  "ErrorCodeMFAVerificationRejected",
+	"mfa_verified_factor_exists":                 "ErrorCodeMFAVerifiedFactorExists",
+	"mfa_webauthn_enroll_not_enabled":            "ErrorCodeMFAWebAuthnEnrollDisabled",
+	"mfa_webauthn_verify_not_enabled":            "ErrorCodeMFAWebAuthnVerifyDisabled",
+	"no_authorization":                           "ErrorCodeNoAuthorization",
+	"not_admin":                                  "ErrorCodeNotAdmin",
+	"oauth_authorization_not_found":              "ErrorCodeOAuthAuthorizationNotFound",
+	"oauth_client_not_found":                     "ErrorCodeOAuthClientNotFound",
+	"oauth_client_state_expired":                 "ErrorCodeOAuthClientStateExpired",
+	"oauth_client_state_not_found":               "ErrorCodeOAuthClientStateNotFound",
+	"oauth_consent_not_found":                    "ErrorCodeOAuthConsentNotFound",
+	"oauth_dynamic_client_registration_disabled": "ErrorCodeOAuthDynamicClientRegistrationDisabled",
+	"oauth_invalid_state":                        "ErrorCodeOAuthInvalidState",
+	"oauth_provider_not_supported":               "ErrorCodeOAuthProviderNotSupported",
+	"otp_disabled":                               "ErrorCodeOTPDisabled",
+	"otp_expired":                                "ErrorCodeOTPExpired",
+	"over_custom_provider_quota":                 "ErrorCodeOverCustomProviderQuota",
+	"over_email_send_rate_limit":                 "ErrorCodeOverEmailSendRateLimit",
+	"over_request_rate_limit":                    "ErrorCodeOverRequestRateLimit",
+	"over_sms_send_rate_limit":                   "ErrorCodeOverSMSSendRateLimit",
+	"phone_exists":                               "ErrorCodePhoneExists",
+	"phone_not_confirmed":                        "ErrorCodePhoneNotConfirmed",
+	"phone_provider_disabled":                    "ErrorCodePhoneProviderDisabled",
+	"provider_disabled":                          "ErrorCodeProviderDisabled",
+	"provider_email_needs_verification":          "ErrorCodeProviderEmailNeedsVerification",
+	"reauthentication_needed":                    "ErrorCodeReauthenticationNeeded",
+	"reauthentication_not_valid":                 "ErrorCodeReauthenticationNotValid",
+	"refresh_token_already_used":                 "ErrorCodeRefreshTokenAlreadyUsed",
+	"refresh_token_not_found":                    "ErrorCodeRefreshTokenNotFound",
+	"request_timeout":                            "ErrorCodeRequestTimeout",
+	"same_password":                              "ErrorCodeSamePassword",
+	"saml_assertion_no_email":                    "ErrorCodeSAMLAssertionNoEmail",
+	"saml_assertion_no_user_id":                  "ErrorCodeSAMLAssertionNoUserID",
+	"saml_entity_id_mismatch":                    "ErrorCodeSAMLEntityIDMismatch",
+	"saml_idp_already_exists":                    "ErrorCodeSAMLIdPAlreadyExists",
+	"saml_idp_not_found":                         "ErrorCodeSAMLIdPNotFound",
+	"saml_metadata_fetch_failed":                 "ErrorCodeSAMLMetadataFetchFailed",
+	"saml_provider_disabled":                     "ErrorCodeSAMLProviderDisabled",
+	"saml_relay_state_expired":                   "ErrorCodeSAMLRelayStateExpired",
+	"saml_relay_state_not_found":                 "ErrorCodeSAMLRelayStateNotFound",
+	"session_expired":                            "ErrorCodeSessionExpired",
+	"session_not_found":                          "ErrorCodeSessionNotFound",
+	"signup_disabled":                            "ErrorCodeSignupDisabled",
+	"single_identity_not_deletable":              "ErrorCodeSingleIdentityNotDeletable",
+	"sms_send_failed":                            "ErrorCodeSMSSendFailed",
+	"sso_domain_already_exists":                  "ErrorCodeSSODomainAlreadyExists",
+	"sso_provider_disabled":                      "ErrorCodeSSOProviderDisabled",
+	"sso_provider_not_found":                     "ErrorCodeSSOProviderNotFound",
+	"too_many_enrolled_mfa_factors":              "ErrorCodeTooManyEnrolledMFAFactors",
+	"unexpected_audience":                        "ErrorCodeUnexpectedAudience",
+	"unexpected_failure":                         "ErrorCodeUnexpectedFailure",
+	"unknown":                                    "ErrorCodeUnknown",
+	"user_already_exists":                        "ErrorCodeUserAlreadyExists",
+	"user_banned":                                "ErrorCodeUserBanned",
+	"user_not_found":                             "ErrorCodeUserNotFound",
+	"user_sso_managed":                           "ErrorCodeUserSSOManaged",
+	"validation_failed":                          "ErrorCodeValidationFailed",
+	"weak_password":                              "ErrorCodeWeakPassword",
+	"web3_provider_disabled":                     "ErrorCodeWeb3ProviderDisabled",
+	"web3_unsupported_chain":                     "ErrorCodeWeb3UnsupportedChain",
+}

internal/api/apierrors/errorcode_test.go

@@ -0,0 +1,151 @@
+package apierrors
+
+import (
+	"flag"
+	"fmt"
+	"go/ast"
+	"go/parser"
+	"go/token"
+	"maps"
+	"os"
+	"slices"
+	"strings"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+var generateFlag = flag.Bool("generate", false, "Run tests that generate code")
+
+func TestErrorCodesMap(t *testing.T) {
+	cur := helpParseErrorCodes(t)
+	gen := errorCodesMap
+
+	for curCode, curName := range cur {
+		genName, ok := gen[curCode]
+		if !ok {
+			t.Fatalf("error code %q: (%v) missing in errorCodesMap",
+				curCode, curName)
+		}
+		if genName != curName {
+			t.Fatalf("error code %q: (%v) has different name (%q) in errorCodesMap",
+				curCode, curName, genName)
+		}
+	}
+	if a, b := len(cur), len(gen); a != b {
+		const msg = "generated code out of sync:" +
+			" errorCodeSlice len(%v) != constant declaration len (%v)"
+		t.Fatalf(msg, a, b)
+	}
+}
+
+func TestGenerate(t *testing.T) {
+	if !*generateFlag {
+		t.SkipNow()
+	}
+
+	ecm := helpParseErrorCodes(t)
+	ecs := slices.Sorted(maps.Keys(ecm))
+
+	var sb strings.Builder
+	sb.WriteString("package apierrors\n\n")
+	sb.WriteString("//go:generate go test -run TestGenerate -args -generate\n")
+	sb.WriteString("//go:generate go fmt\n\n")
+
+	{
+		sb.WriteString("var errorCodesMap = map[string]string{\n")
+		for _, ec := range ecs {
+			fmt.Fprintf(&sb, "\t%q: %q,\n", ec, ecm[ec])
+		}
+		sb.WriteString("}\n\n")
+	}
+
+	os.WriteFile("errorcode_gen.go", []byte(sb.String()), 0644)
+}
+
+func helpParseErrorCodes(t *testing.T) map[string]string {
+	ecm, err := parseErrorCodesOnce()
+	require.NoError(t, err)
+	require.NotEmpty(t, ecm)
+	return maps.Clone(ecm)
+}
+
+var parseErrorCodesOnce = sync.OnceValues(func() (map[string]string, error) {
+	return parseErrorCodes()
+})
+
+func parseErrorCodes() (map[string]string, error) {
+	data, err := os.ReadFile(`errorcode.go`)
+	if err != nil {
+		const msg = "parseErrorCodes: os.ReadFile(`errorcode.go`): %w"
+		return nil, fmt.Errorf(msg, err)
+	}
+	src := string(data)
+
+	fset := token.NewFileSet()
+	f, err := parser.ParseFile(fset, "", src, parser.SkipObjectResolution)
+	if err != nil {
+		const msg = "parseErrorCodes: parser.ParseFile: %w"
+		return nil, fmt.Errorf(msg, err)
+	}
+
+	ecm := make(map[string]string)
+	for declIdx, decl := range f.Decls {
+		if err := parseErrorCodesDecl(ecm, declIdx, decl); err != nil {
+			return nil, fmt.Errorf("parseErrorCodes %w", err)
+		}
+	}
+	return ecm, nil
+}
+
+func parseErrorCodesDecl(ecm map[string]string, decIdx int, decl ast.Decl) error {
+	dec, ok := decl.(*ast.GenDecl)
+	if !ok || dec.Tok != token.CONST {
+		return nil
+	}
+	if n := len(dec.Specs); n == 0 {
+		return fmt.Errorf("decl[%d]: specs are empty", decIdx)
+	}
+	for idx, spec := range dec.Specs {
+		valSpec, ok := spec.(*ast.ValueSpec)
+		if !ok {
+			return fmt.Errorf("const[%d]: unexpected type: %T", idx, spec)
+		}
+		if n := len(valSpec.Names); n != 1 {
+			return fmt.Errorf("const[%d]: unexpected const len: %T", idx, n)
+		}
+
+		constName := valSpec.Names[0].Name
+		if !strings.HasPrefix(constName, "ErrorCode") {
+			return fmt.Errorf("const[%d]: missing ErrorCode prefix: %v", idx, constName)
+		}
+		if n := len(valSpec.Values); n != 1 {
+			return fmt.Errorf("const[%d]: unexpected const value len: %v", idx, n)
+		}
+
+		constExpr := valSpec.Values[0]
+		basicLit, ok := constExpr.(*ast.BasicLit)
+		if !ok {
+			return fmt.Errorf("const[%d]: unexpected const value expr type: %T", idx, constExpr)
+		}
+
+		constValue := basicLit.Value
+		if n := len(constValue); n <= 3 {
+			return fmt.Errorf("const[%d]: unexpected const value string len: %v (%q)",
+				idx, n, constValue)
+		}
+		if constValue[0] != '"' || constValue[len(constValue)-1] != '"' {
+			return fmt.Errorf("const[%d]: unexpected const value string quoting (%q)",
+				idx, constValue)
+		}
+		constValue = constValue[1 : len(constValue)-1]
+
+		if prev, found := ecm[constValue]; found {
+			msg := "const[%d]: duplicate error code: %q: already defined by %q"
+			return fmt.Errorf(msg, idx, constValue, prev)
+		}
+		ecm[constValue] = constName
+	}
+	return nil
+}

internal/api/apierrors/metrics.go

@@ -0,0 +1,81 @@
+package apierrors
+
+import (
+	"context"
+	"errors"
+	"fmt"
+
+	"go.opentelemetry.io/otel"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/metric"
+)
+
+// TODO(cstockton): Don't like how these are global variables here. I think we
+// should probably have a metrics package which is initialized before the api
+// server is created and then passed in as an option to the *API.
+var (
+	errorCodeCounter     metric.Int64Counter
+	errorCodeAttrsByCode = make(map[string]metric.MeasurementOption)
+)
+
+func RecordErrorCode(ctx context.Context, errorCode ErrorCode) {
+	attrs, ok := errorCodeAttrsByCode[errorCode]
+	if !ok {
+		attrs = errorCodeAttrsByCode[ErrorCodeUnknown]
+	}
+	errorCodeCounter.Add(ctx, 1, attrs)
+}
+
+func RecordPostgresCode(ctx context.Context, code string) {
+	attrs := metric.WithAttributeSet(
+		attribute.NewSet(
+			attribute.String("type", "postgres"),
+			attribute.String("error", code),
+		),
+	)
+	errorCodeCounter.Add(ctx, 1, attrs)
+}
+
+func InitMetrics() error {
+	return initMetrics(errorCodesMap)
+}
+
+func initMetrics(ecm map[string]string) error {
+	if len(errorCodesMap) == 0 {
+		const msg = "InitMetrics: errorCodesMap is empty"
+		return errors.New(msg)
+	}
+
+	counter, err := otel.Meter("gotrue").Int64Counter(
+		"global_auth_errors_total",
+		metric.WithDescription("Number of error codes returned by type and error."),
+		metric.WithUnit("{type}"),
+		metric.WithUnit("{error}"),
+	)
+	if err != nil {
+		return fmt.Errorf("InitMetrics: %w", err)
+	}
+
+	// TODO(cstockton): I'm not sure about having a single dimension of
+	// "error_code", as I begin trying to dig into the types of errors we
+	// raise I might want to add a type specifier. For example OAuthError does
+	// not have an auth error code, but may wrap one internally.
+	//
+	// This is really about deciding how to strike the balance between caller
+	// burden and best effort inferrence like we are doing here.
+	errorCodeAttrsByCode[ErrorCodeUnknown] = metric.WithAttributes(
+		attribute.String("error_code", ErrorCodeUnknown),
+	)
+	for code := range ecm {
+		attrs := metric.WithAttributeSet(
+			attribute.NewSet(
+				attribute.String("type", "api"),
+				attribute.String("error", code),
+			),
+		)
+		errorCodeAttrsByCode[code] = attrs
+	}
+
+	errorCodeCounter = counter
+	return nil
+}

internal/api/apierrors/metrics_test.go

@@ -0,0 +1,12 @@
+package apierrors
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestMetrics(t *testing.T) {
+	err := initMetrics(errorCodesMap)
+	require.NoError(t, err)
+}