From 049a4bdceb4b520b1448734cf2c67f114a59b5a2 Mon Sep 17 00:00:00 2001 From: Dan Kortschak Date: Mon, 18 May 2026 07:17:46 +0930 Subject: [PATCH] mito,internal/fb: add -fb flag for filebeat-compatible config validation When -fb is passed, mito validates the run configuration against the same constraints the filebeat CEL input enforces: auth mutual exclusion, per-method field requirements, resource URL, rate limit and retry bounds, max_executions, and rejection of a "secret" key in state. A secret_state config field is provided as the correct alternative, injected into state.secret before execution. Validation uses plain Go structs with yaml tags, not go-ucfg. Behaviour under -fb is not covered by semver compatibility guarantees since the validation rules track upstream filebeat. --- internal/fb/config.go | 403 +++++++++++++++++++++ internal/fb/config_test.go | 724 +++++++++++++++++++++++++++++++++++++ mito.go | 43 ++- testdata/fb_mode.txt | 85 +++++ 4 files changed, 1250 insertions(+), 5 deletions(-) create mode 100644 internal/fb/config.go create mode 100644 internal/fb/config_test.go create mode 100644 testdata/fb_mode.txt diff --git a/internal/fb/config.go b/internal/fb/config.go new file mode 100644 index 0000000..0785b80 --- /dev/null +++ b/internal/fb/config.go @@ -0,0 +1,403 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +// Package fb provides filebeat-compatible configuration validation for +// the mito tool. Validation rules mirror the constraints in the filebeat +// CEL input's config.go and config_auth.go without depending on go-ucfg +// or any elastic-agent-libs package. +// +// Behaviour under -fb is not covered by semver compatibility guarantees. +// Validation rules may change between minor releases to track upstream +// filebeat changes. +package fb + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strings" + "time" + + "github.com/elastic/mito/internal/rc" + "github.com/elastic/mito/lib" +) + +// Config is the filebeat-compatible configuration for mito. It contains +// all the fields from rc.Config plus resource, rate limit, and retry +// sections that filebeat's CEL input expects. Auth field names follow +// filebeat conventions (user/password, not username/password). +type Config struct { + Globals map[string]interface{} `yaml:"globals"` + Regexps map[string]string `yaml:"regexp"` + XSDs map[string]string `yaml:"xsd"` + Auth *authConfig `yaml:"auth"` + HTTPHeaders http.Header `yaml:"http_headers"` + MaxBodySize int64 `yaml:"max_body_size"` + MaxExecutions *int `yaml:"max_executions"` + + Resource *resourceConfig `yaml:"resource"` + SecretState map[string]any `yaml:"secret_state"` +} + +// RC returns the rc.Config subset needed by the mito runtime. Auth is +// converted from filebeat's field names to the lib types mito uses. +func (c *Config) RC() rc.Config { + cfg := rc.Config{ + Globals: c.Globals, + Regexps: c.Regexps, + XSDs: c.XSDs, + HTTPHeaders: c.HTTPHeaders, + MaxBodySize: c.MaxBodySize, + MaxExecutions: c.MaxExecutions, + } + if c.Auth == nil { + return cfg + } + cfg.Auth = &rc.AuthConfig{} + if c.Auth.Basic != nil { + cfg.Auth.Basic = &lib.BasicAuth{ + Username: c.Auth.Basic.User, + Password: c.Auth.Basic.Password, + } + } + if c.Auth.Token != nil { + cfg.Auth.Token = &lib.TokenAuth{ + Type: c.Auth.Token.Type, + Value: c.Auth.Token.Value, + } + } + if c.Auth.OAuth2 != nil { + o := c.Auth.OAuth2 + cfg.Auth.OAuth2 = &rc.OAuth2Config{ + Provider: o.Provider, + ClientID: o.ClientID, + ClientSecret: o.ClientSecret, + EndpointParams: o.EndpointParams, + Password: o.Password, + Scopes: o.Scopes, + TokenURL: o.TokenURL, + User: o.User, + GoogleCredentialsFile: o.GoogleCredentialsFile, + GoogleCredentialsJSON: o.GoogleCredentialsJSON, + GoogleJWTFile: o.GoogleJWTFile, + GoogleJWTJSON: o.GoogleJWTJSON, + GoogleDelegatedAccount: o.GoogleDelegatedAccount, + AzureTenantID: o.AzureTenantID, + AzureResource: o.AzureResource, + } + } + return cfg +} + +// Validate checks the configuration against filebeat CEL input +// constraints. Error messages match filebeat's where possible. +func (c *Config) Validate() error { + if c.Auth != nil { + if err := c.Auth.validate(); err != nil { + return err + } + } + if c.Resource != nil { + if err := c.Resource.validate(); err != nil { + return err + } + } + if c.MaxExecutions != nil && *c.MaxExecutions <= 0 { + return fmt.Errorf("invalid maximum number of executions: %d <= 0", *c.MaxExecutions) + } + return nil +} + +// CheckState inspects state data (the -data JSON) and rejects it if +// it contains a "secret" key at the top level. +func CheckState(state map[string]any) error { + if _, ok := state["secret"]; ok { + return errors.New(`state must not contain a "secret" key: use secret_state instead`) + } + return nil +} + +// authConfig mirrors filebeat's authConfig. Field names and YAML tags +// follow filebeat conventions. +type authConfig struct { + Basic *basicAuthConfig `yaml:"basic"` + Token *tokenAuthConfig `yaml:"token"` + Digest *digestAuthConfig `yaml:"digest"` + File *fileAuthConfig `yaml:"file"` + OAuth2 *oAuth2Config `yaml:"oauth2"` +} + +func (a *authConfig) validate() error { + var n int + if a.Basic != nil { + n++ + } + if a.Token != nil { + n++ + } + if a.Digest != nil { + n++ + } + if a.File != nil { + n++ + } + if a.OAuth2 != nil { + n++ + } + if n > 1 { + return errors.New("only one kind of auth can be enabled") + } + if a.Basic != nil { + if err := a.Basic.validate(); err != nil { + return err + } + } + if a.Token != nil { + if err := a.Token.validate(); err != nil { + return err + } + } + if a.Digest != nil { + if err := a.Digest.validate(); err != nil { + return err + } + } + if a.File != nil { + if err := a.File.validate(); err != nil { + return err + } + } + if a.OAuth2 != nil { + if err := a.OAuth2.validate(); err != nil { + return err + } + } + return nil +} + +type basicAuthConfig struct { + User string `yaml:"user"` + Password string `yaml:"password"` +} + +func (b *basicAuthConfig) validate() error { + if b.User == "" || b.Password == "" { + return errors.New("both user and password must be set") + } + return nil +} + +type tokenAuthConfig struct { + Type string `yaml:"type"` + Value string `yaml:"value"` +} + +func (t *tokenAuthConfig) validate() error { + if t.Type == "" || t.Value == "" { + return errors.New("both type and value must be set") + } + return nil +} + +type digestAuthConfig struct { + User string `yaml:"user"` + Password string `yaml:"password"` +} + +func (d *digestAuthConfig) validate() error { + if d.User == "" || d.Password == "" { + return errors.New("both user and password must be set") + } + return nil +} + +type fileAuthConfig struct { + Path string `yaml:"path"` + RefreshInterval *time.Duration `yaml:"refresh_interval"` +} + +func (f *fileAuthConfig) validate() error { + if f.Path == "" { + return errors.New("path must be set") + } + if f.RefreshInterval != nil && *f.RefreshInterval <= 0 { + return errors.New("refresh_interval must be greater than 0") + } + return nil +} + +type oAuth2Config struct { + Provider string `yaml:"provider"` + + ClientID string `yaml:"client.id"` + ClientSecret *string `yaml:"client.secret"` + EndpointParams url.Values `yaml:"endpoint_params"` + Password string `yaml:"password"` + Scopes []string `yaml:"scopes"` + TokenURL string `yaml:"token_url"` + User string `yaml:"user"` + + GoogleCredentialsFile string `yaml:"google.credentials_file"` + GoogleCredentialsJSON string `yaml:"google.credentials_json"` + GoogleJWTFile string `yaml:"google.jwt_file"` + GoogleJWTJSON string `yaml:"google.jwt_json"` + GoogleDelegatedAccount string `yaml:"google.delegated_account"` + + AzureTenantID string `yaml:"azure.tenant_id"` + AzureResource string `yaml:"azure.resource"` + + OktaJWKFile string `yaml:"okta.jwk_file"` + OktaJWKJSON string `yaml:"okta.jwk_json"` + OktaJWKPEM string `yaml:"okta.jwk_pem"` +} + +func (o *oAuth2Config) validate() error { + switch prov := strings.ToLower(o.Provider); prov { + case "azure": + return o.validateAzure() + case "google": + return o.validateGoogle() + case "okta": + return o.validateOkta() + case "": + if (o.User != "" && o.Password == "") || (o.User == "" && o.Password != "") { + return errors.New("both user and password credentials must be provided") + } + if o.TokenURL == "" || ((o.ClientID == "" || o.ClientSecret == nil) && (o.User == "" || o.Password == "")) { + return errors.New("both token_url and client credentials must be provided") + } + return nil + default: + return fmt.Errorf("unknown provider %q", prov) + } +} + +func (o *oAuth2Config) validateAzure() error { + if o.TokenURL == "" && o.AzureTenantID == "" { + return errors.New("at least one of token_url or tenant_id must be provided") + } + if o.TokenURL != "" && o.AzureTenantID != "" { + return errors.New("only one of token_url and tenant_id can be used") + } + if o.ClientID == "" || o.ClientSecret == nil { + return errors.New("client credentials must be provided") + } + return nil +} + +func (o *oAuth2Config) validateGoogle() error { + if o.TokenURL != "" || o.ClientID != "" || o.ClientSecret != nil || + o.AzureTenantID != "" || o.AzureResource != "" { + return errors.New("none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead") + } + if o.GoogleCredentialsJSON != "" { + if o.GoogleDelegatedAccount != "" { + return errors.New("google.delegated_account can only be provided with a jwt_file") + } + return nil + } + if o.GoogleCredentialsFile != "" { + if o.GoogleDelegatedAccount != "" { + return errors.New("google.delegated_account can only be provided with a jwt_file") + } + return nil + } + if o.GoogleJWTFile != "" || o.GoogleJWTJSON != "" { + return nil + } + return errors.New("no authentication credentials were configured or detected (ADC)") +} + +func (o *oAuth2Config) validateOkta() error { + if o.TokenURL == "" || o.ClientID == "" || len(o.Scopes) == 0 { + return errors.New("okta validation error: token_url, client_id, scopes must be provided") + } + var n int + if o.OktaJWKJSON != "" { + n++ + } + if o.OktaJWKFile != "" { + n++ + } + if o.OktaJWKPEM != "" { + n++ + } + if n != 1 { + return errors.New("okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided") + } + return nil +} + +type resourceConfig struct { + URL string `yaml:"url"` + RateLimit *rateLimit `yaml:"rate_limit"` + Retry *retryConfig `yaml:"retry"` +} + +func (r *resourceConfig) validate() error { + if r.URL == "" { + return errors.New("resource url must be set") + } + if _, err := url.Parse(r.URL); err != nil { + return fmt.Errorf("resource url is not valid: %w", err) + } + if r.RateLimit != nil { + if err := r.RateLimit.validate(); err != nil { + return err + } + } + if r.Retry != nil { + if err := r.Retry.validate(); err != nil { + return err + } + } + return nil +} + +type rateLimit struct { + Limit *float64 `yaml:"limit"` + Burst *int `yaml:"burst"` +} + +func (r *rateLimit) validate() error { + if r.Limit != nil && *r.Limit <= 0 { + return errors.New("limit must be greater than zero") + } + if r.Limit == nil && r.Burst != nil && *r.Burst <= 0 { + return errors.New("burst must be greater than zero if limit is not specified") + } + return nil +} + +type retryConfig struct { + MaxAttempts *int `yaml:"max_attempts"` + WaitMin *time.Duration `yaml:"wait_min"` + WaitMax *time.Duration `yaml:"wait_max"` +} + +func (r *retryConfig) validate() error { + switch { + case r.MaxAttempts != nil && *r.MaxAttempts <= 0: + return errors.New("max_attempts must be greater than zero") + case r.WaitMin != nil && *r.WaitMin <= 0: + return errors.New("wait_min must be greater than zero") + case r.WaitMax != nil && *r.WaitMax <= 0: + return errors.New("wait_max must be greater than zero") + } + return nil +} diff --git a/internal/fb/config_test.go b/internal/fb/config_test.go new file mode 100644 index 0000000..4ab75ca --- /dev/null +++ b/internal/fb/config_test.go @@ -0,0 +1,724 @@ +// Licensed to Elasticsearch B.V. under one or more contributor +// license agreements. See the NOTICE file distributed with +// this work for additional information regarding copyright +// ownership. Elasticsearch B.V. licenses this file to you under +// the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +package fb + +import ( + "testing" + "time" +) + +func ptr[T any](v T) *T { return &v } + +var validateTests = []struct { + name string + cfg Config + wantErr string +}{ + // Auth mutual exclusion. + { + name: "no_auth", + cfg: Config{Resource: &resourceConfig{URL: "http://localhost"}}, + }, + { + name: "basic_auth_valid", + cfg: Config{ + Auth: &authConfig{Basic: &basicAuthConfig{User: "u", Password: "p"}}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "two_auth_methods", + cfg: Config{ + Auth: &authConfig{ + Basic: &basicAuthConfig{User: "u", Password: "p"}, + Token: &tokenAuthConfig{Type: "Bearer", Value: "tok"}, + }, + }, + wantErr: "only one kind of auth can be enabled", + }, + { + name: "three_auth_methods", + cfg: Config{ + Auth: &authConfig{ + Basic: &basicAuthConfig{User: "u", Password: "p"}, + Token: &tokenAuthConfig{Type: "Bearer", Value: "tok"}, + Digest: &digestAuthConfig{User: "u", Password: "p"}, + }, + }, + wantErr: "only one kind of auth can be enabled", + }, + + // Basic auth. + { + name: "basic_missing_user", + cfg: Config{ + Auth: &authConfig{Basic: &basicAuthConfig{Password: "p"}}, + }, + wantErr: "both user and password must be set", + }, + { + name: "basic_missing_password", + cfg: Config{ + Auth: &authConfig{Basic: &basicAuthConfig{User: "u"}}, + }, + wantErr: "both user and password must be set", + }, + { + name: "basic_both_empty", + cfg: Config{ + Auth: &authConfig{Basic: &basicAuthConfig{}}, + }, + wantErr: "both user and password must be set", + }, + + // Token auth. + { + name: "token_valid", + cfg: Config{ + Auth: &authConfig{Token: &tokenAuthConfig{Type: "Bearer", Value: "abc"}}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "token_missing_type", + cfg: Config{ + Auth: &authConfig{Token: &tokenAuthConfig{Value: "abc"}}, + }, + wantErr: "both type and value must be set", + }, + { + name: "token_missing_value", + cfg: Config{ + Auth: &authConfig{Token: &tokenAuthConfig{Type: "Bearer"}}, + }, + wantErr: "both type and value must be set", + }, + + // Digest auth. + { + name: "digest_valid", + cfg: Config{ + Auth: &authConfig{Digest: &digestAuthConfig{User: "u", Password: "p"}}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "digest_missing_user", + cfg: Config{ + Auth: &authConfig{Digest: &digestAuthConfig{Password: "p"}}, + }, + wantErr: "both user and password must be set", + }, + { + name: "digest_missing_password", + cfg: Config{ + Auth: &authConfig{Digest: &digestAuthConfig{User: "u"}}, + }, + wantErr: "both user and password must be set", + }, + + // File auth. + { + name: "file_valid", + cfg: Config{ + Auth: &authConfig{File: &fileAuthConfig{Path: "/etc/token"}}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "file_valid_with_refresh", + cfg: Config{ + Auth: &authConfig{File: &fileAuthConfig{ + Path: "/etc/token", + RefreshInterval: ptr(5 * time.Second), + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "file_missing_path", + cfg: Config{ + Auth: &authConfig{File: &fileAuthConfig{}}, + }, + wantErr: "path must be set", + }, + { + name: "file_zero_refresh", + cfg: Config{ + Auth: &authConfig{File: &fileAuthConfig{ + Path: "/etc/token", + RefreshInterval: ptr(time.Duration(0)), + }}, + }, + wantErr: "refresh_interval must be greater than 0", + }, + { + name: "file_negative_refresh", + cfg: Config{ + Auth: &authConfig{File: &fileAuthConfig{ + Path: "/etc/token", + RefreshInterval: ptr(-time.Second), + }}, + }, + wantErr: "refresh_interval must be greater than 0", + }, + + // OAuth2 — default provider. + { + name: "oauth2_default_client_credentials", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + TokenURL: "https://auth.example.com/token", + ClientID: "id", + ClientSecret: ptr("secret"), + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_default_password_grant", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + TokenURL: "https://auth.example.com/token", + ClientID: "id", + ClientSecret: ptr("secret"), + User: "u", + Password: "p", + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_default_missing_token_url", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + ClientID: "id", + ClientSecret: ptr("secret"), + }}, + }, + wantErr: "both token_url and client credentials must be provided", + }, + { + name: "oauth2_default_user_without_password", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + TokenURL: "https://auth.example.com/token", + User: "u", + }}, + }, + wantErr: "both user and password credentials must be provided", + }, + { + name: "oauth2_default_password_without_user", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + TokenURL: "https://auth.example.com/token", + Password: "p", + }}, + }, + wantErr: "both user and password credentials must be provided", + }, + + // OAuth2 — azure. + { + name: "oauth2_azure_valid_tenant", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "azure", + AzureTenantID: "tenant-id", + ClientID: "id", + ClientSecret: ptr("secret"), + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_azure_valid_token_url", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "azure", + TokenURL: "https://login.microsoftonline.com/tenant/oauth2/token", + ClientID: "id", + ClientSecret: ptr("secret"), + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_azure_both_token_and_tenant", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "azure", + TokenURL: "https://example.com/token", + AzureTenantID: "tenant-id", + ClientID: "id", + ClientSecret: ptr("secret"), + }}, + }, + wantErr: "only one of token_url and tenant_id can be used", + }, + { + name: "oauth2_azure_neither_token_nor_tenant", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "azure", + ClientID: "id", + ClientSecret: ptr("secret"), + }}, + }, + wantErr: "at least one of token_url or tenant_id must be provided", + }, + { + name: "oauth2_azure_missing_client", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "azure", + AzureTenantID: "tenant-id", + }}, + }, + wantErr: "client credentials must be provided", + }, + + // OAuth2 — google. + { + name: "oauth2_google_credentials_json", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + GoogleCredentialsJSON: `{"type":"service_account"}`, + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_google_credentials_file", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + GoogleCredentialsFile: "/path/to/creds.json", + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_google_jwt_file", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + GoogleJWTFile: "/path/to/jwt.json", + GoogleDelegatedAccount: "admin@example.com", + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_google_jwt_json", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + GoogleJWTJSON: `{"type":"service_account"}`, + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_google_no_credentials", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + }}, + }, + wantErr: "no authentication credentials were configured or detected (ADC)", + }, + { + name: "oauth2_google_delegated_with_credentials_json", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + GoogleCredentialsJSON: `{"type":"service_account"}`, + GoogleDelegatedAccount: "admin@example.com", + }}, + }, + wantErr: "google.delegated_account can only be provided with a jwt_file", + }, + { + name: "oauth2_google_delegated_with_credentials_file", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + GoogleCredentialsFile: "/path/to/creds.json", + GoogleDelegatedAccount: "admin@example.com", + }}, + }, + wantErr: "google.delegated_account can only be provided with a jwt_file", + }, + { + name: "oauth2_google_rejects_client_fields", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "google", + TokenURL: "https://example.com/token", + }}, + }, + wantErr: "none of token_url and client credentials can be used, use google.credentials_file, google.jwt_file, google.credentials_json or ADC instead", + }, + + // OAuth2 — okta. + { + name: "oauth2_okta_jwk_json", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "okta", + TokenURL: "https://dev.okta.com/token", + ClientID: "id", + Scopes: []string{"openid"}, + OktaJWKJSON: `{"kty":"RSA"}`, + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_okta_jwk_file", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "okta", + TokenURL: "https://dev.okta.com/token", + ClientID: "id", + Scopes: []string{"openid"}, + OktaJWKFile: "/path/to/jwk.json", + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_okta_jwk_pem", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "okta", + TokenURL: "https://dev.okta.com/token", + ClientID: "id", + Scopes: []string{"openid"}, + OktaJWKPEM: "-----BEGIN PRIVATE KEY-----", + }}, + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "oauth2_okta_missing_token_url", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "okta", + ClientID: "id", + Scopes: []string{"openid"}, + OktaJWKJSON: `{"kty":"RSA"}`, + }}, + }, + wantErr: "okta validation error: token_url, client_id, scopes must be provided", + }, + { + name: "oauth2_okta_missing_scopes", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "okta", + TokenURL: "https://dev.okta.com/token", + ClientID: "id", + OktaJWKJSON: `{"kty":"RSA"}`, + }}, + }, + wantErr: "okta validation error: token_url, client_id, scopes must be provided", + }, + { + name: "oauth2_okta_no_jwk", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "okta", + TokenURL: "https://dev.okta.com/token", + ClientID: "id", + Scopes: []string{"openid"}, + }}, + }, + wantErr: "okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided", + }, + { + name: "oauth2_okta_two_jwk", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "okta", + TokenURL: "https://dev.okta.com/token", + ClientID: "id", + Scopes: []string{"openid"}, + OktaJWKJSON: `{"kty":"RSA"}`, + OktaJWKFile: "/path/to/jwk.json", + }}, + }, + wantErr: "okta validation error: one of okta.jwk_json, okta.jwk_file or okta.jwk_pem must be provided", + }, + + // Unknown provider. + { + name: "oauth2_unknown_provider", + cfg: Config{ + Auth: &authConfig{OAuth2: &oAuth2Config{ + Provider: "unknown", + }}, + }, + wantErr: `unknown provider "unknown"`, + }, + + // Resource URL. + { + name: "resource_valid", + cfg: Config{Resource: &resourceConfig{URL: "http://localhost:8080/api"}}, + }, + { + name: "resource_missing_url", + cfg: Config{Resource: &resourceConfig{}}, + wantErr: "resource url must be set", + }, + + // Rate limit. + { + name: "rate_limit_valid", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + RateLimit: &rateLimit{Limit: ptr(1.0), Burst: ptr(1)}, + }, + }, + }, + { + name: "rate_limit_zero_limit", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + RateLimit: &rateLimit{Limit: ptr(0.0)}, + }, + }, + wantErr: "limit must be greater than zero", + }, + { + name: "rate_limit_negative_limit", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + RateLimit: &rateLimit{Limit: ptr(-1.0)}, + }, + }, + wantErr: "limit must be greater than zero", + }, + { + name: "rate_limit_zero_burst_no_limit", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + RateLimit: &rateLimit{Burst: ptr(0)}, + }, + }, + wantErr: "burst must be greater than zero if limit is not specified", + }, + + // Retry. + { + name: "retry_valid", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + Retry: &retryConfig{ + MaxAttempts: ptr(3), + WaitMin: ptr(time.Second), + WaitMax: ptr(30 * time.Second), + }, + }, + }, + }, + { + name: "retry_zero_max_attempts", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + Retry: &retryConfig{MaxAttempts: ptr(0)}, + }, + }, + wantErr: "max_attempts must be greater than zero", + }, + { + name: "retry_negative_max_attempts", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + Retry: &retryConfig{MaxAttempts: ptr(-1)}, + }, + }, + wantErr: "max_attempts must be greater than zero", + }, + { + name: "retry_zero_wait_min", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + Retry: &retryConfig{WaitMin: ptr(time.Duration(0))}, + }, + }, + wantErr: "wait_min must be greater than zero", + }, + { + name: "retry_zero_wait_max", + cfg: Config{ + Resource: &resourceConfig{ + URL: "http://localhost", + Retry: &retryConfig{WaitMax: ptr(time.Duration(0))}, + }, + }, + wantErr: "wait_max must be greater than zero", + }, + + // Max executions. + { + name: "max_executions_valid", + cfg: Config{ + MaxExecutions: ptr(10), + Resource: &resourceConfig{URL: "http://localhost"}, + }, + }, + { + name: "max_executions_zero", + cfg: Config{MaxExecutions: ptr(0)}, + wantErr: "invalid maximum number of executions: 0 <= 0", + }, + { + name: "max_executions_negative", + cfg: Config{MaxExecutions: ptr(-5)}, + wantErr: "invalid maximum number of executions: -5 <= 0", + }, +} + +func TestValidate(t *testing.T) { + for _, tt := range validateTests { + t.Run(tt.name, func(t *testing.T) { + err := tt.cfg.Validate() + switch { + case tt.wantErr == "" && err != nil: + t.Errorf("unexpected error: %v", err) + case tt.wantErr != "" && err == nil: + t.Errorf("expected error %q, got nil", tt.wantErr) + case tt.wantErr != "" && err != nil: + if got := err.Error(); got != tt.wantErr { + t.Errorf("error mismatch:\n got: %s\nwant: %s", got, tt.wantErr) + } + } + }) + } +} + +var checkStateTests = []struct { + name string + state map[string]any + wantErr string +}{ + { + name: "no_secret", + state: map[string]any{"cursor": "abc"}, + }, + { + name: "empty_state", + state: map[string]any{}, + }, + { + name: "has_secret", + state: map[string]any{"secret": "value"}, + wantErr: `state must not contain a "secret" key: use secret_state instead`, + }, + { + name: "secret_with_other_keys", + state: map[string]any{"cursor": "abc", "secret": map[string]any{"token": "x"}}, + wantErr: `state must not contain a "secret" key: use secret_state instead`, + }, +} + +func TestCheckState(t *testing.T) { + for _, tt := range checkStateTests { + t.Run(tt.name, func(t *testing.T) { + err := CheckState(tt.state) + switch { + case tt.wantErr == "" && err != nil: + t.Errorf("unexpected error: %v", err) + case tt.wantErr != "" && err == nil: + t.Errorf("expected error %q, got nil", tt.wantErr) + case tt.wantErr != "" && err != nil: + if got := err.Error(); got != tt.wantErr { + t.Errorf("error mismatch:\n got: %s\nwant: %s", got, tt.wantErr) + } + } + }) + } +} + +func TestRC(t *testing.T) { + secret := "s3cret" + cfg := Config{ + Globals: map[string]interface{}{"key": "val"}, + Regexps: map[string]string{"foo": "bar"}, + MaxBodySize: 1024, + Auth: &authConfig{ + Basic: &basicAuthConfig{User: "u", Password: "p"}, + }, + } + got := cfg.RC() + if got.Auth == nil { + t.Fatal("expected auth to be set") + } + if got.Auth.Basic == nil { + t.Fatal("expected basic auth to be set") + } + if got.Auth.Basic.Username != "u" || got.Auth.Basic.Password != "p" { + t.Errorf("basic auth mismatch: got %q/%q, want u/p", got.Auth.Basic.Username, got.Auth.Basic.Password) + } + if got.MaxBodySize != 1024 { + t.Errorf("max_body_size mismatch: got %d, want 1024", got.MaxBodySize) + } + + cfg.Auth = &authConfig{ + Token: &tokenAuthConfig{Type: "Bearer", Value: "tok"}, + } + got = cfg.RC() + if got.Auth.Token == nil { + t.Fatal("expected token auth to be set") + } + if got.Auth.Token.Type != "Bearer" || got.Auth.Token.Value != "tok" { + t.Errorf("token auth mismatch: got %q/%q, want Bearer/tok", got.Auth.Token.Type, got.Auth.Token.Value) + } + + cfg.Auth = &authConfig{ + OAuth2: &oAuth2Config{ + Provider: "azure", + ClientID: "id", + ClientSecret: &secret, + TokenURL: "https://example.com/token", + }, + } + got = cfg.RC() + if got.Auth.OAuth2 == nil { + t.Fatal("expected oauth2 to be set") + } + if got.Auth.OAuth2.ClientID != "id" { + t.Errorf("oauth2 client_id mismatch: got %q, want id", got.Auth.OAuth2.ClientID) + } + if got.Auth.OAuth2.ClientSecret == nil || *got.Auth.OAuth2.ClientSecret != secret { + t.Errorf("oauth2 client_secret mismatch") + } +} diff --git a/mito.go b/mito.go index e7eb8b9..23331d7 100644 --- a/mito.go +++ b/mito.go @@ -54,6 +54,7 @@ import ( "google.golang.org/protobuf/proto" "google.golang.org/protobuf/types/known/structpb" + "github.com/elastic/mito/internal/fb" "github.com/elastic/mito/internal/httplog" "github.com/elastic/mito/internal/rc" "github.com/elastic/mito/lib" @@ -80,6 +81,7 @@ func Main() int { fold := flag.Bool("fold", false, "apply constant folding optimisation") dumpState := flag.String("dump", "", "dump eval state ('always' or 'error')") coverage := flag.String("coverage", "", "file to write an execution coverage report to (prefix if multiple executions are run)") + fbMode := flag.Bool("fb", false, "validate configuration against filebeat CEL input constraints (not covered by semver compatibility guarantees; validation rules track upstream filebeat changes)") version := flag.Bool("version", false, "print version and exit") flag.Parse() if *version { @@ -96,6 +98,7 @@ func Main() int { } ctx := context.Background() limit := rate.NewLimiter(1, 1) + var secretState map[string]any if *cfgPath != "" { f, err := os.Open(*cfgPath) if err != nil { @@ -103,13 +106,31 @@ func Main() int { return 2 } defer f.Close() - dec := yaml.NewDecoder(f) + var cfg Config - err = dec.Decode(&cfg) - if err != nil { - fmt.Fprintln(os.Stderr, err) - return 2 + if *fbMode { + dec := yaml.NewDecoder(f) + var fbCfg fb.Config + err = dec.Decode(&fbCfg) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + if err := fbCfg.Validate(); err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + cfg = fbCfg.RC() + secretState = fbCfg.SecretState + } else { + dec := yaml.NewDecoder(f) + err = dec.Decode(&cfg) + if err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } } + if len(cfg.Globals) != 0 { libs = append(libs, lib.Globals(cfg.Globals)) } @@ -219,6 +240,18 @@ func Main() int { fmt.Fprintf(os.Stderr, "failed parsing JSON data from %q: %s", *data, err) return 2 } + if *fbMode { + if state, ok := input.(map[string]any); ok { + if err := fb.CheckState(state); err != nil { + fmt.Fprintln(os.Stderr, err) + return 2 + } + if len(secretState) != 0 { + state["secret"] = secretState + input = state + } + } + } if *maxExecutions > 0 { // Only provide remaining_executions if we have set a limit. input = map[string]interface{}{ diff --git a/testdata/fb_mode.txt b/testdata/fb_mode.txt new file mode 100644 index 0000000..7506322 --- /dev/null +++ b/testdata/fb_mode.txt @@ -0,0 +1,85 @@ +# Test -fb flag rejects invalid auth config (missing password). +! mito -fb -use http -cfg cfg_bad_auth.yaml src.cel +stderr 'both user and password must be set' + +# Test -fb flag rejects missing resource URL. +! mito -fb -use http -cfg cfg_no_resource_url.yaml src.cel +stderr 'resource url must be set' + +# Test -fb flag rejects state with secret key. +! mito -fb -use http -cfg cfg_good.yaml -data state_secret.json src.cel +stderr 'state must not contain a "secret" key' + +# Test -fb flag rejects invalid retry config. +! mito -fb -use http -cfg cfg_bad_retry.yaml src.cel +stderr 'max_attempts must be greater than zero' + +# Test -fb flag rejects two auth methods. +! mito -fb -use http -cfg cfg_two_auth.yaml src.cel +stderr 'only one kind of auth can be enabled' + +# Test -fb flag accepts valid config and runs normally. +serve hello.text +expand src_var.cel src.cel +mito -fb -use http -cfg cfg_good.yaml src.cel +cmp stdout want.txt + +# Test -fb flag wires basic auth into HTTP client. +serve hello.text admin secret +expand src_var.cel src_auth.cel +mito -fb -use http -cfg cfg_good.yaml src_auth.cel +cmp stdout want.txt + +# Test -fb secret_state is injected into state.secret. +mito -fb -cfg cfg_secret_state.yaml -data state_no_secret.json src_secret.cel +cmp stdout want_secret.txt + +-- src_var.cel -- +string(get("${URL}").Body) +-- src.cel -- +"placeholder" +-- hello.text -- +hello +-- cfg_bad_auth.yaml -- +auth: + basic: + user: admin +-- cfg_no_resource_url.yaml -- +resource: + retry: + max_attempts: 3 +-- cfg_bad_retry.yaml -- +resource: + url: "http://localhost" + retry: + max_attempts: 0 +-- cfg_two_auth.yaml -- +auth: + basic: + user: admin + password: secret + token: + type: Bearer + value: abc123 +-- cfg_good.yaml -- +auth: + basic: + user: admin + password: secret +resource: + url: "http://localhost" +-- state_secret.json -- +{"secret": {"token": "do_not_put_this_here"}} +-- state_no_secret.json -- +{"cursor": "abc"} +-- cfg_secret_state.yaml -- +resource: + url: "http://localhost" +secret_state: + api_key: "hunter2" +-- src_secret.cel -- +state.secret.api_key +-- want_secret.txt -- +"hunter2" +-- want.txt -- +"hello\n"