diff --git a/api/v1/common.go b/api/v1/common.go index 87b83df45..4d468cfdd 100644 --- a/api/v1/common.go +++ b/api/v1/common.go @@ -283,6 +283,18 @@ type CustomScraperBase struct { DeleteFields []string `json:"deleteFields,omitempty"` } +// ScraperExclusion specifies patterns for excluding external entities by name. +// Patterns support wildcards via collections.MatchItems (e.g. "system:controller:*"). +type ScraperExclusion struct { + ExternalRoles []string `json:"externalRoles,omitempty" yaml:"externalRoles,omitempty"` + ExternalUsers []string `json:"externalUsers,omitempty" yaml:"externalUsers,omitempty"` + ExternalGroups []string `json:"externalGroups,omitempty" yaml:"externalGroups,omitempty"` +} + +func (e ScraperExclusion) IsEmpty() bool { + return len(e.ExternalRoles) == 0 && len(e.ExternalUsers) == 0 && len(e.ExternalGroups) == 0 +} + type BaseScraper struct { CustomScraperBase `yaml:",inline" json:",inline"` @@ -298,6 +310,9 @@ type BaseScraper struct { // Properties are custom templatable properties for the scraped config items // grouped by the config type. Properties []ConfigProperties `json:"properties,omitempty" template:"true"` + + // Exclude specifies patterns for excluding external entities. + Exclude ScraperExclusion `json:"excludeResources,omitempty" yaml:"excludeResources,omitempty"` } func (base BaseScraper) ApplyPlugins(plugins ...ScrapePluginSpec) BaseScraper { diff --git a/api/v1/zz_generated.deepcopy.go b/api/v1/zz_generated.deepcopy.go index 4fa38e0cc..f314a3d53 100644 --- a/api/v1/zz_generated.deepcopy.go +++ b/api/v1/zz_generated.deepcopy.go @@ -284,6 +284,7 @@ func (in *BaseScraper) DeepCopyInto(out *BaseScraper) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.Exclude.DeepCopyInto(&out.Exclude) } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BaseScraper. @@ -1849,6 +1850,36 @@ func (in *ScrapePluginStatus) DeepCopy() *ScrapePluginStatus { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ScraperExclusion) DeepCopyInto(out *ScraperExclusion) { + *out = *in + if in.ExternalRoles != nil { + in, out := &in.ExternalRoles, &out.ExternalRoles + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalUsers != nil { + in, out := &in.ExternalUsers, &out.ExternalUsers + *out = make([]string, len(*in)) + copy(*out, *in) + } + if in.ExternalGroups != nil { + in, out := &in.ExternalGroups, &out.ExternalGroups + *out = make([]string, len(*in)) + copy(*out, *in) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScraperExclusion. +func (in *ScraperExclusion) DeepCopy() *ScraperExclusion { + if in == nil { + return nil + } + out := new(ScraperExclusion) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *ScraperSpec) DeepCopyInto(out *ScraperSpec) { *out = *in diff --git a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml index d2fc8cdf3..f51d84fb9 100644 --- a/chart/crds/configs.flanksource.com_scrapeconfigs.yaml +++ b/chart/crds/configs.flanksource.com_scrapeconfigs.yaml @@ -142,6 +142,23 @@ spec: items: type: string type: array + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -1110,6 +1127,23 @@ spec: type: object type: array type: object + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object exclusions: properties: activityLogs: @@ -1657,6 +1691,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -2476,6 +2527,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -3888,6 +3956,23 @@ spec: type: object type: object type: array + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -4439,6 +4524,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -5078,6 +5180,23 @@ spec: items: type: string type: array + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -5623,6 +5742,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -6243,6 +6379,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -7062,6 +7215,23 @@ spec: type: object type: object type: array + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -8266,6 +8436,23 @@ spec: type: array type: object type: object + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object exclusions: description: Exclusions excludes certain kubernetes objects from being scraped. @@ -9230,6 +9417,23 @@ spec: required: - cluster type: object + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object files: items: properties: @@ -10073,6 +10277,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object fieldMapping: description: FieldMapping defines how source log fields map to canonical LogLine fields @@ -10937,6 +11158,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -11773,6 +12011,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -12640,6 +12895,23 @@ spec: type: string driver: type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -13170,6 +13442,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties @@ -13922,6 +14211,23 @@ spec: description: A static value or JSONPath expression to use as the description for the resource. type: string + excludeResources: + description: Exclude specifies patterns for excluding external + entities. + properties: + externalGroups: + items: + type: string + type: array + externalRoles: + items: + type: string + type: array + externalUsers: + items: + type: string + type: array + type: object format: description: Format of config item, defaults to JSON, available options are JSON, properties diff --git a/config/schemas/config_aws.schema.json b/config/schemas/config_aws.schema.json index 08af99243..a42923920 100644 --- a/config/schemas/config_aws.schema.json +++ b/config/schemas/config_aws.schema.json @@ -77,6 +77,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string", "description": "ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey." @@ -529,6 +533,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_azure.schema.json b/config/schemas/config_azure.schema.json index 09c0bb166..ec3b93ff8 100644 --- a/config/schemas/config_azure.schema.json +++ b/config/schemas/config_azure.schema.json @@ -77,6 +77,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -584,6 +588,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_azuredevops.schema.json b/config/schemas/config_azuredevops.schema.json index 7820619c2..1fe906dfc 100644 --- a/config/schemas/config_azuredevops.schema.json +++ b/config/schemas/config_azuredevops.schema.json @@ -77,6 +77,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -503,6 +507,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_exec.schema.json b/config/schemas/config_exec.schema.json index 65dbb67e5..1a6d8076a 100644 --- a/config/schemas/config_exec.schema.json +++ b/config/schemas/config_exec.schema.json @@ -370,6 +370,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "script": { "type": "string", "description": "Script is an inline script to run" @@ -758,6 +762,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_file.schema.json b/config/schemas/config_file.schema.json index b8c2f487f..881c86540 100644 --- a/config/schemas/config_file.schema.json +++ b/config/schemas/config_file.schema.json @@ -205,6 +205,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "url": { "type": "string" }, @@ -407,6 +411,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "Tag": { "properties": { "name": { diff --git a/config/schemas/config_gcp.schema.json b/config/schemas/config_gcp.schema.json index b5d93763c..0bf5a9e7e 100644 --- a/config/schemas/config_gcp.schema.json +++ b/config/schemas/config_gcp.schema.json @@ -254,6 +254,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -524,6 +528,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_github.schema.json b/config/schemas/config_github.schema.json index b63bab5b8..c63dfa106 100644 --- a/config/schemas/config_github.schema.json +++ b/config/schemas/config_github.schema.json @@ -254,6 +254,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "repositories": { "items": { "$ref": "#/$defs/GitHubRepository" @@ -517,6 +521,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_githubactions.schema.json b/config/schemas/config_githubactions.schema.json index a05a7f275..39cdbfb89 100644 --- a/config/schemas/config_githubactions.schema.json +++ b/config/schemas/config_githubactions.schema.json @@ -254,6 +254,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "owner": { "type": "string" }, @@ -480,6 +484,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_http.schema.json b/config/schemas/config_http.schema.json index 946ba99a9..508df3d6e 100644 --- a/config/schemas/config_http.schema.json +++ b/config/schemas/config_http.schema.json @@ -287,6 +287,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -592,6 +596,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_kubernetes.schema.json b/config/schemas/config_kubernetes.schema.json index 5c0b4d2e5..b58d1c947 100644 --- a/config/schemas/config_kubernetes.schema.json +++ b/config/schemas/config_kubernetes.schema.json @@ -367,6 +367,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -712,6 +716,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_kubernetesfile.schema.json b/config/schemas/config_kubernetesfile.schema.json index 5ceafc750..c9e7811a6 100644 --- a/config/schemas/config_kubernetesfile.schema.json +++ b/config/schemas/config_kubernetesfile.schema.json @@ -367,6 +367,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -605,6 +609,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_logs.schema.json b/config/schemas/config_logs.schema.json index b07dc3df5..f0dc6bb88 100644 --- a/config/schemas/config_logs.schema.json +++ b/config/schemas/config_logs.schema.json @@ -472,6 +472,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "loki": { "$ref": "#/$defs/LokiConfig", "description": "Loki specifies the Loki configuration for log scraping" @@ -692,6 +696,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_slack.schema.json b/config/schemas/config_slack.schema.json index 19699212d..c16c51041 100644 --- a/config/schemas/config_slack.schema.json +++ b/config/schemas/config_slack.schema.json @@ -453,6 +453,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { @@ -542,6 +567,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "token": { "$ref": "#/$defs/EnvVar", "description": "Slack token" diff --git a/config/schemas/config_sql.schema.json b/config/schemas/config_sql.schema.json index c506ed628..8aa7f6eb0 100644 --- a/config/schemas/config_sql.schema.json +++ b/config/schemas/config_sql.schema.json @@ -461,6 +461,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string", "description": "Connection is either the name of the connection to lookup\nor the connection string itself." @@ -482,6 +486,31 @@ "query" ] }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { diff --git a/config/schemas/config_terraform.schema.json b/config/schemas/config_terraform.schema.json index 827141d80..8ed340cd0 100644 --- a/config/schemas/config_terraform.schema.json +++ b/config/schemas/config_terraform.schema.json @@ -433,6 +433,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "SecretKeySelector": { "properties": { "name": { @@ -548,6 +573,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "state": { "$ref": "#/$defs/TerraformStateSource" } diff --git a/config/schemas/config_trivy.schema.json b/config/schemas/config_trivy.schema.json index eafd43c50..c0b3668d1 100644 --- a/config/schemas/config_trivy.schema.json +++ b/config/schemas/config_trivy.schema.json @@ -307,6 +307,31 @@ "additionalProperties": false, "type": "object" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "Tag": { "properties": { "name": { @@ -479,6 +504,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "version": { "type": "string", "description": "Common Trivy Flags ..." diff --git a/config/schemas/scrape_config.schema.json b/config/schemas/scrape_config.schema.json index f07c50170..bafefec5e 100644 --- a/config/schemas/scrape_config.schema.json +++ b/config/schemas/scrape_config.schema.json @@ -77,6 +77,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string", "description": "ConnectionName of the connection. It'll be used to populate the endpoint, accessKey and secretKey." @@ -333,6 +337,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -495,6 +503,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -811,6 +823,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "awsS3": { "$ref": "#/$defs/AWSS3" }, @@ -1192,6 +1208,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "script": { "type": "string", "description": "Script is an inline script to run" @@ -1397,6 +1417,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "url": { "type": "string" }, @@ -1498,6 +1522,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -1793,6 +1821,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "repositories": { "items": { "$ref": "#/$defs/GitHubRepository" @@ -1901,6 +1933,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "owner": { "type": "string" }, @@ -2050,6 +2086,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -2228,6 +2268,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -2461,6 +2505,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string" }, @@ -2680,6 +2728,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "loki": { "$ref": "#/$defs/LokiConfig", "description": "Loki specifies the Loki configuration for log scraping" @@ -3126,6 +3178,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "sqs": { "$ref": "#/$defs/SQSConfig" }, @@ -3452,6 +3508,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "connection": { "type": "string", "description": "Connection is either the name of the connection to lookup\nor the connection string itself." @@ -3551,6 +3611,31 @@ "type": "object", "description": "ScrapeConfigStatus defines the observed state of ScrapeConfig" }, + "ScraperExclusion": { + "properties": { + "externalRoles": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalUsers": { + "items": { + "type": "string" + }, + "type": "array" + }, + "externalGroups": { + "items": { + "type": "string" + }, + "type": "array" + } + }, + "additionalProperties": false, + "type": "object", + "description": "ScraperExclusion specifies patterns for excluding external entities by name.\nPatterns support wildcards via collections.MatchItems (e.g. \"system:controller:*\")." + }, "ScraperSpec": { "properties": { "logLevel": { @@ -3794,6 +3879,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "token": { "$ref": "#/$defs/EnvVar", "description": "Slack token" @@ -4000,6 +4089,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "state": { "$ref": "#/$defs/TerraformStateSource" } @@ -4176,6 +4269,10 @@ "type": "array", "description": "Properties are custom templatable properties for the scraped config items\ngrouped by the config type." }, + "excludeResources": { + "$ref": "#/$defs/ScraperExclusion", + "description": "Exclude specifies patterns for excluding external entities." + }, "version": { "type": "string", "description": "Common Trivy Flags ..." diff --git a/db/update.go b/db/update.go index 3c3b10f46..f8066c518 100644 --- a/db/update.go +++ b/db/update.go @@ -1270,6 +1270,54 @@ func NewExtractResult() *extractResult { } } +// applyExternalEntityExclusions removes external entities from a ScrapeResult +// whose names match the given exclusion patterns. +// This provides a generic exclusion layer for all scrapers (SQL, file, etc.). +// The Kubernetes scraper applies exclusions earlier during extraction for performance, +// but this function acts as a catch-all for any scraper that returns external entities. +func applyExternalEntityExclusions(result *v1.ScrapeResult, exclusions v1.ScraperExclusion) { + if len(exclusions.ExternalRoles) > 0 && len(result.ExternalRoles) > 0 { + result.ExternalRoles = lo.Filter(result.ExternalRoles, func(r dutyModels.ExternalRole, _ int) bool { + return !collections.MatchItems(r.Name, exclusions.ExternalRoles...) + }) + } + + if len(exclusions.ExternalUsers) > 0 && len(result.ExternalUsers) > 0 { + result.ExternalUsers = lo.Filter(result.ExternalUsers, func(u dutyModels.ExternalUser, _ int) bool { + return !collections.MatchItems(u.Name, exclusions.ExternalUsers...) + }) + } + + if len(exclusions.ExternalGroups) > 0 && len(result.ExternalGroups) > 0 { + result.ExternalGroups = lo.Filter(result.ExternalGroups, func(g dutyModels.ExternalGroup, _ int) bool { + return !collections.MatchItems(g.Name, exclusions.ExternalGroups...) + }) + } + + if len(exclusions.ExternalUsers) > 0 || len(exclusions.ExternalGroups) > 0 || len(exclusions.ExternalRoles) > 0 { + if len(result.ConfigAccess) > 0 { + result.ConfigAccess = lo.Filter(result.ConfigAccess, func(a v1.ExternalConfigAccess, _ int) bool { + for _, alias := range a.ExternalUserAliases { + if collections.MatchItems(alias, exclusions.ExternalUsers...) { + return false + } + } + for _, alias := range a.ExternalRoleAliases { + if collections.MatchItems(alias, exclusions.ExternalRoles...) { + return false + } + } + for _, alias := range a.ExternalGroupAliases { + if collections.MatchItems(alias, exclusions.ExternalGroups...) { + return false + } + } + return true + }) + } + } +} + func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, results []v1.ScrapeResult) (*extractResult, error) { var ( extractResult = NewExtractResult() @@ -1281,6 +1329,11 @@ func extractConfigsAndChangesFromResults(ctx api.ScrapeContext, results []v1.Scr var ci *models.ConfigItem var err error + exclusions := result.BaseScraper.Exclude + if !exclusions.IsEmpty() { + applyExternalEntityExclusions(&result, exclusions) + } + if len(result.ExternalUsers) > 0 { extractResult.externalUsers = append(extractResult.externalUsers, result.ExternalUsers...) } diff --git a/db/update_exclusions_test.go b/db/update_exclusions_test.go new file mode 100644 index 000000000..94b319a96 --- /dev/null +++ b/db/update_exclusions_test.go @@ -0,0 +1,164 @@ +// ABOUTME: Tests for generic external entity exclusion filtering. +// ABOUTME: Verifies that applyExternalEntityExclusions removes matching entities from ScrapeResults. +package db + +import ( + "testing" + + v1 "github.com/flanksource/config-db/api/v1" + "github.com/flanksource/duty/models" +) + +func TestApplyExternalEntityExclusions(t *testing.T) { + t.Run("excludes users by name pattern", func(t *testing.T) { + result := &v1.ScrapeResult{ + ExternalUsers: []models.ExternalUser{ + {Name: "system:node:ip-10-0-1-5"}, + {Name: "alice"}, + {Name: "system:node:ip-10-0-1-6"}, + }, + } + + applyExternalEntityExclusions(result, v1.ScraperExclusion{ + ExternalUsers: []string{"system:node:*"}, + }) + + if len(result.ExternalUsers) != 1 { + t.Fatalf("expected 1 user, got %d", len(result.ExternalUsers)) + } + if result.ExternalUsers[0].Name != "alice" { + t.Fatalf("expected alice, got %s", result.ExternalUsers[0].Name) + } + }) + + t.Run("excludes roles by exact name", func(t *testing.T) { + result := &v1.ScrapeResult{ + ExternalRoles: []models.ExternalRole{ + {Name: "admin"}, + {Name: "system:controller:replicaset-controller"}, + {Name: "viewer"}, + }, + } + + applyExternalEntityExclusions(result, v1.ScraperExclusion{ + ExternalRoles: []string{"system:controller:replicaset-controller"}, + }) + + if len(result.ExternalRoles) != 2 { + t.Fatalf("expected 2 roles, got %d", len(result.ExternalRoles)) + } + }) + + t.Run("excludes groups by wildcard", func(t *testing.T) { + result := &v1.ScrapeResult{ + ExternalGroups: []models.ExternalGroup{ + {Name: "system:authenticated"}, + {Name: "developers"}, + {Name: "system:unauthenticated"}, + }, + } + + applyExternalEntityExclusions(result, v1.ScraperExclusion{ + ExternalGroups: []string{"system:*"}, + }) + + if len(result.ExternalGroups) != 1 { + t.Fatalf("expected 1 group, got %d", len(result.ExternalGroups)) + } + if result.ExternalGroups[0].Name != "developers" { + t.Fatalf("expected developers, got %s", result.ExternalGroups[0].Name) + } + }) + + t.Run("removes config access referencing excluded user aliases", func(t *testing.T) { + result := &v1.ScrapeResult{ + ExternalUsers: []models.ExternalUser{ + {Name: "system:kube-proxy"}, + {Name: "alice"}, + }, + ConfigAccess: []v1.ExternalConfigAccess{ + { + ConfigExternalID: v1.ExternalID{ConfigType: "Test", ExternalID: "1"}, + ExternalUserAliases: []string{"system:kube-proxy"}, + }, + { + ConfigExternalID: v1.ExternalID{ConfigType: "Test", ExternalID: "2"}, + ExternalUserAliases: []string{"alice"}, + }, + }, + } + + applyExternalEntityExclusions(result, v1.ScraperExclusion{ + ExternalUsers: []string{"system:*"}, + }) + + if len(result.ExternalUsers) != 1 { + t.Fatalf("expected 1 user, got %d", len(result.ExternalUsers)) + } + if len(result.ConfigAccess) != 1 { + t.Fatalf("expected 1 config access, got %d", len(result.ConfigAccess)) + } + if result.ConfigAccess[0].ExternalUserAliases[0] != "alice" { + t.Fatalf("expected alice access, got %s", result.ConfigAccess[0].ExternalUserAliases[0]) + } + }) + + t.Run("removes config access referencing excluded role aliases", func(t *testing.T) { + result := &v1.ScrapeResult{ + ConfigAccess: []v1.ExternalConfigAccess{ + { + ConfigExternalID: v1.ExternalID{ConfigType: "Test", ExternalID: "1"}, + ExternalRoleAliases: []string{"system:controller:job-controller"}, + }, + { + ConfigExternalID: v1.ExternalID{ConfigType: "Test", ExternalID: "2"}, + ExternalRoleAliases: []string{"custom-role"}, + }, + }, + } + + applyExternalEntityExclusions(result, v1.ScraperExclusion{ + ExternalRoles: []string{"system:controller:*"}, + }) + + if len(result.ConfigAccess) != 1 { + t.Fatalf("expected 1 config access, got %d", len(result.ConfigAccess)) + } + }) + + t.Run("no-op when exclusions are empty", func(t *testing.T) { + result := &v1.ScrapeResult{ + ExternalUsers: []models.ExternalUser{{Name: "alice"}}, + ExternalGroups: []models.ExternalGroup{{Name: "devs"}}, + ExternalRoles: []models.ExternalRole{{Name: "admin"}}, + } + + applyExternalEntityExclusions(result, v1.ScraperExclusion{}) + + if len(result.ExternalUsers) != 1 || len(result.ExternalGroups) != 1 || len(result.ExternalRoles) != 1 { + t.Fatal("empty exclusions should not filter anything") + } + }) + + t.Run("multiple exclusion patterns", func(t *testing.T) { + result := &v1.ScrapeResult{ + ExternalUsers: []models.ExternalUser{ + {Name: "system:kube-proxy"}, + {Name: "system:node:ip-10-0-1-5"}, + {Name: "alice"}, + {Name: "bot-deployer"}, + }, + } + + applyExternalEntityExclusions(result, v1.ScraperExclusion{ + ExternalUsers: []string{"system:*", "bot-*"}, + }) + + if len(result.ExternalUsers) != 1 { + t.Fatalf("expected 1 user, got %d", len(result.ExternalUsers)) + } + if result.ExternalUsers[0].Name != "alice" { + t.Fatalf("expected alice, got %s", result.ExternalUsers[0].Name) + } + }) +} diff --git a/scrapers/kubernetes/kubernetes.go b/scrapers/kubernetes/kubernetes.go index 63f247409..5e957bdc1 100644 --- a/scrapers/kubernetes/kubernetes.go +++ b/scrapers/kubernetes/kubernetes.go @@ -145,7 +145,7 @@ func ExtractResults(ctx *KubernetesContext, objs []*unstructured.Unstructured) v if ctx.Properties().On(true, "kubernetes.rbac_config_access") { rbacCtx := ctx.ScrapeContext rbacCtx.Context = rbacCtx.WithKubernetes(ctx.config.KubernetesConnection) - rbac = newRBACExtractor(rbacCtx, clusterName, ctx.ScrapeConfig().GetPersistedID()) + rbac = newRBACExtractor(rbacCtx, clusterName, ctx.ScrapeConfig().GetPersistedID(), ctx.config.BaseScraper.Exclude) } ctx.Load(objs) diff --git a/scrapers/kubernetes/rbac.go b/scrapers/kubernetes/rbac.go index 6cad76c0f..07c3b1767 100644 --- a/scrapers/kubernetes/rbac.go +++ b/scrapers/kubernetes/rbac.go @@ -8,6 +8,7 @@ import ( "sync" "time" + "github.com/flanksource/commons/collections" "github.com/flanksource/config-db/api" v1 "github.com/flanksource/config-db/api/v1" "github.com/flanksource/duty/models" @@ -22,6 +23,7 @@ import ( type rbacExtractor struct { clusterName string scraperID *uuid.UUID + exclusions v1.ScraperExclusion roles map[uuid.UUID]models.ExternalRole users map[uuid.UUID]models.ExternalUser groups map[uuid.UUID]models.ExternalGroup @@ -30,6 +32,7 @@ type rbacExtractor struct { roleRules map[string][]rbacRule // key: kind/namespace/name -> rules resourceToKind map[string]string // plural resource name -> Kind (e.g., "pods" -> "Pod") + ignoredRoles map[string]bool // key: kind/namespace/name -> true if role is excluded } type rbacRule struct { @@ -140,7 +143,7 @@ func fetchCRDResourceKinds(ctx api.ScrapeContext, clusterName string) map[string return resourceMap } -func newRBACExtractor(ctx api.ScrapeContext, clusterName string, scraperID *uuid.UUID) *rbacExtractor { +func newRBACExtractor(ctx api.ScrapeContext, clusterName string, scraperID *uuid.UUID, exclusions v1.ScraperExclusion) *rbacExtractor { if scraperID == nil { ctx.Warnf("Ignoring RBAC Extraction due to empty scraperID") return nil @@ -157,19 +160,21 @@ func newRBACExtractor(ctx api.ScrapeContext, clusterName string, scraperID *uuid resourceMap[k] = v } - return newRBACExtractorWithResourceMap(clusterName, scraperID, resourceMap) + return newRBACExtractorWithResourceMap(clusterName, scraperID, resourceMap, exclusions) } -func newRBACExtractorWithResourceMap(clusterName string, scraperID *uuid.UUID, resourceToKind map[string]string) *rbacExtractor { +func newRBACExtractorWithResourceMap(clusterName string, scraperID *uuid.UUID, resourceToKind map[string]string, exclusions v1.ScraperExclusion) *rbacExtractor { return &rbacExtractor{ clusterName: clusterName, scraperID: scraperID, + exclusions: exclusions, roles: make(map[uuid.UUID]models.ExternalRole), users: make(map[uuid.UUID]models.ExternalUser), groups: make(map[uuid.UUID]models.ExternalGroup), roleRules: make(map[string][]rbacRule), seenAccess: make(map[string]struct{}), resourceToKind: resourceToKind, + ignoredRoles: make(map[string]bool), } } @@ -189,6 +194,16 @@ func (r *rbacExtractor) processRole(obj *unstructured.Unstructured) { name := obj.GetName() namespace := obj.GetNamespace() + if len(r.exclusions.ExternalRoles) > 0 && collections.MatchItems(name, r.exclusions.ExternalRoles...) { + key := r.objectKey(kind, namespace, name) + r.ignoredRoles[key] = true + // Still parse and store the rules so bindings can resolve correctly, + // but don't create the ExternalRole entry. + rules := r.parseRules(obj) + r.roleRules[key] = rules + return + } + id := generateRBACID(r.clusterName, kind, namespace, name) alias := KubernetesAlias(r.clusterName, kind, namespace, name) @@ -292,8 +307,12 @@ func (r *rbacExtractor) processRoleBinding(obj *unstructured.Unstructured) { roleNamespace = bindingNamespace } - // Lookup the role's rules; skip if the role was not scraped + // Lookup the role's rules; skip if the role was not scraped or is excluded roleKey := r.objectKey(roleKind, roleNamespace, roleName) + if r.ignoredRoles[roleKey] { + return + } + rules, hasRules := r.roleRules[roleKey] if !hasRules || len(rules) == 0 { return @@ -330,6 +349,18 @@ func (r *rbacExtractor) processRoleBinding(obj *unstructured.Unstructured) { subjName, _ := subjMap["name"].(string) subjNamespace, _ := subjMap["namespace"].(string) + // Skip excluded users (ServiceAccount, User) and groups + switch subjKind { + case "ServiceAccount", "User": + if len(r.exclusions.ExternalUsers) > 0 && collections.MatchItems(subjName, r.exclusions.ExternalUsers...) { + continue + } + case "Group": + if len(r.exclusions.ExternalGroups) > 0 && collections.MatchItems(subjName, r.exclusions.ExternalGroups...) { + continue + } + } + var userAlias, groupAlias string switch subjKind { @@ -501,10 +532,36 @@ func (r *rbacExtractor) getAccess() []v1.ExternalConfigAccess { return r.access } +// pruneOrphanedUsers removes users that have no corresponding access entries. +func (r *rbacExtractor) pruneOrphanedUsers() { + usedAliases := make(map[string]bool) + for _, a := range r.access { + for _, alias := range a.ExternalUserAliases { + usedAliases[alias] = true + } + } + + for id, user := range r.users { + hasAccess := false + for _, alias := range user.Aliases { + if usedAliases[alias] { + hasAccess = true + break + } + } + if !hasAccess { + delete(r.users, id) + } + } +} + func (r *rbacExtractor) results(baseScraper v1.BaseScraper) v1.ScrapeResult { if r == nil { return v1.ScrapeResult{} } + + r.pruneOrphanedUsers() + return v1.ScrapeResult{ BaseScraper: baseScraper, ExternalRoles: r.getRoles(), diff --git a/scrapers/kubernetes/rbac_test.go b/scrapers/kubernetes/rbac_test.go index 0a31a255e..12bdfc0fc 100644 --- a/scrapers/kubernetes/rbac_test.go +++ b/scrapers/kubernetes/rbac_test.go @@ -3,6 +3,7 @@ package kubernetes import ( "time" + v1 "github.com/flanksource/config-db/api/v1" "github.com/google/uuid" "github.com/lib/pq" . "github.com/onsi/ginkgo/v2" @@ -388,6 +389,205 @@ var _ = Describe("RBACExtractor", func() { }) }) + Describe("Exclusions", func() { + var ( + clusterName = "test-cluster" + scraperID = uuid.New() + ) + + Context("role exclusion by exact name", func() { + It("excludes the role and cascades to binding subjects", func() { + exclusions := v1.ScraperExclusion{ + ExternalRoles: []string{"system:controller:job-controller"}, + } + + role := makeClusterRole("system:controller:job-controller", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + binding := makeClusterRoleBinding("job-controller-binding", "ClusterRole", "system:controller:job-controller", []subject{ + {Kind: "ServiceAccount", Name: "job-controller", Namespace: "kube-system"}, + }) + + extractor := testRBACExtractorWithExclusions(clusterName, &scraperID, exclusions) + extractor.processRole(role) + extractor.processRoleBinding(binding) + + Expect(extractor.getRoles()).To(BeEmpty()) + Expect(extractor.getUsers()).To(BeEmpty()) + Expect(extractor.getAccess()).To(BeEmpty()) + }) + }) + + Context("role exclusion by wildcard", func() { + It("excludes multiple roles matching the pattern", func() { + exclusions := v1.ScraperExclusion{ + ExternalRoles: []string{"system:controller:*"}, + } + + role1 := makeClusterRole("system:controller:job-controller", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + role2 := makeClusterRole("system:controller:deployment-controller", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + validRole := makeClusterRole("admin", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"*"}}, + }) + binding1 := makeClusterRoleBinding("b1", "ClusterRole", "system:controller:job-controller", []subject{ + {Kind: "ServiceAccount", Name: "job-controller", Namespace: "kube-system"}, + }) + binding2 := makeClusterRoleBinding("b2", "ClusterRole", "system:controller:deployment-controller", []subject{ + {Kind: "ServiceAccount", Name: "deployment-controller", Namespace: "kube-system"}, + }) + binding3 := makeClusterRoleBinding("b3", "ClusterRole", "admin", []subject{ + {Kind: "User", Name: "admin@example.com"}, + }) + + extractor := testRBACExtractorWithExclusions(clusterName, &scraperID, exclusions) + extractor.processRole(role1) + extractor.processRole(role2) + extractor.processRole(validRole) + extractor.processRoleBinding(binding1) + extractor.processRoleBinding(binding2) + extractor.processRoleBinding(binding3) + + roles := extractor.getRoles() + Expect(roles).To(HaveLen(1)) + Expect(roles[0].Name).To(Equal("admin")) + + users := extractor.getUsers() + Expect(users).To(HaveLen(1)) + Expect(users[0].Name).To(Equal("admin@example.com")) + + access := extractor.getAccess() + Expect(access).To(HaveLen(1)) + }) + }) + + Context("SA referenced by both ignored and non-ignored roles", func() { + It("keeps the SA with only non-ignored access entries", func() { + exclusions := v1.ScraperExclusion{ + ExternalRoles: []string{"system:controller:*"}, + } + + ignoredRole := makeClusterRole("system:controller:foo", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + validRole := makeClusterRole("pod-reader", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + ignoredBinding := makeClusterRoleBinding("b-ignored", "ClusterRole", "system:controller:foo", []subject{ + {Kind: "ServiceAccount", Name: "shared-sa", Namespace: "default"}, + }) + validBinding := makeClusterRoleBinding("b-valid", "ClusterRole", "pod-reader", []subject{ + {Kind: "ServiceAccount", Name: "shared-sa", Namespace: "default"}, + }) + + extractor := testRBACExtractorWithExclusions(clusterName, &scraperID, exclusions) + extractor.processRole(ignoredRole) + extractor.processRole(validRole) + extractor.processRoleBinding(ignoredBinding) + extractor.processRoleBinding(validBinding) + + roles := extractor.getRoles() + Expect(roles).To(HaveLen(1)) + Expect(roles[0].Name).To(Equal("pod-reader")) + + users := extractor.getUsers() + Expect(users).To(HaveLen(1)) + Expect(users[0].Name).To(Equal("shared-sa")) + + access := extractor.getAccess() + Expect(access).To(HaveLen(1)) + + expectedRoleAlias := KubernetesAlias(clusterName, "ClusterRole", "", "pod-reader") + Expect(access[0].ExternalRoleAliases).To(Equal([]string{expectedRoleAlias})) + }) + }) + + Context("SA only referenced by ignored roles", func() { + It("prunes the SA from results", func() { + exclusions := v1.ScraperExclusion{ + ExternalRoles: []string{"system:controller:*"}, + } + + role := makeClusterRole("system:controller:foo", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + binding := makeClusterRoleBinding("b1", "ClusterRole", "system:controller:foo", []subject{ + {Kind: "ServiceAccount", Name: "orphan-sa", Namespace: "kube-system"}, + }) + + extractor := testRBACExtractorWithExclusions(clusterName, &scraperID, exclusions) + extractor.processRole(role) + extractor.processRoleBinding(binding) + + result := extractor.results(v1.BaseScraper{}) + Expect(result.ExternalRoles).To(BeEmpty()) + Expect(result.ExternalUsers).To(BeEmpty()) + Expect(result.ConfigAccess).To(BeEmpty()) + }) + }) + + Context("user exclusion pattern", func() { + It("excludes matching users and their access entries", func() { + exclusions := v1.ScraperExclusion{ + ExternalUsers: []string{"system:kube-*"}, + } + + role := makeClusterRole("view", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + binding := makeClusterRoleBinding("b1", "ClusterRole", "view", []subject{ + {Kind: "User", Name: "system:kube-controller-manager"}, + {Kind: "User", Name: "admin@example.com"}, + }) + + extractor := testRBACExtractorWithExclusions(clusterName, &scraperID, exclusions) + extractor.processRole(role) + extractor.processRoleBinding(binding) + + users := extractor.getUsers() + Expect(users).To(HaveLen(1)) + Expect(users[0].Name).To(Equal("admin@example.com")) + + access := extractor.getAccess() + Expect(access).To(HaveLen(1)) + expectedUserAlias := KubernetesAlias(clusterName, "User", "", "admin@example.com") + Expect(access[0].ExternalUserAliases).To(Equal([]string{expectedUserAlias})) + }) + }) + + Context("group exclusion pattern", func() { + It("excludes matching groups and their access entries", func() { + exclusions := v1.ScraperExclusion{ + ExternalGroups: []string{"system:*"}, + } + + role := makeClusterRole("view", []rbacRuleSpec{ + {APIGroups: []string{""}, Resources: []string{"pods"}, Verbs: []string{"get"}}, + }) + binding := makeClusterRoleBinding("b1", "ClusterRole", "view", []subject{ + {Kind: "Group", Name: "system:authenticated"}, + {Kind: "Group", Name: "developers"}, + }) + + extractor := testRBACExtractorWithExclusions(clusterName, &scraperID, exclusions) + extractor.processRole(role) + extractor.processRoleBinding(binding) + + groups := extractor.getGroups() + Expect(groups).To(HaveLen(1)) + Expect(groups[0].Name).To(Equal("developers")) + + access := extractor.getAccess() + Expect(access).To(HaveLen(1)) + expectedGroupAlias := KubernetesAlias(clusterName, "Group", "", "developers") + Expect(access[0].ExternalGroupAliases).To(Equal([]string{expectedGroupAlias})) + }) + }) + }) + Describe("CRDResourceResolution", func() { It("resolves custom resource types with resourceNames", func() { clusterName := "test-cluster" @@ -406,7 +606,7 @@ var _ = Describe("RBACExtractor", func() { {Kind: "User", Name: "ops@example.com"}, }) - extractor := newRBACExtractorWithResourceMap(clusterName, &scraperID, resourceMap) + extractor := newRBACExtractorWithResourceMap(clusterName, &scraperID, resourceMap, v1.ScraperExclusion{}) extractor.processRole(role) extractor.processRoleBinding(binding) @@ -440,7 +640,15 @@ func testRBACExtractor(clusterName string, scraperID *uuid.UUID) *rbacExtractor for k, v := range builtinResourceKinds { resourceMap[k] = v } - return newRBACExtractorWithResourceMap(clusterName, scraperID, resourceMap) + return newRBACExtractorWithResourceMap(clusterName, scraperID, resourceMap, v1.ScraperExclusion{}) +} + +func testRBACExtractorWithExclusions(clusterName string, scraperID *uuid.UUID, exclusions v1.ScraperExclusion) *rbacExtractor { + resourceMap := make(map[string]string, len(builtinResourceKinds)) + for k, v := range builtinResourceKinds { + resourceMap[k] = v + } + return newRBACExtractorWithResourceMap(clusterName, scraperID, resourceMap, exclusions) } type subject struct {