From 5e26c869ae129e1d0f71a3c1b77669298e2bae27 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 13:46:44 +0000 Subject: [PATCH 01/14] mongodb started refactor --- apps/mongodb.go | 277 ++++++++---------- confgenerator/testdata/feature/golden.csv | 7 +- .../golden/linux-gpu/error | 2 +- .../golden/linux/error | 2 +- .../golden/windows-2012/error | 2 +- .../golden/windows/error | 2 +- .../linux-gpu/feature_tracking_otlp.json | 2 +- .../golden/linux-gpu/features.yaml | 4 + .../golden/linux/feature_tracking_otlp.json | 2 +- .../golden/linux/features.yaml | 4 + .../windows-2012/feature_tracking_otlp.json | 2 +- .../golden/windows-2012/features.yaml | 4 + .../golden/windows/feature_tracking_otlp.json | 2 +- .../golden/windows/features.yaml | 4 + .../logging_processor-mongodb/config.yaml | 1 + .../logging_processor-mongodb/input.log | 13 + .../output_fluentbit.yaml | 193 ++++++++++++ .../output_otel.yaml | 1 + 18 files changed, 363 insertions(+), 161 deletions(-) create mode 100644 transformation_test/testdata/logging_processor-mongodb/config.yaml create mode 100644 transformation_test/testdata/logging_processor-mongodb/input.log create mode 100644 transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml create mode 100644 transformation_test/testdata/logging_processor-mongodb/output_otel.yaml diff --git a/apps/mongodb.go b/apps/mongodb.go index 84ba39f97e..ec5219f478 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -16,12 +16,10 @@ package apps import ( "context" - "fmt" "strings" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit/modify" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/internal/secret" ) @@ -94,20 +92,19 @@ func init() { confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.MetricsReceiver { return &MetricsReceiverMongoDB{} }) } -type LoggingProcessorMongodb struct { - confgenerator.ConfigComponent `yaml:",inline"` +type LoggingProcessorMacroMongodb struct { } -func (*LoggingProcessorMongodb) Type() string { +func (LoggingProcessorMacroMongodb) Type() string { return "mongodb" } -func (p *LoggingProcessorMongodb) Components(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} - c = append(c, p.JsonLogComponents(ctx, tag, uid)...) - c = append(c, p.RegexLogComponents(tag, uid)...) - c = append(c, p.severityParser(ctx, tag, uid)...) + c = append(c, p.JsonLogComponents(ctx)...) + // c = append(c, p.RegexLogComponents()...) + c = append(c, p.severityParser()...) return c } @@ -115,19 +112,19 @@ func (p *LoggingProcessorMongodb) Components(ctx context.Context, tag, uid strin // JsonLogComponents are the fluentbit components for parsing log messages that are json formatted. // these are generally messages from mongo with versions greater than or equal to 4.4 // documentation: https://docs.mongodb.com/v4.4/reference/log-messages/#log-message-format -func (p *LoggingProcessorMongodb) JsonLogComponents(ctx context.Context, tag, uid string) []fluentbit.Component { - c := p.jsonParserWithTimeKey(ctx, tag, uid) +func (p LoggingProcessorMacroMongodb) JsonLogComponents(ctx context.Context) []confgenerator.InternalLoggingProcessor { + c := p.jsonParserWithTimeKey() - c = append(c, p.promoteWiredTiger(tag, uid)...) - c = append(c, p.renames(tag, uid)...) + c = append(c, p.promoteWiredTiger()...) + c = append(c, p.renames()...) return c } // jsonParserWithTimeKey requires promotion of the nested timekey for the json parser so we must // first promote the $date field from the "t" field before declaring the parser -func (p *LoggingProcessorMongodb) jsonParserWithTimeKey(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} jsonParser := &confgenerator.LoggingProcessorParseJson{ ParserShared: confgenerator.ParserShared{ @@ -139,75 +136,75 @@ func (p *LoggingProcessorMongodb) jsonParserWithTimeKey(ctx context.Context, tag }, }, } - jpComponents := jsonParser.Components(ctx, tag, uid) - - // The parserFilterComponent is the actual filter component that configures and defines - // which parser to use. We need the component to determine which parser to use when - // re-parsing below. Each time a parser filter is used, there are 2 filter components right - // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). - // Therefore, the parse filter component is actually the third component in the list. - parserFilterComponent := jpComponents[2] - c = append(c, jpComponents...) - - tempPrefix := "temp_ts_" - timeKey := "time" - // have to bring $date to top level in order for it to be parsed as timeKey - // see https://github.com/fluent/fluent-bit/issues/1013 - liftTs := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Match": tag, - "Operation": "lift", - "Nested_under": "t", - "Add_prefix": tempPrefix, - }, - } - - renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) - renameTs := renameTsOption.Component(tag) - - c = append(c, liftTs, renameTs) - - // IMPORTANT: now that we have lifted the json to top level - // we need to re-parse in order to properly set time at the - // parser level - nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) - parserFilter := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "parser", - "Match": tag, - "Key_Name": "message", - "Reserve_Data": "True", - "Parser": parserFilterComponent.OrderedConfig[0][1], - }, - } - mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) - c = append(c, nestFilters...) - c = append(c, parserFilter) - c = append(c, mergeFilters...) - - removeTimestamp := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "modify", - "Match": tag, - "Remove": timeKey, - }, - } - c = append(c, removeTimestamp) + c = append(c, jsonParser) return c + + // // The parserFilterComponent is the actual filter component that configures and defines + // // which parser to use. We need the component to determine which parser to use when + // // re-parsing below. Each time a parser filter is used, there are 2 filter components right + // // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). + // // Therefore, the parse filter component is actually the third component in the list. + // parserFilterComponent := jpComponents[2] + // c = append(c, jpComponents...) + + // tempPrefix := "temp_ts_" + // timeKey := "time" + // // have to bring $date to top level in order for it to be parsed as timeKey + // // see https://github.com/fluent/fluent-bit/issues/1013 + // liftTs := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Name": "nest", + // "Match": tag, + // "Operation": "lift", + // "Nested_under": "t", + // "Add_prefix": tempPrefix, + // }, + // } + + // renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) + // renameTs := renameTsOption.Component(tag) + + // c = append(c, liftTs, renameTs) + + // // IMPORTANT: now that we have lifted the json to top level + // // we need to re-parse in order to properly set time at the + // // parser level + // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) + // parserFilter := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Name": "parser", + // "Match": tag, + // "Key_Name": "message", + // "Reserve_Data": "True", + // "Parser": parserFilterComponent.OrderedConfig[0][1], + // }, + // } + // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) + // c = append(c, nestFilters...) + // c = append(c, parserFilter) + // c = append(c, mergeFilters...) + + // removeTimestamp := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Name": "modify", + // "Match": tag, + // "Remove": timeKey, + // }, + // } + // c = append(c, removeTimestamp) + + // return c } // severityParser is used by both regex and json parser to ensure an "s" field on the entry gets translated // to a valid logging.googleapis.com/seveirty field -func (p *LoggingProcessorMongodb) severityParser(ctx context.Context, tag, uid string) []fluentbit.Component { - severityComponents := []fluentbit.Component{} - - severityComponents = append(severityComponents, - confgenerator.LoggingProcessorModifyFields{ +func (p LoggingProcessorMacroMongodb) severityParser() []confgenerator.InternalLoggingProcessor { + return []confgenerator.InternalLoggingProcessor{ + &confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ "jsonPayload.severity": { MoveFrom: "jsonPayload.s", @@ -230,14 +227,12 @@ func (p *LoggingProcessorMongodb) severityParser(ctx context.Context, tag, uid s }, InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, - }.Components(ctx, tag, uid)..., - ) - - return severityComponents + }, + } } -func (p *LoggingProcessorMongodb) renames(tag, uid string) []fluentbit.Component { - r := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingProcessor { + r := []confgenerator.InternalLoggingProcessor{} renames := []struct { src string dest string @@ -248,97 +243,81 @@ func (p *LoggingProcessorMongodb) renames(tag, uid string) []fluentbit.Component } for _, rename := range renames { - rename := modify.NewRenameOptions(rename.src, rename.dest) - r = append(r, rename.Component(tag)) + r = append(r, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + rename.src: { + MoveFrom: rename.src, + }, + }, + }) } return r } -func (p *LoggingProcessorMongodb) promoteWiredTiger(tag, uid string) []fluentbit.Component { - // promote messages that are WiredTiger messages and are nested in attr.message - addPrefix := "temp_attributes_" - upNest := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Match": tag, - "Operation": "lift", - "Nested_under": "attr", - "Add_prefix": addPrefix, - }, - } +func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} - hardRenameMessage := modify.NewHardRenameOptions(fmt.Sprintf("%smessage", addPrefix), "msg") - wiredTigerRename := hardRenameMessage.Component(tag) - - renameRemainingAttributes := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Wildcard": fmt.Sprintf("%s*", addPrefix), - "Match": tag, - "Operation": "nest", - "Nest_under": "attributes", - "Remove_prefix": addPrefix, + // promote messages that are WiredTiger messages and are nested in attr.message + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.msg": { + MoveFrom: "jsonPayload.attr.message", + }, }, - } + }) - return []fluentbit.Component{upNest, wiredTigerRename, renameRemainingAttributes} + return c } -func (p *LoggingProcessorMongodb) RegexLogComponents(tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} parseKey := "message" - parser, parserName := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ + parser, _ := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ "message": "string", "id": "integer", "s": "string", "component": "string", "context": "string", - }, fmt.Sprintf("%s_regex", tag), uid) + }, "mongodb_regex", "mongodb") parser.Config["Format"] = "regex" parser.Config["Regex"] = `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$` parser.Config["Key_Name"] = parseKey - nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) - parserFilter := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Match": tag, - "Name": "parser", - "Parser": parserName, - "Reserve_Data": "True", - "Key_Name": parseKey, + // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) + // parserFilter := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Match": tag, + // "Name": "parser", + // "Parser": parserName, + // "Reserve_Data": "True", + // "Key_Name": parseKey, + // }, + // } + // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) + + c = append(c, &confgenerator.LoggingProcessorParseJson{ + ParserShared: confgenerator.ParserShared{ + TimeKey: "timestamp", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", }, - } - mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) - - c = append(c, parser) - c = append(c, nestFilters...) - c = append(c, parserFilter) - c = append(c, mergeFilters...) + }) + // c = append(c, nestFilters...) + // c = append(c, parserFilter) + // c = append(c, mergeFilters...) return c } -type LoggingReceiverMongodb struct { - LoggingProcessorMongodb `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` -} - -func (r *LoggingReceiverMongodb) Components(ctx context.Context, tag string) []fluentbit.Component { - if len(r.ReceiverMixin.IncludePaths) == 0 { - r.ReceiverMixin.IncludePaths = []string{ - // default logging location - "/var/log/mongodb/mongod.log*", - } +func loggingReceiverFilesMixinMongodb() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ + IncludePaths: []string{"/var/log/mongodb/mongod.log*"}, } - c := r.ReceiverMixin.Components(ctx, tag) - c = append(c, r.LoggingProcessorMongodb.Components(ctx, tag, "mongodb")...) - return c } func init() { - confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverMongodb{} }) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroMongodb]( + loggingReceiverFilesMixinMongodb, + ) } diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index 53cfe0f63a..72c30bcbed 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -87,10 +87,6 @@ App,Field,Override, *apps.LoggingReceiverKafka,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, *apps.LoggingReceiverKafka,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, *apps.LoggingReceiverKafka,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, -*apps.LoggingReceiverMongodb,apps.LoggingProcessorMongodb.confgenerator.ConfigComponent.Type, -*apps.LoggingReceiverMongodb,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, -*apps.LoggingReceiverMongodb,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, -*apps.LoggingReceiverMongodb,confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, *apps.LoggingReceiverMysqlError,apps.LoggingProcessorMysqlError.confgenerator.ConfigComponent.Type, *apps.LoggingReceiverMysqlError,confgenerator.LoggingReceiverFilesMixin.BufferInMemory, *apps.LoggingReceiverMysqlError,confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, @@ -255,10 +251,13 @@ App,Field,Override, *confgenerator.PrometheusMetrics,config.[].scrape_configs.scrape_timeout *confgenerator.PrometheusMetrics,config.[].scrape_configs.static_config_target_groups *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink],confgenerator.ConfigComponent.Type, +*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],ReceiverMacro, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError]],ReceiverMacro, diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error index 10751c0cf5..6a0286b285 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error index 10751c0cf5..6a0286b285 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error index 1649e4e323..69c093b42c 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error index 1649e4e323..69c093b42c 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json index 8fe864f885..61a43e8f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml index 8216fe84f1..24cb696bd0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml @@ -18,3 +18,7 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" +- module: logging + feature: receivers:mongodb + key: "[0].include_paths.__length" + value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json index 8fe864f885..61a43e8f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml index 8216fe84f1..24cb696bd0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml @@ -18,3 +18,7 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" +- module: logging + feature: receivers:mongodb + key: "[0].include_paths.__length" + value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json index 8fe864f885..61a43e8f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml index 8216fe84f1..24cb696bd0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml @@ -18,3 +18,7 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" +- module: logging + feature: receivers:mongodb + key: "[0].include_paths.__length" + value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json index 8fe864f885..61a43e8f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml index 8216fe84f1..24cb696bd0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml @@ -18,3 +18,7 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" +- module: logging + feature: receivers:mongodb + key: "[0].include_paths.__length" + value: "1" diff --git a/transformation_test/testdata/logging_processor-mongodb/config.yaml b/transformation_test/testdata/logging_processor-mongodb/config.yaml new file mode 100644 index 0000000000..9c447c9612 --- /dev/null +++ b/transformation_test/testdata/logging_processor-mongodb/config.yaml @@ -0,0 +1 @@ +- type: mongodb diff --git a/transformation_test/testdata/logging_processor-mongodb/input.log b/transformation_test/testdata/logging_processor-mongodb/input.log new file mode 100644 index 0000000000..b0beae6bec --- /dev/null +++ b/transformation_test/testdata/logging_processor-mongodb/input.log @@ -0,0 +1,13 @@ +{"t":{"$date":"2025-07-22T08:30:01.123Z"},"s":"I","c":"CONTROL","id":23352,"ctx":"main","msg":"MongoDB starting","attr":{"pid":10234,"port":27017,"dbPath":"/var/lib/mongodb"}} +{"t":{"$date":"2025-07-22T08:30:01.456Z"},"s":"I","c":"STORAGE","id":20320,"ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1658325001:456789][10234:0x7f9dcd9f3700], txn-recover: Recovering log 1 through 2"}} +{"t":{"$date":"2025-07-22T08:30:01.789Z"},"s":"I","c":"STORAGE","id":22437,"ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":1234}} +{"t":{"$date":"2025-07-22T08:30:03.101Z"},"s":"I","c":"STORAGE","id":4795900,"ctx":"WTCheckpointThread","msg":"WiredTiger checkpoint","attr":{"checkpoint_ts":{"$timestamp":{"t":1658325003,"i":1}}}} +{"t":{"$date":"2025-07-22T08:30:05.234Z"},"s":"I","c":"STORAGE","id":5123400,"ctx":"WTJournalFlusher","msg":"WiredTiger journal flush","attr":{"flush_duration_ms":21}} +{"t":{"$date":"2025-07-22T08:30:06.345Z"},"s":"I","c":"STORAGE","id":22438,"ctx":"WTCacheEvictionWorker-0","msg":"WiredTiger eviction","attr":{"pages_evicted":34,"max_pages":100}} +{"t":{"$date":"2025-07-22T08:30:07.456Z"},"s":"I","c":"COMMAND","id":51803,"ctx":"conn1","msg":"Connection accepted","attr":{"client":"127.0.0.1:37244","connectionId":1,"connectionCount":1}} +{"t":{"$date":"2025-07-22T08:30:08.678Z"},"s":"I","c":"COMMAND","id":51804,"ctx":"conn1","msg":"Received client metadata","attr":{"client":"conn1","doc":{"driver":{"name":"PyMongo","version":"4.2.0"},"os":{"type":"Linux","name":"Ubuntu","architecture":"x86_64","version":"20.04"}}}} +{"t":{"$date":"2025-07-22T08:30:09.111Z"},"s":"I","c":"COMMAND","id":51805,"ctx":"conn1","msg":"Insert command","attr":{"ns":"test.users","ninserted":1,"keysInserted":1,"millis":2}} +{"t":{"$date":"2025-07-22T08:30:09.789Z"},"s":"I","c":"COMMAND","id":51806,"ctx":"conn1","msg":"Query executed","attr":{"ns":"test.users","query":{"name":"Alice"},"docsReturned":1,"keysExamined":1,"millis":1}} +{"t":{"$date":"2025-07-22T08:30:10.000Z"},"s":"W","c":"COMMAND","id":51810,"ctx":"conn1","msg":"Slow query","attr":{"type":"query","ns":"test.logs","query":{"level":"debug"},"planSummary":"COLLSCAN","millis":212}} +{"t":{"$date":"2025-07-22T08:30:11.321Z"},"s":"E","c":"ACCESS","id":20249,"ctx":"conn2","msg":"Authentication failed","attr":{"mechanism":"SCRAM-SHA-256","user":"invalidUser","db":"admin","client":"127.0.0.1:37245","result":"AuthenticationFailed"}} +{"t":{"$date":"2025-07-22T08:30:12.789Z"},"s":"E","c":"NETWORK","id":23213,"ctx":"conn2","msg":"Client metadata parsing failed","attr":{"error":"Missing required field 'driver'"}} \ No newline at end of file diff --git a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml new file mode 100644 index 0000000000..15905075da --- /dev/null +++ b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml @@ -0,0 +1,193 @@ +- entries: + - jsonPayload: + attributes: + dbPath: /var/lib/mongodb + pid: 10234.0 + port: 27017.0 + component: CONTROL + context: main + id: 23352.0 + message: MongoDB starting + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: STORAGE + context: initandlisten + id: 20320.0 + message: "[1658325001:456789][10234:0x7f9dcd9f3700], txn-recover: Recovering log 1 through 2" + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + durationMillis: 1234.0 + component: STORAGE + context: initandlisten + id: 22437.0 + message: WiredTiger opened + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + checkpoint_ts: + $timestamp: + i: 1.0 + t: 1.658325003e+09 + component: STORAGE + context: WTCheckpointThread + id: 4.7959e+06 + message: WiredTiger checkpoint + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + flush_duration_ms: 21.0 + component: STORAGE + context: WTJournalFlusher + id: 5.1234e+06 + message: WiredTiger journal flush + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + max_pages: 100.0 + pages_evicted: 34.0 + component: STORAGE + context: WTCacheEvictionWorker-0 + id: 22438.0 + message: WiredTiger eviction + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + client: 127.0.0.1:37244 + connectionCount: 1.0 + connectionId: 1.0 + component: COMMAND + context: conn1 + id: 51803.0 + message: Connection accepted + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + client: conn1 + doc: + driver: + name: PyMongo + version: 4.2.0 + os: + architecture: x86_64 + name: Ubuntu + type: Linux + version: "20.04" + component: COMMAND + context: conn1 + id: 51804.0 + message: Received client metadata + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + keysInserted: 1.0 + millis: 2.0 + ninserted: 1.0 + ns: test.users + component: COMMAND + context: conn1 + id: 51805.0 + message: Insert command + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + docsReturned: 1.0 + keysExamined: 1.0 + millis: 1.0 + ns: test.users + query: + name: Alice + component: COMMAND + context: conn1 + id: 51806.0 + message: Query executed + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + millis: 212.0 + ns: test.logs + planSummary: COLLSCAN + query: + level: debug + type: query + component: COMMAND + context: conn1 + id: 51810.0 + message: Slow query + severity: W + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + attributes: + client: 127.0.0.1:37245 + db: admin + mechanism: SCRAM-SHA-256 + result: AuthenticationFailed + user: invalidUser + component: ACCESS + context: conn2 + id: 20249.0 + message: Authentication failed + severity: E + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 500.0 + timestamp: now + partialSuccess: true + resource: + labels: {} + type: gce_instance diff --git a/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml b/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml new file mode 100644 index 0000000000..253af10e54 --- /dev/null +++ b/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml @@ -0,0 +1 @@ +- config_error: "not an OTel processor: &apps.LoggingProcessorMongodb{ConfigComponent:confgenerator.ConfigComponent{Type:\"mongodb\"}}" From f620ecb2d0bcb529379e6ee3ea667556c60cf56d Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 14:16:07 +0000 Subject: [PATCH 02/14] updated input log and tests --- apps/mongodb.go | 277 +++--- .../logging_processor-mongodb/input.log | 72 +- .../output_fluentbit.yaml | 931 ++++++++++++++++-- 3 files changed, 1056 insertions(+), 224 deletions(-) diff --git a/apps/mongodb.go b/apps/mongodb.go index ec5219f478..84ba39f97e 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -16,10 +16,12 @@ package apps import ( "context" + "fmt" "strings" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" + "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit/modify" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/internal/secret" ) @@ -92,19 +94,20 @@ func init() { confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.MetricsReceiver { return &MetricsReceiverMongoDB{} }) } -type LoggingProcessorMacroMongodb struct { +type LoggingProcessorMongodb struct { + confgenerator.ConfigComponent `yaml:",inline"` } -func (LoggingProcessorMacroMongodb) Type() string { +func (*LoggingProcessorMongodb) Type() string { return "mongodb" } -func (p LoggingProcessorMacroMongodb) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { - c := []confgenerator.InternalLoggingProcessor{} +func (p *LoggingProcessorMongodb) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + c := []fluentbit.Component{} - c = append(c, p.JsonLogComponents(ctx)...) - // c = append(c, p.RegexLogComponents()...) - c = append(c, p.severityParser()...) + c = append(c, p.JsonLogComponents(ctx, tag, uid)...) + c = append(c, p.RegexLogComponents(tag, uid)...) + c = append(c, p.severityParser(ctx, tag, uid)...) return c } @@ -112,19 +115,19 @@ func (p LoggingProcessorMacroMongodb) Expand(ctx context.Context) []confgenerato // JsonLogComponents are the fluentbit components for parsing log messages that are json formatted. // these are generally messages from mongo with versions greater than or equal to 4.4 // documentation: https://docs.mongodb.com/v4.4/reference/log-messages/#log-message-format -func (p LoggingProcessorMacroMongodb) JsonLogComponents(ctx context.Context) []confgenerator.InternalLoggingProcessor { - c := p.jsonParserWithTimeKey() +func (p *LoggingProcessorMongodb) JsonLogComponents(ctx context.Context, tag, uid string) []fluentbit.Component { + c := p.jsonParserWithTimeKey(ctx, tag, uid) - c = append(c, p.promoteWiredTiger()...) - c = append(c, p.renames()...) + c = append(c, p.promoteWiredTiger(tag, uid)...) + c = append(c, p.renames(tag, uid)...) return c } // jsonParserWithTimeKey requires promotion of the nested timekey for the json parser so we must // first promote the $date field from the "t" field before declaring the parser -func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.InternalLoggingProcessor { - c := []confgenerator.InternalLoggingProcessor{} +func (p *LoggingProcessorMongodb) jsonParserWithTimeKey(ctx context.Context, tag, uid string) []fluentbit.Component { + c := []fluentbit.Component{} jsonParser := &confgenerator.LoggingProcessorParseJson{ ParserShared: confgenerator.ParserShared{ @@ -136,75 +139,75 @@ func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.In }, }, } + jpComponents := jsonParser.Components(ctx, tag, uid) + + // The parserFilterComponent is the actual filter component that configures and defines + // which parser to use. We need the component to determine which parser to use when + // re-parsing below. Each time a parser filter is used, there are 2 filter components right + // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). + // Therefore, the parse filter component is actually the third component in the list. + parserFilterComponent := jpComponents[2] + c = append(c, jpComponents...) + + tempPrefix := "temp_ts_" + timeKey := "time" + // have to bring $date to top level in order for it to be parsed as timeKey + // see https://github.com/fluent/fluent-bit/issues/1013 + liftTs := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "nest", + "Match": tag, + "Operation": "lift", + "Nested_under": "t", + "Add_prefix": tempPrefix, + }, + } - c = append(c, jsonParser) - return c + renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) + renameTs := renameTsOption.Component(tag) + + c = append(c, liftTs, renameTs) + + // IMPORTANT: now that we have lifted the json to top level + // we need to re-parse in order to properly set time at the + // parser level + nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) + parserFilter := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "parser", + "Match": tag, + "Key_Name": "message", + "Reserve_Data": "True", + "Parser": parserFilterComponent.OrderedConfig[0][1], + }, + } + mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) + c = append(c, nestFilters...) + c = append(c, parserFilter) + c = append(c, mergeFilters...) + + removeTimestamp := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "modify", + "Match": tag, + "Remove": timeKey, + }, + } + c = append(c, removeTimestamp) - // // The parserFilterComponent is the actual filter component that configures and defines - // // which parser to use. We need the component to determine which parser to use when - // // re-parsing below. Each time a parser filter is used, there are 2 filter components right - // // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). - // // Therefore, the parse filter component is actually the third component in the list. - // parserFilterComponent := jpComponents[2] - // c = append(c, jpComponents...) - - // tempPrefix := "temp_ts_" - // timeKey := "time" - // // have to bring $date to top level in order for it to be parsed as timeKey - // // see https://github.com/fluent/fluent-bit/issues/1013 - // liftTs := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Name": "nest", - // "Match": tag, - // "Operation": "lift", - // "Nested_under": "t", - // "Add_prefix": tempPrefix, - // }, - // } - - // renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) - // renameTs := renameTsOption.Component(tag) - - // c = append(c, liftTs, renameTs) - - // // IMPORTANT: now that we have lifted the json to top level - // // we need to re-parse in order to properly set time at the - // // parser level - // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) - // parserFilter := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Name": "parser", - // "Match": tag, - // "Key_Name": "message", - // "Reserve_Data": "True", - // "Parser": parserFilterComponent.OrderedConfig[0][1], - // }, - // } - // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) - // c = append(c, nestFilters...) - // c = append(c, parserFilter) - // c = append(c, mergeFilters...) - - // removeTimestamp := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Name": "modify", - // "Match": tag, - // "Remove": timeKey, - // }, - // } - // c = append(c, removeTimestamp) - - // return c + return c } // severityParser is used by both regex and json parser to ensure an "s" field on the entry gets translated // to a valid logging.googleapis.com/seveirty field -func (p LoggingProcessorMacroMongodb) severityParser() []confgenerator.InternalLoggingProcessor { - return []confgenerator.InternalLoggingProcessor{ - &confgenerator.LoggingProcessorModifyFields{ +func (p *LoggingProcessorMongodb) severityParser(ctx context.Context, tag, uid string) []fluentbit.Component { + severityComponents := []fluentbit.Component{} + + severityComponents = append(severityComponents, + confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ "jsonPayload.severity": { MoveFrom: "jsonPayload.s", @@ -227,12 +230,14 @@ func (p LoggingProcessorMacroMongodb) severityParser() []confgenerator.InternalL }, InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, - }, - } + }.Components(ctx, tag, uid)..., + ) + + return severityComponents } -func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingProcessor { - r := []confgenerator.InternalLoggingProcessor{} +func (p *LoggingProcessorMongodb) renames(tag, uid string) []fluentbit.Component { + r := []fluentbit.Component{} renames := []struct { src string dest string @@ -243,81 +248,97 @@ func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingP } for _, rename := range renames { - r = append(r, &confgenerator.LoggingProcessorModifyFields{ - Fields: map[string]*confgenerator.ModifyField{ - rename.src: { - MoveFrom: rename.src, - }, - }, - }) + rename := modify.NewRenameOptions(rename.src, rename.dest) + r = append(r, rename.Component(tag)) } return r } -func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.InternalLoggingProcessor { - c := []confgenerator.InternalLoggingProcessor{} - +func (p *LoggingProcessorMongodb) promoteWiredTiger(tag, uid string) []fluentbit.Component { // promote messages that are WiredTiger messages and are nested in attr.message - c = append(c, &confgenerator.LoggingProcessorModifyFields{ - Fields: map[string]*confgenerator.ModifyField{ - "jsonPayload.msg": { - MoveFrom: "jsonPayload.attr.message", - }, + addPrefix := "temp_attributes_" + upNest := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "nest", + "Match": tag, + "Operation": "lift", + "Nested_under": "attr", + "Add_prefix": addPrefix, }, - }) + } - return c + hardRenameMessage := modify.NewHardRenameOptions(fmt.Sprintf("%smessage", addPrefix), "msg") + wiredTigerRename := hardRenameMessage.Component(tag) + + renameRemainingAttributes := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "nest", + "Wildcard": fmt.Sprintf("%s*", addPrefix), + "Match": tag, + "Operation": "nest", + "Nest_under": "attributes", + "Remove_prefix": addPrefix, + }, + } + + return []fluentbit.Component{upNest, wiredTigerRename, renameRemainingAttributes} } -func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.InternalLoggingProcessor { - c := []confgenerator.InternalLoggingProcessor{} +func (p *LoggingProcessorMongodb) RegexLogComponents(tag, uid string) []fluentbit.Component { + c := []fluentbit.Component{} parseKey := "message" - parser, _ := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ + parser, parserName := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ "message": "string", "id": "integer", "s": "string", "component": "string", "context": "string", - }, "mongodb_regex", "mongodb") + }, fmt.Sprintf("%s_regex", tag), uid) parser.Config["Format"] = "regex" parser.Config["Regex"] = `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$` parser.Config["Key_Name"] = parseKey - // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) - // parserFilter := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Match": tag, - // "Name": "parser", - // "Parser": parserName, - // "Reserve_Data": "True", - // "Key_Name": parseKey, - // }, - // } - // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) - - c = append(c, &confgenerator.LoggingProcessorParseJson{ - ParserShared: confgenerator.ParserShared{ - TimeKey: "timestamp", - TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", + nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) + parserFilter := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Match": tag, + "Name": "parser", + "Parser": parserName, + "Reserve_Data": "True", + "Key_Name": parseKey, }, - }) - // c = append(c, nestFilters...) - // c = append(c, parserFilter) - // c = append(c, mergeFilters...) + } + mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) + + c = append(c, parser) + c = append(c, nestFilters...) + c = append(c, parserFilter) + c = append(c, mergeFilters...) return c } -func loggingReceiverFilesMixinMongodb() confgenerator.LoggingReceiverFilesMixin { - return confgenerator.LoggingReceiverFilesMixin{ - IncludePaths: []string{"/var/log/mongodb/mongod.log*"}, +type LoggingReceiverMongodb struct { + LoggingProcessorMongodb `yaml:",inline"` + ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` +} + +func (r *LoggingReceiverMongodb) Components(ctx context.Context, tag string) []fluentbit.Component { + if len(r.ReceiverMixin.IncludePaths) == 0 { + r.ReceiverMixin.IncludePaths = []string{ + // default logging location + "/var/log/mongodb/mongod.log*", + } } + c := r.ReceiverMixin.Components(ctx, tag) + c = append(c, r.LoggingProcessorMongodb.Components(ctx, tag, "mongodb")...) + return c } func init() { - confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroMongodb]( - loggingReceiverFilesMixinMongodb, - ) + confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverMongodb{} }) } diff --git a/transformation_test/testdata/logging_processor-mongodb/input.log b/transformation_test/testdata/logging_processor-mongodb/input.log index b0beae6bec..bee03125fc 100644 --- a/transformation_test/testdata/logging_processor-mongodb/input.log +++ b/transformation_test/testdata/logging_processor-mongodb/input.log @@ -1,13 +1,59 @@ -{"t":{"$date":"2025-07-22T08:30:01.123Z"},"s":"I","c":"CONTROL","id":23352,"ctx":"main","msg":"MongoDB starting","attr":{"pid":10234,"port":27017,"dbPath":"/var/lib/mongodb"}} -{"t":{"$date":"2025-07-22T08:30:01.456Z"},"s":"I","c":"STORAGE","id":20320,"ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":"[1658325001:456789][10234:0x7f9dcd9f3700], txn-recover: Recovering log 1 through 2"}} -{"t":{"$date":"2025-07-22T08:30:01.789Z"},"s":"I","c":"STORAGE","id":22437,"ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":1234}} -{"t":{"$date":"2025-07-22T08:30:03.101Z"},"s":"I","c":"STORAGE","id":4795900,"ctx":"WTCheckpointThread","msg":"WiredTiger checkpoint","attr":{"checkpoint_ts":{"$timestamp":{"t":1658325003,"i":1}}}} -{"t":{"$date":"2025-07-22T08:30:05.234Z"},"s":"I","c":"STORAGE","id":5123400,"ctx":"WTJournalFlusher","msg":"WiredTiger journal flush","attr":{"flush_duration_ms":21}} -{"t":{"$date":"2025-07-22T08:30:06.345Z"},"s":"I","c":"STORAGE","id":22438,"ctx":"WTCacheEvictionWorker-0","msg":"WiredTiger eviction","attr":{"pages_evicted":34,"max_pages":100}} -{"t":{"$date":"2025-07-22T08:30:07.456Z"},"s":"I","c":"COMMAND","id":51803,"ctx":"conn1","msg":"Connection accepted","attr":{"client":"127.0.0.1:37244","connectionId":1,"connectionCount":1}} -{"t":{"$date":"2025-07-22T08:30:08.678Z"},"s":"I","c":"COMMAND","id":51804,"ctx":"conn1","msg":"Received client metadata","attr":{"client":"conn1","doc":{"driver":{"name":"PyMongo","version":"4.2.0"},"os":{"type":"Linux","name":"Ubuntu","architecture":"x86_64","version":"20.04"}}}} -{"t":{"$date":"2025-07-22T08:30:09.111Z"},"s":"I","c":"COMMAND","id":51805,"ctx":"conn1","msg":"Insert command","attr":{"ns":"test.users","ninserted":1,"keysInserted":1,"millis":2}} -{"t":{"$date":"2025-07-22T08:30:09.789Z"},"s":"I","c":"COMMAND","id":51806,"ctx":"conn1","msg":"Query executed","attr":{"ns":"test.users","query":{"name":"Alice"},"docsReturned":1,"keysExamined":1,"millis":1}} -{"t":{"$date":"2025-07-22T08:30:10.000Z"},"s":"W","c":"COMMAND","id":51810,"ctx":"conn1","msg":"Slow query","attr":{"type":"query","ns":"test.logs","query":{"level":"debug"},"planSummary":"COLLSCAN","millis":212}} -{"t":{"$date":"2025-07-22T08:30:11.321Z"},"s":"E","c":"ACCESS","id":20249,"ctx":"conn2","msg":"Authentication failed","attr":{"mechanism":"SCRAM-SHA-256","user":"invalidUser","db":"admin","client":"127.0.0.1:37245","result":"AuthenticationFailed"}} -{"t":{"$date":"2025-07-22T08:30:12.789Z"},"s":"E","c":"NETWORK","id":23213,"ctx":"conn2","msg":"Client metadata parsing failed","attr":{"error":"Missing required field 'driver'"}} \ No newline at end of file +{"t":{"$date":"2025-07-21T15:00:28.308+00:00"},"s":"I", "c":"CONTROL", "id":23285, "ctx":"main","msg":"Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none'"} +{"t":{"$date":"2025-07-21T15:00:28.309+00:00"},"s":"I", "c":"CONTROL", "id":5945603, "ctx":"main","msg":"Multi threading initialized"} +{"t":{"$date":"2025-07-21T15:00:28.309+00:00"},"s":"I", "c":"NETWORK", "id":4648601, "ctx":"main","msg":"Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set at least one of the related parameters","attr":{"relatedParameters":["tcpFastOpenServer","tcpFastOpenClient","tcpFastOpenQueueSize"]}} +{"t":{"$date":"2025-07-21T15:00:28.310+00:00"},"s":"I", "c":"NETWORK", "id":4915701, "ctx":"main","msg":"Initialized wire specification","attr":{"spec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":25},"incomingInternalClient":{"minWireVersion":0,"maxWireVersion":25},"outgoing":{"minWireVersion":6,"maxWireVersion":25},"isInternalClient":true}}} +{"t":{"$date":"2025-07-21T15:00:28.310+00:00"},"s":"I", "c":"TENANT_M", "id":7091600, "ctx":"main","msg":"Starting TenantMigrationAccessBlockerRegistry"} +{"t":{"$date":"2025-07-21T15:00:28.310+00:00"},"s":"I", "c":"CONTROL", "id":4615611, "ctx":"initandlisten","msg":"MongoDB starting","attr":{"pid":1,"port":27017,"dbPath":"/data/db","architecture":"64-bit","host":"0093111b5a52"}} +{"t":{"$date":"2025-07-21T15:00:28.310+00:00"},"s":"I", "c":"CONTROL", "id":23403, "ctx":"initandlisten","msg":"Build Info","attr":{"buildInfo":{"version":"8.0.11","gitVersion":"bed99f699da6cb2b74262aa6d473446c41476643","openSSLVersion":"OpenSSL 3.0.13 30 Jan 2024","modules":[],"allocator":"tcmalloc-google","environment":{"distmod":"ubuntu2404","distarch":"aarch64","target_arch":"aarch64"}}}} +{"t":{"$date":"2025-07-21T15:00:28.310+00:00"},"s":"I", "c":"CONTROL", "id":51765, "ctx":"initandlisten","msg":"Operating System","attr":{"os":{"name":"Ubuntu","version":"24.04"}}} +{"t":{"$date":"2025-07-21T15:00:28.310+00:00"},"s":"I", "c":"CONTROL", "id":21951, "ctx":"initandlisten","msg":"Options set by command line","attr":{"options":{"net":{"bindIp":"*"}}}} +{"t":{"$date":"2025-07-21T15:00:28.311+00:00"},"s":"I", "c":"STORAGE", "id":22297, "ctx":"initandlisten","msg":"Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem","tags":["startupWarnings"]} +{"t":{"$date":"2025-07-21T15:00:28.311+00:00"},"s":"I", "c":"STORAGE", "id":22315, "ctx":"initandlisten","msg":"Opening WiredTiger","attr":{"config":"create,cache_size=3406M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,remove=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=2000),statistics_log=(wait=0),json_output=(error,message),verbose=[recovery_progress:1,checkpoint_progress:1,compact_progress:1,backup:0,checkpoint:0,compact:0,evict:0,history_store:0,recovery:0,rts:0,salvage:0,tiered:0,timestamp:0,transaction:0,verify:0,log:0],prefetch=(available=true,default=false),"}} +{"t":{"$date":"2025-07-21T15:00:28.542+00:00"},"s":"I", "c":"WTRECOV", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110028,"ts_usec":542739,"thread":"1:0xffffbec76040","session_name":"txn-recover","category":"WT_VERB_RECOVERY_PROGRESS","category_id":34,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"recovery log replay has successfully finished and ran for 0 milliseconds"}}} +{"t":{"$date":"2025-07-21T15:00:28.542+00:00"},"s":"I", "c":"WTRECOV", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110028,"ts_usec":542790,"thread":"1:0xffffbec76040","session_name":"txn-recover","category":"WT_VERB_RECOVERY_PROGRESS","category_id":34,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"Set global recovery timestamp: (0, 0)"}}} +{"t":{"$date":"2025-07-21T15:00:28.542+00:00"},"s":"I", "c":"WTRECOV", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110028,"ts_usec":542801,"thread":"1:0xffffbec76040","session_name":"txn-recover","category":"WT_VERB_RECOVERY_PROGRESS","category_id":34,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"Set global oldest timestamp: (0, 0)"}}} +{"t":{"$date":"2025-07-21T15:00:28.542+00:00"},"s":"I", "c":"WTRECOV", "id":22430, "ctx":"initandlisten","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110028,"ts_usec":542814,"thread":"1:0xffffbec76040","session_name":"txn-recover","category":"WT_VERB_RECOVERY_PROGRESS","category_id":34,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"recovery was completed successfully and took 0ms, including 0ms for the log replay, 0ms for the rollback to stable, and 0ms for the checkpoint."}}} +{"t":{"$date":"2025-07-21T15:00:28.546+00:00"},"s":"I", "c":"STORAGE", "id":4795906, "ctx":"initandlisten","msg":"WiredTiger opened","attr":{"durationMillis":235}} +{"t":{"$date":"2025-07-21T15:00:28.546+00:00"},"s":"I", "c":"RECOVERY", "id":23987, "ctx":"initandlisten","msg":"WiredTiger recoveryTimestamp","attr":{"recoveryTimestamp":{"$timestamp":{"t":0,"i":0}}}} +{"t":{"$date":"2025-07-21T15:00:28.549+00:00"},"s":"I", "c":"STORAGE", "id":9529901, "ctx":"initandlisten","msg":"Initializing durable catalog","attr":{"numRecords":0}} +{"t":{"$date":"2025-07-21T15:00:28.549+00:00"},"s":"I", "c":"STORAGE", "id":9529902, "ctx":"initandlisten","msg":"Retrieving all idents from storage engine"} +{"t":{"$date":"2025-07-21T15:00:28.549+00:00"},"s":"I", "c":"STORAGE", "id":9529903, "ctx":"initandlisten","msg":"Initializing all collections in durable catalog","attr":{"numEntries":0}} +{"t":{"$date":"2025-07-21T15:00:28.550+00:00"},"s":"W", "c":"CONTROL", "id":22120, "ctx":"initandlisten","msg":"Access control is not enabled for the database. Read and write access to data and configuration is unrestricted","tags":["startupWarnings"]} +{"t":{"$date":"2025-07-21T15:00:28.551+00:00"},"s":"W", "c":"CONTROL", "id":9068900, "ctx":"initandlisten","msg":"For customers running the current memory allocator, we suggest changing the contents of the following sysfsFile","attr":{"allocator":"tcmalloc-google","sysfsFile":"/sys/kernel/mm/transparent_hugepage/defrag","currentValue":"madvise","desiredValue":"defer+madvise"},"tags":["startupWarnings"]} +{"t":{"$date":"2025-07-21T15:00:28.551+00:00"},"s":"W", "c":"CONTROL", "id":8640302, "ctx":"initandlisten","msg":"We suggest setting the contents of sysfsFile to 0.","attr":{"sysfsFile":"/sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none","currentValue":511},"tags":["startupWarnings"]} +{"t":{"$date":"2025-07-21T15:00:28.551+00:00"},"s":"W", "c":"NETWORK", "id":5123300, "ctx":"initandlisten","msg":"vm.max_map_count is too low","attr":{"currentValue":262144,"recommendedMinimum":1677720,"maxConns":838860},"tags":["startupWarnings"]} +{"t":{"$date":"2025-07-21T15:00:28.551+00:00"},"s":"W", "c":"CONTROL", "id":8386700, "ctx":"initandlisten","msg":"We suggest setting swappiness to 0 or 1, as swapping can cause performance problems.","attr":{"sysfsFile":"/proc/sys/vm/swappiness","currentValue":60},"tags":["startupWarnings"]} +{"t":{"$date":"2025-07-21T15:00:28.551+00:00"},"s":"I", "c":"STORAGE", "id":20320, "ctx":"initandlisten","msg":"createCollection","attr":{"namespace":"admin.system.version","uuidDisposition":"provided","uuid":{"uuid":{"$uuid":"d2aa3324-da83-4514-8930-750b069145f9"}},"options":{"uuid":{"$uuid":"d2aa3324-da83-4514-8930-750b069145f9"}}}} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"INDEX", "id":20345, "ctx":"initandlisten","msg":"Index build: done building","attr":{"buildUUID":null,"collectionUUID":{"uuid":{"$uuid":"d2aa3324-da83-4514-8930-750b069145f9"}},"namespace":"admin.system.version","index":"_id_","ident":"index-1-3956073341144391669","collectionIdent":"collection-0-3956073341144391669","commitTimestamp":null}} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"REPL", "id":20459, "ctx":"initandlisten","msg":"Setting featureCompatibilityVersion","attr":{"newVersion":"8.0"}} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"REPL", "id":5853300, "ctx":"initandlisten","msg":"current featureCompatibilityVersion value","attr":{"featureCompatibilityVersion":"8.0","context":"setFCV"}} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"NETWORK", "id":4915702, "ctx":"initandlisten","msg":"Updated wire specification","attr":{"oldSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":25},"incomingInternalClient":{"minWireVersion":0,"maxWireVersion":25},"outgoing":{"minWireVersion":6,"maxWireVersion":25},"isInternalClient":true},"newSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":25},"incomingInternalClient":{"minWireVersion":25,"maxWireVersion":25},"outgoing":{"minWireVersion":25,"maxWireVersion":25},"isInternalClient":true}}} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"NETWORK", "id":4915702, "ctx":"initandlisten","msg":"Updated wire specification","attr":{"oldSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":25},"incomingInternalClient":{"minWireVersion":25,"maxWireVersion":25},"outgoing":{"minWireVersion":25,"maxWireVersion":25},"isInternalClient":true},"newSpec":{"incomingExternalClient":{"minWireVersion":0,"maxWireVersion":25},"incomingInternalClient":{"minWireVersion":25,"maxWireVersion":25},"outgoing":{"minWireVersion":25,"maxWireVersion":25},"isInternalClient":true}}} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"REPL", "id":5853300, "ctx":"initandlisten","msg":"current featureCompatibilityVersion value","attr":{"featureCompatibilityVersion":"8.0","context":"startup"}} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"STORAGE", "id":5071100, "ctx":"initandlisten","msg":"Clearing temp directory"} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"CONTROL", "id":6608200, "ctx":"initandlisten","msg":"Initializing cluster server parameters from disk"} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"CONTROL", "id":20536, "ctx":"initandlisten","msg":"Flow Control is enabled on this deployment"} +{"t":{"$date":"2025-07-21T15:00:28.554+00:00"},"s":"I", "c":"FTDC", "id":20625, "ctx":"initandlisten","msg":"Initializing full-time diagnostic data capture","attr":{"dataDirectory":"/data/db/diagnostic.data"}} +{"t":{"$date":"2025-07-21T15:00:28.555+00:00"},"s":"I", "c":"STORAGE", "id":20320, "ctx":"initandlisten","msg":"createCollection","attr":{"namespace":"local.startup_log","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"4db69adb-ed35-47b4-ad78-264f6bf62996"}},"options":{"capped":true,"size":10485760}}} +{"t":{"$date":"2025-07-21T15:00:28.558+00:00"},"s":"I", "c":"INDEX", "id":20345, "ctx":"initandlisten","msg":"Index build: done building","attr":{"buildUUID":null,"collectionUUID":{"uuid":{"$uuid":"4db69adb-ed35-47b4-ad78-264f6bf62996"}},"namespace":"local.startup_log","index":"_id_","ident":"index-3-3956073341144391669","collectionIdent":"collection-2-3956073341144391669","commitTimestamp":null}} +{"t":{"$date":"2025-07-21T15:00:28.558+00:00"},"s":"I", "c":"REPL", "id":6015317, "ctx":"initandlisten","msg":"Setting new configuration state","attr":{"newState":"ConfigReplicationDisabled","oldState":"ConfigPreStart"}} +{"t":{"$date":"2025-07-21T15:00:28.558+00:00"},"s":"I", "c":"STORAGE", "id":22262, "ctx":"initandlisten","msg":"Timestamp monitor starting"} +{"t":{"$date":"2025-07-21T15:00:28.558+00:00"},"s":"I", "c":"STORAGE", "id":7333401, "ctx":"initandlisten","msg":"Starting the DiskSpaceMonitor"} +{"t":{"$date":"2025-07-21T15:00:28.559+00:00"},"s":"I", "c":"NETWORK", "id":23015, "ctx":"listener","msg":"Listening on","attr":{"address":"/tmp/mongodb-27017.sock"}} +{"t":{"$date":"2025-07-21T15:00:28.559+00:00"},"s":"I", "c":"NETWORK", "id":23015, "ctx":"listener","msg":"Listening on","attr":{"address":"0.0.0.0:27017"}} +{"t":{"$date":"2025-07-21T15:00:28.559+00:00"},"s":"I", "c":"NETWORK", "id":23016, "ctx":"listener","msg":"Waiting for connections","attr":{"port":27017,"ssl":"off"}} +{"t":{"$date":"2025-07-21T15:00:28.559+00:00"},"s":"I", "c":"CONTROL", "id":8423403, "ctx":"initandlisten","msg":"mongod startup complete","attr":{"Summary of time elapsed":{"Startup from clean shutdown?":true,"Statistics":{"Set up periodic runner":"0 ms","Set up online certificate status protocol manager":"0 ms","Transport layer setup":"0 ms","Run initial syncer crash recovery":"0 ms","Create storage engine lock file in the data directory":"0 ms","Get metadata describing storage engine":"0 ms","Create storage engine":"238 ms","Write current PID to file":"0 ms","Write a new metadata for storage engine":"0 ms","Initialize FCV before rebuilding indexes":"0 ms","Drop abandoned idents and get back indexes that need to be rebuilt or builds that need to be restarted":"0 ms","Rebuild indexes for collections":"0 ms","Load cluster parameters from disk for a standalone":"0 ms","Build user and roles graph":"0 ms","Set up the background thread pool responsible for waiting for opTimes to be majority committed":"0 ms","Start up the replication coordinator":"0 ms","Ensure the change stream collections on startup contain consistent data":"0 ms","Write startup options to the audit log":"0 ms","Start transport layer":"0 ms","_initAndListen total elapsed time":"249 ms"}}}} +{"t":{"$date":"2025-07-21T15:00:28.559+00:00"},"s":"I", "c":"CONTROL", "id":20712, "ctx":"LogicalSessionCacheReap","msg":"Sessions collection is not set up; waiting until next sessions reap interval","attr":{"error":"NamespaceNotFound: config.system.sessions does not exist"}} +{"t":{"$date":"2025-07-21T15:00:28.559+00:00"},"s":"I", "c":"STORAGE", "id":20320, "ctx":"LogicalSessionCacheRefresh","msg":"createCollection","attr":{"namespace":"config.system.sessions","uuidDisposition":"generated","uuid":{"uuid":{"$uuid":"cc5d7417-27c1-4110-8873-0e83b22c9de4"}},"options":{}}} +{"t":{"$date":"2025-07-21T15:00:28.564+00:00"},"s":"I", "c":"INDEX", "id":20345, "ctx":"LogicalSessionCacheRefresh","msg":"Index build: done building","attr":{"buildUUID":null,"collectionUUID":{"uuid":{"$uuid":"cc5d7417-27c1-4110-8873-0e83b22c9de4"}},"namespace":"config.system.sessions","index":"_id_","ident":"index-5-3956073341144391669","collectionIdent":"collection-4-3956073341144391669","commitTimestamp":null}} +{"t":{"$date":"2025-07-21T15:00:28.564+00:00"},"s":"I", "c":"INDEX", "id":20345, "ctx":"LogicalSessionCacheRefresh","msg":"Index build: done building","attr":{"buildUUID":null,"collectionUUID":{"uuid":{"$uuid":"cc5d7417-27c1-4110-8873-0e83b22c9de4"}},"namespace":"config.system.sessions","index":"lsidTTLIndex","ident":"index-6-3956073341144391669","collectionIdent":"collection-4-3956073341144391669","commitTimestamp":null}} +{"t":{"$date":"2025-07-21T15:00:29.004+00:00"},"s":"W", "c":"CONTROL", "id":636300, "ctx":"ftdc","msg":"Use of deprecated server parameter name","attr":{"deprecatedName":"internalQueryCacheSize","canonicalName":"internalQueryCacheMaxEntriesPerCollection"}} +{"t":{"$date":"2025-07-21T15:00:29.004+00:00"},"s":"W", "c":"CONTROL", "id":636300, "ctx":"ftdc","msg":"Use of deprecated server parameter name","attr":{"deprecatedName":"oplogSamplingLogIntervalSeconds","canonicalName":"collectionSamplingLogIntervalSeconds"}} +{"t":{"$date":"2025-07-21T15:00:29.004+00:00"},"s":"W", "c":"NETWORK", "id":23803, "ctx":"ftdc","msg":"Use of deprecated server parameter 'sslMode', please use 'tlsMode' instead."} +{"t":{"$date":"2025-07-21T15:00:29.004+00:00"},"s":"W", "c":"CONTROL", "id":636300, "ctx":"ftdc","msg":"Use of deprecated server parameter name","attr":{"deprecatedName":"wiredTigerConcurrentReadTransactions","canonicalName":"storageEngineConcurrentReadTransactions"}} +{"t":{"$date":"2025-07-21T15:00:29.004+00:00"},"s":"W", "c":"CONTROL", "id":636300, "ctx":"ftdc","msg":"Use of deprecated server parameter name","attr":{"deprecatedName":"wiredTigerConcurrentWriteTransactions","canonicalName":"storageEngineConcurrentWriteTransactions"}} +{"t":{"$date":"2025-07-21T15:01:28.556+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110088,"ts_usec":555981,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 33, snapshot max: 33 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} +{"t":{"$date":"2025-07-21T15:02:28.572+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110148,"ts_usec":572218,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 35, snapshot max: 35 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} +{"t":{"$date":"2025-07-21T15:03:28.585+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110208,"ts_usec":584957,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 36, snapshot max: 36 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} +{"t":{"$date":"2025-07-21T15:04:28.600+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110268,"ts_usec":600180,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 37, snapshot max: 37 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} +{"t":{"$date":"2025-07-21T15:05:28.612+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110328,"ts_usec":612727,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 38, snapshot max: 38 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} \ No newline at end of file diff --git a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml index 15905075da..e3a9e49c45 100644 --- a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml +++ b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml @@ -1,12 +1,86 @@ - entries: + - jsonPayload: + component: CONTROL + context: main + id: 23285.0 + message: Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none' + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: CONTROL + context: main + id: 5.945603e+06 + message: Multi threading initialized + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + relatedParameters: + - tcpFastOpenServer + - tcpFastOpenClient + - tcpFastOpenQueueSize + component: NETWORK + context: main + id: 4.648601e+06 + message: Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set at least one of the related parameters + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now - jsonPayload: attributes: - dbPath: /var/lib/mongodb - pid: 10234.0 + spec: + incomingExternalClient: + maxWireVersion: 25.0 + minWireVersion: 0.0 + incomingInternalClient: + maxWireVersion: 25.0 + minWireVersion: 0.0 + isInternalClient: true + outgoing: + maxWireVersion: 25.0 + minWireVersion: 6.0 + component: NETWORK + context: main + id: 4.915701e+06 + message: Initialized wire specification + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: TENANT_M + context: main + id: 7.0916e+06 + message: Starting TenantMigrationAccessBlockerRegistry + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + architecture: 64-bit + dbPath: /data/db + host: 0093111b5a52 + pid: 1.0 port: 27017.0 component: CONTROL - context: main - id: 23352.0 + context: initandlisten + id: 4.615611e+06 message: MongoDB starting severity: I labels: @@ -14,12 +88,66 @@ logName: projects/my-project/logs/transformation_test severity: 200.0 timestamp: now + - jsonPayload: + attributes: + buildInfo: + allocator: tcmalloc-google + environment: + distarch: aarch64 + distmod: ubuntu2404 + target_arch: aarch64 + gitVersion: bed99f699da6cb2b74262aa6d473446c41476643 + modules: [] + openSSLVersion: OpenSSL 3.0.13 30 Jan 2024 + version: 8.0.11 + component: CONTROL + context: initandlisten + id: 23403.0 + message: Build Info + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + os: + name: Ubuntu + version: "24.04" + component: CONTROL + context: initandlisten + id: 51765.0 + message: Operating System + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + options: + net: + bindIp: "*" + component: CONTROL + context: initandlisten + id: 21951.0 + message: Options set by command line + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now - jsonPayload: component: STORAGE context: initandlisten - id: 20320.0 - message: "[1658325001:456789][10234:0x7f9dcd9f3700], txn-recover: Recovering log 1 through 2" + id: 22297.0 + message: Using the XFS filesystem is strongly recommended with the WiredTiger storage engine. See http://dochub.mongodb.org/core/prodnotes-filesystem severity: I + tags: + - startupWarnings labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test @@ -27,10 +155,103 @@ timestamp: now - jsonPayload: attributes: - durationMillis: 1234.0 + config: create,cache_size=3406M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,remove=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=2000),statistics_log=(wait=0),json_output=(error,message),verbose=[recovery_progress:1,checkpoint_progress:1,compact_progress:1,backup:0,checkpoint:0,compact:0,evict:0,history_store:0,recovery:0,rts:0,salvage:0,tiered:0,timestamp:0,transaction:0,verify:0,log:0],prefetch=(available=true,default=false), component: STORAGE context: initandlisten - id: 22437.0 + id: 22315.0 + message: Opening WiredTiger + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTRECOV + context: initandlisten + id: 22430.0 + message: + category: WT_VERB_RECOVERY_PROGRESS + category_id: 34.0 + msg: recovery log replay has successfully finished and ran for 0 milliseconds + session_name: txn-recover + thread: 1:0xffffbec76040 + ts_sec: 1.753110028e+09 + ts_usec: 542739.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTRECOV + context: initandlisten + id: 22430.0 + message: + category: WT_VERB_RECOVERY_PROGRESS + category_id: 34.0 + msg: "Set global recovery timestamp: (0, 0)" + session_name: txn-recover + thread: 1:0xffffbec76040 + ts_sec: 1.753110028e+09 + ts_usec: 542790.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTRECOV + context: initandlisten + id: 22430.0 + message: + category: WT_VERB_RECOVERY_PROGRESS + category_id: 34.0 + msg: "Set global oldest timestamp: (0, 0)" + session_name: txn-recover + thread: 1:0xffffbec76040 + ts_sec: 1.753110028e+09 + ts_usec: 542801.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTRECOV + context: initandlisten + id: 22430.0 + message: + category: WT_VERB_RECOVERY_PROGRESS + category_id: 34.0 + msg: recovery was completed successfully and took 0ms, including 0ms for the log replay, 0ms for the rollback to stable, and 0ms for the checkpoint. + session_name: txn-recover + thread: 1:0xffffbec76040 + ts_sec: 1.753110028e+09 + ts_usec: 542814.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + durationMillis: 235.0 + component: STORAGE + context: initandlisten + id: 4.795906e+06 message: WiredTiger opened severity: I labels: @@ -40,14 +261,38 @@ timestamp: now - jsonPayload: attributes: - checkpoint_ts: + recoveryTimestamp: $timestamp: - i: 1.0 - t: 1.658325003e+09 + i: 0.0 + t: 0.0 + component: RECOVERY + context: initandlisten + id: 23987.0 + message: WiredTiger recoveryTimestamp + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + numRecords: 0.0 component: STORAGE - context: WTCheckpointThread - id: 4.7959e+06 - message: WiredTiger checkpoint + context: initandlisten + id: 9.529901e+06 + message: Initializing durable catalog + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: STORAGE + context: initandlisten + id: 9.529902e+06 + message: Retrieving all idents from storage engine severity: I labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -56,25 +301,130 @@ timestamp: now - jsonPayload: attributes: - flush_duration_ms: 21.0 + numEntries: 0.0 component: STORAGE - context: WTJournalFlusher - id: 5.1234e+06 - message: WiredTiger journal flush + context: initandlisten + id: 9.529903e+06 + message: Initializing all collections in durable catalog severity: I labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 timestamp: now + - jsonPayload: + component: CONTROL + context: initandlisten + id: 22120.0 + message: Access control is not enabled for the database. Read and write access to data and configuration is unrestricted + severity: W + tags: + - startupWarnings + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now - jsonPayload: attributes: - max_pages: 100.0 - pages_evicted: 34.0 + allocator: tcmalloc-google + currentValue: madvise + desiredValue: defer+madvise + sysfsFile: /sys/kernel/mm/transparent_hugepage/defrag + component: CONTROL + context: initandlisten + id: 9.0689e+06 + message: For customers running the current memory allocator, we suggest changing the contents of the following sysfsFile + severity: W + tags: + - startupWarnings + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + attributes: + currentValue: 511.0 + sysfsFile: /sys/kernel/mm/transparent_hugepage/khugepaged/max_ptes_none + component: CONTROL + context: initandlisten + id: 8.640302e+06 + message: We suggest setting the contents of sysfsFile to 0. + severity: W + tags: + - startupWarnings + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + attributes: + currentValue: 262144.0 + maxConns: 838860.0 + recommendedMinimum: 1.67772e+06 + component: NETWORK + context: initandlisten + id: 5.1233e+06 + message: vm.max_map_count is too low + severity: W + tags: + - startupWarnings + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + attributes: + currentValue: 60.0 + sysfsFile: /proc/sys/vm/swappiness + component: CONTROL + context: initandlisten + id: 8.3867e+06 + message: We suggest setting swappiness to 0 or 1, as swapping can cause performance problems. + severity: W + tags: + - startupWarnings + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + attributes: + namespace: admin.system.version + options: + uuid: + $uuid: d2aa3324-da83-4514-8930-750b069145f9 + uuid: + uuid: + $uuid: d2aa3324-da83-4514-8930-750b069145f9 + uuidDisposition: provided component: STORAGE - context: WTCacheEvictionWorker-0 - id: 22438.0 - message: WiredTiger eviction + context: initandlisten + id: 20320.0 + message: createCollection + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + collectionIdent: collection-0-3956073341144391669 + collectionUUID: + uuid: + $uuid: d2aa3324-da83-4514-8930-750b069145f9 + ident: index-1-3956073341144391669 + index: _id_ + namespace: admin.system.version + component: INDEX + context: initandlisten + id: 20345.0 + message: "Index build: done building" severity: I labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -83,13 +433,11 @@ timestamp: now - jsonPayload: attributes: - client: 127.0.0.1:37244 - connectionCount: 1.0 - connectionId: 1.0 - component: COMMAND - context: conn1 - id: 51803.0 - message: Connection accepted + newVersion: "8.0" + component: REPL + context: initandlisten + id: 20459.0 + message: Setting featureCompatibilityVersion severity: I labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -98,20 +446,12 @@ timestamp: now - jsonPayload: attributes: - client: conn1 - doc: - driver: - name: PyMongo - version: 4.2.0 - os: - architecture: x86_64 - name: Ubuntu - type: Linux - version: "20.04" - component: COMMAND - context: conn1 - id: 51804.0 - message: Received client metadata + context: setFCV + featureCompatibilityVersion: "8.0" + component: REPL + context: initandlisten + id: 5.8533e+06 + message: current featureCompatibilityVersion value severity: I labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -120,14 +460,32 @@ timestamp: now - jsonPayload: attributes: - keysInserted: 1.0 - millis: 2.0 - ninserted: 1.0 - ns: test.users - component: COMMAND - context: conn1 - id: 51805.0 - message: Insert command + newSpec: + incomingExternalClient: + maxWireVersion: 25.0 + minWireVersion: 0.0 + incomingInternalClient: + maxWireVersion: 25.0 + minWireVersion: 25.0 + isInternalClient: true + outgoing: + maxWireVersion: 25.0 + minWireVersion: 25.0 + oldSpec: + incomingExternalClient: + maxWireVersion: 25.0 + minWireVersion: 0.0 + incomingInternalClient: + maxWireVersion: 25.0 + minWireVersion: 0.0 + isInternalClient: true + outgoing: + maxWireVersion: 25.0 + minWireVersion: 6.0 + component: NETWORK + context: initandlisten + id: 4.915702e+06 + message: Updated wire specification severity: I labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -136,16 +494,32 @@ timestamp: now - jsonPayload: attributes: - docsReturned: 1.0 - keysExamined: 1.0 - millis: 1.0 - ns: test.users - query: - name: Alice - component: COMMAND - context: conn1 - id: 51806.0 - message: Query executed + newSpec: + incomingExternalClient: + maxWireVersion: 25.0 + minWireVersion: 0.0 + incomingInternalClient: + maxWireVersion: 25.0 + minWireVersion: 25.0 + isInternalClient: true + outgoing: + maxWireVersion: 25.0 + minWireVersion: 25.0 + oldSpec: + incomingExternalClient: + maxWireVersion: 25.0 + minWireVersion: 0.0 + incomingInternalClient: + maxWireVersion: 25.0 + minWireVersion: 25.0 + isInternalClient: true + outgoing: + maxWireVersion: 25.0 + minWireVersion: 25.0 + component: NETWORK + context: initandlisten + id: 4.915702e+06 + message: Updated wire specification severity: I labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -154,16 +528,291 @@ timestamp: now - jsonPayload: attributes: - millis: 212.0 - ns: test.logs - planSummary: COLLSCAN - query: - level: debug - type: query - component: COMMAND - context: conn1 - id: 51810.0 - message: Slow query + context: startup + featureCompatibilityVersion: "8.0" + component: REPL + context: initandlisten + id: 5.8533e+06 + message: current featureCompatibilityVersion value + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: STORAGE + context: initandlisten + id: 5.0711e+06 + message: Clearing temp directory + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: CONTROL + context: initandlisten + id: 6.6082e+06 + message: Initializing cluster server parameters from disk + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: CONTROL + context: initandlisten + id: 20536.0 + message: Flow Control is enabled on this deployment + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + dataDirectory: /data/db/diagnostic.data + component: FTDC + context: initandlisten + id: 20625.0 + message: Initializing full-time diagnostic data capture + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + namespace: local.startup_log + options: + capped: true + size: 1.048576e+07 + uuid: + uuid: + $uuid: 4db69adb-ed35-47b4-ad78-264f6bf62996 + uuidDisposition: generated + component: STORAGE + context: initandlisten + id: 20320.0 + message: createCollection + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + collectionIdent: collection-2-3956073341144391669 + collectionUUID: + uuid: + $uuid: 4db69adb-ed35-47b4-ad78-264f6bf62996 + ident: index-3-3956073341144391669 + index: _id_ + namespace: local.startup_log + component: INDEX + context: initandlisten + id: 20345.0 + message: "Index build: done building" + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + newState: ConfigReplicationDisabled + oldState: ConfigPreStart + component: REPL + context: initandlisten + id: 6.015317e+06 + message: Setting new configuration state + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: STORAGE + context: initandlisten + id: 22262.0 + message: Timestamp monitor starting + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: STORAGE + context: initandlisten + id: 7.333401e+06 + message: Starting the DiskSpaceMonitor + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + address: /tmp/mongodb-27017.sock + component: NETWORK + context: listener + id: 23015.0 + message: Listening on + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + address: 0.0.0.0:27017 + component: NETWORK + context: listener + id: 23015.0 + message: Listening on + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + port: 27017.0 + ssl: "off" + component: NETWORK + context: listener + id: 23016.0 + message: Waiting for connections + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + Summary of time elapsed: + Startup from clean shutdown?: true + Statistics: + Build user and roles graph: 0 ms + Create storage engine: 238 ms + Create storage engine lock file in the data directory: 0 ms + Drop abandoned idents and get back indexes that need to be rebuilt or builds that need to be restarted: 0 ms + Ensure the change stream collections on startup contain consistent data: 0 ms + Get metadata describing storage engine: 0 ms + Initialize FCV before rebuilding indexes: 0 ms + Load cluster parameters from disk for a standalone: 0 ms + Rebuild indexes for collections: 0 ms + Run initial syncer crash recovery: 0 ms + Set up online certificate status protocol manager: 0 ms + Set up periodic runner: 0 ms + Set up the background thread pool responsible for waiting for opTimes to be majority committed: 0 ms + Start transport layer: 0 ms + Start up the replication coordinator: 0 ms + Transport layer setup: 0 ms + Write a new metadata for storage engine: 0 ms + Write current PID to file: 0 ms + Write startup options to the audit log: 0 ms + _initAndListen total elapsed time: 249 ms + component: CONTROL + context: initandlisten + id: 8.423403e+06 + message: mongod startup complete + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + error: "NamespaceNotFound: config.system.sessions does not exist" + component: CONTROL + context: LogicalSessionCacheReap + id: 20712.0 + message: Sessions collection is not set up; waiting until next sessions reap interval + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + namespace: config.system.sessions + options: {} + uuid: + uuid: + $uuid: cc5d7417-27c1-4110-8873-0e83b22c9de4 + uuidDisposition: generated + component: STORAGE + context: LogicalSessionCacheRefresh + id: 20320.0 + message: createCollection + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + collectionIdent: collection-4-3956073341144391669 + collectionUUID: + uuid: + $uuid: cc5d7417-27c1-4110-8873-0e83b22c9de4 + ident: index-5-3956073341144391669 + index: _id_ + namespace: config.system.sessions + component: INDEX + context: LogicalSessionCacheRefresh + id: 20345.0 + message: "Index build: done building" + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + collectionIdent: collection-4-3956073341144391669 + collectionUUID: + uuid: + $uuid: cc5d7417-27c1-4110-8873-0e83b22c9de4 + ident: index-6-3956073341144391669 + index: lsidTTLIndex + namespace: config.system.sessions + component: INDEX + context: LogicalSessionCacheRefresh + id: 20345.0 + message: "Index build: done building" + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + attributes: + canonicalName: internalQueryCacheMaxEntriesPerCollection + deprecatedName: internalQueryCacheSize + component: CONTROL + context: ftdc + id: 636300.0 + message: Use of deprecated server parameter name severity: W labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -172,20 +821,136 @@ timestamp: now - jsonPayload: attributes: - client: 127.0.0.1:37245 - db: admin - mechanism: SCRAM-SHA-256 - result: AuthenticationFailed - user: invalidUser - component: ACCESS - context: conn2 - id: 20249.0 - message: Authentication failed - severity: E + canonicalName: collectionSamplingLogIntervalSeconds + deprecatedName: oplogSamplingLogIntervalSeconds + component: CONTROL + context: ftdc + id: 636300.0 + message: Use of deprecated server parameter name + severity: W labels: logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test - severity: 500.0 + severity: 400.0 + timestamp: now + - jsonPayload: + component: NETWORK + context: ftdc + id: 23803.0 + message: Use of deprecated server parameter 'sslMode', please use 'tlsMode' instead. + severity: W + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + attributes: + canonicalName: storageEngineConcurrentReadTransactions + deprecatedName: wiredTigerConcurrentReadTransactions + component: CONTROL + context: ftdc + id: 636300.0 + message: Use of deprecated server parameter name + severity: W + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + attributes: + canonicalName: storageEngineConcurrentWriteTransactions + deprecatedName: wiredTigerConcurrentWriteTransactions + component: CONTROL + context: ftdc + id: 636300.0 + message: Use of deprecated server parameter name + severity: W + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 400.0 + timestamp: now + - jsonPayload: + component: WTCHKPT + context: Checkpointer + id: 22430.0 + message: + category: WT_VERB_CHECKPOINT_PROGRESS + category_id: 7.0 + msg: "saving checkpoint snapshot min: 33, snapshot max: 33 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1" + session_name: WT_SESSION.checkpoint + thread: 1:0xffffab20e680 + ts_sec: 1.753110088e+09 + ts_usec: 555981.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTCHKPT + context: Checkpointer + id: 22430.0 + message: + category: WT_VERB_CHECKPOINT_PROGRESS + category_id: 7.0 + msg: "saving checkpoint snapshot min: 35, snapshot max: 35 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1" + session_name: WT_SESSION.checkpoint + thread: 1:0xffffab20e680 + ts_sec: 1.753110148e+09 + ts_usec: 572218.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTCHKPT + context: Checkpointer + id: 22430.0 + message: + category: WT_VERB_CHECKPOINT_PROGRESS + category_id: 7.0 + msg: "saving checkpoint snapshot min: 36, snapshot max: 36 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1" + session_name: WT_SESSION.checkpoint + thread: 1:0xffffab20e680 + ts_sec: 1.753110208e+09 + ts_usec: 584957.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTCHKPT + context: Checkpointer + id: 22430.0 + message: + category: WT_VERB_CHECKPOINT_PROGRESS + category_id: 7.0 + msg: "saving checkpoint snapshot min: 37, snapshot max: 37 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1" + session_name: WT_SESSION.checkpoint + thread: 1:0xffffab20e680 + ts_sec: 1.753110268e+09 + ts_usec: 600180.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 timestamp: now partialSuccess: true resource: From 9e26aae3496a1bb7d2b23d722fa21e143d3ec615 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 15:46:27 +0000 Subject: [PATCH 03/14] wip --- apps/mongodb.go | 331 +++++++++++++------------ confgenerator/logging_modify_fields.go | 13 +- 2 files changed, 180 insertions(+), 164 deletions(-) diff --git a/apps/mongodb.go b/apps/mongodb.go index 84ba39f97e..25d19d3343 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -16,12 +16,9 @@ package apps import ( "context" - "fmt" "strings" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" - "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit/modify" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" "github.com/GoogleCloudPlatform/ops-agent/internal/secret" ) @@ -94,20 +91,19 @@ func init() { confgenerator.MetricsReceiverTypes.RegisterType(func() confgenerator.MetricsReceiver { return &MetricsReceiverMongoDB{} }) } -type LoggingProcessorMongodb struct { - confgenerator.ConfigComponent `yaml:",inline"` +type LoggingProcessorMacroMongodb struct { } -func (*LoggingProcessorMongodb) Type() string { +func (LoggingProcessorMacroMongodb) Type() string { return "mongodb" } -func (p *LoggingProcessorMongodb) Components(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) Expand(ctx context.Context) []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} - c = append(c, p.JsonLogComponents(ctx, tag, uid)...) - c = append(c, p.RegexLogComponents(tag, uid)...) - c = append(c, p.severityParser(ctx, tag, uid)...) + c = append(c, p.JsonLogComponents(ctx)...) + c = append(c, p.RegexLogComponents()...) + c = append(c, p.severityParser()...) return c } @@ -115,19 +111,19 @@ func (p *LoggingProcessorMongodb) Components(ctx context.Context, tag, uid strin // JsonLogComponents are the fluentbit components for parsing log messages that are json formatted. // these are generally messages from mongo with versions greater than or equal to 4.4 // documentation: https://docs.mongodb.com/v4.4/reference/log-messages/#log-message-format -func (p *LoggingProcessorMongodb) JsonLogComponents(ctx context.Context, tag, uid string) []fluentbit.Component { - c := p.jsonParserWithTimeKey(ctx, tag, uid) +func (p LoggingProcessorMacroMongodb) JsonLogComponents(ctx context.Context) []confgenerator.InternalLoggingProcessor { + c := p.jsonParserWithTimeKey() - c = append(c, p.promoteWiredTiger(tag, uid)...) - c = append(c, p.renames(tag, uid)...) + c = append(c, p.promoteWiredTiger()...) + c = append(c, p.renames()...) return c } // jsonParserWithTimeKey requires promotion of the nested timekey for the json parser so we must // first promote the $date field from the "t" field before declaring the parser -func (p *LoggingProcessorMongodb) jsonParserWithTimeKey(ctx context.Context, tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} jsonParser := &confgenerator.LoggingProcessorParseJson{ ParserShared: confgenerator.ParserShared{ @@ -139,75 +135,101 @@ func (p *LoggingProcessorMongodb) jsonParserWithTimeKey(ctx context.Context, tag }, }, } - jpComponents := jsonParser.Components(ctx, tag, uid) - - // The parserFilterComponent is the actual filter component that configures and defines - // which parser to use. We need the component to determine which parser to use when - // re-parsing below. Each time a parser filter is used, there are 2 filter components right - // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). - // Therefore, the parse filter component is actually the third component in the list. - parserFilterComponent := jpComponents[2] - c = append(c, jpComponents...) - - tempPrefix := "temp_ts_" - timeKey := "time" - // have to bring $date to top level in order for it to be parsed as timeKey - // see https://github.com/fluent/fluent-bit/issues/1013 - liftTs := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Match": tag, - "Operation": "lift", - "Nested_under": "t", - "Add_prefix": tempPrefix, + + c = append(c, jsonParser) + + // // The parserFilterComponent is the actual filter component that configures and defines + // // which parser to use. We need the component to determine which parser to use when + // // re-parsing below. Each time a parser filter is used, there are 2 filter components right + // // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). + // // Therefore, the parse filter component is actually the third component in the list. + // parserFilterComponent := jpComponents[2] + // c = append(c, jpComponents...) + + // tempPrefix := "temp_ts_" + // timeKey := "time" + // // have to bring $date to top level in order for it to be parsed as timeKey + // // see https://github.com/fluent/fluent-bit/issues/1013 + // liftTs := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Name": "nest", + // "Match": tag, + // "Operation": "lift", + // "Nested_under": "t", + // "Add_prefix": tempPrefix, + // }, + // } + + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.time": { + MoveFrom: "jsonPayload.t.$date", + }, }, - } + }) + + // renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) + // renameTs := renameTsOption.Component(tag) + + // c = append(c, liftTs, renameTs) - renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) - renameTs := renameTsOption.Component(tag) - - c = append(c, liftTs, renameTs) - - // IMPORTANT: now that we have lifted the json to top level - // we need to re-parse in order to properly set time at the - // parser level - nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) - parserFilter := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "parser", - "Match": tag, - "Key_Name": "message", - "Reserve_Data": "True", - "Parser": parserFilterComponent.OrderedConfig[0][1], + // // IMPORTANT: now that we have lifted the json to top level + // // we need to re-parse in order to properly set time at the + // // parser level + c = append(c, &confgenerator.LoggingProcessorParseJson{ + ParserShared: confgenerator.ParserShared{ + TimeKey: "time", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", }, - } - mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) - c = append(c, nestFilters...) - c = append(c, parserFilter) - c = append(c, mergeFilters...) - - removeTimestamp := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "modify", - "Match": tag, - "Remove": timeKey, + }) + + // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) + // parserFilter := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Name": "parser", + // "Match": tag, + // "Key_Name": "message", + // "Reserve_Data": "True", + // "Parser": parserFilterComponent.OrderedConfig[0][1], + // }, + // } + // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) + // c = append(c, nestFilters...) + // c = append(c, parserFilter) + // c = append(c, mergeFilters...) + + // removeTimestamp := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Name": "modify", + // "Match": tag, + // "Remove": timeKey, + // }, + // } + // c = append(c, removeTimestamp) + // remove the time field + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.time": { + Remove: true, + }, + "jsonPayload.t": { + Remove: true, + }, }, - } - c = append(c, removeTimestamp) + }) + // return c return c } // severityParser is used by both regex and json parser to ensure an "s" field on the entry gets translated // to a valid logging.googleapis.com/seveirty field -func (p *LoggingProcessorMongodb) severityParser(ctx context.Context, tag, uid string) []fluentbit.Component { - severityComponents := []fluentbit.Component{} - - severityComponents = append(severityComponents, - confgenerator.LoggingProcessorModifyFields{ +func (p LoggingProcessorMacroMongodb) severityParser() []confgenerator.InternalLoggingProcessor { + return []confgenerator.InternalLoggingProcessor{ + &confgenerator.LoggingProcessorModifyFields{ Fields: map[string]*confgenerator.ModifyField{ "jsonPayload.severity": { MoveFrom: "jsonPayload.s", @@ -230,115 +252,108 @@ func (p *LoggingProcessorMongodb) severityParser(ctx context.Context, tag, uid s }, InstrumentationSourceLabel: instrumentationSourceValue(p.Type()), }, - }.Components(ctx, tag, uid)..., - ) - - return severityComponents + }, + } } -func (p *LoggingProcessorMongodb) renames(tag, uid string) []fluentbit.Component { - r := []fluentbit.Component{} +func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingProcessor { + r := []confgenerator.InternalLoggingProcessor{} renames := []struct { src string dest string }{ - {"c", "component"}, - {"ctx", "context"}, - {"msg", "message"}, + {"jsonPayload.c", "jsonPayload.component"}, + {"jsonPayload.ctx", "jsonPayload.context"}, + {"jsonPayload.msg", "jsonPayload.message"}, + {"jsonPayload.attr", "jsonPayload.attributes"}, } for _, rename := range renames { - rename := modify.NewRenameOptions(rename.src, rename.dest) - r = append(r, rename.Component(tag)) + r = append(r, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + rename.dest: { + MoveFrom: rename.src, + }, + }, + }) } return r } -func (p *LoggingProcessorMongodb) promoteWiredTiger(tag, uid string) []fluentbit.Component { - // promote messages that are WiredTiger messages and are nested in attr.message - addPrefix := "temp_attributes_" - upNest := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Match": tag, - "Operation": "lift", - "Nested_under": "attr", - "Add_prefix": addPrefix, - }, - } +func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} - hardRenameMessage := modify.NewHardRenameOptions(fmt.Sprintf("%smessage", addPrefix), "msg") - wiredTigerRename := hardRenameMessage.Component(tag) - - renameRemainingAttributes := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Wildcard": fmt.Sprintf("%s*", addPrefix), - "Match": tag, - "Operation": "nest", - "Nest_under": "attributes", - "Remove_prefix": addPrefix, + // promote messages that are WiredTiger messages and are nested in attr.message + c = append(c, &confgenerator.LoggingProcessorNestWildcard{ + Wildcard: "jsonPayload.msg", + NestUnder: "jsonPayload.message2", + RemovePrefix: "jsonPayload.msg", + }) + + // Remove the attr field if it is empty + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.attr": { + OmitIf: "jsonPayload.attr = {}", + }, }, - } + }) - return []fluentbit.Component{upNest, wiredTigerRename, renameRemainingAttributes} + return c } -func (p *LoggingProcessorMongodb) RegexLogComponents(tag, uid string) []fluentbit.Component { - c := []fluentbit.Component{} - parseKey := "message" - parser, parserName := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ - "message": "string", - "id": "integer", - "s": "string", - "component": "string", - "context": "string", - }, fmt.Sprintf("%s_regex", tag), uid) - parser.Config["Format"] = "regex" - parser.Config["Regex"] = `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$` - parser.Config["Key_Name"] = parseKey - - nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) - parserFilter := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Match": tag, - "Name": "parser", - "Parser": parserName, - "Reserve_Data": "True", - "Key_Name": parseKey, +func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.InternalLoggingProcessor { + c := []confgenerator.InternalLoggingProcessor{} + // parseKey := "message" + // parser, _ := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ + // "message": "string", + // "id": "integer", + // "s": "string", + // "component": "string", + // "context": "string", + // }, "mongodb_regex", "mongodb") + // parser.Config["Format"] = "regex" + // parser.Config["Regex"] = `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$` + // parser.Config["Key_Name"] = parseKey + + // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) + // parserFilter := fluentbit.Component{ + // Kind: "FILTER", + // Config: map[string]string{ + // "Match": tag, + // "Name": "parser", + // "Parser": parserName, + // "Reserve_Data": "True", + // "Key_Name": parseKey, + // }, + // } + // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) + + c = append(c, &confgenerator.LoggingProcessorParseRegex{ + ParserShared: confgenerator.ParserShared{ + TimeKey: "timestamp", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", }, - } - mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) + Regex: `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$`, + Field: "message", + }) - c = append(c, parser) - c = append(c, nestFilters...) - c = append(c, parserFilter) - c = append(c, mergeFilters...) + // c = append(c, nestFilters...) + // c = append(c, parserFilter) + // c = append(c, mergeFilters...) return c } -type LoggingReceiverMongodb struct { - LoggingProcessorMongodb `yaml:",inline"` - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline" validate:"structonly"` -} - -func (r *LoggingReceiverMongodb) Components(ctx context.Context, tag string) []fluentbit.Component { - if len(r.ReceiverMixin.IncludePaths) == 0 { - r.ReceiverMixin.IncludePaths = []string{ - // default logging location - "/var/log/mongodb/mongod.log*", - } +func loggingReceiverFilesMixinMongodb() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ + IncludePaths: []string{"/var/log/mongodb/mongod.log*"}, } - c := r.ReceiverMixin.Components(ctx, tag) - c = append(c, r.LoggingProcessorMongodb.Components(ctx, tag, "mongodb")...) - return c } func init() { - confgenerator.LoggingReceiverTypes.RegisterType(func() confgenerator.LoggingReceiver { return &LoggingReceiverMongodb{} }) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroMongodb]( + loggingReceiverFilesMixinMongodb, + ) } diff --git a/confgenerator/logging_modify_fields.go b/confgenerator/logging_modify_fields.go index c984a35b20..de8a8f5ffd 100644 --- a/confgenerator/logging_modify_fields.go +++ b/confgenerator/logging_modify_fields.go @@ -29,10 +29,11 @@ import ( type ModifyField struct { // Source of value for this field - MoveFrom string `yaml:"move_from" validate:"omitempty,field,excluded_with=CopyFrom StaticValue"` - CopyFrom string `yaml:"copy_from" validate:"omitempty,field,excluded_with=MoveFrom StaticValue"` - StaticValue *string `yaml:"static_value" validate:"excluded_with=MoveFrom CopyFrom DefaultValue"` - DefaultValue *string `yaml:"default_value" validate:"excluded_with=StaticValue"` + MoveFrom string `yaml:"move_from" validate:"omitempty,field,excluded_with=CopyFrom StaticValue Remove"` + CopyFrom string `yaml:"copy_from" validate:"omitempty,field,excluded_with=MoveFrom StaticValue Remove"` + StaticValue *string `yaml:"static_value" validate:"excluded_with=MoveFrom CopyFrom Remove DefaultValue"` + DefaultValue *string `yaml:"default_value" validate:"excluded_with=StaticValue Remove"` + Remove bool `yaml:"remove" validate:"excluded_with=MoveFrom CopyFrom StaticValue DefaultValue"` // OTTL expression with copied value sourceValue ottl.Value `yaml:"-"` @@ -94,7 +95,7 @@ function process(tag, timestamp, record) omitFilters := map[string]*filter.Filter{} for i, dest := range dests { field := p.Fields[dest] - if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil { + if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil && !field.Remove { // Default to modifying field in place field.CopyFrom = dest } @@ -273,7 +274,7 @@ func (p LoggingProcessorModifyFields) statements(_ context.Context) (ottl.Statem for i, dest := range dests { field := p.Fields[dest] - if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil { + if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil && !field.Remove { // Default to modifying field in place field.CopyFrom = dest } From cbe4dbb9b89b18b6ddefdf8503c4f8e1334d5493 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 18:41:55 +0000 Subject: [PATCH 04/14] mongodb first refactor pass --- apps/mongodb.go | 158 +++++------------- confgenerator/logging_modify_fields.go | 13 +- confgenerator/logging_processors.go | 52 ++++++ .../1dda8dfcf80ff2aac28fa0838a4d59b7.lua | 14 ++ .../e568b00a7a9bb9a1c872f142c49757e3.lua | 14 ++ .../e9e7adf2c9870f9e200c5f6638b391a6.lua | 14 ++ .../golden/linux-gpu/fluent_bit_main.conf | 19 ++- .../golden/linux-gpu/fluent_bit_parser.conf | 11 +- .../1dda8dfcf80ff2aac28fa0838a4d59b7.lua | 14 ++ .../e568b00a7a9bb9a1c872f142c49757e3.lua | 14 ++ .../e9e7adf2c9870f9e200c5f6638b391a6.lua | 14 ++ .../golden/linux/fluent_bit_main.conf | 19 ++- .../golden/linux/fluent_bit_parser.conf | 11 +- .../1dda8dfcf80ff2aac28fa0838a4d59b7.lua | 14 ++ .../e568b00a7a9bb9a1c872f142c49757e3.lua | 14 ++ .../e9e7adf2c9870f9e200c5f6638b391a6.lua | 14 ++ .../golden/windows-2012/fluent_bit_main.conf | 19 ++- .../windows-2012/fluent_bit_parser.conf | 11 +- .../1dda8dfcf80ff2aac28fa0838a4d59b7.lua | 14 ++ .../e568b00a7a9bb9a1c872f142c49757e3.lua | 14 ++ .../e9e7adf2c9870f9e200c5f6638b391a6.lua | 14 ++ .../golden/windows/fluent_bit_main.conf | 19 ++- .../golden/windows/fluent_bit_parser.conf | 11 +- .../output_otel.yaml | 2 +- 24 files changed, 342 insertions(+), 171 deletions(-) create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/1dda8dfcf80ff2aac28fa0838a4d59b7.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e568b00a7a9bb9a1c872f142c49757e3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e9e7adf2c9870f9e200c5f6638b391a6.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/1dda8dfcf80ff2aac28fa0838a4d59b7.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e568b00a7a9bb9a1c872f142c49757e3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e9e7adf2c9870f9e200c5f6638b391a6.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/1dda8dfcf80ff2aac28fa0838a4d59b7.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e568b00a7a9bb9a1c872f142c49757e3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e9e7adf2c9870f9e200c5f6638b391a6.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/1dda8dfcf80ff2aac28fa0838a4d59b7.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e568b00a7a9bb9a1c872f142c49757e3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e9e7adf2c9870f9e200c5f6638b391a6.lua diff --git a/apps/mongodb.go b/apps/mongodb.go index 25d19d3343..8a9c9384a8 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -16,6 +16,7 @@ package apps import ( "context" + "fmt" "strings" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" @@ -138,90 +139,29 @@ func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.In c = append(c, jsonParser) - // // The parserFilterComponent is the actual filter component that configures and defines - // // which parser to use. We need the component to determine which parser to use when - // // re-parsing below. Each time a parser filter is used, there are 2 filter components right - // // before it to account for the nest lua script (see confgenerator/fluentbit/parse_deduplication.go). - // // Therefore, the parse filter component is actually the third component in the list. - // parserFilterComponent := jpComponents[2] - // c = append(c, jpComponents...) - - // tempPrefix := "temp_ts_" - // timeKey := "time" - // // have to bring $date to top level in order for it to be parsed as timeKey - // // see https://github.com/fluent/fluent-bit/issues/1013 - // liftTs := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Name": "nest", - // "Match": tag, - // "Operation": "lift", - // "Nested_under": "t", - // "Add_prefix": tempPrefix, - // }, - // } - - c = append(c, &confgenerator.LoggingProcessorModifyFields{ - Fields: map[string]*confgenerator.ModifyField{ - "jsonPayload.time": { - MoveFrom: "jsonPayload.t.$date", - }, - }, + tempPrefix := "temp_ts_" + timeKey := "time" + // have to bring $date to top level in order for it to be parsed as timeKey + // see https://github.com/fluent/fluent-bit/issues/1013 + c = append(c, &confgenerator.LoggingProcessorNestLift{ + NestedUnder: "t", + AddPrefix: tempPrefix, }) - // renameTsOption := modify.NewHardRenameOptions(fmt.Sprintf("%s$date", tempPrefix), timeKey) - // renameTs := renameTsOption.Component(tag) - - // c = append(c, liftTs, renameTs) - - // // IMPORTANT: now that we have lifted the json to top level - // // we need to re-parse in order to properly set time at the - // // parser level - c = append(c, &confgenerator.LoggingProcessorParseJson{ - ParserShared: confgenerator.ParserShared{ - TimeKey: "time", - TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", - }, + c = append(c, &confgenerator.LoggingProcessorHardRename{ + Field: fmt.Sprintf("%s$date", tempPrefix), + NewName: timeKey, }) - // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, "message")) - // parserFilter := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Name": "parser", - // "Match": tag, - // "Key_Name": "message", - // "Reserve_Data": "True", - // "Parser": parserFilterComponent.OrderedConfig[0][1], - // }, - // } - // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) - // c = append(c, nestFilters...) - // c = append(c, parserFilter) - // c = append(c, mergeFilters...) - - // removeTimestamp := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Name": "modify", - // "Match": tag, - // "Remove": timeKey, - // }, - // } - // c = append(c, removeTimestamp) - // remove the time field - c = append(c, &confgenerator.LoggingProcessorModifyFields{ - Fields: map[string]*confgenerator.ModifyField{ - "jsonPayload.time": { - Remove: true, - }, - "jsonPayload.t": { - Remove: true, - }, - }, + // IMPORTANT: now that we have lifted the json to top level + // we need to re-parse in order to properly set time at the + // parser level + c = append(c, jsonParser) + + c = append(c, &confgenerator.LoggingProcessorRemoveField{ + Field: timeKey, }) - // return c return c } @@ -265,7 +205,6 @@ func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingP {"jsonPayload.c", "jsonPayload.component"}, {"jsonPayload.ctx", "jsonPayload.context"}, {"jsonPayload.msg", "jsonPayload.message"}, - {"jsonPayload.attr", "jsonPayload.attributes"}, } for _, rename := range renames { @@ -282,22 +221,24 @@ func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingP } func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.InternalLoggingProcessor { + // promote messages that are WiredTiger messages and are nested in attr.messagey c := []confgenerator.InternalLoggingProcessor{} - // promote messages that are WiredTiger messages and are nested in attr.message - c = append(c, &confgenerator.LoggingProcessorNestWildcard{ - Wildcard: "jsonPayload.msg", - NestUnder: "jsonPayload.message2", - RemovePrefix: "jsonPayload.msg", + addPrefix := "temp_attributes_" + c = append(c, &confgenerator.LoggingProcessorNestLift{ + NestedUnder: "attr", + AddPrefix: addPrefix, }) - // Remove the attr field if it is empty - c = append(c, &confgenerator.LoggingProcessorModifyFields{ - Fields: map[string]*confgenerator.ModifyField{ - "jsonPayload.attr": { - OmitIf: "jsonPayload.attr = {}", - }, - }, + c = append(c, &confgenerator.LoggingProcessorHardRename{ + Field: addPrefix + "message", + NewName: "msg", + }) + + c = append(c, &confgenerator.LoggingProcessorNestWildcard{ + Wildcard: addPrefix + "*", + NestUnder: "attributes", + RemovePrefix: addPrefix, }) return c @@ -305,44 +246,23 @@ func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.Intern func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.InternalLoggingProcessor { c := []confgenerator.InternalLoggingProcessor{} - // parseKey := "message" - // parser, _ := fluentbit.ParserComponentBase("%Y-%m-%dT%H:%M:%S.%L%z", "timestamp", map[string]string{ - // "message": "string", - // "id": "integer", - // "s": "string", - // "component": "string", - // "context": "string", - // }, "mongodb_regex", "mongodb") - // parser.Config["Format"] = "regex" - // parser.Config["Regex"] = `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$` - // parser.Config["Key_Name"] = parseKey - - // nestFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserNestLuaFunction, fmt.Sprintf(fluentbit.ParserNestLuaScriptContents, parseKey)) - // parserFilter := fluentbit.Component{ - // Kind: "FILTER", - // Config: map[string]string{ - // "Match": tag, - // "Name": "parser", - // "Parser": parserName, - // "Reserve_Data": "True", - // "Key_Name": parseKey, - // }, - // } - // mergeFilters := fluentbit.LuaFilterComponents(tag, fluentbit.ParserMergeLuaFunction, fluentbit.ParserMergeLuaScriptContents) c = append(c, &confgenerator.LoggingProcessorParseRegex{ ParserShared: confgenerator.ParserShared{ TimeKey: "timestamp", TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", + Types: map[string]string{ + "message": "string", + "id": "integer", + "s": "string", + "component": "string", + "context": "string", + }, }, Regex: `^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$`, Field: "message", }) - // c = append(c, nestFilters...) - // c = append(c, parserFilter) - // c = append(c, mergeFilters...) - return c } diff --git a/confgenerator/logging_modify_fields.go b/confgenerator/logging_modify_fields.go index de8a8f5ffd..40bafa55a6 100644 --- a/confgenerator/logging_modify_fields.go +++ b/confgenerator/logging_modify_fields.go @@ -29,11 +29,10 @@ import ( type ModifyField struct { // Source of value for this field - MoveFrom string `yaml:"move_from" validate:"omitempty,field,excluded_with=CopyFrom StaticValue Remove"` - CopyFrom string `yaml:"copy_from" validate:"omitempty,field,excluded_with=MoveFrom StaticValue Remove"` - StaticValue *string `yaml:"static_value" validate:"excluded_with=MoveFrom CopyFrom Remove DefaultValue"` - DefaultValue *string `yaml:"default_value" validate:"excluded_with=StaticValue Remove"` - Remove bool `yaml:"remove" validate:"excluded_with=MoveFrom CopyFrom StaticValue DefaultValue"` + MoveFrom string `yaml:"move_from" validate:"omitempty,field,excluded_with=CopyFrom StaticValue"` + CopyFrom string `yaml:"copy_from" validate:"omitempty,field,excluded_with=MoveFrom StaticValue"` + StaticValue *string `yaml:"static_value" validate:"excluded_with=MoveFrom CopyFrom"` + DefaultValue *string `yaml:"default_value" validate:"excluded_with=StaticValue"` // OTTL expression with copied value sourceValue ottl.Value `yaml:"-"` @@ -95,7 +94,7 @@ function process(tag, timestamp, record) omitFilters := map[string]*filter.Filter{} for i, dest := range dests { field := p.Fields[dest] - if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil && !field.Remove { + if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil { // Default to modifying field in place field.CopyFrom = dest } @@ -274,7 +273,7 @@ func (p LoggingProcessorModifyFields) statements(_ context.Context) (ottl.Statem for i, dest := range dests { field := p.Fields[dest] - if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil && !field.Remove { + if field.MoveFrom == "" && field.CopyFrom == "" && field.StaticValue == nil { // Default to modifying field in place field.CopyFrom = dest } diff --git a/confgenerator/logging_processors.go b/confgenerator/logging_processors.go index fcb0c67f15..f003d26fe1 100644 --- a/confgenerator/logging_processors.go +++ b/confgenerator/logging_processors.go @@ -408,6 +408,58 @@ func (p LoggingProcessorNestWildcard) Components(ctx context.Context, tag, uid s } } +type LoggingProcessorRemoveField struct { + Field string +} + +func (p LoggingProcessorRemoveField) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + filter := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "modify", + "Match": tag, + "Remove": p.Field, + }, + } + return []fluentbit.Component{filter} +} + +type LoggingProcessorHardRename struct { + Field string + NewName string +} + +func (p LoggingProcessorHardRename) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + c := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "modify", + "Match": tag, + "Hard_rename": fmt.Sprintf("%s %s", p.Field, p.NewName), + }, + } + return []fluentbit.Component{c} +} + +type LoggingProcessorNestLift struct { + NestedUnder string + AddPrefix string +} + +func (p LoggingProcessorNestLift) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + filter := fluentbit.Component{ + Kind: "FILTER", + Config: map[string]string{ + "Name": "nest", + "Match": tag, + "Operation": "lift", + "Nested_under": p.NestedUnder, + "Add_prefix": p.AddPrefix, + }, + } + return []fluentbit.Component{filter} +} + var LegacyBuiltinProcessors = map[string]LoggingProcessor{ "lib:default_message_parser": &LoggingProcessorParseRegex{ Regex: `^(?.*)$`, diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/1dda8dfcf80ff2aac28fa0838a4d59b7.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/1dda8dfcf80ff2aac28fa0838a4d59b7.lua new file mode 100644 index 0000000000..dbad2430ef --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/1dda8dfcf80ff2aac28fa0838a4d59b7.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["msg"] +end)(); +(function(value) +record["msg"] = value +end)(nil); +local v = __field_0; +(function(value) +record["message"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e568b00a7a9bb9a1c872f142c49757e3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e568b00a7a9bb9a1c872f142c49757e3.lua new file mode 100644 index 0000000000..abd822c4ea --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e568b00a7a9bb9a1c872f142c49757e3.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["c"] +end)(); +(function(value) +record["c"] = value +end)(nil); +local v = __field_0; +(function(value) +record["component"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e9e7adf2c9870f9e200c5f6638b391a6.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e9e7adf2c9870f9e200c5f6638b391a6.lua new file mode 100644 index 0000000000..9c2613b727 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/e9e7adf2c9870f9e200c5f6638b391a6.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["ctx"] +end)(); +(function(value) +record["ctx"] = value +end)(nil); +local v = __field_0; +(function(value) +record["context"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf index 80bbf20465..146a4c9a8f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf @@ -131,8 +131,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb @@ -167,18 +167,21 @@ [FILTER] Match mongodb.mongodb - Name modify - Rename c component + Name lua + call process + script e568b00a7a9bb9a1c872f142c49757e3.lua [FILTER] Match mongodb.mongodb - Name modify - Rename ctx context + Name lua + call process + script e9e7adf2c9870f9e200c5f6638b391a6.lua [FILTER] Match mongodb.mongodb - Name modify - Rename msg message + Name lua + call process + script 1dda8dfcf80ff2aac28fa0838a4d59b7.lua [FILTER] Match mongodb.mongodb @@ -190,8 +193,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb_regex.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf index 83a65ce1bc..6acc5c6818 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf @@ -5,14 +5,19 @@ Time_Key time Types id:integer message:string +[PARSER] + Format json + Name mongodb.mongodb.mongodb + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Key time + Types id:integer message:string + [PARSER] Format regex - Key_Name message - Name mongodb.mongodb_regex.mongodb + Name mongodb.mongodb.mongodb Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp - Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/1dda8dfcf80ff2aac28fa0838a4d59b7.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/1dda8dfcf80ff2aac28fa0838a4d59b7.lua new file mode 100644 index 0000000000..dbad2430ef --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/1dda8dfcf80ff2aac28fa0838a4d59b7.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["msg"] +end)(); +(function(value) +record["msg"] = value +end)(nil); +local v = __field_0; +(function(value) +record["message"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e568b00a7a9bb9a1c872f142c49757e3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e568b00a7a9bb9a1c872f142c49757e3.lua new file mode 100644 index 0000000000..abd822c4ea --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e568b00a7a9bb9a1c872f142c49757e3.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["c"] +end)(); +(function(value) +record["c"] = value +end)(nil); +local v = __field_0; +(function(value) +record["component"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e9e7adf2c9870f9e200c5f6638b391a6.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e9e7adf2c9870f9e200c5f6638b391a6.lua new file mode 100644 index 0000000000..9c2613b727 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/e9e7adf2c9870f9e200c5f6638b391a6.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["ctx"] +end)(); +(function(value) +record["ctx"] = value +end)(nil); +local v = __field_0; +(function(value) +record["context"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf index 80bbf20465..146a4c9a8f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_main.conf @@ -131,8 +131,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb @@ -167,18 +167,21 @@ [FILTER] Match mongodb.mongodb - Name modify - Rename c component + Name lua + call process + script e568b00a7a9bb9a1c872f142c49757e3.lua [FILTER] Match mongodb.mongodb - Name modify - Rename ctx context + Name lua + call process + script e9e7adf2c9870f9e200c5f6638b391a6.lua [FILTER] Match mongodb.mongodb - Name modify - Rename msg message + Name lua + call process + script 1dda8dfcf80ff2aac28fa0838a4d59b7.lua [FILTER] Match mongodb.mongodb @@ -190,8 +193,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb_regex.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf index 83a65ce1bc..6acc5c6818 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf @@ -5,14 +5,19 @@ Time_Key time Types id:integer message:string +[PARSER] + Format json + Name mongodb.mongodb.mongodb + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Key time + Types id:integer message:string + [PARSER] Format regex - Key_Name message - Name mongodb.mongodb_regex.mongodb + Name mongodb.mongodb.mongodb Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp - Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/1dda8dfcf80ff2aac28fa0838a4d59b7.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/1dda8dfcf80ff2aac28fa0838a4d59b7.lua new file mode 100644 index 0000000000..dbad2430ef --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/1dda8dfcf80ff2aac28fa0838a4d59b7.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["msg"] +end)(); +(function(value) +record["msg"] = value +end)(nil); +local v = __field_0; +(function(value) +record["message"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e568b00a7a9bb9a1c872f142c49757e3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e568b00a7a9bb9a1c872f142c49757e3.lua new file mode 100644 index 0000000000..abd822c4ea --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e568b00a7a9bb9a1c872f142c49757e3.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["c"] +end)(); +(function(value) +record["c"] = value +end)(nil); +local v = __field_0; +(function(value) +record["component"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e9e7adf2c9870f9e200c5f6638b391a6.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e9e7adf2c9870f9e200c5f6638b391a6.lua new file mode 100644 index 0000000000..9c2613b727 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/e9e7adf2c9870f9e200c5f6638b391a6.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["ctx"] +end)(); +(function(value) +record["ctx"] = value +end)(nil); +local v = __field_0; +(function(value) +record["context"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf index 9c0791bc45..c8b404dd6f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_main.conf @@ -174,8 +174,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb @@ -210,18 +210,21 @@ [FILTER] Match mongodb.mongodb - Name modify - Rename c component + Name lua + call process + script e568b00a7a9bb9a1c872f142c49757e3.lua [FILTER] Match mongodb.mongodb - Name modify - Rename ctx context + Name lua + call process + script e9e7adf2c9870f9e200c5f6638b391a6.lua [FILTER] Match mongodb.mongodb - Name modify - Rename msg message + Name lua + call process + script 1dda8dfcf80ff2aac28fa0838a4d59b7.lua [FILTER] Match mongodb.mongodb @@ -233,8 +236,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb_regex.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf index 5fa5cce887..98eb4e1cd3 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf @@ -12,14 +12,19 @@ Time_Key time Types id:integer message:string +[PARSER] + Format json + Name mongodb.mongodb.mongodb + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Key time + Types id:integer message:string + [PARSER] Format regex - Key_Name message - Name mongodb.mongodb_regex.mongodb + Name mongodb.mongodb.mongodb Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp - Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/1dda8dfcf80ff2aac28fa0838a4d59b7.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/1dda8dfcf80ff2aac28fa0838a4d59b7.lua new file mode 100644 index 0000000000..dbad2430ef --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/1dda8dfcf80ff2aac28fa0838a4d59b7.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["msg"] +end)(); +(function(value) +record["msg"] = value +end)(nil); +local v = __field_0; +(function(value) +record["message"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e568b00a7a9bb9a1c872f142c49757e3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e568b00a7a9bb9a1c872f142c49757e3.lua new file mode 100644 index 0000000000..abd822c4ea --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e568b00a7a9bb9a1c872f142c49757e3.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["c"] +end)(); +(function(value) +record["c"] = value +end)(nil); +local v = __field_0; +(function(value) +record["component"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e9e7adf2c9870f9e200c5f6638b391a6.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e9e7adf2c9870f9e200c5f6638b391a6.lua new file mode 100644 index 0000000000..9c2613b727 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/e9e7adf2c9870f9e200c5f6638b391a6.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["ctx"] +end)(); +(function(value) +record["ctx"] = value +end)(nil); +local v = __field_0; +(function(value) +record["context"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf index 9c0791bc45..c8b404dd6f 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_main.conf @@ -174,8 +174,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb @@ -210,18 +210,21 @@ [FILTER] Match mongodb.mongodb - Name modify - Rename c component + Name lua + call process + script e568b00a7a9bb9a1c872f142c49757e3.lua [FILTER] Match mongodb.mongodb - Name modify - Rename ctx context + Name lua + call process + script e9e7adf2c9870f9e200c5f6638b391a6.lua [FILTER] Match mongodb.mongodb - Name modify - Rename msg message + Name lua + call process + script 1dda8dfcf80ff2aac28fa0838a4d59b7.lua [FILTER] Match mongodb.mongodb @@ -233,8 +236,8 @@ Key_Name message Match mongodb.mongodb Name parser - Parser mongodb.mongodb_regex.mongodb Reserve_Data True + Parser mongodb.mongodb.mongodb [FILTER] Match mongodb.mongodb diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf index 5fa5cce887..98eb4e1cd3 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf @@ -12,14 +12,19 @@ Time_Key time Types id:integer message:string +[PARSER] + Format json + Name mongodb.mongodb.mongodb + Time_Format %Y-%m-%dT%H:%M:%S.%L%z + Time_Key time + Types id:integer message:string + [PARSER] Format regex - Key_Name message - Name mongodb.mongodb_regex.mongodb + Name mongodb.mongodb.mongodb Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp - Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml b/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml index 253af10e54..deb32a947b 100644 --- a/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml @@ -1 +1 @@ -- config_error: "not an OTel processor: &apps.LoggingProcessorMongodb{ConfigComponent:confgenerator.ConfigComponent{Type:\"mongodb\"}}" +- config_error: "failed generating OTel processor: &confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]{ConfigComponent:confgenerator.ConfigComponent{Type:\"mongodb\"}, ProcessorMacro:apps.LoggingProcessorMacroMongodb{}}, err: unimplemented" From 2aafb052106814f63dca6156566c42804dc2ebb6 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 22 Jul 2025 18:53:53 +0000 Subject: [PATCH 05/14] only export receiver --- apps/mongodb.go | 22 ++++++++++++++----- confgenerator/testdata/feature/golden.csv | 7 +++--- .../golden/linux-gpu/error | 2 +- .../golden/linux/error | 2 +- .../golden/windows-2012/error | 2 +- .../golden/windows/error | 2 +- .../linux-gpu/feature_tracking_otlp.json | 2 +- .../golden/linux-gpu/features.yaml | 4 ---- .../golden/linux-gpu/fluent_bit_parser.conf | 1 + .../golden/linux/feature_tracking_otlp.json | 2 +- .../golden/linux/features.yaml | 4 ---- .../golden/linux/fluent_bit_parser.conf | 1 + .../windows-2012/feature_tracking_otlp.json | 2 +- .../golden/windows-2012/features.yaml | 4 ---- .../windows-2012/fluent_bit_parser.conf | 1 + .../golden/windows/feature_tracking_otlp.json | 2 +- .../golden/windows/features.yaml | 4 ---- .../golden/windows/fluent_bit_parser.conf | 1 + transformation_test/transformation_test.go | 3 ++- 19 files changed, 34 insertions(+), 34 deletions(-) diff --git a/apps/mongodb.go b/apps/mongodb.go index 8a9c9384a8..73a859facd 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -266,14 +266,24 @@ func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.Inter return c } -func loggingReceiverFilesMixinMongodb() confgenerator.LoggingReceiverFilesMixin { - return confgenerator.LoggingReceiverFilesMixin{ - IncludePaths: []string{"/var/log/mongodb/mongod.log*"}, +type LoggingReceiverMacroMongodb struct { + confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` + LoggingProcessorMacroMongodb `yaml:",inline"` + IncludePaths []string `yaml:"include_paths"` +} + +func (r LoggingReceiverMacroMongodb) Expand(ctx context.Context) (confgenerator.InternalLoggingReceiver, []confgenerator.InternalLoggingProcessor) { + if len(r.IncludePaths) == 0 { + r.IncludePaths = []string{"/var/log/mongodb/mongod.log*"} } + + r.LoggingReceiverFilesMixin.IncludePaths = r.IncludePaths + + return &r.LoggingReceiverFilesMixin, r.LoggingProcessorMacroMongodb.Expand(ctx) } func init() { - confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroMongodb]( - loggingReceiverFilesMixinMongodb, - ) + confgenerator.RegisterLoggingReceiverMacro(func() LoggingReceiverMacroMongodb { + return LoggingReceiverMacroMongodb{} + }) } diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index 72c30bcbed..484b1ccc75 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -251,14 +251,15 @@ App,Field,Override, *confgenerator.PrometheusMetrics,config.[].scrape_configs.scrape_timeout *confgenerator.PrometheusMetrics,config.[].scrape_configs.static_config_target_groups *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink],confgenerator.ConfigComponent.Type, -*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroFlink]],confgenerator.ConfigComponent.Type, -*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],ReceiverMacro, -*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxAccess]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroNginxError]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],apps.LoggingReceiverMacroMongodb.confgenerator.LoggingReceiverFilesMixin.BufferInMemory, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],apps.LoggingReceiverMacroMongodb.confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],apps.LoggingReceiverMacroMongodb.confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, +*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],confgenerator.ConfigComponent.Type, diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error index 6a0286b285..10751c0cf5 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error index 6a0286b285..10751c0cf5 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error index 69c093b42c..1649e4e323 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error index 69c093b42c..1649e4e323 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json index 61a43e8f1b..8fe864f885 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml index 24cb696bd0..8216fe84f1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml @@ -18,7 +18,3 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" -- module: logging - feature: receivers:mongodb - key: "[0].include_paths.__length" - value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf index 6acc5c6818..fb691ee6f8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf @@ -18,6 +18,7 @@ Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp + Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json index 61a43e8f1b..8fe864f885 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml index 24cb696bd0..8216fe84f1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/features.yaml @@ -18,7 +18,3 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" -- module: logging - feature: receivers:mongodb - key: "[0].include_paths.__length" - value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf index 6acc5c6818..fb691ee6f8 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/fluent_bit_parser.conf @@ -18,6 +18,7 @@ Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp + Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json index 61a43e8f1b..8fe864f885 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml index 24cb696bd0..8216fe84f1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/features.yaml @@ -18,7 +18,3 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" -- module: logging - feature: receivers:mongodb - key: "[0].include_paths.__length" - value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf index 98eb4e1cd3..80f349ffd2 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/fluent_bit_parser.conf @@ -25,6 +25,7 @@ Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp + Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json index 61a43e8f1b..8fe864f885 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml index 24cb696bd0..8216fe84f1 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/features.yaml @@ -18,7 +18,3 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" -- module: logging - feature: receivers:mongodb - key: "[0].include_paths.__length" - value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf index 98eb4e1cd3..80f349ffd2 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/fluent_bit_parser.conf @@ -25,6 +25,7 @@ Regex ^(?[^ ]*)\s+(?\w)\s+(?[^ ]+)\s+\[(?[^\]]+)]\s+(?.*?) *(?(\d+))?(:?ms)?$ Time_Format %Y-%m-%dT%H:%M:%S.%L%z Time_Key timestamp + Types component:string context:string id:integer message:string s:string [PARSER] Format regex diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index e9c8d4084d..10bf2b11a9 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -33,7 +33,7 @@ import ( "time" logpb "cloud.google.com/go/logging/apiv2/loggingpb" - _ "github.com/GoogleCloudPlatform/ops-agent/apps" + "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" @@ -593,4 +593,5 @@ func sanitizeStacktrace(t *testing.T, input string) string { func init() { // The processors registered here are only meant to be used in transformation tests. confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &confgenerator.LoggingProcessorWindowsEventLogV1{} }) + confgenerator.RegisterLoggingProcessorMacro[apps.LoggingProcessorMacroMongodb]() } From f0ae87dec7af2e8e64cf07489fbe1fe81c5b7a67 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Wed, 23 Jul 2025 14:51:00 +0000 Subject: [PATCH 06/14] cleaned up code --- apps/mongodb.go | 13 +++++-------- confgenerator/logging_modify_fields.go | 2 +- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/apps/mongodb.go b/apps/mongodb.go index 73a859facd..452a6b5017 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -267,19 +267,16 @@ func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.Inter } type LoggingReceiverMacroMongodb struct { - confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` - LoggingProcessorMacroMongodb `yaml:",inline"` - IncludePaths []string `yaml:"include_paths"` + ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` + LoggingProcessorMacroMongodb `yaml:",inline"` } func (r LoggingReceiverMacroMongodb) Expand(ctx context.Context) (confgenerator.InternalLoggingReceiver, []confgenerator.InternalLoggingProcessor) { - if len(r.IncludePaths) == 0 { - r.IncludePaths = []string{"/var/log/mongodb/mongod.log*"} + if len(r.ReceiverMixin.IncludePaths) == 0 { + r.ReceiverMixin.IncludePaths = []string{"/var/log/mongodb/mongod.log*"} } - r.LoggingReceiverFilesMixin.IncludePaths = r.IncludePaths - - return &r.LoggingReceiverFilesMixin, r.LoggingProcessorMacroMongodb.Expand(ctx) + return &r.ReceiverMixin, r.LoggingProcessorMacroMongodb.Expand(ctx) } func init() { diff --git a/confgenerator/logging_modify_fields.go b/confgenerator/logging_modify_fields.go index 40bafa55a6..c984a35b20 100644 --- a/confgenerator/logging_modify_fields.go +++ b/confgenerator/logging_modify_fields.go @@ -31,7 +31,7 @@ type ModifyField struct { // Source of value for this field MoveFrom string `yaml:"move_from" validate:"omitempty,field,excluded_with=CopyFrom StaticValue"` CopyFrom string `yaml:"copy_from" validate:"omitempty,field,excluded_with=MoveFrom StaticValue"` - StaticValue *string `yaml:"static_value" validate:"excluded_with=MoveFrom CopyFrom"` + StaticValue *string `yaml:"static_value" validate:"excluded_with=MoveFrom CopyFrom DefaultValue"` DefaultValue *string `yaml:"default_value" validate:"excluded_with=StaticValue"` // OTTL expression with copied value From 087c68e3b655f40bfa86d757e1ee948dae6d70d4 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Tue, 19 Aug 2025 13:44:39 +0000 Subject: [PATCH 07/14] added new line to input.log --- .../logging_processor-mongodb/input.log | 2 +- .../output_fluentbit.yaml | 79 +++++++++++++++++++ .../output_otel.yaml | 2 +- 3 files changed, 81 insertions(+), 2 deletions(-) diff --git a/transformation_test/testdata/logging_processor-mongodb/input.log b/transformation_test/testdata/logging_processor-mongodb/input.log index bee03125fc..1e3e5eb2ae 100644 --- a/transformation_test/testdata/logging_processor-mongodb/input.log +++ b/transformation_test/testdata/logging_processor-mongodb/input.log @@ -56,4 +56,4 @@ {"t":{"$date":"2025-07-21T15:02:28.572+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110148,"ts_usec":572218,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 35, snapshot max: 35 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} {"t":{"$date":"2025-07-21T15:03:28.585+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110208,"ts_usec":584957,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 36, snapshot max: 36 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} {"t":{"$date":"2025-07-21T15:04:28.600+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110268,"ts_usec":600180,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 37, snapshot max: 37 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} -{"t":{"$date":"2025-07-21T15:05:28.612+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110328,"ts_usec":612727,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 38, snapshot max: 38 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} \ No newline at end of file +{"t":{"$date":"2025-07-21T15:05:28.612+00:00"},"s":"I", "c":"WTCHKPT", "id":22430, "ctx":"Checkpointer","msg":"WiredTiger message","attr":{"message":{"ts_sec":1753110328,"ts_usec":612727,"thread":"1:0xffffab20e680","session_name":"WT_SESSION.checkpoint","category":"WT_VERB_CHECKPOINT_PROGRESS","category_id":7,"verbose_level":"DEBUG_1","verbose_level_id":1,"msg":"saving checkpoint snapshot min: 38, snapshot max: 38 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1"}}} diff --git a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml index e3a9e49c45..5eba361c27 100644 --- a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml +++ b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml @@ -6,6 +6,7 @@ message: Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none' severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -17,6 +18,7 @@ message: Multi threading initialized severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -33,6 +35,7 @@ message: Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set at least one of the related parameters severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -56,6 +59,7 @@ message: Initialized wire specification severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -67,6 +71,7 @@ message: Starting TenantMigrationAccessBlockerRegistry severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -84,6 +89,7 @@ message: MongoDB starting severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -106,6 +112,7 @@ message: Build Info severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -121,6 +128,7 @@ message: Operating System severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -136,6 +144,7 @@ message: Options set by command line severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -149,6 +158,7 @@ tags: - startupWarnings labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -162,6 +172,7 @@ message: Opening WiredTiger severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -182,6 +193,7 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -202,6 +214,7 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -222,6 +235,7 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -242,6 +256,7 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -255,6 +270,7 @@ message: WiredTiger opened severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -271,6 +287,7 @@ message: WiredTiger recoveryTimestamp severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -284,6 +301,7 @@ message: Initializing durable catalog severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -295,6 +313,7 @@ message: Retrieving all idents from storage engine severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -308,6 +327,7 @@ message: Initializing all collections in durable catalog severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -321,6 +341,7 @@ tags: - startupWarnings labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -339,6 +360,7 @@ tags: - startupWarnings labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -355,6 +377,7 @@ tags: - startupWarnings labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -372,6 +395,7 @@ tags: - startupWarnings labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -388,6 +412,7 @@ tags: - startupWarnings labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -408,6 +433,7 @@ message: createCollection severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -427,6 +453,7 @@ message: "Index build: done building" severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -440,6 +467,7 @@ message: Setting featureCompatibilityVersion severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -454,6 +482,7 @@ message: current featureCompatibilityVersion value severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -488,6 +517,7 @@ message: Updated wire specification severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -522,6 +552,7 @@ message: Updated wire specification severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -536,6 +567,7 @@ message: current featureCompatibilityVersion value severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -547,6 +579,7 @@ message: Clearing temp directory severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -558,6 +591,7 @@ message: Initializing cluster server parameters from disk severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -569,6 +603,7 @@ message: Flow Control is enabled on this deployment severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -582,6 +617,7 @@ message: Initializing full-time diagnostic data capture severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -602,6 +638,7 @@ message: createCollection severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -621,6 +658,7 @@ message: "Index build: done building" severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -635,6 +673,7 @@ message: Setting new configuration state severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -646,6 +685,7 @@ message: Timestamp monitor starting severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -657,6 +697,7 @@ message: Starting the DiskSpaceMonitor severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -670,6 +711,7 @@ message: Listening on severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -683,6 +725,7 @@ message: Listening on severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -697,6 +740,7 @@ message: Waiting for connections severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -732,6 +776,7 @@ message: mongod startup complete severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -745,6 +790,7 @@ message: Sessions collection is not set up; waiting until next sessions reap interval severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -763,6 +809,7 @@ message: createCollection severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -782,6 +829,7 @@ message: "Index build: done building" severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -801,6 +849,7 @@ message: "Index build: done building" severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -815,6 +864,7 @@ message: Use of deprecated server parameter name severity: W labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -829,6 +879,7 @@ message: Use of deprecated server parameter name severity: W labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -840,6 +891,7 @@ message: Use of deprecated server parameter 'sslMode', please use 'tlsMode' instead. severity: W labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -854,6 +906,7 @@ message: Use of deprecated server parameter name severity: W labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -868,6 +921,7 @@ message: Use of deprecated server parameter name severity: W labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 @@ -888,6 +942,7 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -908,6 +963,7 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -928,6 +984,7 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 @@ -948,6 +1005,28 @@ verbose_level_id: 1.0 severity: I labels: + compute.googleapis.com/resource_name: hostname + logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb + logName: projects/my-project/logs/transformation_test + severity: 200.0 + timestamp: now + - jsonPayload: + component: WTCHKPT + context: Checkpointer + id: 22430.0 + message: + category: WT_VERB_CHECKPOINT_PROGRESS + category_id: 7.0 + msg: "saving checkpoint snapshot min: 38, snapshot max: 38 snapshot count: 0, oldest timestamp: (0, 0) , meta checkpoint timestamp: (0, 0) base write gen: 1" + session_name: WT_SESSION.checkpoint + thread: 1:0xffffab20e680 + ts_sec: 1.753110328e+09 + ts_usec: 612727.0 + verbose_level: DEBUG_1 + verbose_level_id: 1.0 + severity: I + labels: + compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 diff --git a/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml b/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml index deb32a947b..c83821d3ed 100644 --- a/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml +++ b/transformation_test/testdata/logging_processor-mongodb/output_otel.yaml @@ -1 +1 @@ -- config_error: "failed generating OTel processor: &confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]{ConfigComponent:confgenerator.ConfigComponent{Type:\"mongodb\"}, ProcessorMacro:apps.LoggingProcessorMacroMongodb{}}, err: unimplemented" +- config_error: "processor \"processor0\" has invalid configuration: unimplemented" From ba3d258fd01a7a68a68b19c7e7a3f82941794f48 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 25 Aug 2025 13:27:33 +0000 Subject: [PATCH 08/14] updated mixin files --- apps/mongodb.go | 17 ++++------------- transformation_test/transformation_test.go | 3 +-- 2 files changed, 5 insertions(+), 15 deletions(-) diff --git a/apps/mongodb.go b/apps/mongodb.go index 452a6b5017..89439541d9 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -266,21 +266,12 @@ func (p LoggingProcessorMacroMongodb) RegexLogComponents() []confgenerator.Inter return c } -type LoggingReceiverMacroMongodb struct { - ReceiverMixin confgenerator.LoggingReceiverFilesMixin `yaml:",inline"` - LoggingProcessorMacroMongodb `yaml:",inline"` -} - -func (r LoggingReceiverMacroMongodb) Expand(ctx context.Context) (confgenerator.InternalLoggingReceiver, []confgenerator.InternalLoggingProcessor) { - if len(r.ReceiverMixin.IncludePaths) == 0 { - r.ReceiverMixin.IncludePaths = []string{"/var/log/mongodb/mongod.log*"} +func loggingReceiverFilesMixinMongodb() confgenerator.LoggingReceiverFilesMixin { + return confgenerator.LoggingReceiverFilesMixin{ + IncludePaths: []string{"/var/log/mongodb/mongod.log*"}, } - - return &r.ReceiverMixin, r.LoggingProcessorMacroMongodb.Expand(ctx) } func init() { - confgenerator.RegisterLoggingReceiverMacro(func() LoggingReceiverMacroMongodb { - return LoggingReceiverMacroMongodb{} - }) + confgenerator.RegisterLoggingFilesProcessorMacro[LoggingProcessorMacroMongodb](loggingReceiverFilesMixinMongodb) } diff --git a/transformation_test/transformation_test.go b/transformation_test/transformation_test.go index c471b07b50..236bebdbab 100644 --- a/transformation_test/transformation_test.go +++ b/transformation_test/transformation_test.go @@ -32,7 +32,7 @@ import ( "time" logpb "cloud.google.com/go/logging/apiv2/loggingpb" - "github.com/GoogleCloudPlatform/ops-agent/apps" + _ "github.com/GoogleCloudPlatform/ops-agent/apps" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/fluentbit" "github.com/GoogleCloudPlatform/ops-agent/confgenerator/otel" @@ -595,5 +595,4 @@ func sanitizeStacktrace(t *testing.T, input string) string { func init() { // The processors registered here are only meant to be used in transformation tests. confgenerator.LoggingProcessorTypes.RegisterType(func() confgenerator.LoggingProcessor { return &confgenerator.LoggingProcessorWindowsEventLogV1{} }) - confgenerator.RegisterLoggingProcessorMacro[apps.LoggingProcessorMacroMongodb]() } From 38391cfaa1c91f0b61073cba778121b5204f30d5 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 25 Aug 2025 14:49:19 +0000 Subject: [PATCH 09/14] refactor to remove nest commands --- apps/mongodb.go | 29 ++++--- .../output_fluentbit.yaml | 80 +++++++++++++++++++ 2 files changed, 97 insertions(+), 12 deletions(-) diff --git a/apps/mongodb.go b/apps/mongodb.go index 89439541d9..55472e968f 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -158,8 +158,14 @@ func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.In // parser level c = append(c, jsonParser) - c = append(c, &confgenerator.LoggingProcessorRemoveField{ - Field: timeKey, + timePath := fmt.Sprintf("jsonPayload.%s", timeKey) + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + timePath: { + MoveFrom: timePath, + OmitIf: fmt.Sprintf(`%s =~ ".*"`, timePath), + }, + }, }) return c @@ -205,6 +211,7 @@ func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingP {"jsonPayload.c", "jsonPayload.component"}, {"jsonPayload.ctx", "jsonPayload.context"}, {"jsonPayload.msg", "jsonPayload.message"}, + {"jsonPayload.attr", "jsonPayload.attributes"}, } for _, rename := range renames { @@ -221,13 +228,17 @@ func (p LoggingProcessorMacroMongodb) renames() []confgenerator.InternalLoggingP } func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.InternalLoggingProcessor { - // promote messages that are WiredTiger messages and are nested in attr.messagey + // promote messages that are WiredTiger messages and are nested in attr.message c := []confgenerator.InternalLoggingProcessor{} addPrefix := "temp_attributes_" - c = append(c, &confgenerator.LoggingProcessorNestLift{ - NestedUnder: "attr", - AddPrefix: addPrefix, + + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.temp_attributes_message": { + MoveFrom: "jsonPayload.attr.message", + }, + }, }) c = append(c, &confgenerator.LoggingProcessorHardRename{ @@ -235,12 +246,6 @@ func (p LoggingProcessorMacroMongodb) promoteWiredTiger() []confgenerator.Intern NewName: "msg", }) - c = append(c, &confgenerator.LoggingProcessorNestWildcard{ - Wildcard: addPrefix + "*", - NestUnder: "attributes", - RemovePrefix: addPrefix, - }) - return c } diff --git a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml index 5eba361c27..144876fbee 100644 --- a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml +++ b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml @@ -1,10 +1,12 @@ - entries: - jsonPayload: + attributes: {} component: CONTROL context: main id: 23285.0 message: Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none' severity: I + time: 2025-07-21T15:00:28.308+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -12,11 +14,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: CONTROL context: main id: 5.945603e+06 message: Multi threading initialized severity: I + time: 2025-07-21T15:00:28.309+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -34,6 +38,7 @@ id: 4.648601e+06 message: Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set at least one of the related parameters severity: I + time: 2025-07-21T15:00:28.309+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -58,6 +63,7 @@ id: 4.915701e+06 message: Initialized wire specification severity: I + time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -65,11 +71,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: TENANT_M context: main id: 7.0916e+06 message: Starting TenantMigrationAccessBlockerRegistry severity: I + time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -88,6 +96,7 @@ id: 4.615611e+06 message: MongoDB starting severity: I + time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -111,6 +120,7 @@ id: 23403.0 message: Build Info severity: I + time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -127,6 +137,7 @@ id: 51765.0 message: Operating System severity: I + time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -143,6 +154,7 @@ id: 21951.0 message: Options set by command line severity: I + time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -150,6 +162,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: STORAGE context: initandlisten id: 22297.0 @@ -157,6 +170,7 @@ severity: I tags: - startupWarnings + time: 2025-07-21T15:00:28.311+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -171,6 +185,7 @@ id: 22315.0 message: Opening WiredTiger severity: I + time: 2025-07-21T15:00:28.311+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -178,6 +193,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTRECOV context: initandlisten id: 22430.0 @@ -192,6 +208,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -199,6 +216,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTRECOV context: initandlisten id: 22430.0 @@ -213,6 +231,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -220,6 +239,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTRECOV context: initandlisten id: 22430.0 @@ -234,6 +254,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -241,6 +262,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTRECOV context: initandlisten id: 22430.0 @@ -255,6 +277,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -269,6 +292,7 @@ id: 4.795906e+06 message: WiredTiger opened severity: I + time: 2025-07-21T15:00:28.546+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -286,6 +310,7 @@ id: 23987.0 message: WiredTiger recoveryTimestamp severity: I + time: 2025-07-21T15:00:28.546+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -300,6 +325,7 @@ id: 9.529901e+06 message: Initializing durable catalog severity: I + time: 2025-07-21T15:00:28.549+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -307,11 +333,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: STORAGE context: initandlisten id: 9.529902e+06 message: Retrieving all idents from storage engine severity: I + time: 2025-07-21T15:00:28.549+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -326,6 +354,7 @@ id: 9.529903e+06 message: Initializing all collections in durable catalog severity: I + time: 2025-07-21T15:00:28.549+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -333,6 +362,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: CONTROL context: initandlisten id: 22120.0 @@ -340,6 +370,7 @@ severity: W tags: - startupWarnings + time: 2025-07-21T15:00:28.550+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -359,6 +390,7 @@ severity: W tags: - startupWarnings + time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -376,6 +408,7 @@ severity: W tags: - startupWarnings + time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -394,6 +427,7 @@ severity: W tags: - startupWarnings + time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -411,6 +445,7 @@ severity: W tags: - startupWarnings + time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -432,6 +467,7 @@ id: 20320.0 message: createCollection severity: I + time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -452,6 +488,7 @@ id: 20345.0 message: "Index build: done building" severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -466,6 +503,7 @@ id: 20459.0 message: Setting featureCompatibilityVersion severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -481,6 +519,7 @@ id: 5.8533e+06 message: current featureCompatibilityVersion value severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -516,6 +555,7 @@ id: 4.915702e+06 message: Updated wire specification severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -551,6 +591,7 @@ id: 4.915702e+06 message: Updated wire specification severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -566,6 +607,7 @@ id: 5.8533e+06 message: current featureCompatibilityVersion value severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -573,11 +615,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: STORAGE context: initandlisten id: 5.0711e+06 message: Clearing temp directory severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -585,11 +629,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: CONTROL context: initandlisten id: 6.6082e+06 message: Initializing cluster server parameters from disk severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -597,11 +643,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: CONTROL context: initandlisten id: 20536.0 message: Flow Control is enabled on this deployment severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -616,6 +664,7 @@ id: 20625.0 message: Initializing full-time diagnostic data capture severity: I + time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -637,6 +686,7 @@ id: 20320.0 message: createCollection severity: I + time: 2025-07-21T15:00:28.555+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -657,6 +707,7 @@ id: 20345.0 message: "Index build: done building" severity: I + time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -672,6 +723,7 @@ id: 6.015317e+06 message: Setting new configuration state severity: I + time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -679,11 +731,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: STORAGE context: initandlisten id: 22262.0 message: Timestamp monitor starting severity: I + time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -691,11 +745,13 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: STORAGE context: initandlisten id: 7.333401e+06 message: Starting the DiskSpaceMonitor severity: I + time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -710,6 +766,7 @@ id: 23015.0 message: Listening on severity: I + time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -724,6 +781,7 @@ id: 23015.0 message: Listening on severity: I + time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -739,6 +797,7 @@ id: 23016.0 message: Waiting for connections severity: I + time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -775,6 +834,7 @@ id: 8.423403e+06 message: mongod startup complete severity: I + time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -789,6 +849,7 @@ id: 20712.0 message: Sessions collection is not set up; waiting until next sessions reap interval severity: I + time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -808,6 +869,7 @@ id: 20320.0 message: createCollection severity: I + time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -828,6 +890,7 @@ id: 20345.0 message: "Index build: done building" severity: I + time: 2025-07-21T15:00:28.564+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -848,6 +911,7 @@ id: 20345.0 message: "Index build: done building" severity: I + time: 2025-07-21T15:00:28.564+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -863,6 +927,7 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W + time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -878,6 +943,7 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W + time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -885,11 +951,13 @@ severity: 400.0 timestamp: now - jsonPayload: + attributes: {} component: NETWORK context: ftdc id: 23803.0 message: Use of deprecated server parameter 'sslMode', please use 'tlsMode' instead. severity: W + time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -905,6 +973,7 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W + time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -920,6 +989,7 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W + time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -927,6 +997,7 @@ severity: 400.0 timestamp: now - jsonPayload: + attributes: {} component: WTCHKPT context: Checkpointer id: 22430.0 @@ -941,6 +1012,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:01:28.556+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -948,6 +1020,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTCHKPT context: Checkpointer id: 22430.0 @@ -962,6 +1035,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:02:28.572+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -969,6 +1043,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTCHKPT context: Checkpointer id: 22430.0 @@ -983,6 +1058,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:03:28.585+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -990,6 +1066,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTCHKPT context: Checkpointer id: 22430.0 @@ -1004,6 +1081,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:04:28.600+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb @@ -1011,6 +1089,7 @@ severity: 200.0 timestamp: now - jsonPayload: + attributes: {} component: WTCHKPT context: Checkpointer id: 22430.0 @@ -1025,6 +1104,7 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I + time: 2025-07-21T15:05:28.612+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb From 87897ed243cbbb95a804310a0bef24f57e1cf710 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 25 Aug 2025 15:30:40 +0000 Subject: [PATCH 10/14] added timestamp parser --- apps/mongodb.go | 38 ++-- confgenerator/logging_processors.go | 21 +++ .../output_fluentbit.yaml | 177 ++++++------------ 3 files changed, 94 insertions(+), 142 deletions(-) diff --git a/apps/mongodb.go b/apps/mongodb.go index 55472e968f..063cdaf131 100644 --- a/apps/mongodb.go +++ b/apps/mongodb.go @@ -16,7 +16,6 @@ package apps import ( "context" - "fmt" "strings" "github.com/GoogleCloudPlatform/ops-agent/confgenerator" @@ -126,45 +125,36 @@ func (p LoggingProcessorMacroMongodb) JsonLogComponents(ctx context.Context) []c func (p LoggingProcessorMacroMongodb) jsonParserWithTimeKey() []confgenerator.InternalLoggingProcessor { c := []confgenerator.InternalLoggingProcessor{} - jsonParser := &confgenerator.LoggingProcessorParseJson{ + c = append(c, &confgenerator.LoggingProcessorParseJson{ ParserShared: confgenerator.ParserShared{ - TimeKey: "time", - TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", Types: map[string]string{ "id": "integer", "message": "string", }, }, - } - - c = append(c, jsonParser) + }) - tempPrefix := "temp_ts_" - timeKey := "time" // have to bring $date to top level in order for it to be parsed as timeKey // see https://github.com/fluent/fluent-bit/issues/1013 - c = append(c, &confgenerator.LoggingProcessorNestLift{ - NestedUnder: "t", - AddPrefix: tempPrefix, + c = append(c, &confgenerator.LoggingProcessorModifyFields{ + Fields: map[string]*confgenerator.ModifyField{ + "jsonPayload.time": { + MoveFrom: "jsonPayload.t.$date", + }, + }, }) - c = append(c, &confgenerator.LoggingProcessorHardRename{ - Field: fmt.Sprintf("%s$date", tempPrefix), - NewName: timeKey, + c = append(c, &confgenerator.LoggingProcessorRemoveField{ + Field: "t", }) // IMPORTANT: now that we have lifted the json to top level // we need to re-parse in order to properly set time at the // parser level - c = append(c, jsonParser) - - timePath := fmt.Sprintf("jsonPayload.%s", timeKey) - c = append(c, &confgenerator.LoggingProcessorModifyFields{ - Fields: map[string]*confgenerator.ModifyField{ - timePath: { - MoveFrom: timePath, - OmitIf: fmt.Sprintf(`%s =~ ".*"`, timePath), - }, + c = append(c, &confgenerator.LoggingProcessorParseTimestamp{ + ParserShared: confgenerator.ParserShared{ + TimeKey: "time", + TimeFormat: "%Y-%m-%dT%H:%M:%S.%L%z", }, }) diff --git a/confgenerator/logging_processors.go b/confgenerator/logging_processors.go index 697ce89c93..3c71176c53 100644 --- a/confgenerator/logging_processors.go +++ b/confgenerator/logging_processors.go @@ -477,6 +477,27 @@ func (p LoggingProcessorRemoveField) Components(ctx context.Context, tag, uid st return []fluentbit.Component{filter} } +type LoggingProcessorParseTimestamp struct { + ConfigComponent `yaml:",inline"` + ParserShared `yaml:",inline"` +} + +func (p LoggingProcessorParseTimestamp) Components(ctx context.Context, tag, uid string) []fluentbit.Component { + parser, parserName := p.ParserShared.Component(tag, uid) + + parser.Config["Format"] = "regex" + parser.Config["Regex"] = fmt.Sprintf("^(?<%s>.*)$", p.TimeKey) + + components := []fluentbit.Component{} + components = append(components, parser) + + components = append(components, + fluentbit.ParserFilterComponents(tag, p.TimeKey, []string{parserName}, false)..., + ) + + return components +} + type LoggingProcessorHardRename struct { Field string NewName string diff --git a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml index 144876fbee..aecf1933cb 100644 --- a/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml +++ b/transformation_test/testdata/logging_processor-mongodb/output_fluentbit.yaml @@ -6,13 +6,12 @@ id: 23285.0 message: Automatically disabling TLS 1.0, to force-enable TLS 1.0 specify --sslDisabledProtocols 'none' severity: I - time: 2025-07-21T15:00:28.308+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.308000000Z - jsonPayload: attributes: {} component: CONTROL @@ -20,13 +19,12 @@ id: 5.945603e+06 message: Multi threading initialized severity: I - time: 2025-07-21T15:00:28.309+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.309000000Z - jsonPayload: attributes: relatedParameters: @@ -38,13 +36,12 @@ id: 4.648601e+06 message: Implicit TCP FastOpen unavailable. If TCP FastOpen is required, set at least one of the related parameters severity: I - time: 2025-07-21T15:00:28.309+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.309000000Z - jsonPayload: attributes: spec: @@ -63,13 +60,12 @@ id: 4.915701e+06 message: Initialized wire specification severity: I - time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.310000000Z - jsonPayload: attributes: {} component: TENANT_M @@ -77,13 +73,12 @@ id: 7.0916e+06 message: Starting TenantMigrationAccessBlockerRegistry severity: I - time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.310000000Z - jsonPayload: attributes: architecture: 64-bit @@ -96,13 +91,12 @@ id: 4.615611e+06 message: MongoDB starting severity: I - time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.310000000Z - jsonPayload: attributes: buildInfo: @@ -120,13 +114,12 @@ id: 23403.0 message: Build Info severity: I - time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.310000000Z - jsonPayload: attributes: os: @@ -137,13 +130,12 @@ id: 51765.0 message: Operating System severity: I - time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.310000000Z - jsonPayload: attributes: options: @@ -154,13 +146,12 @@ id: 21951.0 message: Options set by command line severity: I - time: 2025-07-21T15:00:28.310+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.310000000Z - jsonPayload: attributes: {} component: STORAGE @@ -170,13 +161,12 @@ severity: I tags: - startupWarnings - time: 2025-07-21T15:00:28.311+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.311000000Z - jsonPayload: attributes: config: create,cache_size=3406M,session_max=33000,eviction=(threads_min=4,threads_max=4),config_base=false,statistics=(fast),log=(enabled=true,remove=true,path=journal,compressor=snappy),builtin_extension_config=(zstd=(compression_level=6)),file_manager=(close_idle_time=600,close_scan_interval=10,close_handle_minimum=2000),statistics_log=(wait=0),json_output=(error,message),verbose=[recovery_progress:1,checkpoint_progress:1,compact_progress:1,backup:0,checkpoint:0,compact:0,evict:0,history_store:0,recovery:0,rts:0,salvage:0,tiered:0,timestamp:0,transaction:0,verify:0,log:0],prefetch=(available=true,default=false), @@ -185,13 +175,12 @@ id: 22315.0 message: Opening WiredTiger severity: I - time: 2025-07-21T15:00:28.311+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.311000000Z - jsonPayload: attributes: {} component: WTRECOV @@ -208,13 +197,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.542000000Z - jsonPayload: attributes: {} component: WTRECOV @@ -231,13 +219,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.542000000Z - jsonPayload: attributes: {} component: WTRECOV @@ -254,13 +241,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.542000000Z - jsonPayload: attributes: {} component: WTRECOV @@ -277,13 +263,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:00:28.542+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.542000000Z - jsonPayload: attributes: durationMillis: 235.0 @@ -292,13 +277,12 @@ id: 4.795906e+06 message: WiredTiger opened severity: I - time: 2025-07-21T15:00:28.546+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.546000000Z - jsonPayload: attributes: recoveryTimestamp: @@ -310,13 +294,12 @@ id: 23987.0 message: WiredTiger recoveryTimestamp severity: I - time: 2025-07-21T15:00:28.546+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.546000000Z - jsonPayload: attributes: numRecords: 0.0 @@ -325,13 +308,12 @@ id: 9.529901e+06 message: Initializing durable catalog severity: I - time: 2025-07-21T15:00:28.549+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.549000000Z - jsonPayload: attributes: {} component: STORAGE @@ -339,13 +321,12 @@ id: 9.529902e+06 message: Retrieving all idents from storage engine severity: I - time: 2025-07-21T15:00:28.549+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.549000000Z - jsonPayload: attributes: numEntries: 0.0 @@ -354,13 +335,12 @@ id: 9.529903e+06 message: Initializing all collections in durable catalog severity: I - time: 2025-07-21T15:00:28.549+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.549000000Z - jsonPayload: attributes: {} component: CONTROL @@ -370,13 +350,12 @@ severity: W tags: - startupWarnings - time: 2025-07-21T15:00:28.550+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.550000000Z - jsonPayload: attributes: allocator: tcmalloc-google @@ -390,13 +369,12 @@ severity: W tags: - startupWarnings - time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.551000000Z - jsonPayload: attributes: currentValue: 511.0 @@ -408,13 +386,12 @@ severity: W tags: - startupWarnings - time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.551000000Z - jsonPayload: attributes: currentValue: 262144.0 @@ -427,13 +404,12 @@ severity: W tags: - startupWarnings - time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.551000000Z - jsonPayload: attributes: currentValue: 60.0 @@ -445,13 +421,12 @@ severity: W tags: - startupWarnings - time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.551000000Z - jsonPayload: attributes: namespace: admin.system.version @@ -467,13 +442,12 @@ id: 20320.0 message: createCollection severity: I - time: 2025-07-21T15:00:28.551+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.551000000Z - jsonPayload: attributes: collectionIdent: collection-0-3956073341144391669 @@ -488,13 +462,12 @@ id: 20345.0 message: "Index build: done building" severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: newVersion: "8.0" @@ -503,13 +476,12 @@ id: 20459.0 message: Setting featureCompatibilityVersion severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: context: setFCV @@ -519,13 +491,12 @@ id: 5.8533e+06 message: current featureCompatibilityVersion value severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: newSpec: @@ -555,13 +526,12 @@ id: 4.915702e+06 message: Updated wire specification severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: newSpec: @@ -591,13 +561,12 @@ id: 4.915702e+06 message: Updated wire specification severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: context: startup @@ -607,13 +576,12 @@ id: 5.8533e+06 message: current featureCompatibilityVersion value severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: {} component: STORAGE @@ -621,13 +589,12 @@ id: 5.0711e+06 message: Clearing temp directory severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: {} component: CONTROL @@ -635,13 +602,12 @@ id: 6.6082e+06 message: Initializing cluster server parameters from disk severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: {} component: CONTROL @@ -649,13 +615,12 @@ id: 20536.0 message: Flow Control is enabled on this deployment severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: dataDirectory: /data/db/diagnostic.data @@ -664,13 +629,12 @@ id: 20625.0 message: Initializing full-time diagnostic data capture severity: I - time: 2025-07-21T15:00:28.554+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.554000000Z - jsonPayload: attributes: namespace: local.startup_log @@ -686,13 +650,12 @@ id: 20320.0 message: createCollection severity: I - time: 2025-07-21T15:00:28.555+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.555000000Z - jsonPayload: attributes: collectionIdent: collection-2-3956073341144391669 @@ -707,13 +670,12 @@ id: 20345.0 message: "Index build: done building" severity: I - time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.558000000Z - jsonPayload: attributes: newState: ConfigReplicationDisabled @@ -723,13 +685,12 @@ id: 6.015317e+06 message: Setting new configuration state severity: I - time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.558000000Z - jsonPayload: attributes: {} component: STORAGE @@ -737,13 +698,12 @@ id: 22262.0 message: Timestamp monitor starting severity: I - time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.558000000Z - jsonPayload: attributes: {} component: STORAGE @@ -751,13 +711,12 @@ id: 7.333401e+06 message: Starting the DiskSpaceMonitor severity: I - time: 2025-07-21T15:00:28.558+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.558000000Z - jsonPayload: attributes: address: /tmp/mongodb-27017.sock @@ -766,13 +725,12 @@ id: 23015.0 message: Listening on severity: I - time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.559000000Z - jsonPayload: attributes: address: 0.0.0.0:27017 @@ -781,13 +739,12 @@ id: 23015.0 message: Listening on severity: I - time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.559000000Z - jsonPayload: attributes: port: 27017.0 @@ -797,13 +754,12 @@ id: 23016.0 message: Waiting for connections severity: I - time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.559000000Z - jsonPayload: attributes: Summary of time elapsed: @@ -834,13 +790,12 @@ id: 8.423403e+06 message: mongod startup complete severity: I - time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.559000000Z - jsonPayload: attributes: error: "NamespaceNotFound: config.system.sessions does not exist" @@ -849,13 +804,12 @@ id: 20712.0 message: Sessions collection is not set up; waiting until next sessions reap interval severity: I - time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.559000000Z - jsonPayload: attributes: namespace: config.system.sessions @@ -869,13 +823,12 @@ id: 20320.0 message: createCollection severity: I - time: 2025-07-21T15:00:28.559+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.559000000Z - jsonPayload: attributes: collectionIdent: collection-4-3956073341144391669 @@ -890,13 +843,12 @@ id: 20345.0 message: "Index build: done building" severity: I - time: 2025-07-21T15:00:28.564+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.564000000Z - jsonPayload: attributes: collectionIdent: collection-4-3956073341144391669 @@ -911,13 +863,12 @@ id: 20345.0 message: "Index build: done building" severity: I - time: 2025-07-21T15:00:28.564+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:00:28.564000000Z - jsonPayload: attributes: canonicalName: internalQueryCacheMaxEntriesPerCollection @@ -927,13 +878,12 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W - time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:29.004000000Z - jsonPayload: attributes: canonicalName: collectionSamplingLogIntervalSeconds @@ -943,13 +893,12 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W - time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:29.004000000Z - jsonPayload: attributes: {} component: NETWORK @@ -957,13 +906,12 @@ id: 23803.0 message: Use of deprecated server parameter 'sslMode', please use 'tlsMode' instead. severity: W - time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:29.004000000Z - jsonPayload: attributes: canonicalName: storageEngineConcurrentReadTransactions @@ -973,13 +921,12 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W - time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:29.004000000Z - jsonPayload: attributes: canonicalName: storageEngineConcurrentWriteTransactions @@ -989,13 +936,12 @@ id: 636300.0 message: Use of deprecated server parameter name severity: W - time: 2025-07-21T15:00:29.004+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 400.0 - timestamp: now + timestamp: 2025-07-21T15:00:29.004000000Z - jsonPayload: attributes: {} component: WTCHKPT @@ -1012,13 +958,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:01:28.556+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:01:28.556000000Z - jsonPayload: attributes: {} component: WTCHKPT @@ -1035,13 +980,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:02:28.572+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:02:28.572000000Z - jsonPayload: attributes: {} component: WTCHKPT @@ -1058,13 +1002,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:03:28.585+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:03:28.585000000Z - jsonPayload: attributes: {} component: WTCHKPT @@ -1081,13 +1024,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:04:28.600+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:04:28.600000000Z - jsonPayload: attributes: {} component: WTCHKPT @@ -1104,13 +1046,12 @@ verbose_level: DEBUG_1 verbose_level_id: 1.0 severity: I - time: 2025-07-21T15:05:28.612+00:00 labels: compute.googleapis.com/resource_name: hostname logging.googleapis.com/instrumentation_source: agent.googleapis.com/mongodb logName: projects/my-project/logs/transformation_test severity: 200.0 - timestamp: now + timestamp: 2025-07-21T15:05:28.612000000Z partialSuccess: true resource: labels: {} From 75b549243399efd588915c999f8e412c31c47cff Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 25 Aug 2025 15:31:42 +0000 Subject: [PATCH 11/14] remove LoggingProcessorNestLift --- confgenerator/logging_processors.go | 19 ------------------- 1 file changed, 19 deletions(-) diff --git a/confgenerator/logging_processors.go b/confgenerator/logging_processors.go index 3c71176c53..392d69c7da 100644 --- a/confgenerator/logging_processors.go +++ b/confgenerator/logging_processors.go @@ -515,25 +515,6 @@ func (p LoggingProcessorHardRename) Components(ctx context.Context, tag, uid str return []fluentbit.Component{c} } -type LoggingProcessorNestLift struct { - NestedUnder string - AddPrefix string -} - -func (p LoggingProcessorNestLift) Components(ctx context.Context, tag, uid string) []fluentbit.Component { - filter := fluentbit.Component{ - Kind: "FILTER", - Config: map[string]string{ - "Name": "nest", - "Match": tag, - "Operation": "lift", - "Nested_under": p.NestedUnder, - "Add_prefix": p.AddPrefix, - }, - } - return []fluentbit.Component{filter} -} - var LegacyBuiltinProcessors = map[string]LoggingProcessor{ "lib:default_message_parser": &LoggingProcessorParseRegex{ Regex: `^(?.*)$`, From 1bba7a35ed5519329c5c920d41a5c61f54359012 Mon Sep 17 00:00:00 2001 From: Dylan Strohschein Date: Mon, 25 Aug 2025 15:35:56 +0000 Subject: [PATCH 12/14] updated golden files --- confgenerator/logging_processors.go | 3 +- confgenerator/testdata/feature/golden.csv | 7 ++- .../golden/linux-gpu/error | 2 +- .../golden/linux/error | 2 +- .../golden/windows-2012/error | 2 +- .../golden/windows/error | 2 +- .../golden/linux-gpu/error | 2 +- .../golden/linux/error | 2 +- .../golden/windows-2012/error | 2 +- .../golden/windows/error | 2 +- .../08a90af0796513028c1d99ecc6e59ef3.lua | 14 ++++++ .../0b7b2df85fd188c3e6583425ed73e5b3.lua | 17 +++++++ .../6b37080b5cf99b41252dbda2dfb4bf7a.lua | 22 +++++++++ .../eb8352830870355ccbd0d44ad0273fda.lua | 22 +++++++++ .../linux-gpu/feature_tracking_otlp.json | 2 +- .../golden/linux-gpu/features.yaml | 4 ++ .../golden/linux-gpu/fluent_bit_main.conf | 47 ++++++++----------- .../golden/linux-gpu/fluent_bit_parser.conf | 14 +++--- .../08a90af0796513028c1d99ecc6e59ef3.lua | 14 ++++++ .../0b7b2df85fd188c3e6583425ed73e5b3.lua | 17 +++++++ .../6b37080b5cf99b41252dbda2dfb4bf7a.lua | 22 +++++++++ .../eb8352830870355ccbd0d44ad0273fda.lua | 22 +++++++++ .../golden/linux/feature_tracking_otlp.json | 2 +- .../golden/linux/features.yaml | 4 ++ .../golden/linux/fluent_bit_main.conf | 47 ++++++++----------- .../golden/linux/fluent_bit_parser.conf | 14 +++--- .../08a90af0796513028c1d99ecc6e59ef3.lua | 14 ++++++ .../0b7b2df85fd188c3e6583425ed73e5b3.lua | 17 +++++++ .../6b37080b5cf99b41252dbda2dfb4bf7a.lua | 22 +++++++++ .../eb8352830870355ccbd0d44ad0273fda.lua | 22 +++++++++ .../windows-2012/feature_tracking_otlp.json | 2 +- .../golden/windows-2012/features.yaml | 4 ++ .../golden/windows-2012/fluent_bit_main.conf | 47 ++++++++----------- .../windows-2012/fluent_bit_parser.conf | 14 +++--- .../08a90af0796513028c1d99ecc6e59ef3.lua | 14 ++++++ .../0b7b2df85fd188c3e6583425ed73e5b3.lua | 17 +++++++ .../6b37080b5cf99b41252dbda2dfb4bf7a.lua | 22 +++++++++ .../eb8352830870355ccbd0d44ad0273fda.lua | 22 +++++++++ .../golden/windows/feature_tracking_otlp.json | 2 +- .../golden/windows/features.yaml | 4 ++ .../golden/windows/fluent_bit_main.conf | 47 ++++++++----------- .../golden/windows/fluent_bit_parser.conf | 14 +++--- 42 files changed, 432 insertions(+), 162 deletions(-) create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/08a90af0796513028c1d99ecc6e59ef3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/0b7b2df85fd188c3e6583425ed73e5b3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/6b37080b5cf99b41252dbda2dfb4bf7a.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/eb8352830870355ccbd0d44ad0273fda.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/08a90af0796513028c1d99ecc6e59ef3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/0b7b2df85fd188c3e6583425ed73e5b3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/6b37080b5cf99b41252dbda2dfb4bf7a.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux/eb8352830870355ccbd0d44ad0273fda.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/08a90af0796513028c1d99ecc6e59ef3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/0b7b2df85fd188c3e6583425ed73e5b3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/6b37080b5cf99b41252dbda2dfb4bf7a.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows-2012/eb8352830870355ccbd0d44ad0273fda.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/08a90af0796513028c1d99ecc6e59ef3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/0b7b2df85fd188c3e6583425ed73e5b3.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/6b37080b5cf99b41252dbda2dfb4bf7a.lua create mode 100644 confgenerator/testdata/goldens/logging-receiver_mongodb/golden/windows/eb8352830870355ccbd0d44ad0273fda.lua diff --git a/confgenerator/logging_processors.go b/confgenerator/logging_processors.go index 392d69c7da..957ba16933 100644 --- a/confgenerator/logging_processors.go +++ b/confgenerator/logging_processors.go @@ -478,8 +478,7 @@ func (p LoggingProcessorRemoveField) Components(ctx context.Context, tag, uid st } type LoggingProcessorParseTimestamp struct { - ConfigComponent `yaml:",inline"` - ParserShared `yaml:",inline"` + ParserShared `yaml:",inline"` } func (p LoggingProcessorParseTimestamp) Components(ctx context.Context, tag, uid string) []fluentbit.Component { diff --git a/confgenerator/testdata/feature/golden.csv b/confgenerator/testdata/feature/golden.csv index 4536eff7c8..77cdad4a39 100644 --- a/confgenerator/testdata/feature/golden.csv +++ b/confgenerator/testdata/feature/golden.csv @@ -164,6 +164,7 @@ App,Field,Override, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroHbaseSystem],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroJettyAccess],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroKafka],confgenerator.ConfigComponent.Type, +*confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlError],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlGeneral],confgenerator.ConfigComponent.Type, *confgenerator.loggingProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlSlow],confgenerator.ConfigComponent.Type, @@ -199,6 +200,8 @@ App,Field,Override, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroJettyAccess]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroKafka]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroKafka]],confgenerator.ConfigComponent.Type, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],ReceiverMacro, +*confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMongodb]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlError]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlError]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroMysqlGeneral]],ReceiverMacro, @@ -223,7 +226,3 @@ App,Field,Override, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroTomcatSystem]],confgenerator.ConfigComponent.Type, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroVarnish]],ReceiverMacro, *confgenerator.loggingReceiverMacroAdapter[*github.com/GoogleCloudPlatform/ops-agent/confgenerator.loggingFilesProcessorMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingProcessorMacroVarnish]],confgenerator.ConfigComponent.Type, -*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],apps.LoggingReceiverMacroMongodb.confgenerator.LoggingReceiverFilesMixin.BufferInMemory, -*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],apps.LoggingReceiverMacroMongodb.confgenerator.LoggingReceiverFilesMixin.RecordLogFilePath, -*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],apps.LoggingReceiverMacroMongodb.confgenerator.LoggingReceiverFilesMixin.WildcardRefreshInterval, -*confgenerator.loggingReceiverMacroAdapter[github.com/GoogleCloudPlatform/ops-agent/apps.LoggingReceiverMacroMongodb],confgenerator.ConfigComponent.Type, diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error index 9fa315a8aa..8d63ba522e 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error index 9fa315a8aa..8d63ba522e 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error index 3c370321e7..28d618e049 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error index 3c370321e7..28d618e049 100644 --- a/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-otel-unsupported_processor/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error index 7e31a8e3b8..62a4691e19 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux-gpu/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error index 7e31a8e3b8..62a4691e19 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/linux/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error index 4c2d67d198..ba7d9defad 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows-2012/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error index 4c2d67d198..ba7d9defad 100644 --- a/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error +++ b/confgenerator/testdata/goldens/invalid-logging-processor_unsupported_type/golden/windows/error @@ -1 +1 @@ -logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file +logging processor with type "unsupported_type" is not supported. Supported logging processor types: [apache_access, apache_error, cassandra_debug, cassandra_gc, cassandra_system, couchdb, elasticsearch_gc, elasticsearch_json, exclude_logs, flink, hbase_system, iis_access, jetty_access, kafka, modify_fields, mongodb, mysql_error, mysql_general, mysql_slow, nginx_access, nginx_error, oracledb_alert, oracledb_audit, parse_json, parse_multiline, parse_regex, postgresql_general, rabbitmq, redis, saphana, solr_system, tomcat_access, tomcat_system, varnish, wildfly_system]. \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/08a90af0796513028c1d99ecc6e59ef3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/08a90af0796513028c1d99ecc6e59ef3.lua new file mode 100644 index 0000000000..208adf4ef0 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/08a90af0796513028c1d99ecc6e59ef3.lua @@ -0,0 +1,14 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +return record["attr"] +end)(); +(function(value) +record["attr"] = value +end)(nil); +local v = __field_0; +(function(value) +record["attributes"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/0b7b2df85fd188c3e6583425ed73e5b3.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/0b7b2df85fd188c3e6583425ed73e5b3.lua new file mode 100644 index 0000000000..4a0b4a0fcd --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/0b7b2df85fd188c3e6583425ed73e5b3.lua @@ -0,0 +1,17 @@ + +function parser_nest(tag, timestamp, record) + local nestedRecord = {} + local parseKey = "time" + for k, v in pairs(record) do + if k ~= parseKey then + nestedRecord[k] = v + end + end + + local result = {} + result[parseKey] = record[parseKey] + result["logging.googleapis.com/__tmp"] = nestedRecord + + return 2, timestamp, result +end + diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/6b37080b5cf99b41252dbda2dfb4bf7a.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/6b37080b5cf99b41252dbda2dfb4bf7a.lua new file mode 100644 index 0000000000..7f4e6b4300 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/6b37080b5cf99b41252dbda2dfb4bf7a.lua @@ -0,0 +1,22 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +if record["attr"] == nil +then +return nil +end +return record["attr"]["message"] +end)(); +(function(value) +if record["attr"] == nil +then +record["attr"] = {} +end +record["attr"]["message"] = value +end)(nil); +local v = __field_0; +(function(value) +record["temp_attributes_message"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/eb8352830870355ccbd0d44ad0273fda.lua b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/eb8352830870355ccbd0d44ad0273fda.lua new file mode 100644 index 0000000000..52221c0468 --- /dev/null +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/eb8352830870355ccbd0d44ad0273fda.lua @@ -0,0 +1,22 @@ + +function process(tag, timestamp, record) +local __field_0 = (function() +if record["t"] == nil +then +return nil +end +return record["t"]["$date"] +end)(); +(function(value) +if record["t"] == nil +then +record["t"] = {} +end +record["t"]["$date"] = value +end)(nil); +local v = __field_0; +(function(value) +record["time"] = value +end)(v) +return 2, timestamp, record +end diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json index 8fe864f885..61a43e8f1b 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/feature_tracking_otlp.json @@ -1 +1 @@ -{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file +{"resourceMetrics":[{"resource":{"attributes":[{"key":"k","value":{"stringValue":"v"}}]},"scopeMetrics":[{"scope":{},"metrics":[{"name":"agent.googleapis.com/agent/internal/ops/feature_tracking","gauge":{"dataPoints":[{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"metrics"}},{"key":"feature","value":{"stringValue":"service:pipelines"}},{"key":"key","value":{"stringValue":"default_pipeline_overridden"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"global"}},{"key":"feature","value":{"stringValue":"default:self_log"}},{"key":"key","value":{"stringValue":"default_self_log_file_collection"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"service:otel_logging"}},{"key":"key","value":{"stringValue":"otel_logging_supported_config"}},{"key":"value","value":{"stringValue":"false"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].enabled"}},{"key":"value","value":{"stringValue":"true"}}],"asInt":"1"},{"attributes":[{"key":"module","value":{"stringValue":"logging"}},{"key":"feature","value":{"stringValue":"receivers:mongodb"}},{"key":"key","value":{"stringValue":"[0].include_paths.__length"}},{"key":"value","value":{"stringValue":"1"}}],"asInt":"1"}]}}]}]}]} \ No newline at end of file diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml index 8216fe84f1..24cb696bd0 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/features.yaml @@ -18,3 +18,7 @@ feature: receivers:mongodb key: "[0].enabled" value: "true" +- module: logging + feature: receivers:mongodb + key: "[0].include_paths.__length" + value: "1" diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf index b0e44a4fcc..9b4175b9ea 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_main.conf @@ -110,25 +110,24 @@ script 5fc5f42c16c9e1ab8292e3d42f74f3be.lua [FILTER] - Add_prefix temp_ts_ - Match mongodb.mongodb - Name nest - Nested_under t - Operation lift + Match mongodb.mongodb + Name lua + call process + script eb8352830870355ccbd0d44ad0273fda.lua [FILTER] - Hard_rename temp_ts_$date time - Match mongodb.mongodb - Name modify + Match mongodb.mongodb + Name modify + Remove t [FILTER] Match mongodb.mongodb Name lua call parser_nest - script b4a0dead382dce7b4fe011d3f59fdb6d.lua + script 0b7b2df85fd188c3e6583425ed73e5b3.lua [FILTER] - Key_Name message + Key_Name time Match mongodb.mongodb Name parser Reserve_Data True @@ -142,29 +141,15 @@ [FILTER] Match mongodb.mongodb - Name modify - Remove time - -[FILTER] - Add_prefix temp_attributes_ - Match mongodb.mongodb - Name nest - Nested_under attr - Operation lift + Name lua + call process + script 6b37080b5cf99b41252dbda2dfb4bf7a.lua [FILTER] Hard_rename temp_attributes_message msg Match mongodb.mongodb Name modify -[FILTER] - Match mongodb.mongodb - Name nest - Nest_under attributes - Operation nest - Remove_prefix temp_attributes_ - Wildcard temp_attributes_* - [FILTER] Match mongodb.mongodb Name lua @@ -183,6 +168,12 @@ call process script 1dda8dfcf80ff2aac28fa0838a4d59b7.lua +[FILTER] + Match mongodb.mongodb + Name lua + call process + script 08a90af0796513028c1d99ecc6e59ef3.lua + [FILTER] Match mongodb.mongodb Name lua @@ -194,7 +185,7 @@ Match mongodb.mongodb Name parser Reserve_Data True - Parser mongodb.mongodb.11 + Parser mongodb.mongodb.10 [FILTER] Match mongodb.mongodb diff --git a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf index dbd172c02d..309943611d 100644 --- a/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf +++ b/confgenerator/testdata/goldens/logging-receiver_mongodb/golden/linux-gpu/fluent_bit_parser.conf @@ -1,20 +1,18 @@ [PARSER] - Format json - Name mongodb.mongodb.0 - Time_Format %Y-%m-%dT%H:%M:%S.%L%z - Time_Key time - Types id:integer message:string + Format json + Name mongodb.mongodb.0 + Types id:integer message:string [PARSER] - Format json + Format regex Name mongodb.mongodb.3 + Regex ^(?