diff --git a/pkg/cognito.go b/pkg/cognito.go new file mode 100644 index 00000000..6ca3135e --- /dev/null +++ b/pkg/cognito.go @@ -0,0 +1,33 @@ +package main + +import ( + "encoding/json" + "fmt" + "time" + + "github.com/grafana/loki/v3/pkg/logproto" +) + +// Parses a Cognito Record and returns a logproto.Entry +func parseCognitoRecord(record Record) (logproto.Entry, error) { + timestamp := time.Now() + if record.Error != nil { + return logproto.Entry{}, record.Error + } + document, err := json.Marshal(record.Content) + if err != nil { + return logproto.Entry{}, fmt.Errorf("failed to marshal cognito record content: %w", err) + } + if val, ok := record.Content["eventTimestamp"]; ok { + sec, nsec, err := getUnixSecNsec(val.(string)) + if err != nil { + return logproto.Entry{}, fmt.Errorf("failed to parse cognito eventTimestamp %q: %w", val, err) + } + + timestamp = time.Unix(sec, nsec).UTC() + } + return logproto.Entry{ + Line: string(document), + Timestamp: timestamp, + }, nil +} diff --git a/pkg/cognito_test.go b/pkg/cognito_test.go new file mode 100644 index 00000000..12acef2c --- /dev/null +++ b/pkg/cognito_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "compress/gzip" + "os" + "testing" +) + +func TestParseCognitoJson(t *testing.T) { + records := make(chan Record) + jsonStream := NewJSONStream(records) + file, err := os.Open("../testdata/cognito-log-file.json.gz") + if err != nil { + t.Error(err) + } + gzipReader, err := gzip.NewReader(file) + if err != nil { + t.Error(err) + } + go jsonStream.Start(gzipReader, 0) + + for record := range jsonStream.records { + if record.Error != nil { + t.Error(record.Error) + } + _, err := parseCognitoRecord(record) + if err != nil { + t.Error(err) + } + } +} diff --git a/pkg/s3.go b/pkg/s3.go index 3e6ba54b..9c51a66b 100644 --- a/pkg/s3.go +++ b/pkg/s3.go @@ -55,6 +55,7 @@ const ( GuardDutyLogType string = "GuardDuty" MskLogType string = "KafkaBrokerLogs" S3AccessLogType string = "S3AccessLogs" + CognitoServiceLogsType string = "AWSCognitoServiceLogs" ) var ( @@ -92,6 +93,10 @@ var ( // source: https://docs.aws.amazon.com/AmazonS3/latest/userguide/ServerLogs.html // format: aws-account-id/region/bucket-name/year/month/day/timestamp-hash // example: 123456789012/us-west-2/amzn-s3-demo-source-bucket/2023/03/01/2023-03-01-21-32-16-E568B2907131C0C0 + // AWS Cognito Service Logs + // source: https://docs.aws.amazon.com/cognito/latest/developerguide/monitoring.html + // format: my-bucket/AWSLogs/aws-account-id/AWSCognitoServiceLogs/region/cognito-pool-id/event-type/random-string.log.gz + // example: my-bucket/AWSLogs/116911669293/AWSCognitoServiceLogs/eu-central-1/eu-central-1-xxx/USER_AUTH_EVENTS/123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-xxx_userAuthEvents_INFO_S3_yyy_cognito-eu-central-1-xxx_USER_AUTH_EVENTS_20251216_ccc.log.gz defaultFilenameRegex = regexp.MustCompile(`AWSLogs\/(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/(?:health_check_log_)?\d+\_(?:elasticloadbalancing|vpcflowlogs)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?Papp|net)\.*?)?(?P[a-zA-Z0-9\-]+)`) defaultTimestampRegex = regexp.MustCompile(`(?P\d+-\d+-\d+T\d+:\d+:\d+(?:\.\d+Z)?)`) cloudtrailFilenameRegex = regexp.MustCompile(`AWSLogs\/(?Po-[a-z0-9]{10,32})?\/?(?P\d+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[\w-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/\d+\_(?:CloudTrail|CloudTrail-Digest)_(?:\w+-\w+-(?:\w+-)?\d)_(?:(?:app|nlb|net)\.*?)?.+_(?P[a-zA-Z0-9\-]+)`) @@ -104,6 +109,7 @@ var ( mskTimestampRegex = regexp.MustCompile(`^\[(?P\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2},\d{3})\]`) s3AccessLogFilenameRegex = regexp.MustCompile(`(?P\d+)\/(?P[\w-]+)\/(?P[a-zA-Z0-9\-]+)\/(?P\d+)\/(?P\d+)\/(?P\d+)\/[a-zA-Z0-9\-]+$`) s3AccessLogTimestampRegex = regexp.MustCompile(`\[(?P\d+\/\w+\/\d+:\d+:\d+:\d+ [\+-]\d+)\]`) + cognitoServiceLogsRegex = regexp.MustCompile(`AWSLogs\/(?P\d+)\/AWSCognitoServiceLogs\/(?P[\w-]+)\/(?P[a-zA-Z0-9_\-]+)\/(?P[a-zA-Z0-9_\-]+)\/(?P\d+_aws_cognito_service_logs_[a-zA-Z0-9_\-]+\.log\.gz)`) parsers = map[string]parserConfig{ FlowLogType: { logTypeLabel: "s3_vpc_flow", @@ -169,6 +175,11 @@ var ( timestampRegex: s3AccessLogTimestampRegex, timestampType: "string", }, + CognitoServiceLogsType: { + logTypeLabel: "s3_cognito_service_logs", + filenameRegex: cognitoServiceLogsRegex, + ownerLabelKey: "account_id", + }, } ) @@ -221,7 +232,7 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io. ls = applyLabels(ls) // extract the timestamp of the nested event and sends the rest as raw json - if labels["type"] == CloudTrailLogType || labels["type"] == GuardDutyLogType { + if labels["type"] == CloudTrailLogType || labels["type"] == GuardDutyLogType || labels["type"] == CognitoServiceLogsType { records := make(chan Record) jsonStream := NewJSONStream(records) go jsonStream.Start(reader, parser.skipHeaderCount) @@ -230,11 +241,18 @@ func parseS3Log(ctx context.Context, b *batch, labels map[string]string, obj io. if record.Error != nil { return record.Error } - trailEntry, err := parseCloudtrailRecord(record) + + var objEntry logproto.Entry + if labels["type"] == CognitoServiceLogsType { + objEntry, err = parseCognitoRecord(record) + } else { + objEntry, err = parseCloudtrailRecord(record) + } + if err != nil { return err } - if err := b.add(ctx, entry{ls, trailEntry}); err != nil { + if err := b.add(ctx, entry{ls, objEntry}); err != nil { return err } } diff --git a/pkg/s3_test.go b/pkg/s3_test.go index ddcf80bb..bea95034 100644 --- a/pkg/s3_test.go +++ b/pkg/s3_test.go @@ -624,6 +624,38 @@ func Test_getLabels(t *testing.T) { }, wantErr: true, }, + { + name: "cognito_service_logs", + args: args{ + record: events.S3EventRecord{ + AWSRegion: "eu-central-1", + S3: events.S3Entity{ + Bucket: events.S3Bucket{ + Name: "my-bucket", + OwnerIdentity: events.S3UserIdentity{ + PrincipalID: "test", + }, + }, + Object: events.S3Object{ + Key: "my-bucket/AWSLogs/123456789012/AWSCognitoServiceLogs/eu-central-1/eu-central-1-manual-testing/USER_AUTH_EVENTS/123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-manual-testing_userAuthEvents_INFO_S3_07fc3956-72c4-4de4-b7b2-70a6dbe22034_cognito-eu-central-1-manual-testing_USER_AUTH_EVENTS_20251216_8c1fea12.log.gz", + }, + }, + }, + }, + want: map[string]string{ + "account_id": "123456789012", + "bucket": "my-bucket", + "bucket_owner": "test", + "bucket_region": "eu-central-1", + "cognito_pool_id": "eu-central-1-manual-testing", + "event_type_id": "USER_AUTH_EVENTS", + "key": "my-bucket/AWSLogs/123456789012/AWSCognitoServiceLogs/eu-central-1/eu-central-1-manual-testing/USER_AUTH_EVENTS/123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-manual-testing_userAuthEvents_INFO_S3_07fc3956-72c4-4de4-b7b2-70a6dbe22034_cognito-eu-central-1-manual-testing_USER_AUTH_EVENTS_20251216_8c1fea12.log.gz", + "region": "eu-central-1", + "src": "123456789012_aws_cognito_service_logs_eu-central-1_eu-central-1-manual-testing_userAuthEvents_INFO_S3_07fc3956-72c4-4de4-b7b2-70a6dbe22034_cognito-eu-central-1-manual-testing_USER_AUTH_EVENTS_20251216_8c1fea12.log.gz", + "type": CognitoServiceLogsType, + }, + wantErr: false, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -900,6 +932,29 @@ func Test_parseS3Log(t *testing.T) { expectedLog: `level=warn msg="timestamp type of no_type parser unknown, using current time"` + "\n", wantErr: false, }, + { + name: "cognitoservicelogs", + args: args{ + batchSize: 131072, // Set large enough we don't try and send to promtail + filename: "../testdata/cognito-log-file.json.gz", + b: &batch{ + streams: map[string]*logproto.Stream{}, + processor: process, + }, + labels: map[string]string{ + "type": CognitoServiceLogsType, + "src": "source", + "account_id": "123456789", + }, + }, + expectedLen: 1, + expectedStream: `{__aws_log_type="s3_cognito_service_logs", __aws_s3_cognito_service_logs="source", __aws_s3_cognito_service_logs_owner="123456789"}`, + expectedTimestamps: []time.Time{ + time.Date(2025, time.December, 16, 8, 47, 05, 0, time.UTC), + time.Date(2025, time.December, 16, 8, 47, 10, 0, time.UTC), + }, + wantErr: false, + }, } parsers["no_type"] = parserConfig{ logTypeLabel: "s3_waf", diff --git a/testdata/cognito-log-file.json.gz b/testdata/cognito-log-file.json.gz new file mode 100644 index 00000000..28fd27b3 Binary files /dev/null and b/testdata/cognito-log-file.json.gz differ