From f83f4ab1d7a13732baef91470f749c9bd3d9d341 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Fri, 3 Jun 2022 16:01:12 -0400 Subject: [PATCH 01/11] Eventbridge Cloudtrail Cloudformation Template --- .../eventbridge-single-region.yaml | 121 +++++++++++ aws_eventbridge/eventbridge.yaml | 203 ++++++++++++++++++ aws_eventbridge/release.sh | 49 +++++ 3 files changed, 373 insertions(+) create mode 100644 aws_eventbridge/eventbridge-single-region.yaml create mode 100644 aws_eventbridge/eventbridge.yaml create mode 100755 aws_eventbridge/release.sh diff --git a/aws_eventbridge/eventbridge-single-region.yaml b/aws_eventbridge/eventbridge-single-region.yaml new file mode 100644 index 00000000..6ef41301 --- /dev/null +++ b/aws_eventbridge/eventbridge-single-region.yaml @@ -0,0 +1,121 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Datadog AWS Streams +Parameters: + ApiKey: + Description: >- + Your Datadog API Key + Type: String + AllowedPattern: .+ + ConstraintDescription: ApiKey is required + ServiceRoleArn: + Description: >- + The arn for the service role used by kinesis firehose + Type: String + AllowedPattern: .+ + ConstraintDescription: ServiceRoleArn is required + EventbridgeRoleArn: + Description: >- + The arn for the eventbridge used by the eventbridge + Type: String + AllowedPattern: .+ + ConstraintDescription: StreamRoleArn is required + DdSite: + Description: >- + Define your Datadog Site to send data to. For the Datadog EU site, set to datadoghq.eu + Type: String + AllowedPattern: .+ + Default: 'datadoghq.com' + ConstraintDescription: DdSite is required +Conditions: + EUDatacenter: !Equals [ !Ref DdSite, 'datadoghq.eu' ] + US5Datacenter: !Equals [ !Ref DdSite, 'us5.datadoghq.com' ] + US3Datacenter: !Equals [ !Ref DdSite, 'us3.datadoghq.com' ] + Staging: !Equals [ !Ref DdSite, 'datad0g.com' ] +Resources: + DatadogCloudtrailLogs: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: "datadog-cloudtrail-stream" + RetentionInDays: 14 + HTTPLogStream: + Type: AWS::Logs::LogStream + Properties: + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "http_endpoint_delivery" + S3Backup: + Type: AWS::Logs::LogStream + Properties: + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "s3_backup" + DatadogCloudtrailBackupBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub "datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-${AWS::Region}" + DatadogCloudtrailKinesisFirehose: + Type: AWS::KinesisFirehose::DeliveryStream + Properties: + DeliveryStreamName: "datadog-cloudtrail-stream" + DeliveryStreamType: "DirectPut" + HttpEndpointDestinationConfiguration: + BufferingHints: + SizeInMBs: 4 + IntervalInSeconds: 60 + EndpointConfiguration: + Url: + !If + - Staging + - "https://aws-kinesis-http-intake.logs.datad0g.com/v1/input" + - !If + - EUDatacenter + - "https://aws-kinesis-http-intake.logs.datadoghq.eu/v1/input" + - !If + - US5Datacenter + - "https://aws-kinesis-http-intake.logs.us5.datadoghq.com/v1/input" + - !If + - US3Datacenter + - "https://aws-kinesis-http-intake.logs.us3.datadoghq.com/v1/input" + - "https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input" + Name: "Kinesis intake" + AccessKey: !Ref ApiKey + CloudWatchLoggingOptions: + Enabled: True + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "http_endpoint_delivery" + RoleARN: !Ref ServiceRoleArn + RetryOptions: + DurationInSeconds: 60 + S3BackupMode: "FailedDataOnly" + S3Configuration: + RoleARN: !Ref ServiceRoleArn + BucketARN: !GetAtt DatadogCloudtrailBackupBucket.Arn + ErrorOutputPrefix: "datadog_cloudtrail" + BufferingHints: + SizeInMBs: 4 + IntervalInSeconds: 60 + CompressionFormat: "GZIP" + CloudWatchLoggingOptions: + Enabled: True + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "s3_backup" + Tags: + - Key: "Team" + Value: "aws-integration" + - Key: "StreamAccountID" + Value: !Ref "AWS::AccountId" + DatadogCloudtrailEventbridgeRule: + Type: AWS::Events::Rule + Properties: + Description: "Eventbridge Rule to Forward Cloudtrail Events to Datadog" + EventBusName: "default" + EventPattern: + detail-type: + - AWS API Call via CloudTrail + Name: "datadog-cloudtrail" + RoleArn: !Ref EventbridgeRoleArn + State: ENABLED + Targets: + - Arn: !GetAtt + - DatadogCloudtrailKinesisFirehose + - Arn + Id: Id123 + RoleArn: !Ref EventbridgeRoleArn diff --git a/aws_eventbridge/eventbridge.yaml b/aws_eventbridge/eventbridge.yaml new file mode 100644 index 00000000..8837eeb5 --- /dev/null +++ b/aws_eventbridge/eventbridge.yaml @@ -0,0 +1,203 @@ +AWSTemplateFormatVersion: "2010-09-09" +Parameters: + ApiKey: + Description: >- + Your Datadog API Key + Type: String + AllowedPattern: .+ + ConstraintDescription: ApiKey is required + NoEcho: true + Regions: + Description: >- + Comma separated list of regions to enable metric streaming + Type: CommaDelimitedList + ConstraintDescription: Regions is required + Default: '' + DdSite: + Type: String + Default: datadoghq.com + Description: Define your Datadog Site to send data to. For example, datadoghq.eu or us5.datadoghq.com + AllowedPattern: .+ + ConstraintDescription: DdSite is required +Resources: + DatadogEventbridgeStackSetAdministrationRole: + Type: AWS::IAM::Role + Properties: + RoleName: "DatadogEventbridgeStackSetAdministrationRole" + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - "cloudformation.amazonaws.com" + Action: sts:AssumeRole + Path: / + Policies: + - PolicyName: DatadogEventbridgeCfnStackSetAssumeRole + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - cloudformation:* + Resource: "*" + DatadogEventbridgeStackSetExecutionRole: + Type: AWS::IAM::Role + Properties: + RoleName: "DatadogEventbridgeStackSetExecutionRole" + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + AWS: !GetAtt DatadogEventbridgeStackSetAdministrationRole.Arn + Action: sts:AssumeRole + Path: / + Policies: + - PolicyName: DatadogEventbridgeCfnStackAssumeRole + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Action: + - "s3:*" + Resource: + - "arn:aws:s3:::cf-templates-*" + - Effect: Allow + Action: + - "cloudformation:*" + Resource: + - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stack/StackSet-DatadogEventbridge-*" + - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stackset/DatadogEventbridge*" + - Effect: Allow + Action: + - "sns:Publish" + Resource: + - "arn:aws:sns:*:*:CfnNotificationSNSTopic" + - Effect: Allow + Action: + - iam:GetRole + - iam:PassRole + Resource: "*" + - Effect: Allow + Action: + - s3:CreateBucket + - s3:DeleteBucket + Resource: + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*" + - Effect: Allow + Action: + - logs:CreateLogGroup + - logs:DeleteLogGroup + - logs:PutRetentionPolicy + - logs:CreateLogStream + - logs:DeleteLogStream + - logs:DescribeLogStreams + Resource: + - !Sub "arn:aws:logs:*:${AWS::AccountId}:log-group:datadog-cloudtrail-stream*" + - Effect: Allow + Action: + - firehose:CreateDeliveryStream + - firehose:DescribeDeliveryStream + - firehose:DeleteDeliveryStream + Resource: + - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-cloudtrail-stream" + - Effect: Allow + Action: + - events:DescribeRule + - events:PutRule + - events:DeleteRule + - events:PutTargets + - events:RemoveTargets + Resource: + - !Sub "arn:aws:events:*:${AWS::AccountId}:rule/datadog-cloudtrail" + ServiceRole: + Type: "AWS::IAM::Role" + Properties: + RoleName: "DatadogCloudtrailServiceRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: "Allow" + Principal: + Service: + - "firehose.amazonaws.com" + Action: + - 'sts:AssumeRole' + Path: / + Policies: + - PolicyName: "datadog_cloudtrail_stream_s3_policy" + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: "Allow" + Action: + - "s3:AbortMultipartUpload" + - "s3:GetBucketLocation" + - "s3:GetObject" + - "s3:ListBucket" + - "s3:ListBucketMultipartUploads" + - "s3:PutObject" + Resource: + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*" + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*/*" + DatadogEventbridgeRole: + Type: AWS::IAM::Role + Properties: + RoleName: "DatadogEventbridgeRole" + AssumeRolePolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Principal: + Service: + - events.amazonaws.com + Action: + - "sts:AssumeRole" + Path: / + Policies: + - PolicyName: "datadog_eventbridge_invoke_firehose_policy" + PolicyDocument: + Version: 2012-10-17 + Statement: + - Effect: Allow + Action: + - "firehose:PutRecord" + - "firehose:PutRecordBatch" + Resource: + - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-cloudtrail-stream" + Description: A cloudtrail stream role + DatadogEventbridgeStackSet: + Type: AWS::CloudFormation::StackSet + Properties: + StackSetName: DatadogEventbridge + PermissionModel: SELF_MANAGED + AdministrationRoleARN: !GetAtt DatadogEventbridgeStackSetAdministrationRole.Arn + ExecutionRoleName: !Ref DatadogEventbridgeStackSetExecutionRole + StackInstancesGroup: + - DeploymentTargets: + Accounts: + - !Ref "AWS::AccountId" + Regions: !Ref Regions + TemplateURL: "https://s3.amazonaws.com//aws/eventbridge-single-region.yaml" + Parameters: + - ParameterKey: ApiKey + ParameterValue: !Ref ApiKey + - ParameterKey: ServiceRoleArn + ParameterValue: !GetAtt ServiceRole.Arn + - ParameterKey: EventbridgeRoleArn + ParameterValue: !GetAtt DatadogEventbridgeRole.Arn + - ParameterKey: DdSite + ParameterValue: !Ref DdSite +Metadata: + AWS::CloudFormation::Interface: + ParameterGroups: + - Label: + default: Required + Parameters: + - ApiKey + - DdSite + - Regions + - Label: + default: Optional diff --git a/aws_eventbridge/release.sh b/aws_eventbridge/release.sh new file mode 100755 index 00000000..95eb7a59 --- /dev/null +++ b/aws_eventbridge/release.sh @@ -0,0 +1,49 @@ +#!/bin/bash + +# Usage: ./release.sh + +set -e + +# Read the S3 bucket +if [ -z "$1" ]; then + echo "Must specify a S3 bucket to publish the template" + exit 1 +else + BUCKET=$1 +fi + +# Upload templates to a private bucket -- useful for testing +if [[ $# -eq 2 ]] && [[ $2 = "--private" ]]; then + PRIVATE_TEMPLATE=true +else + PRIVATE_TEMPLATE=false +fi + +# Confirm to proceed +for i in *.yaml; do + [ -f "$i" ] || break + echo "About to upload $i to s3://${BUCKET}/aws/$i" +done +read -p "Continue (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi + +# Update bucket placeholder +# Use datadog-cloudformation-template as the s3 template for production +cp eventbridge.yaml eventbridge.yaml.bak +perl -pi -e "s//${BUCKET}/g" eventbridge.yaml +trap 'mv eventbridge.yaml.bak eventbridge.yaml' EXIT + +# Upload +if [ "$PRIVATE_TEMPLATE" = true ] ; then + aws s3 cp . s3://${BUCKET}/aws --recursive --exclude "*" --include "*.yaml" +else + aws s3 cp . s3://${BUCKET}/aws --recursive --exclude "*" --include "*.yaml" \ + --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers +fi +echo "Done uploading the template, and here is the CloudFormation quick launch URL" +echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloudtrail-eventbridge&templateURL=https://${BUCKET}.s3.amazonaws.com/aws/eventbridge.yaml" + +echo "Done!" From 182362b3e0610b1e336ad16e017e59f036ba2077 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Tue, 21 Jun 2022 17:10:13 -0400 Subject: [PATCH 02/11] Rename resources --- .../eventbridge-single-region.yaml | 14 ++++----- aws_eventbridge/eventbridge.yaml | 30 +++++++++---------- 2 files changed, 22 insertions(+), 22 deletions(-) diff --git a/aws_eventbridge/eventbridge-single-region.yaml b/aws_eventbridge/eventbridge-single-region.yaml index 6ef41301..422ae7c6 100644 --- a/aws_eventbridge/eventbridge-single-region.yaml +++ b/aws_eventbridge/eventbridge-single-region.yaml @@ -7,13 +7,13 @@ Parameters: Type: String AllowedPattern: .+ ConstraintDescription: ApiKey is required - ServiceRoleArn: + DatadogCloudtrailServiceRoleArn: Description: >- The arn for the service role used by kinesis firehose Type: String AllowedPattern: .+ - ConstraintDescription: ServiceRoleArn is required - EventbridgeRoleArn: + ConstraintDescription: DatadogCloudtrailServiceRoleArn is required + DatadogCloudtrailEventbridgeRoleArn: Description: >- The arn for the eventbridge used by the eventbridge Type: String @@ -81,12 +81,12 @@ Resources: Enabled: True LogGroupName: !Ref DatadogCloudtrailLogs LogStreamName: "http_endpoint_delivery" - RoleARN: !Ref ServiceRoleArn + RoleARN: !Ref DatadogCloudtrailServiceRoleArn RetryOptions: DurationInSeconds: 60 S3BackupMode: "FailedDataOnly" S3Configuration: - RoleARN: !Ref ServiceRoleArn + RoleARN: !Ref DatadogCloudtrailServiceRoleArn BucketARN: !GetAtt DatadogCloudtrailBackupBucket.Arn ErrorOutputPrefix: "datadog_cloudtrail" BufferingHints: @@ -111,11 +111,11 @@ Resources: detail-type: - AWS API Call via CloudTrail Name: "datadog-cloudtrail" - RoleArn: !Ref EventbridgeRoleArn + RoleArn: !Ref DatadogCloudtrailEventbridgeRoleArn State: ENABLED Targets: - Arn: !GetAtt - DatadogCloudtrailKinesisFirehose - Arn Id: Id123 - RoleArn: !Ref EventbridgeRoleArn + RoleArn: !Ref DatadogCloudtrailEventbridgeRoleArn diff --git a/aws_eventbridge/eventbridge.yaml b/aws_eventbridge/eventbridge.yaml index 8837eeb5..9949a6b0 100644 --- a/aws_eventbridge/eventbridge.yaml +++ b/aws_eventbridge/eventbridge.yaml @@ -20,10 +20,10 @@ Parameters: AllowedPattern: .+ ConstraintDescription: DdSite is required Resources: - DatadogEventbridgeStackSetAdministrationRole: + DatadogCloudtrailEventbridgeStackSetAdministrationRole: Type: AWS::IAM::Role Properties: - RoleName: "DatadogEventbridgeStackSetAdministrationRole" + RoleName: "DatadogCloudtrailEventbridgeStackSetAdministrationRole" AssumeRolePolicyDocument: Version: '2012-10-17' Statement: @@ -42,16 +42,16 @@ Resources: Action: - cloudformation:* Resource: "*" - DatadogEventbridgeStackSetExecutionRole: + DatadogCloudtrailEventbridgeStackSetExecutionRole: Type: AWS::IAM::Role Properties: - RoleName: "DatadogEventbridgeStackSetExecutionRole" + RoleName: "DatadogCloudtrailEventbridgeStackSetExecutionRole" AssumeRolePolicyDocument: Version: '2012-10-17' Statement: - Effect: Allow Principal: - AWS: !GetAtt DatadogEventbridgeStackSetAdministrationRole.Arn + AWS: !GetAtt DatadogCloudtrailEventbridgeStackSetAdministrationRole.Arn Action: sts:AssumeRole Path: / Policies: @@ -112,7 +112,7 @@ Resources: - events:RemoveTargets Resource: - !Sub "arn:aws:events:*:${AWS::AccountId}:rule/datadog-cloudtrail" - ServiceRole: + DatadogCloudtrailServiceRole: Type: "AWS::IAM::Role" Properties: RoleName: "DatadogCloudtrailServiceRole" @@ -142,10 +142,10 @@ Resources: Resource: - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*" - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*/*" - DatadogEventbridgeRole: + DatadogCloudtrailEventbridgeRole: Type: AWS::IAM::Role Properties: - RoleName: "DatadogEventbridgeRole" + RoleName: "DatadogCloudtrailEventbridgeRole" AssumeRolePolicyDocument: Version: 2012-10-17 Statement: @@ -168,13 +168,13 @@ Resources: Resource: - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-cloudtrail-stream" Description: A cloudtrail stream role - DatadogEventbridgeStackSet: + DatadogCloudtrailEventbridgeStackSet: Type: AWS::CloudFormation::StackSet Properties: StackSetName: DatadogEventbridge PermissionModel: SELF_MANAGED - AdministrationRoleARN: !GetAtt DatadogEventbridgeStackSetAdministrationRole.Arn - ExecutionRoleName: !Ref DatadogEventbridgeStackSetExecutionRole + AdministrationRoleARN: !GetAtt DatadogCloudtrailEventbridgeStackSetAdministrationRole.Arn + ExecutionRoleName: !Ref DatadogCloudtrailEventbridgeStackSetExecutionRole StackInstancesGroup: - DeploymentTargets: Accounts: @@ -184,10 +184,10 @@ Resources: Parameters: - ParameterKey: ApiKey ParameterValue: !Ref ApiKey - - ParameterKey: ServiceRoleArn - ParameterValue: !GetAtt ServiceRole.Arn - - ParameterKey: EventbridgeRoleArn - ParameterValue: !GetAtt DatadogEventbridgeRole.Arn + - ParameterKey: DatadogCloudtrailServiceRoleArn + ParameterValue: !GetAtt DatadogCloudtrailServiceRole.Arn + - ParameterKey: DatadogCloudtrailEventbridgeRoleArn + ParameterValue: !GetAtt DatadogCloudtrailEventbridgeRole.Arn - ParameterKey: DdSite ParameterValue: !Ref DdSite Metadata: From 8f32fc5adf76709c9bed0084784bfa95cbb79f8d Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Wed, 22 Jun 2022 15:19:50 -0400 Subject: [PATCH 03/11] Rename resources --- .../eventbridge-single-region.yaml | 14 ++++++++----- aws_eventbridge/eventbridge.yaml | 20 +++++++++---------- 2 files changed, 19 insertions(+), 15 deletions(-) diff --git a/aws_eventbridge/eventbridge-single-region.yaml b/aws_eventbridge/eventbridge-single-region.yaml index 422ae7c6..fbd9d631 100644 --- a/aws_eventbridge/eventbridge-single-region.yaml +++ b/aws_eventbridge/eventbridge-single-region.yaml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Streams +Description: Datadog AWS Cloudtrail Eventbright Logs Integration Parameters: ApiKey: Description: >- @@ -35,7 +35,7 @@ Resources: DatadogCloudtrailLogs: Type: AWS::Logs::LogGroup Properties: - LogGroupName: "datadog-cloudtrail-stream" + LogGroupName: "datadog-eventbridge-cloudtrail-stream" RetentionInDays: 14 HTTPLogStream: Type: AWS::Logs::LogStream @@ -50,11 +50,11 @@ Resources: DatadogCloudtrailBackupBucket: Type: AWS::S3::Bucket Properties: - BucketName: !Sub "datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-${AWS::Region}" + BucketName: !Sub "datadog-aws-cloudtrail-backup-${AWS::AccountId}-${AWS::Region}" DatadogCloudtrailKinesisFirehose: Type: AWS::KinesisFirehose::DeliveryStream Properties: - DeliveryStreamName: "datadog-cloudtrail-stream" + DeliveryStreamName: "datadog-eventbridge-cloudtrail-stream" DeliveryStreamType: "DirectPut" HttpEndpointDestinationConfiguration: BufferingHints: @@ -110,7 +110,11 @@ Resources: EventPattern: detail-type: - AWS API Call via CloudTrail - Name: "datadog-cloudtrail" + - AWS Insight via CloudTrail + - AWS Console Sign In via CloudTrail + - AWS Console Action via CloudTrail + - AWS Service Event via CloudTrail + Name: "datadog-eventbridge-cloudtrail" RoleArn: !Ref DatadogCloudtrailEventbridgeRoleArn State: ENABLED Targets: diff --git a/aws_eventbridge/eventbridge.yaml b/aws_eventbridge/eventbridge.yaml index 9949a6b0..c50520dc 100644 --- a/aws_eventbridge/eventbridge.yaml +++ b/aws_eventbridge/eventbridge.yaml @@ -68,8 +68,8 @@ Resources: Action: - "cloudformation:*" Resource: - - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stack/StackSet-DatadogEventbridge-*" - - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stackset/DatadogEventbridge*" + - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stack/StackSet-DatadogCloudtrailEventbridge-*" + - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stackset/DatadogCloudtrailEventbridge*" - Effect: Allow Action: - "sns:Publish" @@ -85,7 +85,7 @@ Resources: - s3:CreateBucket - s3:DeleteBucket Resource: - - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*" + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-*" - Effect: Allow Action: - logs:CreateLogGroup @@ -95,14 +95,14 @@ Resources: - logs:DeleteLogStream - logs:DescribeLogStreams Resource: - - !Sub "arn:aws:logs:*:${AWS::AccountId}:log-group:datadog-cloudtrail-stream*" + - !Sub "arn:aws:logs:*:${AWS::AccountId}:log-group:datadog-eventbridge-cloudtrail-stream*" - Effect: Allow Action: - firehose:CreateDeliveryStream - firehose:DescribeDeliveryStream - firehose:DeleteDeliveryStream Resource: - - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-cloudtrail-stream" + - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-eventbridge-cloudtrail-stream" - Effect: Allow Action: - events:DescribeRule @@ -111,7 +111,7 @@ Resources: - events:PutTargets - events:RemoveTargets Resource: - - !Sub "arn:aws:events:*:${AWS::AccountId}:rule/datadog-cloudtrail" + - !Sub "arn:aws:events:*:${AWS::AccountId}:rule/datadog-eventbridge-cloudtrail" DatadogCloudtrailServiceRole: Type: "AWS::IAM::Role" Properties: @@ -140,8 +140,8 @@ Resources: - "s3:ListBucketMultipartUploads" - "s3:PutObject" Resource: - - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*" - - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-stream-backup-${AWS::AccountId}-*/*" + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-*" + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-*/*" DatadogCloudtrailEventbridgeRole: Type: AWS::IAM::Role Properties: @@ -166,12 +166,12 @@ Resources: - "firehose:PutRecord" - "firehose:PutRecordBatch" Resource: - - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-cloudtrail-stream" + - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-eventbridge-cloudtrail-stream" Description: A cloudtrail stream role DatadogCloudtrailEventbridgeStackSet: Type: AWS::CloudFormation::StackSet Properties: - StackSetName: DatadogEventbridge + StackSetName: DatadogCloudtrailEventbridge PermissionModel: SELF_MANAGED AdministrationRoleARN: !GetAtt DatadogCloudtrailEventbridgeStackSetAdministrationRole.Arn ExecutionRoleName: !Ref DatadogCloudtrailEventbridgeStackSetExecutionRole From 33462f450ca7f01eb81d16a2222b09e3b0ad9a51 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Wed, 29 Jun 2022 14:50:19 -0400 Subject: [PATCH 04/11] Some changes re: code review --- .../eventbridge-single-region.yaml | 38 ++++++++----------- aws_eventbridge/eventbridge.yaml | 21 +++++----- 2 files changed, 27 insertions(+), 32 deletions(-) diff --git a/aws_eventbridge/eventbridge-single-region.yaml b/aws_eventbridge/eventbridge-single-region.yaml index fbd9d631..ae67988a 100644 --- a/aws_eventbridge/eventbridge-single-region.yaml +++ b/aws_eventbridge/eventbridge-single-region.yaml @@ -1,5 +1,5 @@ AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Cloudtrail Eventbright Logs Integration +Description: Datadog AWS Cloudtrail Eventbridge Logs Integration Parameters: ApiKey: Description: >- @@ -19,18 +19,23 @@ Parameters: Type: String AllowedPattern: .+ ConstraintDescription: StreamRoleArn is required - DdSite: + DatadogSite: Description: >- - Define your Datadog Site to send data to. For the Datadog EU site, set to datadoghq.eu + Define your Datadog Site to send data to. Type: String - AllowedPattern: .+ Default: 'datadoghq.com' - ConstraintDescription: DdSite is required + ConstraintDescription: DatadogSite is required + AllowedValues: + - datadoghq.com + - datadoghq.eu + - us3.datadoghq.com + - us5.datadoghq.com + - ddog-gov.com Conditions: - EUDatacenter: !Equals [ !Ref DdSite, 'datadoghq.eu' ] - US5Datacenter: !Equals [ !Ref DdSite, 'us5.datadoghq.com' ] - US3Datacenter: !Equals [ !Ref DdSite, 'us3.datadoghq.com' ] - Staging: !Equals [ !Ref DdSite, 'datad0g.com' ] + EUDatacenter: !Equals [ !Ref DatadogSite, 'datadoghq.eu' ] + US5Datacenter: !Equals [ !Ref DatadogSite, 'us5.datadoghq.com' ] + US3Datacenter: !Equals [ !Ref DatadogSite, 'us3.datadoghq.com' ] + Staging: !Equals [ !Ref DatadogSite, 'datad0g.com' ] Resources: DatadogCloudtrailLogs: Type: AWS::Logs::LogGroup @@ -61,20 +66,7 @@ Resources: SizeInMBs: 4 IntervalInSeconds: 60 EndpointConfiguration: - Url: - !If - - Staging - - "https://aws-kinesis-http-intake.logs.datad0g.com/v1/input" - - !If - - EUDatacenter - - "https://aws-kinesis-http-intake.logs.datadoghq.eu/v1/input" - - !If - - US5Datacenter - - "https://aws-kinesis-http-intake.logs.us5.datadoghq.com/v1/input" - - !If - - US3Datacenter - - "https://aws-kinesis-http-intake.logs.us3.datadoghq.com/v1/input" - - "https://aws-kinesis-http-intake.logs.datadoghq.com/v1/input" + Url: !Sub "https://aws-kinesis-http-intake.logs.${DatadogSite}/v1/input" Name: "Kinesis intake" AccessKey: !Ref ApiKey CloudWatchLoggingOptions: diff --git a/aws_eventbridge/eventbridge.yaml b/aws_eventbridge/eventbridge.yaml index c50520dc..86f8c0ba 100644 --- a/aws_eventbridge/eventbridge.yaml +++ b/aws_eventbridge/eventbridge.yaml @@ -13,12 +13,17 @@ Parameters: Type: CommaDelimitedList ConstraintDescription: Regions is required Default: '' - DdSite: + DatadogSite: Type: String Default: datadoghq.com - Description: Define your Datadog Site to send data to. For example, datadoghq.eu or us5.datadoghq.com - AllowedPattern: .+ - ConstraintDescription: DdSite is required + Description: Define your Datadog Site to send data to. + AllowedValues: + - datadoghq.com + - datadoghq.eu + - us3.datadoghq.com + - us5.datadoghq.com + - ddog-gov.com + ConstraintDescription: DatadogSite is required Resources: DatadogCloudtrailEventbridgeStackSetAdministrationRole: Type: AWS::IAM::Role @@ -188,8 +193,8 @@ Resources: ParameterValue: !GetAtt DatadogCloudtrailServiceRole.Arn - ParameterKey: DatadogCloudtrailEventbridgeRoleArn ParameterValue: !GetAtt DatadogCloudtrailEventbridgeRole.Arn - - ParameterKey: DdSite - ParameterValue: !Ref DdSite + - ParameterKey: DatadogSite + ParameterValue: !Ref DatadogSite Metadata: AWS::CloudFormation::Interface: ParameterGroups: @@ -197,7 +202,5 @@ Metadata: default: Required Parameters: - ApiKey - - DdSite + - DatadogSite - Regions - - Label: - default: Optional From 8a99677db6ebf5fe39826a0b0406e427650b1bc2 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Wed, 29 Jun 2022 15:53:48 -0400 Subject: [PATCH 05/11] Removed unused conditions --- aws_eventbridge/eventbridge-single-region.yaml | 5 ----- 1 file changed, 5 deletions(-) diff --git a/aws_eventbridge/eventbridge-single-region.yaml b/aws_eventbridge/eventbridge-single-region.yaml index ae67988a..f8db6860 100644 --- a/aws_eventbridge/eventbridge-single-region.yaml +++ b/aws_eventbridge/eventbridge-single-region.yaml @@ -31,11 +31,6 @@ Parameters: - us3.datadoghq.com - us5.datadoghq.com - ddog-gov.com -Conditions: - EUDatacenter: !Equals [ !Ref DatadogSite, 'datadoghq.eu' ] - US5Datacenter: !Equals [ !Ref DatadogSite, 'us5.datadoghq.com' ] - US3Datacenter: !Equals [ !Ref DatadogSite, 'us3.datadoghq.com' ] - Staging: !Equals [ !Ref DatadogSite, 'datad0g.com' ] Resources: DatadogCloudtrailLogs: Type: AWS::Logs::LogGroup From 46ab6d1c9e0e222319c6718ee6addc22d5b831e3 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Thu, 30 Jun 2022 13:44:13 -0400 Subject: [PATCH 06/11] Rename dir --- .../eventbridge-single-region.yaml | 0 {aws_eventbridge => aws_cloudtrail_eventbridge}/eventbridge.yaml | 0 {aws_eventbridge => aws_cloudtrail_eventbridge}/release.sh | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {aws_eventbridge => aws_cloudtrail_eventbridge}/eventbridge-single-region.yaml (100%) rename {aws_eventbridge => aws_cloudtrail_eventbridge}/eventbridge.yaml (100%) rename {aws_eventbridge => aws_cloudtrail_eventbridge}/release.sh (100%) diff --git a/aws_eventbridge/eventbridge-single-region.yaml b/aws_cloudtrail_eventbridge/eventbridge-single-region.yaml similarity index 100% rename from aws_eventbridge/eventbridge-single-region.yaml rename to aws_cloudtrail_eventbridge/eventbridge-single-region.yaml diff --git a/aws_eventbridge/eventbridge.yaml b/aws_cloudtrail_eventbridge/eventbridge.yaml similarity index 100% rename from aws_eventbridge/eventbridge.yaml rename to aws_cloudtrail_eventbridge/eventbridge.yaml diff --git a/aws_eventbridge/release.sh b/aws_cloudtrail_eventbridge/release.sh similarity index 100% rename from aws_eventbridge/release.sh rename to aws_cloudtrail_eventbridge/release.sh From 2ff5d38534f7c546e4b9d1a9a79fbe27f0ac8a75 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Fri, 1 Jul 2022 11:01:43 -0400 Subject: [PATCH 07/11] Block public access on S3 backup bucket, and add kms encryption --- .../eventbridge-single-region.yaml | 9 +++++++++ aws_cloudtrail_eventbridge/eventbridge.yaml | 8 ++++++++ 2 files changed, 17 insertions(+) diff --git a/aws_cloudtrail_eventbridge/eventbridge-single-region.yaml b/aws_cloudtrail_eventbridge/eventbridge-single-region.yaml index f8db6860..cc40840b 100644 --- a/aws_cloudtrail_eventbridge/eventbridge-single-region.yaml +++ b/aws_cloudtrail_eventbridge/eventbridge-single-region.yaml @@ -51,6 +51,15 @@ Resources: Type: AWS::S3::Bucket Properties: BucketName: !Sub "datadog-aws-cloudtrail-backup-${AWS::AccountId}-${AWS::Region}" + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: "aws:kms" + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true DatadogCloudtrailKinesisFirehose: Type: AWS::KinesisFirehose::DeliveryStream Properties: diff --git a/aws_cloudtrail_eventbridge/eventbridge.yaml b/aws_cloudtrail_eventbridge/eventbridge.yaml index 86f8c0ba..ca90ff1e 100644 --- a/aws_cloudtrail_eventbridge/eventbridge.yaml +++ b/aws_cloudtrail_eventbridge/eventbridge.yaml @@ -89,6 +89,8 @@ Resources: Action: - s3:CreateBucket - s3:DeleteBucket + - s3:PutBucketPublicAccessBlock + - s3:PutEncryptionConfiguration Resource: - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-*" - Effect: Allow @@ -117,6 +119,12 @@ Resources: - events:RemoveTargets Resource: - !Sub "arn:aws:events:*:${AWS::AccountId}:rule/datadog-eventbridge-cloudtrail" + # To get object from encrypted s3 buckets. Use PermissionsBoundaryArn to limit access if needed. + # https://aws.amazon.com/premiumsupport/knowledge-center/s3-troubleshoot-403/#AWS_KMS_encryption + - Effect: Allow + Action: + - "kms:Decrypt" + Resource: "*" DatadogCloudtrailServiceRole: Type: "AWS::IAM::Role" Properties: From c6d810dde32d3a9ed33403824db72945493d336d Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Fri, 1 Jul 2022 12:27:59 -0400 Subject: [PATCH 08/11] Remove unneeded permission --- aws_cloudtrail_eventbridge/eventbridge.yaml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/aws_cloudtrail_eventbridge/eventbridge.yaml b/aws_cloudtrail_eventbridge/eventbridge.yaml index ca90ff1e..b85a6e3a 100644 --- a/aws_cloudtrail_eventbridge/eventbridge.yaml +++ b/aws_cloudtrail_eventbridge/eventbridge.yaml @@ -119,12 +119,6 @@ Resources: - events:RemoveTargets Resource: - !Sub "arn:aws:events:*:${AWS::AccountId}:rule/datadog-eventbridge-cloudtrail" - # To get object from encrypted s3 buckets. Use PermissionsBoundaryArn to limit access if needed. - # https://aws.amazon.com/premiumsupport/knowledge-center/s3-troubleshoot-403/#AWS_KMS_encryption - - Effect: Allow - Action: - - "kms:Decrypt" - Resource: "*" DatadogCloudtrailServiceRole: Type: "AWS::IAM::Role" Properties: From 6d160797c750a103069f76702a788b239ce8937b Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Wed, 6 Jul 2022 09:37:56 -0400 Subject: [PATCH 09/11] Rename templates --- ...ngle-region.yaml => cloudtrail-eventbridge-single-region.yaml} | 0 .../{eventbridge.yaml => cloudtrail-eventbridge.yaml} | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename aws_cloudtrail_eventbridge/{eventbridge-single-region.yaml => cloudtrail-eventbridge-single-region.yaml} (100%) rename aws_cloudtrail_eventbridge/{eventbridge.yaml => cloudtrail-eventbridge.yaml} (100%) diff --git a/aws_cloudtrail_eventbridge/eventbridge-single-region.yaml b/aws_cloudtrail_eventbridge/cloudtrail-eventbridge-single-region.yaml similarity index 100% rename from aws_cloudtrail_eventbridge/eventbridge-single-region.yaml rename to aws_cloudtrail_eventbridge/cloudtrail-eventbridge-single-region.yaml diff --git a/aws_cloudtrail_eventbridge/eventbridge.yaml b/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml similarity index 100% rename from aws_cloudtrail_eventbridge/eventbridge.yaml rename to aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml From b1bd346ad725018a5698d792409bd6219615acd5 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Tue, 19 Jul 2022 16:35:05 -0400 Subject: [PATCH 10/11] Update CF template to use StackSets with native regions --- .../cloudtrail-eventbridge-single-region.yaml | 121 -------- .../cloudtrail-eventbridge.yaml | 259 ++++++++---------- aws_cloudtrail_eventbridge/release.sh | 8 +- 3 files changed, 118 insertions(+), 270 deletions(-) delete mode 100644 aws_cloudtrail_eventbridge/cloudtrail-eventbridge-single-region.yaml diff --git a/aws_cloudtrail_eventbridge/cloudtrail-eventbridge-single-region.yaml b/aws_cloudtrail_eventbridge/cloudtrail-eventbridge-single-region.yaml deleted file mode 100644 index cc40840b..00000000 --- a/aws_cloudtrail_eventbridge/cloudtrail-eventbridge-single-region.yaml +++ /dev/null @@ -1,121 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Cloudtrail Eventbridge Logs Integration -Parameters: - ApiKey: - Description: >- - Your Datadog API Key - Type: String - AllowedPattern: .+ - ConstraintDescription: ApiKey is required - DatadogCloudtrailServiceRoleArn: - Description: >- - The arn for the service role used by kinesis firehose - Type: String - AllowedPattern: .+ - ConstraintDescription: DatadogCloudtrailServiceRoleArn is required - DatadogCloudtrailEventbridgeRoleArn: - Description: >- - The arn for the eventbridge used by the eventbridge - Type: String - AllowedPattern: .+ - ConstraintDescription: StreamRoleArn is required - DatadogSite: - Description: >- - Define your Datadog Site to send data to. - Type: String - Default: 'datadoghq.com' - ConstraintDescription: DatadogSite is required - AllowedValues: - - datadoghq.com - - datadoghq.eu - - us3.datadoghq.com - - us5.datadoghq.com - - ddog-gov.com -Resources: - DatadogCloudtrailLogs: - Type: AWS::Logs::LogGroup - Properties: - LogGroupName: "datadog-eventbridge-cloudtrail-stream" - RetentionInDays: 14 - HTTPLogStream: - Type: AWS::Logs::LogStream - Properties: - LogGroupName: !Ref DatadogCloudtrailLogs - LogStreamName: "http_endpoint_delivery" - S3Backup: - Type: AWS::Logs::LogStream - Properties: - LogGroupName: !Ref DatadogCloudtrailLogs - LogStreamName: "s3_backup" - DatadogCloudtrailBackupBucket: - Type: AWS::S3::Bucket - Properties: - BucketName: !Sub "datadog-aws-cloudtrail-backup-${AWS::AccountId}-${AWS::Region}" - BucketEncryption: - ServerSideEncryptionConfiguration: - - ServerSideEncryptionByDefault: - SSEAlgorithm: "aws:kms" - PublicAccessBlockConfiguration: - BlockPublicAcls: true - BlockPublicPolicy: true - IgnorePublicAcls: true - RestrictPublicBuckets: true - DatadogCloudtrailKinesisFirehose: - Type: AWS::KinesisFirehose::DeliveryStream - Properties: - DeliveryStreamName: "datadog-eventbridge-cloudtrail-stream" - DeliveryStreamType: "DirectPut" - HttpEndpointDestinationConfiguration: - BufferingHints: - SizeInMBs: 4 - IntervalInSeconds: 60 - EndpointConfiguration: - Url: !Sub "https://aws-kinesis-http-intake.logs.${DatadogSite}/v1/input" - Name: "Kinesis intake" - AccessKey: !Ref ApiKey - CloudWatchLoggingOptions: - Enabled: True - LogGroupName: !Ref DatadogCloudtrailLogs - LogStreamName: "http_endpoint_delivery" - RoleARN: !Ref DatadogCloudtrailServiceRoleArn - RetryOptions: - DurationInSeconds: 60 - S3BackupMode: "FailedDataOnly" - S3Configuration: - RoleARN: !Ref DatadogCloudtrailServiceRoleArn - BucketARN: !GetAtt DatadogCloudtrailBackupBucket.Arn - ErrorOutputPrefix: "datadog_cloudtrail" - BufferingHints: - SizeInMBs: 4 - IntervalInSeconds: 60 - CompressionFormat: "GZIP" - CloudWatchLoggingOptions: - Enabled: True - LogGroupName: !Ref DatadogCloudtrailLogs - LogStreamName: "s3_backup" - Tags: - - Key: "Team" - Value: "aws-integration" - - Key: "StreamAccountID" - Value: !Ref "AWS::AccountId" - DatadogCloudtrailEventbridgeRule: - Type: AWS::Events::Rule - Properties: - Description: "Eventbridge Rule to Forward Cloudtrail Events to Datadog" - EventBusName: "default" - EventPattern: - detail-type: - - AWS API Call via CloudTrail - - AWS Insight via CloudTrail - - AWS Console Sign In via CloudTrail - - AWS Console Action via CloudTrail - - AWS Service Event via CloudTrail - Name: "datadog-eventbridge-cloudtrail" - RoleArn: !Ref DatadogCloudtrailEventbridgeRoleArn - State: ENABLED - Targets: - - Arn: !GetAtt - - DatadogCloudtrailKinesisFirehose - - Arn - Id: Id123 - RoleArn: !Ref DatadogCloudtrailEventbridgeRoleArn diff --git a/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml b/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml index b85a6e3a..1a5eac2d 100644 --- a/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml +++ b/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml @@ -1,4 +1,5 @@ -AWSTemplateFormatVersion: "2010-09-09" +AWSTemplateFormatVersion: 2010-09-09 +Description: Datadog AWS Cloudtrail Eventbridge Logs Integration Parameters: ApiKey: Description: >- @@ -6,123 +7,89 @@ Parameters: Type: String AllowedPattern: .+ ConstraintDescription: ApiKey is required - NoEcho: true - Regions: - Description: >- - Comma separated list of regions to enable metric streaming - Type: CommaDelimitedList - ConstraintDescription: Regions is required - Default: '' DatadogSite: + Description: >- + Define your Datadog Site to send data to. Type: String - Default: datadoghq.com - Description: Define your Datadog Site to send data to. + Default: 'datadoghq.com' + ConstraintDescription: DatadogSite is required AllowedValues: - datadoghq.com - datadoghq.eu - us3.datadoghq.com - us5.datadoghq.com - ddog-gov.com - ConstraintDescription: DatadogSite is required Resources: - DatadogCloudtrailEventbridgeStackSetAdministrationRole: + DatadogCloudtrailLogs: + Type: AWS::Logs::LogGroup + Properties: + LogGroupName: "datadog-eventbridge-cloudtrail-stream" + RetentionInDays: 14 + HTTPLogStream: + Type: AWS::Logs::LogStream + Properties: + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "http_endpoint_delivery" + S3Backup: + Type: AWS::Logs::LogStream + Properties: + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "s3_backup" + DatadogCloudtrailBackupBucket: + Type: AWS::S3::Bucket + Properties: + BucketName: !Sub "datadog-aws-cloudtrail-backup-${AWS::AccountId}-${AWS::Region}" + BucketEncryption: + ServerSideEncryptionConfiguration: + - ServerSideEncryptionByDefault: + SSEAlgorithm: "aws:kms" + PublicAccessBlockConfiguration: + BlockPublicAcls: true + BlockPublicPolicy: true + IgnorePublicAcls: true + RestrictPublicBuckets: true + DatadogCloudtrailEventbridgeRole: Type: AWS::IAM::Role Properties: - RoleName: "DatadogCloudtrailEventbridgeStackSetAdministrationRole" + RoleName: !Sub "DatadogCloudtrailEventbridgeRole-${AWS::Region}" AssumeRolePolicyDocument: - Version: '2012-10-17' + Version: 2012-10-17 Statement: - Effect: Allow Principal: Service: - - "cloudformation.amazonaws.com" - Action: sts:AssumeRole + - events.amazonaws.com + Action: + - "sts:AssumeRole" Path: / Policies: - - PolicyName: DatadogEventbridgeCfnStackSetAssumeRole + - PolicyName: "datadog_eventbridge_invoke_firehose_policy" PolicyDocument: - Version: '2012-10-17' + Version: 2012-10-17 Statement: - Effect: Allow Action: - - cloudformation:* - Resource: "*" - DatadogCloudtrailEventbridgeStackSetExecutionRole: - Type: AWS::IAM::Role - Properties: - RoleName: "DatadogCloudtrailEventbridgeStackSetExecutionRole" - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - AWS: !GetAtt DatadogCloudtrailEventbridgeStackSetAdministrationRole.Arn - Action: sts:AssumeRole - Path: / - Policies: - - PolicyName: DatadogEventbridgeCfnStackAssumeRole + - "firehose:PutRecord" + - "firehose:PutRecordBatch" + Resource: + - !Sub "arn:aws:firehose:${AWS::Region}:${AWS::AccountId}:deliverystream/datadog-eventbridge-cloudtrail-stream" + - PolicyName: "datadog_eventbridge_log_policy" PolicyDocument: - Version: '2012-10-17' + Version: 2012-10-17 Statement: - Effect: Allow Action: - - "s3:*" + - "logs:CreateLogGroup" + - "logs:CreateLogStream" + - "logs:PutLogEvents" + - "logs:DescribeLogStreams" Resource: - - "arn:aws:s3:::cf-templates-*" - - Effect: Allow - Action: - - "cloudformation:*" - Resource: - - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stack/StackSet-DatadogCloudtrailEventbridge-*" - - !Sub "arn:aws:cloudformation:*:${AWS::AccountId}:stackset/DatadogCloudtrailEventbridge*" - - Effect: Allow - Action: - - "sns:Publish" - Resource: - - "arn:aws:sns:*:*:CfnNotificationSNSTopic" - - Effect: Allow - Action: - - iam:GetRole - - iam:PassRole - Resource: "*" - - Effect: Allow - Action: - - s3:CreateBucket - - s3:DeleteBucket - - s3:PutBucketPublicAccessBlock - - s3:PutEncryptionConfiguration - Resource: - - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-*" - - Effect: Allow - Action: - - logs:CreateLogGroup - - logs:DeleteLogGroup - - logs:PutRetentionPolicy - - logs:CreateLogStream - - logs:DeleteLogStream - - logs:DescribeLogStreams - Resource: - - !Sub "arn:aws:logs:*:${AWS::AccountId}:log-group:datadog-eventbridge-cloudtrail-stream*" - - Effect: Allow - Action: - - firehose:CreateDeliveryStream - - firehose:DescribeDeliveryStream - - firehose:DeleteDeliveryStream - Resource: - - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-eventbridge-cloudtrail-stream" - - Effect: Allow - Action: - - events:DescribeRule - - events:PutRule - - events:DeleteRule - - events:PutTargets - - events:RemoveTargets - Resource: - - !Sub "arn:aws:events:*:${AWS::AccountId}:rule/datadog-eventbridge-cloudtrail" + - !Sub "arn:aws:firehose:${AWS::Region}:${AWS::AccountId}:deliverystream/datadog-eventbridge-cloudtrail-stream" + Description: A cloudtrail stream role DatadogCloudtrailServiceRole: Type: "AWS::IAM::Role" Properties: - RoleName: "DatadogCloudtrailServiceRole" + RoleName: !Sub "DatadogCloudtrailServiceRole-${AWS::Region}" AssumeRolePolicyDocument: Version: 2012-10-17 Statement: @@ -147,62 +114,64 @@ Resources: - "s3:ListBucketMultipartUploads" - "s3:PutObject" Resource: - - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-*" - - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-*/*" - DatadogCloudtrailEventbridgeRole: - Type: AWS::IAM::Role + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-${AWS::Region}" + - !Sub "arn:aws:s3:::datadog-aws-cloudtrail-backup-${AWS::AccountId}-${AWS::Region}/*" + DatadogCloudtrailKinesisFirehose: + Type: AWS::KinesisFirehose::DeliveryStream Properties: - RoleName: "DatadogCloudtrailEventbridgeRole" - AssumeRolePolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Principal: - Service: - - events.amazonaws.com - Action: - - "sts:AssumeRole" - Path: / - Policies: - - PolicyName: "datadog_eventbridge_invoke_firehose_policy" - PolicyDocument: - Version: 2012-10-17 - Statement: - - Effect: Allow - Action: - - "firehose:PutRecord" - - "firehose:PutRecordBatch" - Resource: - - !Sub "arn:aws:firehose:*:${AWS::AccountId}:deliverystream/datadog-eventbridge-cloudtrail-stream" - Description: A cloudtrail stream role - DatadogCloudtrailEventbridgeStackSet: - Type: AWS::CloudFormation::StackSet + DeliveryStreamName: "datadog-eventbridge-cloudtrail-stream" + DeliveryStreamType: "DirectPut" + HttpEndpointDestinationConfiguration: + BufferingHints: + SizeInMBs: 4 + IntervalInSeconds: 60 + EndpointConfiguration: + Url: !Sub "https://aws-kinesis-http-intake.logs.${DatadogSite}/v1/input" + Name: "Kinesis intake" + AccessKey: !Ref ApiKey + CloudWatchLoggingOptions: + Enabled: True + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "http_endpoint_delivery" + RoleARN: !GetAtt DatadogCloudtrailServiceRole.Arn + RetryOptions: + DurationInSeconds: 60 + S3BackupMode: "FailedDataOnly" + S3Configuration: + RoleARN: !GetAtt DatadogCloudtrailServiceRole.Arn + BucketARN: !GetAtt DatadogCloudtrailBackupBucket.Arn + ErrorOutputPrefix: "datadog_cloudtrail" + BufferingHints: + SizeInMBs: 4 + IntervalInSeconds: 60 + CompressionFormat: "GZIP" + CloudWatchLoggingOptions: + Enabled: True + LogGroupName: !Ref DatadogCloudtrailLogs + LogStreamName: "s3_backup" + Tags: + - Key: "Team" + Value: "aws-integration" + - Key: "StreamAccountID" + Value: !Ref "AWS::AccountId" + DatadogCloudtrailEventbridgeRule: + Type: AWS::Events::Rule Properties: - StackSetName: DatadogCloudtrailEventbridge - PermissionModel: SELF_MANAGED - AdministrationRoleARN: !GetAtt DatadogCloudtrailEventbridgeStackSetAdministrationRole.Arn - ExecutionRoleName: !Ref DatadogCloudtrailEventbridgeStackSetExecutionRole - StackInstancesGroup: - - DeploymentTargets: - Accounts: - - !Ref "AWS::AccountId" - Regions: !Ref Regions - TemplateURL: "https://s3.amazonaws.com//aws/eventbridge-single-region.yaml" - Parameters: - - ParameterKey: ApiKey - ParameterValue: !Ref ApiKey - - ParameterKey: DatadogCloudtrailServiceRoleArn - ParameterValue: !GetAtt DatadogCloudtrailServiceRole.Arn - - ParameterKey: DatadogCloudtrailEventbridgeRoleArn - ParameterValue: !GetAtt DatadogCloudtrailEventbridgeRole.Arn - - ParameterKey: DatadogSite - ParameterValue: !Ref DatadogSite -Metadata: - AWS::CloudFormation::Interface: - ParameterGroups: - - Label: - default: Required - Parameters: - - ApiKey - - DatadogSite - - Regions + Description: "Eventbridge Rule to Forward Cloudtrail Events to Datadog" + EventBusName: "default" + EventPattern: + detail-type: + - AWS API Call via CloudTrail + - AWS Insight via CloudTrail + - AWS Console Sign In via CloudTrail + - AWS Console Action via CloudTrail + - AWS Service Event via CloudTrail + Name: "datadog-eventbridge-cloudtrail" + RoleArn: !GetAtt DatadogCloudtrailEventbridgeRole.Arn + State: ENABLED + Targets: + - Arn: !GetAtt + - DatadogCloudtrailKinesisFirehose + - Arn + Id: Id123 + RoleArn: !GetAtt DatadogCloudtrailEventbridgeRole.Arn diff --git a/aws_cloudtrail_eventbridge/release.sh b/aws_cloudtrail_eventbridge/release.sh index 95eb7a59..b56b2095 100755 --- a/aws_cloudtrail_eventbridge/release.sh +++ b/aws_cloudtrail_eventbridge/release.sh @@ -32,9 +32,9 @@ fi # Update bucket placeholder # Use datadog-cloudformation-template as the s3 template for production -cp eventbridge.yaml eventbridge.yaml.bak -perl -pi -e "s//${BUCKET}/g" eventbridge.yaml -trap 'mv eventbridge.yaml.bak eventbridge.yaml' EXIT +cp cloudtrail-eventbridge.yaml cloudtrail-eventbridge.yaml.bak +perl -pi -e "s//${BUCKET}/g" cloudtrail-eventbridge.yaml +trap 'mv cloudtrail-eventbridge.yaml.bak cloudtrail-eventbridge.yaml' EXIT # Upload if [ "$PRIVATE_TEMPLATE" = true ] ; then @@ -44,6 +44,6 @@ else --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers fi echo "Done uploading the template, and here is the CloudFormation quick launch URL" -echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloudtrail-eventbridge&templateURL=https://${BUCKET}.s3.amazonaws.com/aws/eventbridge.yaml" +echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloudtrail-eventbridge&templateURL=https://${BUCKET}.s3.amazonaws.com/aws/cloudtrail-eventbridge.yaml" echo "Done!" From 95336d8ad6e1f17bd91bff04d637a88a887c53b0 Mon Sep 17 00:00:00 2001 From: Ivan Klishch Date: Thu, 21 Jul 2022 10:36:01 -0400 Subject: [PATCH 11/11] Added NoEcho to the API Key --- aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml b/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml index 1a5eac2d..710bb150 100644 --- a/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml +++ b/aws_cloudtrail_eventbridge/cloudtrail-eventbridge.yaml @@ -7,6 +7,7 @@ Parameters: Type: String AllowedPattern: .+ ConstraintDescription: ApiKey is required + NoEcho: true DatadogSite: Description: >- Define your Datadog Site to send data to.