From f34de7f1af2ee058ee7964a86d2843ff86ed3bee Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Thu, 22 Aug 2024 16:50:03 -0400 Subject: [PATCH 01/10] add a custom IAM resource to add product specific permsissions --- aws_datadog_iam/datadog_custom_iam.yaml | 187 ++++++++++++++++++++++++ 1 file changed, 187 insertions(+) create mode 100644 aws_datadog_iam/datadog_custom_iam.yaml diff --git a/aws_datadog_iam/datadog_custom_iam.yaml b/aws_datadog_iam/datadog_custom_iam.yaml new file mode 100644 index 00000000..126d0446 --- /dev/null +++ b/aws_datadog_iam/datadog_custom_iam.yaml @@ -0,0 +1,187 @@ +AWSTemplateFormatVersion: 2010-09-09 +Description: Datadog AWS Integration Custom IAM Resource +Parameters: + DatadogIntegrationManagedPolicy: + Type: String + Description: The name of the IAM role that has been integrated with Datadog. + Default: DatadogIntegrationManagedPolicy + Product: + Type: String + Description: The product to update the IAM role for. + AllowedValues: + - CloudCost + ProductVariables: + Type: String + Description: The variables to be used in the product statement as a JSON string + +Resources: + LambdaExecutionRoleDatadogCustomIAM: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + + LambdaExecutionRolePolicyDatadogCustomIAMCall: + Type: AWS::IAM::Policy + Properties: + PolicyName: "dd-custom-iam-policy" + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: "Allow" + Action: + - "iam:GetPolicy" + - "iam:GetPolicyVersion" + - "iam:CreatePolicyVersion" + - "iam:DeletePolicyVersion" + Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/${DatadogIntegrationManagedPolicy}" + Condition: + StringEquals: + "aws:userid": + !Join [":", [!GetAtt LambdaExecutionRoleDatadogCustomIAM.RoleId, !Ref DatadogCustomIAMFunction]] + Roles: + - !Ref LambdaExecutionRoleDatadogCustomIAM + + DatadogCCMCustomIAM: + Type: "Custom::DatadogCustomIAM" + DependsOn: LambdaExecutionRolePolicyDatadogCustomIAMCall + Properties: + ServiceToken: !GetAtt DatadogCustomIAMFunction.Arn + DatadogIntegrationManagedPolicy: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/${DatadogIntegrationManagedPolicy}" + Product: !Ref Product + ProductVariables: !Ref ProductVariables + + DatadogCustomIAMFunction: + Type: AWS::Lambda::Function + Properties: + Role: !GetAtt LambdaExecutionRoleDatadogCustomIAM.Arn + Description: "A function to modify the IAM role integrated with Datadog." + Handler: "index.handler" + Runtime: "python3.11" + Timeout: 30 + Code: + ZipFile: | + import boto3 + import cfnresponse + import json + import secrets + + def getCCMPolicyStatement(product_variables): + return [ + { + "Sid": "ReadCloudCostBucket", + "Effect": "Allow", + "Action": [ + "s3:ListBucket" + ], + "Resource": "arn:aws:s3:::{product_variables["DDCloudCostBucketName"]}" + }, + { + "Sid": "GetBill", + "Effect": "Allow", + "Action": [ + "s3:GetObject" + ], + "Resource": "arn:aws:s3:::{product_variables["DDCloudCostBucketName"]}/{product_variables["DDCloudCostBucketPrefix"]}/{product_variables["DDCloudCostReportName"]}/*" + }, + { + "Sid": "CheckAccuracy", + "Effect": "Allow", + "Action": [ + "ce:Get*" + ], + "Resource": "*" + }, + { + "Sid": "ListCURs", + "Effect": "Allow", + "Action": [ + "cur:DescribeReportDefinitions" + ], + "Resource": "*" + }, + { + "Sid": "ListOrganizations", + "Effect": "Allow", + "Action": [ + "organizations:Describe*", + "organizations:List*" + ], + "Resource": "*" + } + ] + + PRODUCT_TO_STATEMENTS = { + "CloudCost": { + "statementsGenerator": getCCMPolicyStatement, + "variables": ["DDCloudCostBucketName", "DDCloudCostBucketPrefix", "DDCloudCostReportName"] + } + } + + def modify_policy(policy_json, method, statements, logical_resource_id, product): + policy_statements = policy_json['Statement'] + policy_statements_by_id = {i['Sid']:i for i in policy_statements} + if method == 'Update' or method == 'Create': + for statement in statements: + policy_statements_by_id[statement['Sid']] = statement + elif method == 'Delete': + statement_ids = list(policy_statements_by_id.keys()) + for statement_id in statement_ids: + if statement_id.startswith(f"{logical_resource_id}{product}"): + del policy_statements_by_id[statement_id] + else: + raise ValueError('Invalid method') + policy_json['Statement'] = list(policy_statements_by_id.values()) + return policy_json + + def prepare_statements(product, logical_resource_id, product_variables): + product_data = PRODUCT_TO_STATEMENTS.get(product) + statements = product_data["statementsGenerator"](product_variables) + for statement in statements: + statement['Sid'] = f"{logical_resource_id}{product}{statement.get('Sid', secrets.token_hex(nbytes=3))}" + return statements + + def get_policy(iam, policy_arn): + resp = iam.get_policy(PolicyArn=policy_arn) + version = resp["Policy"]["DefaultVersionId"] + policy_resp = iam.get_policy_version(PolicyArn=policy_arn, VersionId=version) + return (policy_resp["PolicyVersion"]["Document"], version) + + def validate_variables(product_variables, product): + if not all([var in product_variables for var in PRODUCT_TO_STATEMENTS[product]['variables']]): + raise ValueError('Invalid product variables, missing required keys') + + def modify_iam(event, context): + method = event['RequestType'] + policy_arn = event['ResourceProperties']['DatadogIntegrationManagedPolicy'] + product = event['ResourceProperties']['Product'] + product_variables = json.loads(event['ResourceProperties']['ProductVariables']) + logical_resource_id = event['ResourceProperties']['LogicalResourceId'] + + validate_variables(product_variables, product) + statements = prepare_statements(product, logical_resource_id, product_variables) + + iam = boto3.client('iam') + policy_json, policy_version = get_policy(iam, policy_arn) + modified_policy_json = modify_policy(policy_json, method, statements, logical_resource_id, product) + iam.create_policy_version(PolicyArn=policy_arn, PolicyDocument=json.dumps(modified_policy_json), SetAsDefault=True) + iam.delete_policy_version(PolicyArn=policy_arn, VersionId=policy_version) + + def handler(event, context): + print(event) + try: + modify_iam(event, context) + cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, context.log_stream_name) + except Exception as e: + print(e) + cfnresponse.send(event, context, cfnresponse.FAILED, {}, context.log_stream_name) From 75a5943481fc69ff234db9e7f253973105e3223b Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Thu, 22 Aug 2024 16:53:06 -0400 Subject: [PATCH 02/10] add ccm specific cfn --- aws_ccm/datadog_ccm.yaml | 98 ++++++++++++++++++++++++++++++++++++++++ 1 file changed, 98 insertions(+) create mode 100644 aws_ccm/datadog_ccm.yaml diff --git a/aws_ccm/datadog_ccm.yaml b/aws_ccm/datadog_ccm.yaml new file mode 100644 index 00000000..879b07d6 --- /dev/null +++ b/aws_ccm/datadog_ccm.yaml @@ -0,0 +1,98 @@ + +Parameters: + DDCloudCostBucketName: + Type: String + Description: The name of the S3 bucket for storing the cost and usage reports. + DDCloudCostBucketRegion: + Type: String + Description: The AWS region where the S3 bucket is located. + Default: us-east-1 + DDCloudCostBucketPrefix: + Type: String + Description: The prefix to be added to the cost and usage report files in the S3 bucket. + DDCloudCostReportName: + Type: String + Description: The name of the cost and usage report. + DatadogIntegrationRole: + Type: String + Description: The name of the IAM role that has been integrated with Datadog. + Default: DatadogIntegrationRole + CreateCloudCostBucket: + Type: String + Default: true + AllowedValues: + - true + - false + Description: Whether to create the S3 bucket for storing the cost and usage reports. + +Conditions: + ShouldCreateCloudCostBucket: + Fn::Equals: + - Ref: CreateCloudCostBucket + - true + +Resources: + DDCloudCostPermissions: + Type: AWS::CloudFormation::Stack + Properties: + TemplateURL: "https://.s3.amazonaws.com/aws_datatdog_iam//datadog_custom_iam.yaml" + Parameters: + Product: CloudCost + ProductVariables: !Sub '{"DDCloudCostReportName": "${DDCloudCostReportName}", "DDCloudCostBucketPrefix": "${DDCloudCostBucketPrefix}", "DDCloudCostBucketName": "${DDCloudCostBucketName}" }' + + DDCloudCostBucket: + Condition: ShouldCreateCloudCostBucket + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref DDCloudCostBucketName + + DDCloudCostBucketAccessPolicy: + Condition: ShouldCreateCloudCostBucket + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref DDCloudCostBucket + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - billingreports.amazonaws.com + - bcm-data-exports.amazonaws.com + Action: + - s3:PutObject + - s3:GetBucketPolicy + Resource: + - !Sub ${DDCloudCostBucket.Arn} + - !Sub ${DDCloudCostBucket.Arn}/* + Condition: + StringLike: + aws:SourceArn: + - !Sub arn:aws:cur:us-east-1:${AWS::AccountId}:definition/* + - !Sub arn:aws:bcm-data-exports:us-east-1:${AWS::AccountId}:export/* + aws:SourceAccount: !Sub ${AWS::AccountId} + + DDCURReportDefinition: + DependsOn: S3ClientBucketAccessPolicy + Type: AWS::BCMDataExports::Export + Properties: + Export: + DataQuery: + QueryStatement: "SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT" + TableConfigurations: + COST_AND_USAGE_REPORT: + INCLUDE_RESOURCES: "TRUE" + INCLUDE_SPLIT_COST_ALLOCATION_DATA: "TRUE" + TIME_GRANULARITY: "HOURLY" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref DDCloudCostBucketName + S3Region: !Ref DDCloudCostBucketRegion + S3Prefix: !Ref DDCloudCostBucketPrefix + S3OutputConfigurations: + Compression: "GZIP" + Format: "textORcsv" + Overwrite: "CREATE_NEW_REPORT" + Name: !Ref DDCloudCostReportName + RefreshCadence: + Frequency: "DAILY" \ No newline at end of file From 78ee49a42789f8a36f142446241a658b0e8cff50 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Thu, 22 Aug 2024 17:03:44 -0400 Subject: [PATCH 03/10] separate the custom resource and the caller --- aws_ccm/datadog_ccm.yaml | 17 ++++++++++++++--- aws_datadog_iam/datadog_custom_iam.yaml | 21 +++++---------------- 2 files changed, 19 insertions(+), 19 deletions(-) diff --git a/aws_ccm/datadog_ccm.yaml b/aws_ccm/datadog_ccm.yaml index 879b07d6..aac478da 100644 --- a/aws_ccm/datadog_ccm.yaml +++ b/aws_ccm/datadog_ccm.yaml @@ -17,6 +17,10 @@ Parameters: Type: String Description: The name of the IAM role that has been integrated with Datadog. Default: DatadogIntegrationRole + DatadogIntegrationManagedPolicy: + Type: String + Description: The name of the IAM policy used in the role that has been integrated with Datadog. + Default: DatadogIntegrationManagedPolicy CreateCloudCostBucket: Type: String Default: true @@ -32,13 +36,20 @@ Conditions: - true Resources: - DDCloudCostPermissions: + DatadogCustomIAM: Type: AWS::CloudFormation::Stack Properties: TemplateURL: "https://.s3.amazonaws.com/aws_datatdog_iam//datadog_custom_iam.yaml" Parameters: - Product: CloudCost - ProductVariables: !Sub '{"DDCloudCostReportName": "${DDCloudCostReportName}", "DDCloudCostBucketPrefix": "${DDCloudCostBucketPrefix}", "DDCloudCostBucketName": "${DDCloudCostBucketName}" }' + DatadogIntegrationManagedPolicy: !Ref DatadogIntegrationManagedPolicy + + DatadogCCMCustomIAM: + Type: "Custom::DatadogCustomIAM" + Properties: + ServiceToken: !GetAtt DatadogCustomIAM.DatadogCustomIAMFunctionArn + DatadogIntegrationManagedPolicy: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/${DatadogIntegrationManagedPolicy}" + Product: CloudCost + ProductVariables: !Sub '{"DDCloudCostReportName": "${DDCloudCostReportName}", "DDCloudCostBucketPrefix": "${DDCloudCostBucketPrefix}", "DDCloudCostBucketName": "${DDCloudCostBucketName}" }' DDCloudCostBucket: Condition: ShouldCreateCloudCostBucket diff --git a/aws_datadog_iam/datadog_custom_iam.yaml b/aws_datadog_iam/datadog_custom_iam.yaml index 126d0446..68c23e6e 100644 --- a/aws_datadog_iam/datadog_custom_iam.yaml +++ b/aws_datadog_iam/datadog_custom_iam.yaml @@ -5,14 +5,6 @@ Parameters: Type: String Description: The name of the IAM role that has been integrated with Datadog. Default: DatadogIntegrationManagedPolicy - Product: - Type: String - Description: The product to update the IAM role for. - AllowedValues: - - CloudCost - ProductVariables: - Type: String - Description: The variables to be used in the product statement as a JSON string Resources: LambdaExecutionRoleDatadogCustomIAM: @@ -52,14 +44,6 @@ Resources: Roles: - !Ref LambdaExecutionRoleDatadogCustomIAM - DatadogCCMCustomIAM: - Type: "Custom::DatadogCustomIAM" - DependsOn: LambdaExecutionRolePolicyDatadogCustomIAMCall - Properties: - ServiceToken: !GetAtt DatadogCustomIAMFunction.Arn - DatadogIntegrationManagedPolicy: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/${DatadogIntegrationManagedPolicy}" - Product: !Ref Product - ProductVariables: !Ref ProductVariables DatadogCustomIAMFunction: Type: AWS::Lambda::Function @@ -185,3 +169,8 @@ Resources: except Exception as e: print(e) cfnresponse.send(event, context, cfnresponse.FAILED, {}, context.log_stream_name) + +Outputs: + DatadogCustomIAMFunctionArn: + Description: "The ARN of the Lambda function to modify the IAM role integrated with Datadog." + Value: !GetAtt DatadogCustomIAMFunction.Arn \ No newline at end of file From 500fa806d4734370ad69f9be0a0008e32772742f Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Thu, 22 Aug 2024 17:20:34 -0400 Subject: [PATCH 04/10] add lambda permissions to only have cfn call the lambda --- aws_ccm/datadog_ccm.yaml | 1 + aws_datadog_iam/datadog_custom_iam.yaml | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/aws_ccm/datadog_ccm.yaml b/aws_ccm/datadog_ccm.yaml index aac478da..98015b64 100644 --- a/aws_ccm/datadog_ccm.yaml +++ b/aws_ccm/datadog_ccm.yaml @@ -42,6 +42,7 @@ Resources: TemplateURL: "https://.s3.amazonaws.com/aws_datatdog_iam//datadog_custom_iam.yaml" Parameters: DatadogIntegrationManagedPolicy: !Ref DatadogIntegrationManagedPolicy + CallerStackId: !Ref "AWS::StackId" DatadogCCMCustomIAM: Type: "Custom::DatadogCustomIAM" diff --git a/aws_datadog_iam/datadog_custom_iam.yaml b/aws_datadog_iam/datadog_custom_iam.yaml index 68c23e6e..9e793c8b 100644 --- a/aws_datadog_iam/datadog_custom_iam.yaml +++ b/aws_datadog_iam/datadog_custom_iam.yaml @@ -5,6 +5,9 @@ Parameters: Type: String Description: The name of the IAM role that has been integrated with Datadog. Default: DatadogIntegrationManagedPolicy + CallerStackId: + Type: String + Description: The stack ID of the calling stack. Resources: LambdaExecutionRoleDatadogCustomIAM: @@ -44,6 +47,13 @@ Resources: Roles: - !Ref LambdaExecutionRoleDatadogCustomIAM + DatadogCustomIAMFunctionInvokePermission: + Type: AWS::Lambda::Permission + Properties: + FunctionName: !GetAtt DatadogCustomIAMFunction.Arn + Action: 'lambda:InvokeFunction' + Principal: "cloudformation.amazonaws.com" + SourceArn: !Ref CallerStackId DatadogCustomIAMFunction: Type: AWS::Lambda::Function From ee975ee6d712190d0fdc85e7527a8c1266854b98 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Thu, 22 Aug 2024 18:38:54 -0400 Subject: [PATCH 05/10] fix code syntax errors --- aws_datadog_iam/datadog_custom_iam.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/aws_datadog_iam/datadog_custom_iam.yaml b/aws_datadog_iam/datadog_custom_iam.yaml index 9e793c8b..c7908e1d 100644 --- a/aws_datadog_iam/datadog_custom_iam.yaml +++ b/aws_datadog_iam/datadog_custom_iam.yaml @@ -78,7 +78,7 @@ Resources: "Action": [ "s3:ListBucket" ], - "Resource": "arn:aws:s3:::{product_variables["DDCloudCostBucketName"]}" + "Resource": f"arn:aws:s3:::{product_variables['DDCloudCostBucketName']}" }, { "Sid": "GetBill", @@ -86,7 +86,7 @@ Resources: "Action": [ "s3:GetObject" ], - "Resource": "arn:aws:s3:::{product_variables["DDCloudCostBucketName"]}/{product_variables["DDCloudCostBucketPrefix"]}/{product_variables["DDCloudCostReportName"]}/*" + "Resource": f"arn:aws:s3:::{product_variables['DDCloudCostBucketName']}/{product_variables['DDCloudCostBucketPrefix']}/{product_variables['DDCloudCostReportName']}/*" }, { "Sid": "CheckAccuracy", @@ -160,7 +160,7 @@ Resources: policy_arn = event['ResourceProperties']['DatadogIntegrationManagedPolicy'] product = event['ResourceProperties']['Product'] product_variables = json.loads(event['ResourceProperties']['ProductVariables']) - logical_resource_id = event['ResourceProperties']['LogicalResourceId'] + logical_resource_id = event['LogicalResourceId'] validate_variables(product_variables, product) statements = prepare_statements(product, logical_resource_id, product_variables) From 2398068daaa21e32e693b901dd32c90383181d47 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Fri, 23 Aug 2024 10:11:17 -0400 Subject: [PATCH 06/10] remove lambda permissions since it didn't change anything --- aws_ccm/datadog_ccm.yaml | 1 - aws_datadog_iam/datadog_custom_iam.yaml | 11 ----------- 2 files changed, 12 deletions(-) diff --git a/aws_ccm/datadog_ccm.yaml b/aws_ccm/datadog_ccm.yaml index 98015b64..aac478da 100644 --- a/aws_ccm/datadog_ccm.yaml +++ b/aws_ccm/datadog_ccm.yaml @@ -42,7 +42,6 @@ Resources: TemplateURL: "https://.s3.amazonaws.com/aws_datatdog_iam//datadog_custom_iam.yaml" Parameters: DatadogIntegrationManagedPolicy: !Ref DatadogIntegrationManagedPolicy - CallerStackId: !Ref "AWS::StackId" DatadogCCMCustomIAM: Type: "Custom::DatadogCustomIAM" diff --git a/aws_datadog_iam/datadog_custom_iam.yaml b/aws_datadog_iam/datadog_custom_iam.yaml index c7908e1d..76636ade 100644 --- a/aws_datadog_iam/datadog_custom_iam.yaml +++ b/aws_datadog_iam/datadog_custom_iam.yaml @@ -5,9 +5,6 @@ Parameters: Type: String Description: The name of the IAM role that has been integrated with Datadog. Default: DatadogIntegrationManagedPolicy - CallerStackId: - Type: String - Description: The stack ID of the calling stack. Resources: LambdaExecutionRoleDatadogCustomIAM: @@ -47,14 +44,6 @@ Resources: Roles: - !Ref LambdaExecutionRoleDatadogCustomIAM - DatadogCustomIAMFunctionInvokePermission: - Type: AWS::Lambda::Permission - Properties: - FunctionName: !GetAtt DatadogCustomIAMFunction.Arn - Action: 'lambda:InvokeFunction' - Principal: "cloudformation.amazonaws.com" - SourceArn: !Ref CallerStackId - DatadogCustomIAMFunction: Type: AWS::Lambda::Function Properties: From 6ee299fd3e9b7661d1c778838df41e303fd2735b Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Fri, 23 Aug 2024 11:57:39 -0400 Subject: [PATCH 07/10] fix issues with the CCM template --- aws_ccm/datadog_ccm.yaml | 88 ++++++++++++++++++++++++++++++++++++---- 1 file changed, 80 insertions(+), 8 deletions(-) diff --git a/aws_ccm/datadog_ccm.yaml b/aws_ccm/datadog_ccm.yaml index aac478da..50b06bd7 100644 --- a/aws_ccm/datadog_ccm.yaml +++ b/aws_ccm/datadog_ccm.yaml @@ -13,10 +13,6 @@ Parameters: DDCloudCostReportName: Type: String Description: The name of the cost and usage report. - DatadogIntegrationRole: - Type: String - Description: The name of the IAM role that has been integrated with Datadog. - Default: DatadogIntegrationRole DatadogIntegrationManagedPolicy: Type: String Description: The name of the IAM policy used in the role that has been integrated with Datadog. @@ -83,13 +79,88 @@ Resources: - !Sub arn:aws:bcm-data-exports:us-east-1:${AWS::AccountId}:export/* aws:SourceAccount: !Sub ${AWS::AccountId} + # DDCURReportDefinition can only be done after the bucket is created + # and has the right policies. The bucket is only created if ShouldCreateCloudCostBucket + # is true. DependsOn does not work with Conditions, so we use a WaitCondition + # There are 2 wait handles, + # - BucketWaitHandle: for the bucket if bucket should be created + # - EmptyWaitHandle: for empty wait condition if bucket should not be created + # BucketWaitCondition to join them and be used as a dependency for DDCURReportDefinition + # More info here - https://garbe.io/blog/2017/07/17/cloudformation-hacks/ + BucketWaitHandle: + Condition: ShouldCreateCloudCostBucket + DependsOn: DDCloudCostBucketAccessPolicy + Type: "AWS::CloudFormation::WaitConditionHandle" + + EmptyWaitHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + + BucketWaitCondition: + Type: "AWS::CloudFormation::WaitCondition" + Properties: + Handle: !If [ShouldCreateCloudCostBucket, !Ref BucketWaitHandle, !Ref EmptyWaitHandle] + Timeout: "1" + Count: 0 + DDCURReportDefinition: - DependsOn: S3ClientBucketAccessPolicy + DependsOn: BucketWaitCondition Type: AWS::BCMDataExports::Export Properties: Export: DataQuery: - QueryStatement: "SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_resource_id, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment, split_line_item_actual_usage, split_line_item_net_split_cost, split_line_item_net_unused_cost, split_line_item_parent_resource_id, split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT" + QueryStatement: > + SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, + bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, + bill_payer_account_id, bill_payer_account_name, cost_category, discount, + discount_bundled_discount, discount_total_discount, identity_line_item_id, + identity_time_interval, line_item_availability_zone, line_item_blended_cost, + line_item_blended_rate, line_item_currency_code, line_item_legal_entity, + line_item_line_item_description, line_item_line_item_type, + line_item_net_unblended_cost, line_item_net_unblended_rate, + line_item_normalization_factor, line_item_normalized_usage_amount, + line_item_operation, line_item_product_code, line_item_resource_id, + line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, + line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, + line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, + pricing_currency, pricing_lease_contract_length, pricing_offering_class, + pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, + pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, + product_comment, product_fee_code, product_fee_description, product_from_location, + product_from_location_type, product_from_region_code, product_instance_family, + product_instance_type, product_instancesku, product_location, product_location_type, + product_operation, product_pricing_unit, product_product_family, product_region_code, + product_servicecode, product_sku, product_to_location, product_to_location_type, + product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, + reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, + reservation_effective_cost, reservation_end_time, reservation_modification_status, + reservation_net_amortized_upfront_cost_for_usage, + reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, + reservation_net_recurring_fee_for_usage, + reservation_net_unused_amortized_upfront_fee_for_billing_period, + reservation_net_unused_recurring_fee, reservation_net_upfront_value, + reservation_normalized_units_per_reservation, reservation_number_of_reservations, + reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, + reservation_start_time, reservation_subscription_id, + reservation_total_reserved_normalized_units, reservation_total_reserved_units, + reservation_units_per_reservation, + reservation_unused_amortized_upfront_fee_for_billing_period, + reservation_unused_normalized_unit_quantity, reservation_unused_quantity, + reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, + savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, + savings_plan_instance_type_family, + savings_plan_net_amortized_upfront_commitment_for_billing_period, + savings_plan_net_recurring_commitment_for_billing_period, + savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, + savings_plan_payment_option, savings_plan_purchase_term, + savings_plan_recurring_commitment_for_billing_period, savings_plan_region, + savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, + savings_plan_savings_plan_rate, savings_plan_start_time, + savings_plan_total_commitment_to_date, savings_plan_used_commitment, + split_line_item_actual_usage, split_line_item_net_split_cost, + split_line_item_net_unused_cost, split_line_item_parent_resource_id, + split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, + split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, + split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT TableConfigurations: COST_AND_USAGE_REPORT: INCLUDE_RESOURCES: "TRUE" @@ -102,8 +173,9 @@ Resources: S3Prefix: !Ref DDCloudCostBucketPrefix S3OutputConfigurations: Compression: "GZIP" - Format: "textORcsv" + Format: "TEXT_OR_CSV" Overwrite: "CREATE_NEW_REPORT" + OutputType: "CUSTOM" Name: !Ref DDCloudCostReportName RefreshCadence: - Frequency: "DAILY" \ No newline at end of file + Frequency: "SYNCHRONOUS" \ No newline at end of file From 2b72c628b73588189143a884072767875048432d Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Wed, 28 Aug 2024 09:54:30 -0400 Subject: [PATCH 08/10] remove custom IAM and add a ccm policy to the DD integration role --- aws_ccm/datadog_ccm.yaml | 53 ++++--- aws_datadog_iam/datadog_custom_iam.yaml | 175 ------------------------ 2 files changed, 35 insertions(+), 193 deletions(-) delete mode 100644 aws_datadog_iam/datadog_custom_iam.yaml diff --git a/aws_ccm/datadog_ccm.yaml b/aws_ccm/datadog_ccm.yaml index 50b06bd7..e9fa2425 100644 --- a/aws_ccm/datadog_ccm.yaml +++ b/aws_ccm/datadog_ccm.yaml @@ -6,17 +6,15 @@ Parameters: DDCloudCostBucketRegion: Type: String Description: The AWS region where the S3 bucket is located. - Default: us-east-1 DDCloudCostBucketPrefix: Type: String Description: The prefix to be added to the cost and usage report files in the S3 bucket. DDCloudCostReportName: Type: String Description: The name of the cost and usage report. - DatadogIntegrationManagedPolicy: + DatadogIntegrationRole: Type: String - Description: The name of the IAM policy used in the role that has been integrated with Datadog. - Default: DatadogIntegrationManagedPolicy + Description: The name of the IAM role that has been integrated with Datadog. CreateCloudCostBucket: Type: String Default: true @@ -32,21 +30,40 @@ Conditions: - true Resources: - DatadogCustomIAM: - Type: AWS::CloudFormation::Stack + DatadogCCMIAMPolicy: + Type: AWS::IAM::RolePolicy Properties: - TemplateURL: "https://.s3.amazonaws.com/aws_datatdog_iam//datadog_custom_iam.yaml" - Parameters: - DatadogIntegrationManagedPolicy: !Ref DatadogIntegrationManagedPolicy - - DatadogCCMCustomIAM: - Type: "Custom::DatadogCustomIAM" - Properties: - ServiceToken: !GetAtt DatadogCustomIAM.DatadogCustomIAMFunctionArn - DatadogIntegrationManagedPolicy: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/${DatadogIntegrationManagedPolicy}" - Product: CloudCost - ProductVariables: !Sub '{"DDCloudCostReportName": "${DDCloudCostReportName}", "DDCloudCostBucketPrefix": "${DDCloudCostBucketPrefix}", "DDCloudCostBucketName": "${DDCloudCostBucketName}" }' - + PolicyName: datadog-ccm-iam-policy + RoleName: !Ref DatadogIntegrationRole + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: ReadCloudCostBucket + Effect: Allow + Action: + - s3:ListBucket + Resource: !Sub "arn:aws:s3:::${DDCloudCostBucketName}" + - Sid: GetBill + Effect: Allow + Action: + - s3:GetObject + Resource: !Sub "arn:aws:s3:::${DDCloudCostBucketName}/${DDCloudCostBucketPrefix}/${DDCloudCostReportName}/*" + - Sid: CheckAccuracy + Effect: Allow + Action: + - ce:Get* + Resource: "*" + - Sid: ListCURs + Effect: Allow + Action: + - cur:DescribeReportDefinitions + Resource: "*" + - Sid: ListOrganizations + Effect: Allow + Action: + - organizations:Describe* + - organizations:List* + Resource: "*" DDCloudCostBucket: Condition: ShouldCreateCloudCostBucket Type: AWS::S3::Bucket diff --git a/aws_datadog_iam/datadog_custom_iam.yaml b/aws_datadog_iam/datadog_custom_iam.yaml deleted file mode 100644 index 76636ade..00000000 --- a/aws_datadog_iam/datadog_custom_iam.yaml +++ /dev/null @@ -1,175 +0,0 @@ -AWSTemplateFormatVersion: 2010-09-09 -Description: Datadog AWS Integration Custom IAM Resource -Parameters: - DatadogIntegrationManagedPolicy: - Type: String - Description: The name of the IAM role that has been integrated with Datadog. - Default: DatadogIntegrationManagedPolicy - -Resources: - LambdaExecutionRoleDatadogCustomIAM: - Type: AWS::IAM::Role - Properties: - AssumeRolePolicyDocument: - Version: '2012-10-17' - Statement: - - Effect: Allow - Principal: - Service: - - lambda.amazonaws.com - Action: - - sts:AssumeRole - Path: "/" - ManagedPolicyArns: - - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" - - LambdaExecutionRolePolicyDatadogCustomIAMCall: - Type: AWS::IAM::Policy - Properties: - PolicyName: "dd-custom-iam-policy" - PolicyDocument: - Version: "2012-10-17" - Statement: - - Effect: "Allow" - Action: - - "iam:GetPolicy" - - "iam:GetPolicyVersion" - - "iam:CreatePolicyVersion" - - "iam:DeletePolicyVersion" - Resource: !Sub "arn:${AWS::Partition}:iam::${AWS::AccountId}:policy/${DatadogIntegrationManagedPolicy}" - Condition: - StringEquals: - "aws:userid": - !Join [":", [!GetAtt LambdaExecutionRoleDatadogCustomIAM.RoleId, !Ref DatadogCustomIAMFunction]] - Roles: - - !Ref LambdaExecutionRoleDatadogCustomIAM - - DatadogCustomIAMFunction: - Type: AWS::Lambda::Function - Properties: - Role: !GetAtt LambdaExecutionRoleDatadogCustomIAM.Arn - Description: "A function to modify the IAM role integrated with Datadog." - Handler: "index.handler" - Runtime: "python3.11" - Timeout: 30 - Code: - ZipFile: | - import boto3 - import cfnresponse - import json - import secrets - - def getCCMPolicyStatement(product_variables): - return [ - { - "Sid": "ReadCloudCostBucket", - "Effect": "Allow", - "Action": [ - "s3:ListBucket" - ], - "Resource": f"arn:aws:s3:::{product_variables['DDCloudCostBucketName']}" - }, - { - "Sid": "GetBill", - "Effect": "Allow", - "Action": [ - "s3:GetObject" - ], - "Resource": f"arn:aws:s3:::{product_variables['DDCloudCostBucketName']}/{product_variables['DDCloudCostBucketPrefix']}/{product_variables['DDCloudCostReportName']}/*" - }, - { - "Sid": "CheckAccuracy", - "Effect": "Allow", - "Action": [ - "ce:Get*" - ], - "Resource": "*" - }, - { - "Sid": "ListCURs", - "Effect": "Allow", - "Action": [ - "cur:DescribeReportDefinitions" - ], - "Resource": "*" - }, - { - "Sid": "ListOrganizations", - "Effect": "Allow", - "Action": [ - "organizations:Describe*", - "organizations:List*" - ], - "Resource": "*" - } - ] - - PRODUCT_TO_STATEMENTS = { - "CloudCost": { - "statementsGenerator": getCCMPolicyStatement, - "variables": ["DDCloudCostBucketName", "DDCloudCostBucketPrefix", "DDCloudCostReportName"] - } - } - - def modify_policy(policy_json, method, statements, logical_resource_id, product): - policy_statements = policy_json['Statement'] - policy_statements_by_id = {i['Sid']:i for i in policy_statements} - if method == 'Update' or method == 'Create': - for statement in statements: - policy_statements_by_id[statement['Sid']] = statement - elif method == 'Delete': - statement_ids = list(policy_statements_by_id.keys()) - for statement_id in statement_ids: - if statement_id.startswith(f"{logical_resource_id}{product}"): - del policy_statements_by_id[statement_id] - else: - raise ValueError('Invalid method') - policy_json['Statement'] = list(policy_statements_by_id.values()) - return policy_json - - def prepare_statements(product, logical_resource_id, product_variables): - product_data = PRODUCT_TO_STATEMENTS.get(product) - statements = product_data["statementsGenerator"](product_variables) - for statement in statements: - statement['Sid'] = f"{logical_resource_id}{product}{statement.get('Sid', secrets.token_hex(nbytes=3))}" - return statements - - def get_policy(iam, policy_arn): - resp = iam.get_policy(PolicyArn=policy_arn) - version = resp["Policy"]["DefaultVersionId"] - policy_resp = iam.get_policy_version(PolicyArn=policy_arn, VersionId=version) - return (policy_resp["PolicyVersion"]["Document"], version) - - def validate_variables(product_variables, product): - if not all([var in product_variables for var in PRODUCT_TO_STATEMENTS[product]['variables']]): - raise ValueError('Invalid product variables, missing required keys') - - def modify_iam(event, context): - method = event['RequestType'] - policy_arn = event['ResourceProperties']['DatadogIntegrationManagedPolicy'] - product = event['ResourceProperties']['Product'] - product_variables = json.loads(event['ResourceProperties']['ProductVariables']) - logical_resource_id = event['LogicalResourceId'] - - validate_variables(product_variables, product) - statements = prepare_statements(product, logical_resource_id, product_variables) - - iam = boto3.client('iam') - policy_json, policy_version = get_policy(iam, policy_arn) - modified_policy_json = modify_policy(policy_json, method, statements, logical_resource_id, product) - iam.create_policy_version(PolicyArn=policy_arn, PolicyDocument=json.dumps(modified_policy_json), SetAsDefault=True) - iam.delete_policy_version(PolicyArn=policy_arn, VersionId=policy_version) - - def handler(event, context): - print(event) - try: - modify_iam(event, context) - cfnresponse.send(event, context, cfnresponse.SUCCESS, {}, context.log_stream_name) - except Exception as e: - print(e) - cfnresponse.send(event, context, cfnresponse.FAILED, {}, context.log_stream_name) - -Outputs: - DatadogCustomIAMFunctionArn: - Description: "The ARN of the Lambda function to modify the IAM role integrated with Datadog." - Value: !GetAtt DatadogCustomIAMFunction.Arn \ No newline at end of file From a6fdf33f78f6dcf53c3b0835a5c4649238fc2714 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Wed, 28 Aug 2024 11:08:45 -0400 Subject: [PATCH 09/10] add readme and release script --- aws_cloud_cost/README.md | 26 ++++++++++ .../datadog_cloud_cost_setup.yaml | 3 +- aws_cloud_cost/release.sh | 48 +++++++++++++++++++ 3 files changed, 76 insertions(+), 1 deletion(-) create mode 100644 aws_cloud_cost/README.md rename aws_ccm/datadog_ccm.yaml => aws_cloud_cost/datadog_cloud_cost_setup.yaml (98%) create mode 100755 aws_cloud_cost/release.sh diff --git a/aws_cloud_cost/README.md b/aws_cloud_cost/README.md new file mode 100644 index 00000000..67912098 --- /dev/null +++ b/aws_cloud_cost/README.md @@ -0,0 +1,26 @@ +# Datadog AWS Cloud Cost Setup + +[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-cloud-cost-setup&templateURL=https://datadog-cloudformation-template.s3.amazonaws.com/aws/datadog_cloud_cost_setup.yaml) + +## AWS Resources + +This template creates the following AWS resources required for setting up Cloud Cost Management in Datadog. + +- An S3 Bucket (if not using an existing one) + - With Bucket policies +- Cost and Usage Report using the COST_AND_USAGE_REPORT table configurations +- IAM policy needed to access the bucket and the CUR + - Attaches the policy to the main Datadog integration role + +## Publishing the template +Use the release script to upload the template to a S3 bucket following the example below. Make sure you have correct access credentials before launching the script. + +``` +./release +``` + +Use an optional argument `--private` to prevent granting public access to the uploaded template (good for testing purposes). +The uploaded template file can be found at `/aws/datadog_cloud_cost_setup.yaml` key on the chosen S3 bucket. + + + diff --git a/aws_ccm/datadog_ccm.yaml b/aws_cloud_cost/datadog_cloud_cost_setup.yaml similarity index 98% rename from aws_ccm/datadog_ccm.yaml rename to aws_cloud_cost/datadog_cloud_cost_setup.yaml index e9fa2425..422356de 100644 --- a/aws_ccm/datadog_ccm.yaml +++ b/aws_cloud_cost/datadog_cloud_cost_setup.yaml @@ -21,7 +21,7 @@ Parameters: AllowedValues: - true - false - Description: Whether to create the S3 bucket for storing the cost and usage reports. + Description: Should the S3 bucket for storing the cost and usage reports be created, or use an existing bucket. Conditions: ShouldCreateCloudCostBucket: @@ -64,6 +64,7 @@ Resources: - organizations:Describe* - organizations:List* Resource: "*" + DDCloudCostBucket: Condition: ShouldCreateCloudCostBucket Type: AWS::S3::Bucket diff --git a/aws_cloud_cost/release.sh b/aws_cloud_cost/release.sh new file mode 100755 index 00000000..fcbf6418 --- /dev/null +++ b/aws_cloud_cost/release.sh @@ -0,0 +1,48 @@ +#!/usr/bin/env bash + +# Usage: ./release.sh + +set -e + +# Read the S3 bucket +if [ -z "$1" ]; then + echo "Must specify a S3 bucket to publish the template" + exit 1 +else + BUCKET=$1 +fi + +# Upload templates to a private bucket -- useful for testing +if [[ $# -eq 2 ]] && [[ $2 = "--private" ]]; then + PRIVATE_TEMPLATE=true +else + PRIVATE_TEMPLATE=false +fi + +# Confirm to proceed +for i in *.yaml; do + [ -f "$i" ] || break + echo "About to upload $i to s3://${BUCKET}/aws/$i" +done +read -p "Continue (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi + +# Update bucket placeholder +cp datadog_cloud_cost_setup.yaml datadog_cloud_cost_setup.yaml.bak +perl -pi -e "s//${BUCKET}/g" datadog_cloud_cost_setup.yaml +trap 'mv datadog_cloud_cost_setup.yaml.bak datadog_cloud_cost_setup.yaml' EXIT + +# Upload +if [ "$PRIVATE_TEMPLATE" = true ] ; then + aws s3 cp . s3://${BUCKET}/aws --recursive --exclude "*" --include "*.yaml" +else + aws s3 cp . s3://${BUCKET}/aws --recursive --exclude "*" --include "*.yaml" \ + --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers +fi +echo "Done uploading the template, and here is the CloudFormation quick launch URL" +echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-cloud-cost-setup&templateURL=https://${BUCKET}.s3.amazonaws.com/aws/datadog_cloud_cost_setup.yaml" + +echo "Done!" From 513145b91faf28e74ea3442eaa8081997382a886 Mon Sep 17 00:00:00 2001 From: Ashwin Venkatesan Date: Wed, 28 Aug 2024 12:45:59 -0400 Subject: [PATCH 10/10] add release version and update release script --- aws_cloud_cost/README.md | 4 +- ...atadog_cloud_cost_setup.yaml => main.yaml} | 4 +- aws_cloud_cost/release.sh | 43 +++++++++++++++---- aws_cloud_cost/version.txt | 1 + 4 files changed, 41 insertions(+), 11 deletions(-) rename aws_cloud_cost/{datadog_cloud_cost_setup.yaml => main.yaml} (98%) create mode 100644 aws_cloud_cost/version.txt diff --git a/aws_cloud_cost/README.md b/aws_cloud_cost/README.md index 67912098..671ad648 100644 --- a/aws_cloud_cost/README.md +++ b/aws_cloud_cost/README.md @@ -1,6 +1,6 @@ # Datadog AWS Cloud Cost Setup -[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-cloud-cost-setup&templateURL=https://datadog-cloudformation-template.s3.amazonaws.com/aws/datadog_cloud_cost_setup.yaml) +[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://datadog-cloudformation-template.s3.amazonaws.com/aws_cloud_cost/v0.0.1/main.yaml) ## AWS Resources @@ -20,7 +20,7 @@ Use the release script to upload the template to a S3 bucket following the examp ``` Use an optional argument `--private` to prevent granting public access to the uploaded template (good for testing purposes). -The uploaded template file can be found at `/aws/datadog_cloud_cost_setup.yaml` key on the chosen S3 bucket. +The uploaded template file can be found at `/aws_cloud_cost/main.yaml` key on the chosen S3 bucket. diff --git a/aws_cloud_cost/datadog_cloud_cost_setup.yaml b/aws_cloud_cost/main.yaml similarity index 98% rename from aws_cloud_cost/datadog_cloud_cost_setup.yaml rename to aws_cloud_cost/main.yaml index 422356de..b967a04b 100644 --- a/aws_cloud_cost/datadog_cloud_cost_setup.yaml +++ b/aws_cloud_cost/main.yaml @@ -1,4 +1,6 @@ - +# version: +AWSTemplateFormatVersion: 2010-09-09 +Description: Datadog Cloud Cost Management Setup Parameters: DDCloudCostBucketName: Type: String diff --git a/aws_cloud_cost/release.sh b/aws_cloud_cost/release.sh index fcbf6418..b07d246a 100755 --- a/aws_cloud_cost/release.sh +++ b/aws_cloud_cost/release.sh @@ -1,4 +1,4 @@ -#!/usr/bin/env bash +#!/bin/bash # Usage: ./release.sh @@ -12,6 +12,31 @@ else BUCKET=$1 fi +# Read the version +VERSION=$(head -n 1 version.txt) + +# Confirm the bucket for the current release doesn't already exist so we don't overwrite it +set +e +EXIT_CODE=0 +response=$(aws s3api head-object \ + --bucket "${BUCKET}" \ + --key "aws_cloud_cost/${VERSION}/main.yaml" > /dev/null 2>&1) + +if [[ ${?} -eq 0 ]]; then + echo "S3 bucket path ${BUCKET}/aws_cloud_cost/${VERSION} already exists. Please up the version." + exit 1 +fi +set -e + +# Confirm that the readme link points to the current release +set +e +response=$(grep -q "aws_cloud_cost/${VERSION}/main.yaml" README.md) +if [[ ${?} -ne 0 ]]; then + echo "README.md does not point to the current release. Please update the link." + exit 1 +fi +set -e + # Upload templates to a private bucket -- useful for testing if [[ $# -eq 2 ]] && [[ $2 = "--private" ]]; then PRIVATE_TEMPLATE=true @@ -22,7 +47,7 @@ fi # Confirm to proceed for i in *.yaml; do [ -f "$i" ] || break - echo "About to upload $i to s3://${BUCKET}/aws/$i" + echo "About to upload $i to s3://${BUCKET}/aws_cloud_cost/${VERSION}/$i" done read -p "Continue (y/n)?" CONT if [ "$CONT" != "y" ]; then @@ -31,18 +56,20 @@ if [ "$CONT" != "y" ]; then fi # Update bucket placeholder -cp datadog_cloud_cost_setup.yaml datadog_cloud_cost_setup.yaml.bak -perl -pi -e "s//${BUCKET}/g" datadog_cloud_cost_setup.yaml -trap 'mv datadog_cloud_cost_setup.yaml.bak datadog_cloud_cost_setup.yaml' EXIT +cp main.yaml main.yaml.bak +perl -pi -e "s//${BUCKET}/g" main.yaml +perl -pi -e "s//${VERSION}/g" main.yaml + +trap 'mv main.yaml.bak main.yaml' EXIT # Upload if [ "$PRIVATE_TEMPLATE" = true ] ; then - aws s3 cp . s3://${BUCKET}/aws --recursive --exclude "*" --include "*.yaml" + aws s3 cp . s3://${BUCKET}/aws_cloud_cost/${VERSION} --recursive --exclude "*" --include "*.yaml" else - aws s3 cp . s3://${BUCKET}/aws --recursive --exclude "*" --include "*.yaml" \ + aws s3 cp . s3://${BUCKET}/aws_cloud_cost/${VERSION} --recursive --exclude "*" --include "*.yaml" \ --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers fi echo "Done uploading the template, and here is the CloudFormation quick launch URL" -echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-cloud-cost-setup&templateURL=https://${BUCKET}.s3.amazonaws.com/aws/datadog_cloud_cost_setup.yaml" +echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://${BUCKET}.s3.amazonaws.com/aws_cloud_cost/${VERSION}/main.yaml" echo "Done!" diff --git a/aws_cloud_cost/version.txt b/aws_cloud_cost/version.txt new file mode 100644 index 00000000..95e94cdd --- /dev/null +++ b/aws_cloud_cost/version.txt @@ -0,0 +1 @@ +v0.0.1 \ No newline at end of file