add cfn custom iam resource and CCM specific template

aws_cloud_cost/README.md

@@ -0,0 +1,26 @@
+# Datadog AWS Cloud Cost Setup
+
+[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://datadog-cloudformation-template.s3.amazonaws.com/aws_cloud_cost/v0.0.1/main.yaml)
+
+## AWS Resources
+
+This template creates the following AWS resources required for setting up Cloud Cost Management in Datadog.
+
+- An S3 Bucket (if not using an existing one)
+    - With Bucket policies
+- Cost and Usage Report using the COST_AND_USAGE_REPORT table configurations
+- IAM policy needed to access the bucket and the CUR
+    - Attaches the policy to the main Datadog integration role
+
+## Publishing the template
+Use the release script to upload the template to a S3 bucket following the example below. Make sure you have correct access credentials before launching the script.
+
+```
+./release <bucket_name>
+```
+
+Use an optional argument `--private` to prevent granting public access to the uploaded template (good for testing purposes). 
+The uploaded template file can be found at `/aws_cloud_cost/main.yaml` key on the chosen S3 bucket.
+
+
+

aws_cloud_cost/main.yaml

@@ -0,0 +1,201 @@
+# version: <VERSION_PLACEHOLDER>
+AWSTemplateFormatVersion: 2010-09-09
+Description: Datadog Cloud Cost Management Setup
+Parameters:
+  DDCloudCostBucketName:
+    Type: String
+    Description: The name of the S3 bucket for storing the cost and usage reports.
+  DDCloudCostBucketRegion:
+    Type: String
+    Description: The AWS region where the S3 bucket is located.
+  DDCloudCostBucketPrefix:
+    Type: String
+    Description: The prefix to be added to the cost and usage report files in the S3 bucket.
+  DDCloudCostReportName:
+    Type: String
+    Description: The name of the cost and usage report.
+  DatadogIntegrationRole:
+    Type: String
+    Description: The name of the IAM role that has been integrated with Datadog.
+  CreateCloudCostBucket:
+    Type: String
+    Default: true
+    AllowedValues:
+      - true
+      - false
+    Description: Should the S3 bucket for storing the cost and usage reports be created, or use an existing bucket.
+
+Conditions:
+  ShouldCreateCloudCostBucket:
+    Fn::Equals:
+      - Ref: CreateCloudCostBucket
+      - true
+
+Resources:
+  DatadogCCMIAMPolicy:
+    Type: AWS::IAM::RolePolicy
+    Properties:
+      PolicyName: datadog-ccm-iam-policy
+      RoleName: !Ref DatadogIntegrationRole
+      PolicyDocument:
+        Version: "2012-10-17"
+        Statement:
+          - Sid: ReadCloudCostBucket
+            Effect: Allow
+            Action:
+              - s3:ListBucket
+            Resource: !Sub "arn:aws:s3:::${DDCloudCostBucketName}"
+          - Sid: GetBill
+            Effect: Allow
+            Action:
+              - s3:GetObject
+            Resource: !Sub "arn:aws:s3:::${DDCloudCostBucketName}/${DDCloudCostBucketPrefix}/${DDCloudCostReportName}/*"
+          - Sid: CheckAccuracy
+            Effect: Allow
+            Action:
+              - ce:Get*
+            Resource: "*"
+          - Sid: ListCURs
+            Effect: Allow
+            Action:
+              - cur:DescribeReportDefinitions
+            Resource: "*"
+          - Sid: ListOrganizations
+            Effect: Allow
+            Action:
+              - organizations:Describe*
+              - organizations:List*
+            Resource: "*"
+
+  DDCloudCostBucket:
+    Condition: ShouldCreateCloudCostBucket
+    Type: AWS::S3::Bucket
+    Properties:
+      BucketName: !Ref DDCloudCostBucketName
+
+  DDCloudCostBucketAccessPolicy:
+    Condition: ShouldCreateCloudCostBucket
+    Type: AWS::S3::BucketPolicy
+    Properties:
+      Bucket: !Ref DDCloudCostBucket
+      PolicyDocument:
+        Version: '2012-10-17'
+        Statement:
+          - Effect: Allow
+            Principal:
+              Service:
+                - billingreports.amazonaws.com
+                - bcm-data-exports.amazonaws.com
+            Action:
+              - s3:PutObject
+              - s3:GetBucketPolicy
+            Resource:
+              - !Sub ${DDCloudCostBucket.Arn}
+              - !Sub ${DDCloudCostBucket.Arn}/*
+            Condition:
+              StringLike:
+                aws:SourceArn:
+                  - !Sub arn:aws:cur:us-east-1:${AWS::AccountId}:definition/*
+                  - !Sub arn:aws:bcm-data-exports:us-east-1:${AWS::AccountId}:export/*
+                aws:SourceAccount: !Sub ${AWS::AccountId}
+
+  # DDCURReportDefinition can only be done after the bucket is created 
+  # and has the right policies. The bucket is only created if ShouldCreateCloudCostBucket
+  # is true. DependsOn does not work with Conditions, so we use a WaitCondition
+  # There are 2 wait handles, 
+  # - BucketWaitHandle: for the bucket if bucket should be created
+  # - EmptyWaitHandle: for empty wait condition if bucket should not be created
+  # BucketWaitCondition to join them and be used as a dependency for DDCURReportDefinition
+  # More info here - https://garbe.io/blog/2017/07/17/cloudformation-hacks/
+  BucketWaitHandle: 
+    Condition: ShouldCreateCloudCostBucket
+    DependsOn: DDCloudCostBucketAccessPolicy
+    Type: "AWS::CloudFormation::WaitConditionHandle"
+
+  EmptyWaitHandle: 
+    Type: "AWS::CloudFormation::WaitConditionHandle"
+
+  BucketWaitCondition: 
+    Type: "AWS::CloudFormation::WaitCondition"
+    Properties: 
+      Handle: !If [ShouldCreateCloudCostBucket, !Ref BucketWaitHandle, !Ref EmptyWaitHandle]
+      Timeout: "1"
+      Count: 0
+
+  DDCURReportDefinition:
+    DependsOn: BucketWaitCondition
+    Type: AWS::BCMDataExports::Export
+    Properties:
+      Export: 
+        DataQuery:
+          QueryStatement: >
+            SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date,
+            bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity,
+            bill_payer_account_id, bill_payer_account_name, cost_category, discount, 
+            discount_bundled_discount, discount_total_discount, identity_line_item_id, 
+            identity_time_interval, line_item_availability_zone, line_item_blended_cost, 
+            line_item_blended_rate, line_item_currency_code, line_item_legal_entity, 
+            line_item_line_item_description, line_item_line_item_type, 
+            line_item_net_unblended_cost, line_item_net_unblended_rate, 
+            line_item_normalization_factor, line_item_normalized_usage_amount, 
+            line_item_operation, line_item_product_code, line_item_resource_id, 
+            line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, 
+            line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, 
+            line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type,
+            pricing_currency, pricing_lease_contract_length, pricing_offering_class, 
+            pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, 
+            pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, 
+            product_comment, product_fee_code, product_fee_description, product_from_location, 
+            product_from_location_type, product_from_region_code, product_instance_family, 
+            product_instance_type, product_instancesku, product_location, product_location_type, 
+            product_operation, product_pricing_unit, product_product_family, product_region_code, 
+            product_servicecode, product_sku, product_to_location, product_to_location_type, 
+            product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, 
+            reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, 
+            reservation_effective_cost, reservation_end_time, reservation_modification_status, 
+            reservation_net_amortized_upfront_cost_for_usage, 
+            reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, 
+            reservation_net_recurring_fee_for_usage, 
+            reservation_net_unused_amortized_upfront_fee_for_billing_period, 
+            reservation_net_unused_recurring_fee, reservation_net_upfront_value, 
+            reservation_normalized_units_per_reservation, reservation_number_of_reservations, 
+            reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, 
+            reservation_start_time, reservation_subscription_id, 
+            reservation_total_reserved_normalized_units, reservation_total_reserved_units, 
+            reservation_units_per_reservation, 
+            reservation_unused_amortized_upfront_fee_for_billing_period, 
+            reservation_unused_normalized_unit_quantity, reservation_unused_quantity, 
+            reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, 
+            savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, 
+            savings_plan_instance_type_family, 
+            savings_plan_net_amortized_upfront_commitment_for_billing_period, 
+            savings_plan_net_recurring_commitment_for_billing_period, 
+            savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, 
+            savings_plan_payment_option, savings_plan_purchase_term, 
+            savings_plan_recurring_commitment_for_billing_period, savings_plan_region, 
+            savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, 
+            savings_plan_savings_plan_rate, savings_plan_start_time, 
+            savings_plan_total_commitment_to_date, savings_plan_used_commitment, 
+            split_line_item_actual_usage, split_line_item_net_split_cost, 
+            split_line_item_net_unused_cost, split_line_item_parent_resource_id, 
+            split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, 
+            split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, 
+            split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT
+          TableConfigurations: 
+            COST_AND_USAGE_REPORT: 
+              INCLUDE_RESOURCES: "TRUE"
+              INCLUDE_SPLIT_COST_ALLOCATION_DATA: "TRUE"
+              TIME_GRANULARITY: "HOURLY"
+        DestinationConfigurations:
+          S3Destination:
+            S3Bucket: !Ref DDCloudCostBucketName
+            S3Region: !Ref DDCloudCostBucketRegion
+            S3Prefix: !Ref DDCloudCostBucketPrefix
+            S3OutputConfigurations:
+              Compression: "GZIP"
+              Format: "TEXT_OR_CSV"
+              Overwrite: "CREATE_NEW_REPORT"
+              OutputType: "CUSTOM"
+        Name: !Ref DDCloudCostReportName
+        RefreshCadence: 
+          Frequency: "SYNCHRONOUS"

aws_cloud_cost/release.sh

@@ -0,0 +1,75 @@
+#!/bin/bash
+
+# Usage: ./release.sh <S3_Bucket>
+
+set -e
+
+# Read the S3 bucket
+if [ -z "$1" ]; then
+    echo "Must specify a S3 bucket to publish the template"
+    exit 1
+else
+    BUCKET=$1
+fi
+
+# Read the version
+VERSION=$(head -n 1 version.txt)
+
+# Confirm the bucket for the current release doesn't already exist so we don't overwrite it
+set +e
+EXIT_CODE=0
+response=$(aws s3api head-object \
+    --bucket "${BUCKET}" \
+    --key "aws_cloud_cost/${VERSION}/main.yaml" > /dev/null 2>&1)
+
+if [[ ${?} -eq 0 ]]; then
+    echo "S3 bucket path ${BUCKET}/aws_cloud_cost/${VERSION} already exists. Please up the version."
+    exit 1
+fi
+set -e
+
+# Confirm that the readme link points to the current release
+set +e
+response=$(grep -q "aws_cloud_cost/${VERSION}/main.yaml" README.md)
+if [[ ${?} -ne 0 ]]; then
+    echo "README.md does not point to the current release. Please update the link."
+    exit 1
+fi
+set -e
+
+# Upload templates to a private bucket -- useful for testing
+if [[ $# -eq 2 ]] && [[ $2 = "--private" ]]; then
+    PRIVATE_TEMPLATE=true
+else
+    PRIVATE_TEMPLATE=false
+fi
+
+# Confirm to proceed
+for i in *.yaml; do
+    [ -f "$i" ] || break
+    echo "About to upload $i to s3://${BUCKET}/aws_cloud_cost/${VERSION}/$i"
+done
+read -p "Continue (y/n)?" CONT
+if [ "$CONT" != "y" ]; then
+  echo "Exiting"
+  exit 1
+fi
+
+# Update bucket placeholder
+cp main.yaml main.yaml.bak
+perl -pi -e "s/<BUCKET_PLACEHOLDER>/${BUCKET}/g" main.yaml
+perl -pi -e "s/<VERSION_PLACEHOLDER>/${VERSION}/g" main.yaml
+
+trap 'mv main.yaml.bak main.yaml' EXIT
+
+# Upload
+if [ "$PRIVATE_TEMPLATE" = true ] ; then
+    aws s3 cp . s3://${BUCKET}/aws_cloud_cost/${VERSION} --recursive --exclude "*" --include "*.yaml"
+else
+    aws s3 cp . s3://${BUCKET}/aws_cloud_cost/${VERSION} --recursive --exclude "*" --include "*.yaml" \
+        --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers
+fi
+echo "Done uploading the template, and here is the CloudFormation quick launch URL"
+echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://${BUCKET}.s3.amazonaws.com/aws_cloud_cost/${VERSION}/main.yaml"
+
+echo "Done!"

aws_cloud_cost/version.txt

@@ -0,0 +1 @@
+v0.0.1