diff --git a/aws_cloud_cost/README.md b/aws_cloud_cost/README.md new file mode 100644 index 00000000..671ad648 --- /dev/null +++ b/aws_cloud_cost/README.md @@ -0,0 +1,26 @@ +# Datadog AWS Cloud Cost Setup + +[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://datadog-cloudformation-template.s3.amazonaws.com/aws_cloud_cost/v0.0.1/main.yaml) + +## AWS Resources + +This template creates the following AWS resources required for setting up Cloud Cost Management in Datadog. + +- An S3 Bucket (if not using an existing one) + - With Bucket policies +- Cost and Usage Report using the COST_AND_USAGE_REPORT table configurations +- IAM policy needed to access the bucket and the CUR + - Attaches the policy to the main Datadog integration role + +## Publishing the template +Use the release script to upload the template to a S3 bucket following the example below. Make sure you have correct access credentials before launching the script. + +``` +./release +``` + +Use an optional argument `--private` to prevent granting public access to the uploaded template (good for testing purposes). +The uploaded template file can be found at `/aws_cloud_cost/main.yaml` key on the chosen S3 bucket. + + + diff --git a/aws_cloud_cost/main.yaml b/aws_cloud_cost/main.yaml new file mode 100644 index 00000000..b967a04b --- /dev/null +++ b/aws_cloud_cost/main.yaml @@ -0,0 +1,201 @@ +# version: +AWSTemplateFormatVersion: 2010-09-09 +Description: Datadog Cloud Cost Management Setup +Parameters: + DDCloudCostBucketName: + Type: String + Description: The name of the S3 bucket for storing the cost and usage reports. + DDCloudCostBucketRegion: + Type: String + Description: The AWS region where the S3 bucket is located. + DDCloudCostBucketPrefix: + Type: String + Description: The prefix to be added to the cost and usage report files in the S3 bucket. + DDCloudCostReportName: + Type: String + Description: The name of the cost and usage report. + DatadogIntegrationRole: + Type: String + Description: The name of the IAM role that has been integrated with Datadog. + CreateCloudCostBucket: + Type: String + Default: true + AllowedValues: + - true + - false + Description: Should the S3 bucket for storing the cost and usage reports be created, or use an existing bucket. + +Conditions: + ShouldCreateCloudCostBucket: + Fn::Equals: + - Ref: CreateCloudCostBucket + - true + +Resources: + DatadogCCMIAMPolicy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyName: datadog-ccm-iam-policy + RoleName: !Ref DatadogIntegrationRole + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: ReadCloudCostBucket + Effect: Allow + Action: + - s3:ListBucket + Resource: !Sub "arn:aws:s3:::${DDCloudCostBucketName}" + - Sid: GetBill + Effect: Allow + Action: + - s3:GetObject + Resource: !Sub "arn:aws:s3:::${DDCloudCostBucketName}/${DDCloudCostBucketPrefix}/${DDCloudCostReportName}/*" + - Sid: CheckAccuracy + Effect: Allow + Action: + - ce:Get* + Resource: "*" + - Sid: ListCURs + Effect: Allow + Action: + - cur:DescribeReportDefinitions + Resource: "*" + - Sid: ListOrganizations + Effect: Allow + Action: + - organizations:Describe* + - organizations:List* + Resource: "*" + + DDCloudCostBucket: + Condition: ShouldCreateCloudCostBucket + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref DDCloudCostBucketName + + DDCloudCostBucketAccessPolicy: + Condition: ShouldCreateCloudCostBucket + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref DDCloudCostBucket + PolicyDocument: + Version: '2012-10-17' + Statement: + - Effect: Allow + Principal: + Service: + - billingreports.amazonaws.com + - bcm-data-exports.amazonaws.com + Action: + - s3:PutObject + - s3:GetBucketPolicy + Resource: + - !Sub ${DDCloudCostBucket.Arn} + - !Sub ${DDCloudCostBucket.Arn}/* + Condition: + StringLike: + aws:SourceArn: + - !Sub arn:aws:cur:us-east-1:${AWS::AccountId}:definition/* + - !Sub arn:aws:bcm-data-exports:us-east-1:${AWS::AccountId}:export/* + aws:SourceAccount: !Sub ${AWS::AccountId} + + # DDCURReportDefinition can only be done after the bucket is created + # and has the right policies. The bucket is only created if ShouldCreateCloudCostBucket + # is true. DependsOn does not work with Conditions, so we use a WaitCondition + # There are 2 wait handles, + # - BucketWaitHandle: for the bucket if bucket should be created + # - EmptyWaitHandle: for empty wait condition if bucket should not be created + # BucketWaitCondition to join them and be used as a dependency for DDCURReportDefinition + # More info here - https://garbe.io/blog/2017/07/17/cloudformation-hacks/ + BucketWaitHandle: + Condition: ShouldCreateCloudCostBucket + DependsOn: DDCloudCostBucketAccessPolicy + Type: "AWS::CloudFormation::WaitConditionHandle" + + EmptyWaitHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + + BucketWaitCondition: + Type: "AWS::CloudFormation::WaitCondition" + Properties: + Handle: !If [ShouldCreateCloudCostBucket, !Ref BucketWaitHandle, !Ref EmptyWaitHandle] + Timeout: "1" + Count: 0 + + DDCURReportDefinition: + DependsOn: BucketWaitCondition + Type: AWS::BCMDataExports::Export + Properties: + Export: + DataQuery: + QueryStatement: > + SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, + bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, + bill_payer_account_id, bill_payer_account_name, cost_category, discount, + discount_bundled_discount, discount_total_discount, identity_line_item_id, + identity_time_interval, line_item_availability_zone, line_item_blended_cost, + line_item_blended_rate, line_item_currency_code, line_item_legal_entity, + line_item_line_item_description, line_item_line_item_type, + line_item_net_unblended_cost, line_item_net_unblended_rate, + line_item_normalization_factor, line_item_normalized_usage_amount, + line_item_operation, line_item_product_code, line_item_resource_id, + line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, + line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, + line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, + pricing_currency, pricing_lease_contract_length, pricing_offering_class, + pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, + pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, + product_comment, product_fee_code, product_fee_description, product_from_location, + product_from_location_type, product_from_region_code, product_instance_family, + product_instance_type, product_instancesku, product_location, product_location_type, + product_operation, product_pricing_unit, product_product_family, product_region_code, + product_servicecode, product_sku, product_to_location, product_to_location_type, + product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, + reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, + reservation_effective_cost, reservation_end_time, reservation_modification_status, + reservation_net_amortized_upfront_cost_for_usage, + reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, + reservation_net_recurring_fee_for_usage, + reservation_net_unused_amortized_upfront_fee_for_billing_period, + reservation_net_unused_recurring_fee, reservation_net_upfront_value, + reservation_normalized_units_per_reservation, reservation_number_of_reservations, + reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, + reservation_start_time, reservation_subscription_id, + reservation_total_reserved_normalized_units, reservation_total_reserved_units, + reservation_units_per_reservation, + reservation_unused_amortized_upfront_fee_for_billing_period, + reservation_unused_normalized_unit_quantity, reservation_unused_quantity, + reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, + savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, + savings_plan_instance_type_family, + savings_plan_net_amortized_upfront_commitment_for_billing_period, + savings_plan_net_recurring_commitment_for_billing_period, + savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, + savings_plan_payment_option, savings_plan_purchase_term, + savings_plan_recurring_commitment_for_billing_period, savings_plan_region, + savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, + savings_plan_savings_plan_rate, savings_plan_start_time, + savings_plan_total_commitment_to_date, savings_plan_used_commitment, + split_line_item_actual_usage, split_line_item_net_split_cost, + split_line_item_net_unused_cost, split_line_item_parent_resource_id, + split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, + split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, + split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT + TableConfigurations: + COST_AND_USAGE_REPORT: + INCLUDE_RESOURCES: "TRUE" + INCLUDE_SPLIT_COST_ALLOCATION_DATA: "TRUE" + TIME_GRANULARITY: "HOURLY" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref DDCloudCostBucketName + S3Region: !Ref DDCloudCostBucketRegion + S3Prefix: !Ref DDCloudCostBucketPrefix + S3OutputConfigurations: + Compression: "GZIP" + Format: "TEXT_OR_CSV" + Overwrite: "CREATE_NEW_REPORT" + OutputType: "CUSTOM" + Name: !Ref DDCloudCostReportName + RefreshCadence: + Frequency: "SYNCHRONOUS" \ No newline at end of file diff --git a/aws_cloud_cost/release.sh b/aws_cloud_cost/release.sh new file mode 100755 index 00000000..b07d246a --- /dev/null +++ b/aws_cloud_cost/release.sh @@ -0,0 +1,75 @@ +#!/bin/bash + +# Usage: ./release.sh + +set -e + +# Read the S3 bucket +if [ -z "$1" ]; then + echo "Must specify a S3 bucket to publish the template" + exit 1 +else + BUCKET=$1 +fi + +# Read the version +VERSION=$(head -n 1 version.txt) + +# Confirm the bucket for the current release doesn't already exist so we don't overwrite it +set +e +EXIT_CODE=0 +response=$(aws s3api head-object \ + --bucket "${BUCKET}" \ + --key "aws_cloud_cost/${VERSION}/main.yaml" > /dev/null 2>&1) + +if [[ ${?} -eq 0 ]]; then + echo "S3 bucket path ${BUCKET}/aws_cloud_cost/${VERSION} already exists. Please up the version." + exit 1 +fi +set -e + +# Confirm that the readme link points to the current release +set +e +response=$(grep -q "aws_cloud_cost/${VERSION}/main.yaml" README.md) +if [[ ${?} -ne 0 ]]; then + echo "README.md does not point to the current release. Please update the link." + exit 1 +fi +set -e + +# Upload templates to a private bucket -- useful for testing +if [[ $# -eq 2 ]] && [[ $2 = "--private" ]]; then + PRIVATE_TEMPLATE=true +else + PRIVATE_TEMPLATE=false +fi + +# Confirm to proceed +for i in *.yaml; do + [ -f "$i" ] || break + echo "About to upload $i to s3://${BUCKET}/aws_cloud_cost/${VERSION}/$i" +done +read -p "Continue (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi + +# Update bucket placeholder +cp main.yaml main.yaml.bak +perl -pi -e "s//${BUCKET}/g" main.yaml +perl -pi -e "s//${VERSION}/g" main.yaml + +trap 'mv main.yaml.bak main.yaml' EXIT + +# Upload +if [ "$PRIVATE_TEMPLATE" = true ] ; then + aws s3 cp . s3://${BUCKET}/aws_cloud_cost/${VERSION} --recursive --exclude "*" --include "*.yaml" +else + aws s3 cp . s3://${BUCKET}/aws_cloud_cost/${VERSION} --recursive --exclude "*" --include "*.yaml" \ + --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers +fi +echo "Done uploading the template, and here is the CloudFormation quick launch URL" +echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://${BUCKET}.s3.amazonaws.com/aws_cloud_cost/${VERSION}/main.yaml" + +echo "Done!" diff --git a/aws_cloud_cost/version.txt b/aws_cloud_cost/version.txt new file mode 100644 index 00000000..95e94cdd --- /dev/null +++ b/aws_cloud_cost/version.txt @@ -0,0 +1 @@ +v0.0.1 \ No newline at end of file