diff --git a/aws_cloud_cost_cur2/CHANGELOG.md b/aws_cloud_cost_cur2/CHANGELOG.md new file mode 100644 index 00000000..b7f9a5d6 --- /dev/null +++ b/aws_cloud_cost_cur2/CHANGELOG.md @@ -0,0 +1,3 @@ +## 0.0.1 + +Initial version diff --git a/aws_cloud_cost_cur2/README.md b/aws_cloud_cost_cur2/README.md new file mode 100644 index 00000000..a1da7db2 --- /dev/null +++ b/aws_cloud_cost_cur2/README.md @@ -0,0 +1,49 @@ +# Datadog AWS Cloud Cost Setup (CUR 2.0) + +[![Launch Stack](https://s3.amazonaws.com/cloudformation-examples/cloudformation-launch-stack.png)](ADDLINK) + +## Overview + +This template sets up AWS Cloud Cost Management for Datadog using **CUR 2.0 (Data Exports)**. It creates the necessary AWS resources and configures the Datadog integration automatically. + +## AWS Resources + +This template creates the following: + +- **S3 Bucket** (optional) - For storing cost and usage data + - Bucket policy allowing `bcm-data-exports.amazonaws.com` to write reports +- **BCM Data Export** (optional) - CUR 2.0 export with hourly granularity in Parquet format +- **IAM Policy** - Grants Datadog read access to the S3 bucket and Cost Explorer APIs + - Attached to your existing Datadog integration role +- **Lambda Function** - Calls the Datadog API to configure CCM with the data export details + +## Parameters + +| Parameter | Description | Required | +|-----------|-------------|----------| +| `DatadogApiKey` | Datadog API key | Yes | +| `DatadogAppKey` | Datadog APP key | Yes | +| `DatadogSite` | Datadog site (e.g., datadoghq.com) | No (default: datadoghq.com) | +| `CloudCostBucketName` | S3 bucket name for cost reports | Yes | +| `CloudCostBucketRegion` | AWS region of the S3 bucket | Yes | +| `CloudCostReportPrefix` | Prefix for report files in S3 | Yes | +| `CloudCostReportName` | Name of the data export | Yes | +| `DatadogIntegrationRole` | Name of your Datadog integration IAM role | Yes | +| `CreateCloudCostReport` | Create a new data export (true/false) | No (default: true) | +| `CreateCloudCostBucket` | Create a new S3 bucket (true/false) | No (default: true) | + +## Publishing the Template + +Use the release script to upload the template to an S3 bucket: + +```sh +./release.sh +``` + +Use `--private` to prevent granting public access (useful for testing): + +```sh +./release.sh --private +``` + +The template will be available at `s3:///aws_cloud_cost_cur2/v0.0.1/main.yaml`. diff --git a/aws_cloud_cost_cur2/datadog_ccm_api_call.py b/aws_cloud_cost_cur2/datadog_ccm_api_call.py new file mode 100644 index 00000000..0c45c7c9 --- /dev/null +++ b/aws_cloud_cost_cur2/datadog_ccm_api_call.py @@ -0,0 +1,130 @@ +#!/usr/bin/env python3 + +import json +import logging +import urllib.request +import urllib.error +import cfnresponse + +LOGGER = logging.getLogger() +LOGGER.setLevel(logging.INFO) + +API_CALL_SOURCE_HEADER_VALUE = "cfn-ccm-cur2" + + +def get_datadog_account_uuid(event): + """Get the Datadog account UUID for this AWS account.""" + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + account_id = event["ResourceProperties"]["AccountId"] + + url = f"https://api.{api_url}/api/v2/integration/aws/accounts?aws_account_id={account_id}" + headers = { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + } + request = urllib.request.Request(url, headers=headers) + request.get_method = lambda: "GET" + try: + response = urllib.request.urlopen(request) + data = json.loads(response.read()) + if len(data.get("data", [])) == 0: + return None, "No Datadog integration found for this AWS account" + if len(data["data"]) > 1: + return None, "Multiple Datadog integrations found for this AWS account" + return data["data"][0]["id"], None + except urllib.error.HTTPError as e: + error_body = e.read().decode("utf-8") + return None, f"Failed to get account: {e.code} - {error_body}" + + +def patch_ccm_config(event, uuid): + """PATCH the Datadog account with CCM data export config.""" + api_key = event["ResourceProperties"]["APIKey"] + app_key = event["ResourceProperties"]["APPKey"] + api_url = event["ResourceProperties"]["ApiURL"] + bucket_name = event["ResourceProperties"]["BucketName"] + bucket_region = event["ResourceProperties"]["BucketRegion"] + report_name = event["ResourceProperties"]["ReportName"] + report_prefix = event["ResourceProperties"]["ReportPrefix"] + + url = f"https://api.{api_url}/api/v2/integration/aws/accounts/{uuid}" + headers = { + "DD-API-KEY": api_key, + "DD-APPLICATION-KEY": app_key, + "Dd-Aws-Api-Call-Source": API_CALL_SOURCE_HEADER_VALUE, + "Content-Type": "application/json", + "Accept": "application/json", + } + + payload = { + "data": { + "type": "account", + "attributes": { + "ccm_config": { + "data_export_configs": [ + { + "report_name": report_name, + "report_prefix": report_prefix, + "report_type": "CUR2.0", + "bucket_name": bucket_name, + "bucket_region": bucket_region, + } + ] + } + } + } + } + + data = json.dumps(payload).encode("utf-8") + request = urllib.request.Request(url, data=data, headers=headers) + request.get_method = lambda: "PATCH" + + try: + response = urllib.request.urlopen(request) + return response.getcode(), json.loads(response.read()) + except urllib.error.HTTPError as e: + error_body = e.read().decode("utf-8") + return e.code, {"error": error_body} + + +def handler(event, context): + """Handle Lambda event from AWS CloudFormation.""" + LOGGER.info(f"Received event: {json.dumps(event)}") + + if event["RequestType"] == "Delete": + # On delete, we don't remove the CCM config - just succeed + LOGGER.info("Delete request - no action needed for CCM config") + cfnresponse.send(event, context, cfnresponse.SUCCESS, {"Message": "Delete successful"}) + return + + try: + # Get the Datadog account UUID + uuid, error = get_datadog_account_uuid(event) + if error: + LOGGER.error(f"Failed to get account UUID: {error}") + cfnresponse.send(event, context, cfnresponse.FAILED, {"Message": error}) + return + + LOGGER.info(f"Found Datadog account UUID: {uuid}") + + # PATCH the CCM config + status_code, response_data = patch_ccm_config(event, uuid) + + if status_code == 200: + LOGGER.info("Successfully configured CCM data export") + cfnresponse.send(event, context, cfnresponse.SUCCESS, { + "Message": "CCM data export configured successfully", + "AccountUUID": uuid, + }) + else: + error_msg = f"API returned {status_code}: {response_data}" + LOGGER.error(error_msg) + cfnresponse.send(event, context, cfnresponse.FAILED, {"Message": error_msg}) + + except Exception as e: + LOGGER.exception("Exception during processing") + cfnresponse.send(event, context, cfnresponse.FAILED, {"Message": str(e)}) + diff --git a/aws_cloud_cost_cur2/main.yaml b/aws_cloud_cost_cur2/main.yaml new file mode 100644 index 00000000..46c7314a --- /dev/null +++ b/aws_cloud_cost_cur2/main.yaml @@ -0,0 +1,298 @@ +# version: 0.0.1 +AWSTemplateFormatVersion: 2010-09-09 +Description: Datadog Cloud Cost Management Setup +Parameters: + DatadogApiKey: + Type: String + Description: API key for the Datadog account. + NoEcho: true + MinLength: 1 + DatadogAppKey: + Type: String + Description: APP key for the Datadog account. + NoEcho: true + MinLength: 1 + DatadogSite: + Type: String + Default: datadoghq.com + Description: Define your Datadog Site to send data to. + AllowedValues: + - datadoghq.com + - datadoghq.eu + - us3.datadoghq.com + - us5.datadoghq.com + - ap1.datadoghq.com + - ap2.datadoghq.com + - ddog-gov.com + CloudCostBucketName: + Type: String + Description: The name of the S3 bucket for storing the cost and usage reports. + MinLength: 1 + CloudCostBucketRegion: + Type: String + Description: The AWS region where the S3 bucket is located. + MinLength: 1 + CloudCostReportPrefix: + Type: String + Description: The prefix to be added to the cost and usage report files in the S3 bucket. + MinLength: 1 + CloudCostReportName: + Type: String + Description: The name of the cost and usage report. + MinLength: 1 + DatadogIntegrationRole: + Type: String + Description: The name of the IAM role that has been integrated with Datadog. + MinLength: 1 + CreateCloudCostReport: + Type: String + Default: true + AllowedValues: + - true + - false + Description: Whether a new Cost and Usage Report is created, or an existing one is used. + CreateCloudCostBucket: + Type: String + Default: true + AllowedValues: + - true + - false + Description: Whether the S3 bucket for storing the Cost and Usage Report is created, or an existing one is used. Ignored if CreateCloudCostReport is false. + +Conditions: + ShouldCreateCloudCostReport: + Fn::Equals: + - Ref: CreateCloudCostReport + - true + ShouldCreateCloudCostBucket: + Fn::And: + - Fn::Equals: + - Ref: CreateCloudCostBucket + - true + - !Condition ShouldCreateCloudCostReport # If CUR already exists, we don't need to create the bucket + +Resources: + DatadogCCMIAMPolicy: + Type: AWS::IAM::RolePolicy + Properties: + PolicyName: !Sub "datadog-ccm-iam-policy--${CloudCostBucketName}-${CloudCostReportPrefix}-${CloudCostReportName}" + RoleName: !Ref DatadogIntegrationRole + PolicyDocument: + Version: "2012-10-17" + Statement: + - Sid: ReadCloudCostBucket + Effect: Allow + Action: + - s3:ListBucket + Resource: !Sub "arn:${AWS::Partition}:s3:::${CloudCostBucketName}" + - Sid: GetBill + Effect: Allow + Action: + - s3:GetObject + Resource: !Sub "arn:${AWS::Partition}:s3:::${CloudCostBucketName}/${CloudCostReportPrefix}/${CloudCostReportName}/*" + - Sid: CheckAccuracy + Effect: Allow + Action: + - ce:Get* + Resource: "*" + - Sid: ListCURs + Effect: Allow + Action: + - cur:DescribeReportDefinitions + Resource: "*" + - Sid: ListOrganizations + Effect: Allow + Action: + - organizations:Describe* + - organizations:List* + Resource: "*" + + DDCloudCostBucket: + Condition: ShouldCreateCloudCostBucket + Type: AWS::S3::Bucket + Properties: + BucketName: !Ref CloudCostBucketName + + DDCloudCostBucketAccessPolicy: + Condition: ShouldCreateCloudCostBucket + Type: AWS::S3::BucketPolicy + Properties: + Bucket: !Ref DDCloudCostBucket + PolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - billingreports.amazonaws.com + - bcm-data-exports.amazonaws.com + Action: + - s3:PutObject + - s3:GetBucketPolicy + Resource: + - !Sub ${DDCloudCostBucket.Arn} + - !Sub ${DDCloudCostBucket.Arn}/* + Condition: + StringLike: + aws:SourceArn: + - !Sub arn:*:cur:*:${AWS::AccountId}:definition/* + - !Sub arn:*:bcm-data-exports:*:${AWS::AccountId}:export/* + aws:SourceAccount: !Sub ${AWS::AccountId} + + # DDCURReportDefinition can only be done after the bucket is created + # and has the right policies. The bucket is only created if ShouldCreateCloudCostBucket + # is true. DependsOn does not work with Conditions, so we use a WaitCondition + # There are 2 wait handles, + # - BucketWaitHandle: for the bucket if bucket should be created + # - EmptyWaitHandle: for empty wait condition if bucket should not be created + # BucketWaitCondition to join them and be used as a dependency for DDCURReportDefinition + # More info here - https://garbe.io/blog/2017/07/17/cloudformation-hacks/ + BucketWaitHandle: + Condition: ShouldCreateCloudCostBucket + DependsOn: DDCloudCostBucketAccessPolicy + Type: "AWS::CloudFormation::WaitConditionHandle" + + EmptyWaitHandle: + Type: "AWS::CloudFormation::WaitConditionHandle" + + BucketWaitCondition: + Type: "AWS::CloudFormation::WaitCondition" + Properties: + Handle: + !If [ + ShouldCreateCloudCostBucket, + !Ref BucketWaitHandle, + !Ref EmptyWaitHandle, + ] + Timeout: "1" + Count: 0 + + DDCURReportDefinition: + Condition: ShouldCreateCloudCostReport + DependsOn: BucketWaitCondition + Type: AWS::BCMDataExports::Export + Properties: + Export: + DataQuery: + QueryStatement: > + SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, + bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, + bill_payer_account_id, bill_payer_account_name, cost_category, discount, + discount_bundled_discount, discount_total_discount, identity_line_item_id, + identity_time_interval, line_item_availability_zone, line_item_blended_cost, + line_item_blended_rate, line_item_currency_code, line_item_legal_entity, + line_item_line_item_description, line_item_line_item_type, + line_item_net_unblended_cost, line_item_net_unblended_rate, + line_item_normalization_factor, line_item_normalized_usage_amount, + line_item_operation, line_item_product_code, line_item_resource_id, + line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, + line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, + line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, + pricing_currency, pricing_lease_contract_length, pricing_offering_class, + pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, + pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, + product_comment, product_fee_code, product_fee_description, product_from_location, + product_from_location_type, product_from_region_code, product_instance_family, + product_instance_type, product_instancesku, product_location, product_location_type, + product_operation, product_pricing_unit, product_product_family, product_region_code, + product_servicecode, product_sku, product_to_location, product_to_location_type, + product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, + reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, + reservation_effective_cost, reservation_end_time, reservation_modification_status, + reservation_net_amortized_upfront_cost_for_usage, + reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, + reservation_net_recurring_fee_for_usage, + reservation_net_unused_amortized_upfront_fee_for_billing_period, + reservation_net_unused_recurring_fee, reservation_net_upfront_value, + reservation_normalized_units_per_reservation, reservation_number_of_reservations, + reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, + reservation_start_time, reservation_subscription_id, + reservation_total_reserved_normalized_units, reservation_total_reserved_units, + reservation_units_per_reservation, + reservation_unused_amortized_upfront_fee_for_billing_period, + reservation_unused_normalized_unit_quantity, reservation_unused_quantity, + reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, + savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, + savings_plan_instance_type_family, + savings_plan_net_amortized_upfront_commitment_for_billing_period, + savings_plan_net_recurring_commitment_for_billing_period, + savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, + savings_plan_payment_option, savings_plan_purchase_term, + savings_plan_recurring_commitment_for_billing_period, savings_plan_region, + savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, + savings_plan_savings_plan_rate, savings_plan_start_time, + savings_plan_total_commitment_to_date, savings_plan_used_commitment, + split_line_item_actual_usage, split_line_item_net_split_cost, + split_line_item_net_unused_cost, split_line_item_parent_resource_id, + split_line_item_public_on_demand_split_cost, split_line_item_public_on_demand_unused_cost, + split_line_item_reserved_usage, split_line_item_split_cost, split_line_item_split_usage, + split_line_item_split_usage_ratio, split_line_item_unused_cost FROM COST_AND_USAGE_REPORT + TableConfigurations: + COST_AND_USAGE_REPORT: + INCLUDE_RESOURCES: "TRUE" + INCLUDE_SPLIT_COST_ALLOCATION_DATA: "TRUE" + TIME_GRANULARITY: "HOURLY" + DestinationConfigurations: + S3Destination: + S3Bucket: !Ref CloudCostBucketName + S3Region: !Ref CloudCostBucketRegion + S3Prefix: !Ref CloudCostReportPrefix + S3OutputConfigurations: + Compression: "PARQUET" + Format: "PARQUET" + Overwrite: "CREATE_NEW_REPORT" + OutputType: "CUSTOM" + Name: !Ref CloudCostReportName + RefreshCadence: + Frequency: "SYNCHRONOUS" + + # Lambda function to call Datadog API and configure CCM + DatadogCCMAPIHandlerLambdaExecutionRole: + Type: AWS::IAM::Role + Properties: + AssumeRolePolicyDocument: + Version: "2012-10-17" + Statement: + - Effect: Allow + Principal: + Service: + - lambda.amazonaws.com + Action: + - sts:AssumeRole + Path: "/" + ManagedPolicyArns: + - !Sub "arn:${AWS::Partition}:iam::aws:policy/service-role/AWSLambdaBasicExecutionRole" + + DatadogCCMAPICallFunction: + Type: AWS::Lambda::Function + Properties: + Description: "A function to call the Datadog API to configure CCM data export." + Role: !GetAtt DatadogCCMAPIHandlerLambdaExecutionRole.Arn + Handler: "index.handler" + LoggingConfig: + ApplicationLogLevel: "INFO" + LogFormat: "JSON" + Runtime: "python3.14" + Timeout: 30 + Code: + ZipFile: | + + + DatadogCCMConfiguration: + Type: Custom::DatadogCCMConfiguration + DependsOn: DatadogCCMIAMPolicy + Properties: + ServiceToken: !GetAtt DatadogCCMAPICallFunction.Arn + APIKey: !Ref DatadogApiKey + APPKey: !Ref DatadogAppKey + ApiURL: !Ref DatadogSite + AccountId: !Ref AWS::AccountId + BucketName: !Ref CloudCostBucketName + BucketRegion: !Ref CloudCostBucketRegion + ReportName: !Ref CloudCostReportName + ReportPrefix: !Ref CloudCostReportPrefix + +Outputs: + DatadogAccountUUID: + Description: The Datadog AWS Integration Account UUID + Value: !GetAtt DatadogCCMConfiguration.AccountUUID diff --git a/aws_cloud_cost_cur2/release.sh b/aws_cloud_cost_cur2/release.sh new file mode 100755 index 00000000..e6aa07a4 --- /dev/null +++ b/aws_cloud_cost_cur2/release.sh @@ -0,0 +1,83 @@ +#!/bin/bash + +# Usage: ./release.sh + +set -e + +# Read the S3 bucket +if [ -z "$1" ]; then + echo "Must specify a S3 bucket to publish the template" + exit 1 +else + BUCKET=$1 +fi + +# Read the version +VERSION=$(head -n 1 version.txt) + +# Confirm the bucket for the current release doesn't already exist so we don't overwrite it +set +e +EXIT_CODE=0 +response=$(aws s3api head-object \ + --bucket "${BUCKET}" \ + --key "aws_cloud_cost_cur2/${VERSION}/main.yaml" > /dev/null 2>&1) + +if [[ ${?} -eq 0 ]]; then + echo "S3 bucket path ${BUCKET}/aws_cloud_cost_cur2/${VERSION} already exists. Please up the version." + exit 1 +fi +set -e + +# Confirm that the readme link points to the current release +set +e +response=$(grep -q "aws_cloud_cost_cur2/${VERSION}/main.yaml" README.md) +if [[ ${?} -ne 0 ]]; then + echo "README.md does not point to the current release. Please update the link." + exit 1 +fi +set -e + +# Upload templates to a private bucket -- useful for testing +if [[ $# -eq 2 ]] && [[ $2 = "--private" ]]; then + PRIVATE_TEMPLATE=true +else + PRIVATE_TEMPLATE=false +fi + +# Confirm to proceed +for i in *.yaml; do + [ -f "$i" ] || break + echo "About to upload $i to s3://${BUCKET}/aws_cloud_cost_cur2/${VERSION}/$i" +done +read -p "Continue (y/n)?" CONT +if [ "$CONT" != "y" ]; then + echo "Exiting" + exit 1 +fi + +# Update bucket placeholder +cp main.yaml main.yaml.bak +perl -pi -e "s//${BUCKET}/g" main.yaml +perl -pi -e "s//${VERSION}/g" main.yaml + +# Replace ZIPFILE_PLACEHOLDER with the contents of the Python file +perl -i -pe ' + BEGIN { $p = do { local $/; } } + /^(\s+)/ && ( + $_ = join("\n", map { $1 . $_ } split(/\n/, $p)) . "\n" + ) +' main.yaml < datadog_ccm_api_call.py + +trap 'mv main.yaml.bak main.yaml' EXIT + +# Upload +if [ "$PRIVATE_TEMPLATE" = true ] ; then + aws s3 cp . s3://${BUCKET}/aws_cloud_cost_cur2/${VERSION} --recursive --exclude "*" --include "*.yaml" +else + aws s3 cp . s3://${BUCKET}/aws_cloud_cost_cur2/${VERSION} --recursive --exclude "*" --include "*.yaml" \ + --grants read=uri=http://acs.amazonaws.com/groups/global/AllUsers +fi +echo "Done uploading the template, and here is the CloudFormation quick launch URL" +echo "https://console.aws.amazon.com/cloudformation/home#/stacks/create/review?stackName=datadog-aws-cloud-cost&templateURL=https://${BUCKET}.s3.amazonaws.com/aws_cloud_cost_cur2/${VERSION}/main.yaml" + +echo "Done!" diff --git a/aws_cloud_cost_cur2/version.txt b/aws_cloud_cost_cur2/version.txt new file mode 100644 index 00000000..95e94cdd --- /dev/null +++ b/aws_cloud_cost_cur2/version.txt @@ -0,0 +1 @@ +v0.0.1 \ No newline at end of file