Cloudformation for CUR 2.0 (Standard data export)

0

[Update] Complete stack (replace variables before executing):

AWSTemplateFormatVersion: '2010-09-09'

Resources:
  S3ClientBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: <bucket name>
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        IgnorePublicAcls: true
        RestrictPublicBuckets: true
  S3ClientBucketAccessPolicy:
    Type: "AWS::S3::BucketPolicy"
    Properties:
      Bucket: <bucket name>
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - billingreports.amazonaws.com
                - bcm-data-exports.amazonaws.com
            Action:
              - s3:PutObject
              - s3:GetBucketPolicy
            Resource:
              - arn:aws:s3:::<bucket name>
              - arn:aws:s3:::<bucket name>/*
            Condition:
              StringLike:
                aws:SourceArn:
                  - arn:aws:cur:us-east-1:<account id>:definition/*
                  - arn:aws:bcm-data-exports:us-east-1:<account id>:export/*
                aws:SourceAccount: '<account id>'
  CURReportDefinition:
    Type: AWS::BCMDataExports::Export
    Properties:
      Export: 
        DataQuery:
          QueryStatement: "SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment FROM COST_AND_USAGE_REPORT"
          TableConfigurations: 
            COST_AND_USAGE_REPORT: 
              INCLUDE_RESOURCES: "FALSE"
              INCLUDE_SPLIT_COST_ALLOCATION_DATA: "FALSE"
              TIME_GRANULARITY: "DAILY"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: <bucket name>
            S3Region: "us-east-1"
            S3Prefix: "cur/"
            S3OutputConfigurations:
              Compression: "PARQUET"
              Format: "PARQUET"
              OutputType: "CUSTOM"
              Overwrite: "OVERWRITE_REPORT"
        Name: testcur
        RefreshCadence: 
          Frequency: "SYNCHRONOUS"

Executing the above template fails with Event name: CreateExport, Error code: ValidationException and errorMessage": "S3 bucket permission validation failed".

I have created a CUR using the following CFN template:

[..]
 CostAndUsageReport:
    Type: AWS::CUR::ReportDefinition
    Properties:
      AdditionalSchemaElements: 
        - RESOURCES
      Compression: Parquet
      Format: Parquet
      RefreshClosedReports: true
      ReportName: my-fancy-report-name
      ReportVersioning: CREATE_NEW_REPORT
      S3Bucket: my-fancy-bucket-name
      S3Prefix: cur
      S3Region: us-east-1
      TimeUnit: DAILY

It shows up as "Legacy CUR export" in the UI. A manually created report's type is "Standard data export".

How do I create a "Standard data export" (CUR 2.0) with Cloudformation?

asked 3 months ago505 views
3 Answers
2

Hello.

CUR2.0 is a relatively new feature announced last year.
https://aws.amazon.com/about-aws/whats-new/2023/11/aws-billing-cost-management-data-exports/?nc1=h_ls

I think that the related CloudFormation is from the following document, but I have not tried whether it is possible to create CUR2.0.
https://docs.aws.amazon.com/ja_jp/AWSCloudFormation/latest/UserGuide/aws-resource-bcmdataexports-export.html

profile picture
EXPERT
answered 3 months ago
profile picture
EXPERT
reviewed 3 months ago
  • Do you have an example for an AWS::BCMDataExports::Export? The documentation is meh (e.g. QueryStatement - The query statement and TableConfigurations - The table configuration).

1
Accepted Answer

Please use this CloudFormation template.

AWSTemplateFormatVersion: '2010-09-09'
Description: 'AWS CUR 2.0 Report Definition'

Parameters:
# ------------------------------------------------------------#
# Parameters
# ------------------------------------------------------------# 
  S3Name:
    Default: kobayashi-cur
    Type: String

Resources:
  S3ClientBucket:
    Type: AWS::S3::Bucket
    Properties:
      BucketName: !Sub ${S3Name}-${AWS::Region}-${AWS::AccountId}
      BucketEncryption:
        ServerSideEncryptionConfiguration:
          - ServerSideEncryptionByDefault:
              SSEAlgorithm: AES256
      PublicAccessBlockConfiguration:
        IgnorePublicAcls: true
        RestrictPublicBuckets: true

  S3ClientBucketAccessPolicy:
    Type: AWS::S3::BucketPolicy
    Properties:
      Bucket: !Ref S3ClientBucket
      PolicyDocument:
        Version: '2012-10-17'
        Statement:
          - Effect: Allow
            Principal:
              Service:
                - billingreports.amazonaws.com
                - bcm-data-exports.amazonaws.com
            Action:
              - s3:PutObject
              - s3:GetBucketPolicy
            Resource:
              - !Sub ${S3ClientBucket.Arn}
              - !Sub ${S3ClientBucket.Arn}/*
            Condition:
              StringLike:
                aws:SourceArn:
                  - !Sub arn:aws:cur:us-east-1:${AWS::AccountId}:definition/*
                  - !Sub arn:aws:bcm-data-exports:us-east-1:${AWS::AccountId}:export/*
                aws:SourceAccount: !Sub ${AWS::AccountId}


  CURReportDefinition:
    DependsOn: S3ClientBucketAccessPolicy
    Type: AWS::BCMDataExports::Export
    Properties:
      Export: 
        DataQuery:
          QueryStatement: "SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment FROM COST_AND_USAGE_REPORT"
          TableConfigurations: 
            COST_AND_USAGE_REPORT: 
              INCLUDE_RESOURCES: "FALSE"
              INCLUDE_SPLIT_COST_ALLOCATION_DATA: "FALSE"
              TIME_GRANULARITY: "DAILY"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: !Ref S3ClientBucket
            S3Region: "us-east-1"
            S3Prefix: "cur/"
            S3OutputConfigurations:
              Compression: "PARQUET"
              Format: "PARQUET"
              OutputType: "CUSTOM"
              Overwrite: "OVERWRITE_REPORT"
        Name: testcur
        RefreshCadence: 
          Frequency: "SYNCHRONOUS"
profile picture
EXPERT
answered 3 months ago
profile picture
EXPERT
reviewed 3 months ago
  • Thank you! The "DependsOn" was missing in my template :(

1

I was able to create it using the CloudFormation template below.
I checked the parameters from the "CreateExport" event that was output to CloudTrail and tried creating it.

AWSTemplateFormatVersion: '2010-09-09'
Description: 'AWS CUR 2.0 Report Definition'

Resources:
  CURReportDefinition:
    Type: AWS::BCMDataExports::Export
    Properties:
      Export: 
        DataQuery:
          QueryStatement: "SELECT bill_bill_type, bill_billing_entity, bill_billing_period_end_date, bill_billing_period_start_date, bill_invoice_id, bill_invoicing_entity, bill_payer_account_id, bill_payer_account_name, cost_category, discount, discount_bundled_discount, discount_total_discount, identity_line_item_id, identity_time_interval, line_item_availability_zone, line_item_blended_cost, line_item_blended_rate, line_item_currency_code, line_item_legal_entity, line_item_line_item_description, line_item_line_item_type, line_item_net_unblended_cost, line_item_net_unblended_rate, line_item_normalization_factor, line_item_normalized_usage_amount, line_item_operation, line_item_product_code, line_item_tax_type, line_item_unblended_cost, line_item_unblended_rate, line_item_usage_account_id, line_item_usage_account_name, line_item_usage_amount, line_item_usage_end_date, line_item_usage_start_date, line_item_usage_type, pricing_currency, pricing_lease_contract_length, pricing_offering_class, pricing_public_on_demand_cost, pricing_public_on_demand_rate, pricing_purchase_option, pricing_rate_code, pricing_rate_id, pricing_term, pricing_unit, product, product_comment, product_fee_code, product_fee_description, product_from_location, product_from_location_type, product_from_region_code, product_instance_family, product_instance_type, product_instancesku, product_location, product_location_type, product_operation, product_pricing_unit, product_product_family, product_region_code, product_servicecode, product_sku, product_to_location, product_to_location_type, product_to_region_code, product_usagetype, reservation_amortized_upfront_cost_for_usage, reservation_amortized_upfront_fee_for_billing_period, reservation_availability_zone, reservation_effective_cost, reservation_end_time, reservation_modification_status, reservation_net_amortized_upfront_cost_for_usage, reservation_net_amortized_upfront_fee_for_billing_period, reservation_net_effective_cost, reservation_net_recurring_fee_for_usage, reservation_net_unused_amortized_upfront_fee_for_billing_period, reservation_net_unused_recurring_fee, reservation_net_upfront_value, reservation_normalized_units_per_reservation, reservation_number_of_reservations, reservation_recurring_fee_for_usage, reservation_reservation_a_r_n, reservation_start_time, reservation_subscription_id, reservation_total_reserved_normalized_units, reservation_total_reserved_units, reservation_units_per_reservation, reservation_unused_amortized_upfront_fee_for_billing_period, reservation_unused_normalized_unit_quantity, reservation_unused_quantity, reservation_unused_recurring_fee, reservation_upfront_value, resource_tags, savings_plan_amortized_upfront_commitment_for_billing_period, savings_plan_end_time, savings_plan_instance_type_family, savings_plan_net_amortized_upfront_commitment_for_billing_period, savings_plan_net_recurring_commitment_for_billing_period, savings_plan_net_savings_plan_effective_cost, savings_plan_offering_type, savings_plan_payment_option, savings_plan_purchase_term, savings_plan_recurring_commitment_for_billing_period, savings_plan_region, savings_plan_savings_plan_a_r_n, savings_plan_savings_plan_effective_cost, savings_plan_savings_plan_rate, savings_plan_start_time, savings_plan_total_commitment_to_date, savings_plan_used_commitment FROM COST_AND_USAGE_REPORT"
          TableConfigurations: 
            COST_AND_USAGE_REPORT: 
              INCLUDE_RESOURCES: "FALSE"
              INCLUDE_SPLIT_COST_ALLOCATION_DATA: "FALSE"
              TIME_GRANULARITY: "DAILY"
        DestinationConfigurations:
          S3Destination:
            S3Bucket: your-s3-bucket-name
            S3Region: "us-east-1"
            S3Prefix: "cur/"
            S3OutputConfigurations:
              Compression: "PARQUET"
              Format: "PARQUET"
              OutputType: "CUSTOM"
              Overwrite: "OVERWRITE_REPORT"
        Name: testcur
        RefreshCadence: 
          Frequency: "SYNCHRONOUS"
profile picture
EXPERT
answered 3 months ago
profile picture
EXPERT
reviewed 3 months ago
  • The S3 bucket policy looks like this:

    {
        "Version": "2012-10-17",
        "Statement": [
            {
                "Sid": "EnableAWSDataExportsToWriteToS3AndCheckPolicy",
                "Effect": "Allow",
                "Principal": {
                    "Service": [
                        "billingreports.amazonaws.com",
                        "bcm-data-exports.amazonaws.com"
                    ]
                },
                "Action": [
                    "s3:PutObject",
                    "s3:GetBucketPolicy"
                ],
                "Resource": [
                    "arn:aws:s3:::your-s3-bucket-name",
                    "arn:aws:s3:::your-s3-bucket-name/*"
                ],
                "Condition": {
                    "StringLike": {
                        "aws:SourceArn": [
                            "arn:aws:cur:us-east-1:111111111111:definition/*",
                            "arn:aws:bcm-data-exports:us-east-1:111111111111:export/*"
                        ],
                        "aws:SourceAccount": "11111111111"
                    }
                }
            }
        ]
    }
    
  • Can you post your BucketPolicy as well please? I am getting "errorCode": "ValidationException" and "errorMessage": "S3 bucket permission validation failed". I did try @Riku_Kobayashi's policy as well.

    I have updated my question with the template which fails.

You are not logged in. Log in to post an answer.

A good answer clearly answers the question and provides constructive feedback and encourages professional growth in the question asker.

Guidelines for Answering Questions