From 1ec609d404b74427c3b49a6c126eb1c40d0cf39b Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 12 May 2026 16:18:18 -0500 Subject: [PATCH 1/7] Buildkite improvements --- .buildkite/hooks/pre-command | 25 --- .buildkite/pipeline.yml | 74 --------- .buildkite/scripts/run_models.sh | 24 --- integration_tests/ci/sample.profiles.yml | 50 ------ integration_tests/ci/test_scenarios.yml | 44 +++++ integration_tests/dbt_project.yml | 199 +++-------------------- 6 files changed, 69 insertions(+), 347 deletions(-) delete mode 100644 .buildkite/hooks/pre-command delete mode 100644 .buildkite/pipeline.yml delete mode 100644 .buildkite/scripts/run_models.sh delete mode 100644 integration_tests/ci/sample.profiles.yml create mode 100644 integration_tests/ci/test_scenarios.yml diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command deleted file mode 100644 index 81b7b002..00000000 --- a/.buildkite/hooks/pre-command +++ /dev/null @@ -1,25 +0,0 @@ -#!/bin/bash - -set -e - -# Export secrets for Docker containers. -# Restrict exposing secrets only to the steps that need them -export GCLOUD_SERVICE_KEY=$(gcloud secrets versions access latest --secret="GCLOUD_SERVICE_KEY" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_HOST" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_USER=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_USER" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_PASS" --project="dbt-package-testing-363917") -export CI_POSTGRES_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_POSTGRES_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_DBNAME=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_DBNAME" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_HOST" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_PASS" --project="dbt-package-testing-363917") -export CI_REDSHIFT_DBT_USER=$(gcloud secrets versions access latest --secret="CI_REDSHIFT_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ACCOUNT=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ACCOUNT" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_DATABASE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_DATABASE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_PASS=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_PASS" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_ROLE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_ROLE" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_USER" --project="dbt-package-testing-363917") -export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml deleted file mode 100644 index 26036027..00000000 --- a/.buildkite/pipeline.yml +++ /dev/null @@ -1,74 +0,0 @@ -steps: - - label: ":postgres: Run Tests - Postgres" - key: "run-dbt-postgres" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_POSTGRES_DBT_DBNAME" - - "CI_POSTGRES_DBT_HOST" - - "CI_POSTGRES_DBT_PASS" - - "CI_POSTGRES_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh postgres - - - label: ":snowflake-db: Run Tests - Snowflake" - key: "run_dbt_snowflake" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_SNOWFLAKE_DBT_ACCOUNT" - - "CI_SNOWFLAKE_DBT_DATABASE" - - "CI_SNOWFLAKE_DBT_PASS" - - "CI_SNOWFLAKE_DBT_ROLE" - - "CI_SNOWFLAKE_DBT_USER" - - "CI_SNOWFLAKE_DBT_WAREHOUSE" - commands: | - bash .buildkite/scripts/run_models.sh snowflake - - - label: ":gcloud: Run Tests - BigQuery" - key: "run_dbt_bigquery" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "GCLOUD_SERVICE_KEY" - commands: | - bash .buildkite/scripts/run_models.sh bigquery - - - label: ":amazon-redshift: Run Tests - Redshift" - key: "run_dbt_redshift" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_REDSHIFT_DBT_DBNAME" - - "CI_REDSHIFT_DBT_HOST" - - "CI_REDSHIFT_DBT_PASS" - - "CI_REDSHIFT_DBT_USER" - commands: | - bash .buildkite/scripts/run_models.sh redshift - - - label: ":databricks: Run Tests - Databricks" - key: "run_dbt_databricks" - plugins: - - docker#v3.13.0: - image: "python:3.10.13" - shell: [ "/bin/bash", "-e", "-c" ] - environment: - - "BASH_ENV=/tmp/.bashrc" - - "CI_DATABRICKS_DBT_HOST" - - "CI_DATABRICKS_DBT_HTTP_PATH" - - "CI_DATABRICKS_DBT_TOKEN" - - "CI_DATABRICKS_DBT_CATALOG" - commands: | - bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/.buildkite/scripts/run_models.sh b/.buildkite/scripts/run_models.sh deleted file mode 100644 index 1f11e11f..00000000 --- a/.buildkite/scripts/run_models.sh +++ /dev/null @@ -1,24 +0,0 @@ -#!/bin/bash - -set -euo pipefail - -apt-get update -apt-get install libsasl2-dev - -python3 -m venv venv -. venv/bin/activate -pip install --upgrade pip setuptools -pip install -r integration_tests/requirements.txt -mkdir -p ~/.dbt -cp integration_tests/ci/sample.profiles.yml ~/.dbt/profiles.yml - -db=$1 -echo `pwd` -cd integration_tests -dbt deps -dbt seed --target "$db" --full-refresh -dbt source freshness --target "$db" || echo "...Only verifying freshness runs…" -dbt run --target "$db" --full-refresh --vars '{ad_reporting__facebook_ads_enabled: true, facebook_ads__using_demographics_country: true, facebook_ads__using_demographics_region: true, ad_reporting__google_ads_enabled: true, ad_reporting__amazon_ads_enabled: false, ad_reporting__apple_search_ads_enabled: false, ad_reporting__linkedin_ads_enabled: true, ad_reporting__microsoft_ads_enabled: false, ad_reporting__pinterest_ads_enabled: false, ad_reporting__reddit_ads_enabled: false, ad_reporting__snapchat_ads_enabled: false, ad_reporting__tiktok_ads_enabled: false, ad_reporting__twitter_ads_enabled: false}' -dbt test --target "$db" --vars '{ad_reporting__facebook_ads_enabled: true, facebook_ads__using_demographics_country: true, facebook_ads__using_demographics_region: true, ad_reporting__google_ads_enabled: true, ad_reporting__amazon_ads_enabled: false, ad_reporting__apple_search_ads_enabled: false, ad_reporting__linkedin_ads_enabled: true, ad_reporting__microsoft_ads_enabled: false, ad_reporting__pinterest_ads_enabled: false, ad_reporting__reddit_ads_enabled: false, ad_reporting__snapchat_ads_enabled: false, ad_reporting__tiktok_ads_enabled: false, ad_reporting__twitter_ads_enabled: false}' - -dbt run-operation fivetran_utils.drop_schemas_automation --target "$db" diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml deleted file mode 100644 index 0d935d40..00000000 --- a/integration_tests/ci/sample.profiles.yml +++ /dev/null @@ -1,50 +0,0 @@ - -# HEY! This file is used in the dbt package integrations tests with Buildkite. -# You should __NEVER__ check credentials into version control. Thanks for reading :) - -integration_tests: - target: redshift - outputs: - redshift: - type: redshift - host: "{{ env_var('CI_REDSHIFT_DBT_HOST') }}" - user: "{{ env_var('CI_REDSHIFT_DBT_USER') }}" - pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" - dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" - port: 5439 - schema: ad_reporting_integration_tests_9 - threads: 8 - bigquery: - type: bigquery - method: service-account-json - project: 'dbt-package-testing' - schema: ad_reporting_integration_tests_9 - threads: 8 - keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" - snowflake: - type: snowflake - account: "{{ env_var('CI_SNOWFLAKE_DBT_ACCOUNT') }}" - user: "{{ env_var('CI_SNOWFLAKE_DBT_USER') }}" - password: "{{ env_var('CI_SNOWFLAKE_DBT_PASS') }}" - role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" - database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" - warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: ad_reporting_integration_tests_9 - threads: 8 - postgres: - type: postgres - host: "{{ env_var('CI_POSTGRES_DBT_HOST') }}" - user: "{{ env_var('CI_POSTGRES_DBT_USER') }}" - pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" - dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" - port: 5432 - schema: ad_reporting_integration_tests_9 - threads: 8 - databricks: - catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" - host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" - http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: ad_reporting_integration_tests_9 - threads: 8 - token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" - type: databricks \ No newline at end of file diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml new file mode 100644 index 00000000..397e05b0 --- /dev/null +++ b/integration_tests/ci/test_scenarios.yml @@ -0,0 +1,44 @@ +# Only set if not using default adapter ranges +# dbt_adapter_versions: +# snowflake: ">=1.3.0,<2.0.0" +# bigquery: ">=1.3.0,<2.0.0" +# postgres: ">=1.3.0,<2.0.0" +# redshift: ">=1.3.0,<2.0.0" +# databricks: ">=1.6.0,<2.0.0" + +schema_variable_name: "ad_reporting_schema" + +additional_schema_variables: + - "amazon_ads_schema" + - "apple_search_ads_schema" + - "facebook_ads_schema" + - "google_ads_schema" + - "linkedin_ads_schema" + - "microsoft_ads_schema" + - "pinterest_ads_schema" + - "reddit_ads_schema" + - "snapchat_ads_schema" + - "tiktok_ads_schema" + - "twitter_ads_schema" + +include_databricks_sql: false +include_sqlserver: false +include_dbt_compile: true + +test_scenarios: + - name: "Default" + vars: + ad_reporting__facebook_ads_enabled: true + facebook_ads__using_demographics_country: true + facebook_ads__using_demographics_region: true + ad_reporting__google_ads_enabled: true + ad_reporting__amazon_ads_enabled: false + ad_reporting__apple_search_ads_enabled: false + ad_reporting__linkedin_ads_enabled: true + ad_reporting__microsoft_ads_enabled: false + ad_reporting__pinterest_ads_enabled: false + ad_reporting__reddit_ads_enabled: false + ad_reporting__snapchat_ads_enabled: false + ad_reporting__tiktok_ads_enabled: false + ad_reporting__twitter_ads_enabled: false + include_incremental: false \ No newline at end of file diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 3d89fe31..1a2a9200 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -402,7 +402,7 @@ vars: ############################ END PASSTHROUGHS ############################ amazon_ads: - amazon_ads_schema: ad_reporting_integration_tests_9 + amazon_ads_schema: ad_reporting_integration_tests amazon_ads_ad_group_history_identifier: "amazon_ads_ad_group_history_data" amazon_ads_ad_group_level_report_identifier: "amazon_ads_ad_group_level_report_data" amazon_ads_advertised_product_report_identifier: "amazon_ads_advertised_product_report_data" @@ -416,7 +416,7 @@ vars: amazon_ads_search_term_ad_keyword_report_identifier: "amazon_ads_search_term_ad_keyword_report_data" apple_search_ads: - apple_search_ads_schema: ad_reporting_integration_tests_9 + apple_search_ads_schema: ad_reporting_integration_tests apple_search_ads_organization_identifier: "apple_search_organization_data" apple_search_ads_campaign_history_identifier: "apple_search_campaign_history_data" apple_search_ads_campaign_report_identifier: "apple_search_campaign_report_data" @@ -430,7 +430,7 @@ vars: facebook_ads: - facebook_ads_schema: ad_reporting_integration_tests_9 + facebook_ads_schema: ad_reporting_integration_tests facebook_ads_account_history_identifier: "facebook_ads_account_history_data" facebook_ads_ad_history_identifier: "facebook_ads_ad_history_data" facebook_ads_ad_set_history_identifier: "facebook_ads_ad_set_history_data" @@ -446,7 +446,7 @@ vars: google_ads: - google_ads_schema: ad_reporting_integration_tests_9 + google_ads_schema: ad_reporting_integration_tests google_ads_ad_stats_identifier: "google_ads_ad_stats_data" google_ads_ad_history_identifier: "google_ads_ad_history_data" google_ads_ad_group_history_identifier: "google_ads_ad_group_history_data" @@ -465,7 +465,7 @@ vars: linkedin: - linkedin_ads_schema: ad_reporting_integration_tests_9 + linkedin_ads_schema: ad_reporting_integration_tests linkedin_ads_account_history_identifier: "linkedin_ad_account_history_data" linkedin_ads_ad_analytics_by_creative_identifier: "linkedin_ad_analytics_by_creative_data" linkedin_ads_campaign_group_history_identifier: "linkedin_ad_campaign_group_history_data" @@ -478,7 +478,7 @@ vars: microsoft_ads: - microsoft_ads_schema: ad_reporting_integration_tests_9 + microsoft_ads_schema: ad_reporting_integration_tests microsoft_ads_account_history_identifier: "microsoft_ads_account_history_data" microsoft_ads_account_performance_daily_report_identifier: "microsoft_ads_account_performance_daily_report_data" microsoft_ads_ad_group_history_identifier: "microsoft_ads_ad_group_history_data" @@ -494,7 +494,7 @@ vars: pinterest: - pinterest_schema: ad_reporting_integration_tests_9 + pinterest_schema: ad_reporting_integration_tests pinterest_ads_ad_group_history_identifier: "pinterest_ad_group_history_data" pinterest_ads_campaign_history_identifier: "pinterest_campaign_history_data" pinterest_ads_pin_promotion_history_identifier: "pinterest_pin_promotion_history_data" @@ -511,7 +511,7 @@ vars: reddit_ads: - reddit_ads_schema: ad_reporting_integration_tests_9 + reddit_ads_schema: ad_reporting_integration_tests reddit_ads_account_identifier: "reddit_ads_account_data" reddit_ads_account_report_identifier: "reddit_ads_account_report_data" reddit_ads_ad_identifier: "reddit_ads_ad_data" @@ -530,7 +530,7 @@ vars: snapchat_ads: - snapchat_ads_schema: ad_reporting_integration_tests_9 + snapchat_ads_schema: ad_reporting_integration_tests snapchat_ads_ad_account_history_identifier: "snapchat_ad_account_history_data" snapchat_ads_ad_history_identifier: "snapchat_ad_history_data" snapchat_ads_ad_hourly_report_identifier: "snapchat_ad_hourly_report_data" @@ -545,7 +545,7 @@ vars: tiktok_ads: - tiktok_ads_schema: ad_reporting_integration_tests_9 + tiktok_ads_schema: ad_reporting_integration_tests tiktok_ads_adgroup_history_identifier: "tiktok_adgroup_history_data" tiktok_ads_ad_history_identifier: "tiktok_ad_history_data" tiktok_ads_advertiser_identifier: "tiktok_advertiser_data" @@ -557,7 +557,7 @@ vars: twitter_ads: - twitter_ads_schema: ad_reporting_integration_tests_9 + twitter_ads_schema: ad_reporting_integration_tests twitter_ads_account_history_identifier: "twitter_account_history_data" twitter_ads_campaign_history_identifier: "twitter_campaign_history_data" twitter_ads_line_item_history_identifier: "twitter_line_item_history_data" @@ -595,80 +595,37 @@ seeds: microsoft_ads_ad_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" - spend: "float" microsoft_ads_ad_group_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" - spend: "float" microsoft_ads_account_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" - spend: "float" microsoft_ads_keyword_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" - spend: "float" microsoft_ads_keyword_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" - spend: "float" microsoft_ads_search_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" - spend: "float" microsoft_ads_campaign_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" - spend: "float" microsoft_ads_geographic_performance_daily_report_data: +column_types: account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" @@ -742,104 +699,51 @@ seeds: pinterest_ad_group_history_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_campaign_history_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_history_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_ad_group_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_advertiser_history_data: +column_types: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_advertiser_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_campaign_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_keyword_history_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_keyword_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_targeting_report_data: +column_types: campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" @@ -848,91 +752,41 @@ seeds: google_ads_ad_group_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_campaign_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_keyword_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_campaign_history_data: +column_types: - customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_group_history_data: +column_types: - customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_history_data: +column_types: - customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_group_criterion_history_data: +column_types: - customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_search_term_keyword_stats_data: +column_types: ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" @@ -1043,12 +897,10 @@ seeds: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" - click_through_rate: "float" amazon_ads_advertised_product_report_data: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" - click_through_rate: "float" amazon_ads_search_term_ad_keyword_report_data: +column_types: _fivetran_synced: "timestamp" @@ -1058,7 +910,6 @@ seeds: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" - click_through_rate: "float" keyword_bid: "float" From b2cd5504f5425268c90aa301da2d0c9f5451d78b Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 12 May 2026 16:26:10 -0500 Subject: [PATCH 2/7] Update dbt_project.yml --- integration_tests/dbt_project.yml | 179 +++++++++++++++++++++++++++--- 1 file changed, 164 insertions(+), 15 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 1a2a9200..b05e548d 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -595,37 +595,80 @@ seeds: microsoft_ads_ad_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + last_modified_time: "timestamp" + modified_time: "timestamp" + spend: "float" microsoft_ads_ad_group_performance_daily_report_data: +column_types: + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + last_modified_time: "timestamp" + modified_time: "timestamp" + spend: "float" microsoft_ads_account_performance_daily_report_data: +column_types: + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + last_modified_time: "timestamp" + modified_time: "timestamp" + spend: "float" microsoft_ads_keyword_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + last_modified_time: "timestamp" + modified_time: "timestamp" + spend: "float" microsoft_ads_keyword_performance_daily_report_data: +column_types: + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + last_modified_time: "timestamp" + modified_time: "timestamp" + spend: "float" microsoft_ads_search_performance_daily_report_data: +column_types: + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + last_modified_time: "timestamp" + modified_time: "timestamp" + spend: "float" microsoft_ads_campaign_performance_daily_report_data: +column_types: + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + last_modified_time: "timestamp" + modified_time: "timestamp" + spend: "float" microsoft_ads_geographic_performance_daily_report_data: +column_types: account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" @@ -699,51 +742,104 @@ seeds: pinterest_ad_group_history_data: +column_types: + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_campaign_history_data: +column_types: + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_history_data: +column_types: + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_report_data: +column_types: - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_ad_group_report_data: +column_types: - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_advertiser_history_data: +column_types: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_advertiser_report_data: +column_types: + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_campaign_report_data: +column_types: + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_keyword_history_data: +column_types: + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_keyword_report_data: +column_types: - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_targeting_report_data: +column_types: campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" @@ -752,41 +848,91 @@ seeds: google_ads_ad_group_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_campaign_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_keyword_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_campaign_history_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_group_history_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_history_data: +column_types: - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_group_criterion_history_data: +column_types: + customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_search_term_keyword_stats_data: +column_types: ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" @@ -897,10 +1043,12 @@ seeds: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" + click_through_rate: "float" amazon_ads_advertised_product_report_data: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" + click_through_rate: "float" amazon_ads_search_term_ad_keyword_report_data: +column_types: _fivetran_synced: "timestamp" @@ -910,6 +1058,7 @@ seeds: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" + click_through_rate: "float" keyword_bid: "float" @@ -976,4 +1125,4 @@ seeds: campaign_id: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar(255)' }}" flags: send_anonymous_usage_stats: False - use_colors: True \ No newline at end of file + use_colors: True From fa5991ed581cb21782045f14936483a8f84f1c6e Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Tue, 12 May 2026 22:31:54 -0500 Subject: [PATCH 3/7] Update test_scenarios.yml --- integration_tests/ci/test_scenarios.yml | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 397e05b0..149e9982 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -6,10 +6,9 @@ # redshift: ">=1.3.0,<2.0.0" # databricks: ">=1.6.0,<2.0.0" -schema_variable_name: "ad_reporting_schema" +schema_variable_name: "amazon_ads_schema" additional_schema_variables: - - "amazon_ads_schema" - "apple_search_ads_schema" - "facebook_ads_schema" - "google_ads_schema" @@ -41,4 +40,4 @@ test_scenarios: ad_reporting__snapchat_ads_enabled: false ad_reporting__tiktok_ads_enabled: false ad_reporting__twitter_ads_enabled: false - include_incremental: false \ No newline at end of file + include_incremental: false From a584180de45b7ae17b88efc7df225d04f4d51b29 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 15 May 2026 17:55:26 -0500 Subject: [PATCH 4/7] column types --- integration_tests/dbt_project.yml | 65 +++---------------------------- 1 file changed, 6 insertions(+), 59 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index b05e548d..970fb6ca 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -848,105 +848,55 @@ seeds: google_ads_ad_group_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_campaign_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_keyword_stats_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_campaign_history_data: +column_types: customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_group_history_data: +column_types: - customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_history_data: +column_types: - customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" + id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_ad_group_criterion_history_data: +column_types: - customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" base_campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_criterion_criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - base_adgroup_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" google_ads_search_term_keyword_stats_data: +column_types: ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" customer_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_bid_modifier_history_data: + google_ads_campaign_bid_modifier_history_data: +column_types: campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" criterion_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_budget_history_data: + google_ads_campaign_budget_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_criterion_history_data: + google_ads_campaign_criterion_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" @@ -957,7 +907,7 @@ seeds: user_interest_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" user_list_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" mobile_app_category_constant_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_bidding_strategy_history_data: + google_ads_campaign_bidding_strategy_history_data: +column_types: campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" @@ -1043,12 +993,10 @@ seeds: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" - click_through_rate: "float" amazon_ads_advertised_product_report_data: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" - click_through_rate: "float" amazon_ads_search_term_ad_keyword_report_data: +column_types: _fivetran_synced: "timestamp" @@ -1058,7 +1006,6 @@ seeds: +column_types: _fivetran_synced: "timestamp" campaign_budget_amount: "float" - click_through_rate: "float" keyword_bid: "float" From 2d3d830c084fe5289741ab762c5a29e68d751667 Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 15 May 2026 18:03:18 -0500 Subject: [PATCH 5/7] column types --- integration_tests/dbt_project.yml | 100 +++--------------------------- 1 file changed, 8 insertions(+), 92 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 970fb6ca..f246a9d3 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -595,79 +595,43 @@ seeds: microsoft_ads_ad_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" modified_time: "timestamp" - spend: "float" microsoft_ads_ad_group_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" spend: "float" microsoft_ads_account_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" spend: "float" microsoft_ads_keyword_history_data: +column_types: id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" modified_time: "timestamp" - spend: "float" microsoft_ads_keyword_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" spend: "float" microsoft_ads_search_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" spend: "float" microsoft_ads_campaign_performance_daily_report_data: +column_types: - id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" account_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - ad_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'INT64' if target.type == 'bigquery' else 'bigint' }}" - last_modified_time: "timestamp" - modified_time: "timestamp" spend: "float" microsoft_ads_geographic_performance_daily_report_data: +column_types: @@ -744,105 +708,57 @@ seeds: +column_types: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_campaign_history_data: +column_types: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_history_data: +column_types: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_ad_group_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_advertiser_history_data: +column_types: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_advertiser_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_campaign_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_keyword_history_data: +column_types: id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_keyword_report_data: +column_types: - id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - ad_account_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - keyword_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_targeting_report_data: +column_types: + ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" + pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ### Google Seeds google_ads_ad_group_stats_data: From 63c9d38608496f28c7dcb3272e64ed892fffbaad Mon Sep 17 00:00:00 2001 From: Catherine Fritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 15 May 2026 18:11:54 -0500 Subject: [PATCH 6/7] column types --- integration_tests/dbt_project.yml | 6 ------ 1 file changed, 6 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f246a9d3..2cfa537c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -654,11 +654,8 @@ seeds: linkedin_ad_creative_history_data: +column_types: created_time: timestamp - last_modified_time: timestamp - status: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}" intended_status: "{{ 'string' if target.type in ['bigquery','spark','databricks'] else 'varchar' }}" last_modified_at: timestamp - created_at: timestamp linkedin_ad_analytics_by_campaign_data: +column_types: day: timestamp @@ -755,10 +752,7 @@ seeds: advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" pinterest_pin_promotion_targeting_report_data: +column_types: - ad_group_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - advertiser_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" campaign_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" - pin_promotion_id: "{{ 'int64' if target.type == 'bigquery' else 'bigint' }}" ### Google Seeds google_ads_ad_group_stats_data: From e5ce2f4fc63fb2174cf629044678c935eb68822d Mon Sep 17 00:00:00 2001 From: fivetran-catfritz <111930712+fivetran-catfritz@users.noreply.github.com> Date: Fri, 15 May 2026 19:04:26 -0500 Subject: [PATCH 7/7] Update test_scenarios.yml --- integration_tests/ci/test_scenarios.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/integration_tests/ci/test_scenarios.yml b/integration_tests/ci/test_scenarios.yml index 149e9982..94c9bb2b 100644 --- a/integration_tests/ci/test_scenarios.yml +++ b/integration_tests/ci/test_scenarios.yml @@ -14,7 +14,7 @@ additional_schema_variables: - "google_ads_schema" - "linkedin_ads_schema" - "microsoft_ads_schema" - - "pinterest_ads_schema" + - "pinterest_schema" - "reddit_ads_schema" - "snapchat_ads_schema" - "tiktok_ads_schema"