From 7c06eaf714f2364b008ff551ef93e93648081e85 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 3 Mar 2023 13:48:53 -0800 Subject: [PATCH 01/21] utils --- dbt_project.yml | 5 ++- models/src_shopify.yml | 2 +- models/tmp/stg_shopify__discount_code_tmp.sql | 26 ++-------------- .../tmp/stg_shopify__order_adjustment_tmp.sql | 29 ++--------------- .../stg_shopify__order_line_refund_tmp.sql | 30 ++---------------- models/tmp/stg_shopify__refund_tmp.sql | 31 ++----------------- packages.yml | 7 +++-- 7 files changed, 19 insertions(+), 111 deletions(-) diff --git a/dbt_project.yml b/dbt_project.yml index 7305b66..37d3369 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -52,4 +52,7 @@ vars: order_pass_through_columns: [] product_pass_through_columns: [] product_variant_pass_through_columns: [] - transaction_pass_through_columns: [] \ No newline at end of file + transaction_pass_through_columns: [] + + shopify_union_databases: [] + shopify_union_schemas: [] \ No newline at end of file diff --git a/models/src_shopify.yml b/models/src_shopify.yml index 3133b12..7c9554a 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -1,7 +1,7 @@ version: 2 sources: - - name: shopify + - name: shopify # only if you're using a singular connector schema: "{{ var('shopify_schema', 'shopify') }}" database: "{% if target.type != 'spark'%}{{ var('shopify_database', target.database) }}{% endif %}" tables: diff --git a/models/tmp/stg_shopify__discount_code_tmp.sql b/models/tmp/stg_shopify__discount_code_tmp.sql index eba47d3..9795304 100644 --- a/models/tmp/stg_shopify__discount_code_tmp.sql +++ b/models/tmp/stg_shopify__discount_code_tmp.sql @@ -1,14 +1,5 @@ -- this model will be all NULL until you create a discount code in Shopify -{%- set source_relation = adapter.get_relation( - database=source('shopify', 'discount_code').database, - schema=source('shopify', 'discount_code').schema, - identifier=source('shopify', 'discount_code').name) -%} - -{% set table_exists=source_relation is not none %} - -{% if table_exists %} - {{ fivetran_utils.union_data( table_identifier='discount_code', @@ -17,21 +8,8 @@ default_database=target.database, default_schema='shopify', default_variable='discount_code_source', + column_macro=get_discount_code_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) -}} - -{% else %} - -select - cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, - cast(null as {{ dbt.type_string() }}) as code, - cast(null as {{ dbt.type_timestamp() }}) as created_at, - cast(null as {{ dbt.type_int() }}) as id, - cast(null as {{ dbt.type_int() }}) as price_rule_id, - cast(null as {{ dbt.type_timestamp() }}) as updated_at, - cast(null as {{ dbt.type_float() }}) as usage_count, - cast(null as {{ dbt.type_string() }}) as _dbt_source_relation - -{% endif %} \ No newline at end of file +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_adjustment_tmp.sql b/models/tmp/stg_shopify__order_adjustment_tmp.sql index 6ac4801..c815d9f 100644 --- a/models/tmp/stg_shopify__order_adjustment_tmp.sql +++ b/models/tmp/stg_shopify__order_adjustment_tmp.sql @@ -1,14 +1,5 @@ -- this model will be all NULL until you have made an order adjustment in Shopify -{%- set source_relation = adapter.get_relation( - database=source('shopify', 'order_adjustment').database, - schema=source('shopify', 'order_adjustment').schema, - identifier=source('shopify', 'order_adjustment').name) -%} - -{% set table_exists=source_relation is not none %} - -{% if table_exists %} - {{ fivetran_utils.union_data( table_identifier='order_adjustment', @@ -17,24 +8,8 @@ default_database=target.database, default_schema='shopify', default_variable='order_adjustment_source', + column_macro=get_order_adjustment_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) -}} - -{% else %} - -select - cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, - cast(null as {{ dbt.type_numeric() }}) as id, - cast(null as {{ dbt.type_numeric() }}) as order_id, - cast(null as {{ dbt.type_numeric() }}) as refund_id, - cast(null as {{ dbt.type_float() }}) as amount, - cast(null as {{ dbt.type_string() }}) as amount_set, - cast(null as {{ dbt.type_float() }}) as tax_amount, - cast(null as {{ dbt.type_string() }}) as tax_amount_set, - cast(null as {{ dbt.type_string() }}) as kind, - cast(null as {{ dbt.type_string() }}) as reason, - cast(null as {{ dbt.type_string() }}) as _dbt_source_relation - -{% endif %} \ No newline at end of file +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__order_line_refund_tmp.sql b/models/tmp/stg_shopify__order_line_refund_tmp.sql index 188bb1f..4a8f063 100644 --- a/models/tmp/stg_shopify__order_line_refund_tmp.sql +++ b/models/tmp/stg_shopify__order_line_refund_tmp.sql @@ -1,14 +1,5 @@ -- this model will be all NULL until you have made an order line refund in Shopify -{%- set source_relation = adapter.get_relation( - database=source('shopify', 'order_line_refund').database, - schema=source('shopify', 'order_line_refund').schema, - identifier=source('shopify', 'order_line_refund').name) -%} - -{% set table_exists=source_relation is not none %} - -{% if table_exists %} - {{ fivetran_utils.union_data( table_identifier='order_line_refund', @@ -17,25 +8,8 @@ default_database=target.database, default_schema='shopify', default_variable='order_line_refund_source', + column_macro=get_order_line_refund_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) -}} - -{% else %} - -select - cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, - cast(null as {{ dbt.type_numeric() }}) as id, - cast(null as {{ dbt.type_numeric() }}) as location_id, - cast(null as {{ dbt.type_numeric() }}) as order_line_id, - cast(null as {{ dbt.type_numeric() }}) as subtotal, - cast(null as {{ dbt.type_string() }}) as subtotal_set, - cast(null as {{ dbt.type_numeric() }}) as total_tax, - cast(null as {{ dbt.type_string() }}) as total_tax_set, - cast(null as {{ dbt.type_float() }}) as quantity, - cast(null as {{ dbt.type_numeric() }}) as refund_id, - cast(null as {{ dbt.type_string() }}) as restock_type, - cast(null as {{ dbt.type_string() }}) as _dbt_source_relation - -{% endif %} \ No newline at end of file +}} \ No newline at end of file diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index d57c0ce..7f857f1 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -1,14 +1,4 @@ ---To disable this model, set the shopify__using_refund variable within your dbt_project.yml file to False. -{{ config(enabled=var('shopify__using_refund', True)) }} - -{%- set source_relation = adapter.get_relation( - database=source('shopify', 'refund').database, - schema=source('shopify', 'refund').schema, - identifier=source('shopify', 'refund').name) -%} - -{% set table_exists=source_relation is not none %} - -{% if table_exists %} +-- this model will be all NULL until you create a refund in Shopify {{ fivetran_utils.union_data( @@ -18,23 +8,8 @@ default_database=target.database, default_schema='shopify', default_variable='refund_source', + column_macro=get_refund_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) -}} - -{% else %} - -select - cast(null as {{ dbt.type_timestamp() }}) as _fivetran_synced, - cast(null as {{ dbt.type_timestamp() }}) as created_at, - cast(null as {{ dbt.type_numeric() }}) as id, - cast(null as {{ dbt.type_string() }}) as note, - cast(null as {{ dbt.type_numeric() }}) as order_id, - cast(null as {{ dbt.type_timestamp() }}) as processed_at, - cast(null as boolean) as restock, - cast(null as {{ dbt.type_string() }}) as total_duties_set, - cast(null as {{ dbt.type_numeric() }}) as user_id, - cast(null as {{ dbt.type_string() }}) as _dbt_source_relation - -{% endif %} \ No newline at end of file +}} \ No newline at end of file diff --git a/packages.yml b/packages.yml index c2758dd..ba03cb0 100644 --- a/packages.yml +++ b/packages.yml @@ -1,6 +1,9 @@ packages: -- package: fivetran/fivetran_utils - version: [">=0.4.0", "<0.5.0"] +# - package: fivetran/fivetran_utils +# version: [">=0.4.0", "<0.5.0"] +# - local: ../dbt_fivetran_utils +- git: https://github.com/fivetran/dbt_fivetran_utils.git + revision: feature/expand-union-data - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] - package: calogica/dbt_expectations From bfe37a13b5773f9de54702ccfc6cce7177255fed Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 3 Mar 2023 13:58:24 -0800 Subject: [PATCH 02/21] utils --- CHANGELOG.md | 8 ++++++++ dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 73dce21..75de3fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,11 @@ +# dbt_shopify_source v0.8.2 + +## Bug Fixes +- The fivetan_utils.union_data macro has been expanded to handle cases where none of the provided Shopify schemas or databases contain an expected source table. + +## Contributors +- @dfagnan - https://github.com/fivetran/dbt_shopify_source/issues/57 + # dbt_shopify_source v0.8.1 ## Bug Fixes diff --git a/dbt_project.yml b/dbt_project.yml index 37d3369..99abc4a 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,5 +1,5 @@ name: 'shopify_source' -version: '0.8.1' +version: '0.8.2' config-version: 2 require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index f11f6d9..a612eeb 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'shopify_source_integration_tests' -version: '0.8.1' +version: '0.8.2' profile: 'integration_tests' config-version: 2 From 5600bb0ffd37d00e9bd68068ca72ee390f25928c Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 3 Mar 2023 14:06:29 -0800 Subject: [PATCH 03/21] trweaks --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 75de3fe..ab4cd26 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_shopify_source v0.8.2 ## Bug Fixes -- The fivetan_utils.union_data macro has been expanded to handle cases where none of the provided Shopify schemas or databases contain an expected source table. +- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will create an all-null version of the staging model. ## Contributors - @dfagnan - https://github.com/fivetran/dbt_shopify_source/issues/57 From f8c88053676743ab85981826704c55e0f26ab910 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 3 Mar 2023 14:11:14 -0800 Subject: [PATCH 04/21] decision log --- CHANGELOG.md | 2 +- DECISIONLOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ab4cd26..6eacc89 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_shopify_source v0.8.2 ## Bug Fixes -- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will create an all-null version of the staging model. +- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will create an all-null version of the staging model ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). ## Contributors - @dfagnan - https://github.com/fivetran/dbt_shopify_source/issues/57 diff --git a/DECISIONLOG.md b/DECISIONLOG.md index d4653ab..938ab0d 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -2,7 +2,7 @@ In creating this package, which is meant for a wide range of use cases, we had to take opinionated stances on a few different questions we came across during development. We've consolidated significant choices we made here, and will continue to update as the package evolves. -## Creating Empty Tables for Refunds, Order Line Refunds, and Order Adjustments +## Creating Empty Tables for Refunds, Order Line Refunds, and Order Adjustments -- todo: update this after confirming the plan forward Source tables related to `refunds`, `order_line_refunds`, and `order_adjustments` are created in the Shopify schema dyanmically. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. From e99aff9d9c6735cce2778b549b66051a7fd82cec Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 8 Mar 2023 13:02:30 -0800 Subject: [PATCH 05/21] joe feedback --- CHANGELOG.md | 2 +- DECISIONLOG.md | 2 +- dbt_project.yml | 3 --- models/src_shopify.yml | 2 +- models/tmp/stg_shopify__discount_code_tmp.sql | 1 - models/tmp/stg_shopify__order_adjustment_tmp.sql | 1 - models/tmp/stg_shopify__refund_tmp.sql | 1 - 7 files changed, 3 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6eacc89..740637c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -4,7 +4,7 @@ - The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will create an all-null version of the staging model ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). ## Contributors -- @dfagnan - https://github.com/fivetran/dbt_shopify_source/issues/57 +- [@dfagnan](https://github.com/dfagnan) (Issue https://github.com/fivetran/dbt_shopify_source/issues/57) # dbt_shopify_source v0.8.1 diff --git a/DECISIONLOG.md b/DECISIONLOG.md index 938ab0d..c3fc615 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -2,7 +2,7 @@ In creating this package, which is meant for a wide range of use cases, we had to take opinionated stances on a few different questions we came across during development. We've consolidated significant choices we made here, and will continue to update as the package evolves. -## Creating Empty Tables for Refunds, Order Line Refunds, and Order Adjustments -- todo: update this after confirming the plan forward +## Creating Empty Tables for Refunds, Order Line Refunds, and Order Adjustments Source tables related to `refunds`, `order_line_refunds`, and `order_adjustments` are created in the Shopify schema dyanmically. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. diff --git a/dbt_project.yml b/dbt_project.yml index 99abc4a..e3f84ec 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -53,6 +53,3 @@ vars: product_pass_through_columns: [] product_variant_pass_through_columns: [] transaction_pass_through_columns: [] - - shopify_union_databases: [] - shopify_union_schemas: [] \ No newline at end of file diff --git a/models/src_shopify.yml b/models/src_shopify.yml index 7c9554a..c587e74 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -1,7 +1,7 @@ version: 2 sources: - - name: shopify # only if you're using a singular connector + - name: shopify # This source will only be used if you are using a single Shopify source connector. If multiple sources are being unioned, their tables will be directly referenced via adatper.get_relation schema: "{{ var('shopify_schema', 'shopify') }}" database: "{% if target.type != 'spark'%}{{ var('shopify_database', target.database) }}{% endif %}" tables: diff --git a/models/tmp/stg_shopify__discount_code_tmp.sql b/models/tmp/stg_shopify__discount_code_tmp.sql index 9795304..eca13f4 100644 --- a/models/tmp/stg_shopify__discount_code_tmp.sql +++ b/models/tmp/stg_shopify__discount_code_tmp.sql @@ -8,7 +8,6 @@ default_database=target.database, default_schema='shopify', default_variable='discount_code_source', - column_macro=get_discount_code_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) diff --git a/models/tmp/stg_shopify__order_adjustment_tmp.sql b/models/tmp/stg_shopify__order_adjustment_tmp.sql index c815d9f..608f386 100644 --- a/models/tmp/stg_shopify__order_adjustment_tmp.sql +++ b/models/tmp/stg_shopify__order_adjustment_tmp.sql @@ -8,7 +8,6 @@ default_database=target.database, default_schema='shopify', default_variable='order_adjustment_source', - column_macro=get_order_adjustment_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) diff --git a/models/tmp/stg_shopify__refund_tmp.sql b/models/tmp/stg_shopify__refund_tmp.sql index 7f857f1..7491be0 100644 --- a/models/tmp/stg_shopify__refund_tmp.sql +++ b/models/tmp/stg_shopify__refund_tmp.sql @@ -8,7 +8,6 @@ default_database=target.database, default_schema='shopify', default_variable='refund_source', - column_macro=get_refund_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) From a2174cbbb8508476d6d9598b3e1bc65fe7f497a4 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 8 Mar 2023 15:10:38 -0800 Subject: [PATCH 06/21] dynamic not_nulls --- models/stg_shopify.yml | 96 ++++++++++++++++++++++++++++-------------- 1 file changed, 64 insertions(+), 32 deletions(-) diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index 0741a75..04e9565 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -667,7 +667,9 @@ models: description: The payment gateway used by the checkout. - name: checkout_id description: The ID for the checkout. - - not_null + tests: + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: landing_site_base_url description: The URL for the page where the customer entered the shop. - name: location_id @@ -753,11 +755,13 @@ models: - name: collection_id description: ID referencing the `collection` the product belongs to. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: product_id description: ID referencing the `product`. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: source_relation description: "{{ doc('source_relation') }}" @@ -782,7 +786,8 @@ models: - name: collection_id description: The ID for the collection. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: published_at description: The time and date (ISO 8601 format) when the collection was made visible. Returns null for a hidden collection. - name: published_scope @@ -828,11 +833,13 @@ models: - name: index description: Index (starting from 1) representing when the tag was placed on the customer. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: customer_id description: ID of the customer being tagged. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: value description: Value of the tag. - name: source_relation @@ -857,7 +864,8 @@ models: - name: discount_code_id description: The ID for the discount code. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: price_rule_id description: The ID for the price rule that this discount code belongs to. - name: updated_at @@ -884,7 +892,8 @@ models: - name: fulfillment_id description: The ID for the fulfillment. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: location_id description: The unique identifier of the location that the fulfillment was processed at. - name: name @@ -957,7 +966,8 @@ models: - name: inventory_item_id description: The ID of the inventory item. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: province_code_of_origin description: The province code (ISO 3166-2 alpha-2) of where the item came from. The province code is only used if the shipping provider for the inventory item is Canada Post. - name: is_shipping_required @@ -991,11 +1001,13 @@ models: - name: inventory_item_id description: The ID of the inventory item associated with the inventory level. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: location_id description: The ID of the location that the inventory level belongs to. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: updated_at description: The date and time (ISO 8601 format) when the inventory level was last modified. - name: source_relation @@ -1033,7 +1045,8 @@ models: - name: location_id description: The ID of the location. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: is_legacy description: > Boolean representing whether this is a fulfillment service location. If true, then the location is a fulfillment service location. @@ -1076,7 +1089,8 @@ models: - name: metafield_id description: The unique ID of the metafield. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: key description: The key of the metafield. Keys can be up to 64 characters long and can contain alphanumeric characters, hyphens, underscores, and periods. - name: namespace @@ -1112,11 +1126,13 @@ models: - name: name description: Name of the attribute. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: order_id description: ID referencing the order the note attribute belongs to. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: value description: Value of the attribute. - name: source_relation @@ -1145,7 +1161,8 @@ models: - name: order_shipping_line_id description: A globally-unique identifier. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: order_id description: ID of the associated order. - name: phone @@ -1177,11 +1194,13 @@ models: - name: index description: Index (from 1) representing the order of shipping lines per order. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: order_shipping_line_id description: ID of the order shipping line this record is associated with. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: price description: The amount of tax, in shop currency, after discounts and before returns. - name: price_set @@ -1207,11 +1226,13 @@ models: - name: index description: Index (starting from 1) representing when the tag was placed on the order. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: order_id description: ID of the order being tagged. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: value description: Value of the tag. - name: source_relation @@ -1230,12 +1251,14 @@ models: description: "{{ doc('_fivetran_synced') }}" - name: key description: Key of the tag pair. - tests: - - not_null + tests: + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: order_id description: ID of the order url being tagged. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: value description: Value of the tag. - name: source_relation @@ -1272,7 +1295,8 @@ models: - name: price_rule_id description: The ID for the price rule. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: is_once_per_customer description: Boolean representing whether the generated discount code will be valid only for a single use per customer. This is tracked using customer ID. - name: prereq_min_quantity @@ -1341,7 +1365,8 @@ models: - name: product_image_id description: Unique numeric identifier of the product image. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: position description: The order of the product image in the list. The first product image is at position 1 and is the "main" image for the product. - name: product_id @@ -1371,11 +1396,13 @@ models: - name: index description: Index (starting from 1) representing when the tag was placed on the product. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: product_id description: ID of the product being tagged. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: value description: Value of the tag. - name: source_relation @@ -1444,7 +1471,8 @@ models: - name: shop_id description: The ID for the shop. A 64-bit unsigned integer. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: latitude description: The latitude of the shop's location. - name: longitude @@ -1520,7 +1548,8 @@ models: - name: transaction_id description: The ID of the transaction. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: order_id description: The ID of the order that the tender transaction belongs to. - name: payment_method @@ -1563,7 +1592,8 @@ models: - name: checkout_id description: ID of the checkout. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: code description: The discount code. - name: created_at @@ -1593,11 +1623,13 @@ models: - name: code description: This property returns the discount code that was entered at checkout. Otherwise this property returns the title of the discount that was applied. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: order_id description: Associated order ID. tests: - - not_null + - dbt_expectations.expect_column_values_to_not_be_null: + row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) - name: type description: The type of discount - `fixed_amount`, `percentage`, or `shipping`. - name: source_relation From 672e0bcd9857610126b35bbd4063b3a22f4ae7a8 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 8 Mar 2023 15:38:33 -0800 Subject: [PATCH 07/21] remove macro --- models/tmp/stg_shopify__order_line_refund_tmp.sql | 1 - 1 file changed, 1 deletion(-) diff --git a/models/tmp/stg_shopify__order_line_refund_tmp.sql b/models/tmp/stg_shopify__order_line_refund_tmp.sql index 4a8f063..1d57168 100644 --- a/models/tmp/stg_shopify__order_line_refund_tmp.sql +++ b/models/tmp/stg_shopify__order_line_refund_tmp.sql @@ -8,7 +8,6 @@ default_database=target.database, default_schema='shopify', default_variable='order_line_refund_source', - column_macro=get_order_line_refund_columns(), union_schema_variable='shopify_union_schemas', union_database_variable='shopify_union_databases' ) From e894259352fbe79a5683fbb27cb087d792ffe940 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 9 Mar 2023 12:25:25 -0800 Subject: [PATCH 08/21] tweaks --- CHANGELOG.md | 2 +- DECISIONLOG.md | 8 ++-- README.md | 4 +- models/stg_shopify.yml | 90 ++++++++++++++---------------------------- 4 files changed, 38 insertions(+), 66 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 740637c..6e91236 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_shopify_source v0.8.2 ## Bug Fixes -- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will create an all-null version of the staging model ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). +- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) with just one string column (`_dbt_source_relation`). ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). ## Contributors - [@dfagnan](https://github.com/dfagnan) (Issue https://github.com/fivetran/dbt_shopify_source/issues/57) diff --git a/DECISIONLOG.md b/DECISIONLOG.md index c3fc615..249d368 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -2,13 +2,13 @@ In creating this package, which is meant for a wide range of use cases, we had to take opinionated stances on a few different questions we came across during development. We've consolidated significant choices we made here, and will continue to update as the package evolves. -## Creating Empty Tables for Refunds, Order Line Refunds, and Order Adjustments +## Creating Empty Staging Models for Non-Existent Source Tables -Source tables related to `refunds`, `order_line_refunds`, and `order_adjustments` are created in the Shopify schema dyanmically. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. +Empty source tables are created in the Shopify schema dyanmically if they do not exist in your raw source schema. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. -Thus, the source package will create empty (1 row of all `NULL` fields) staging models if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once `refund`, `order_line_refund`, or `order_adjustment` exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). +Thus, the source package will will return **completely** empty staging models (ie `limit 0`) with just one string column (`_dbt_source_relation`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -> In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution. +> In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. ## Keeping Deleted Entities diff --git a/README.md b/README.md index 6294dcc..97c70d7 100644 --- a/README.md +++ b/README.md @@ -128,7 +128,7 @@ models: shopify_source: +schema: my_new_schema_name # leave blank for just the target_schema ``` -### Change the source table references +### Change the source table references (not available if unioning multiple Shopify connectors) If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: > IMPORTANT: See this project's [`src_shopify.yml`](https://github.com/fivetran/dbt_shopify_source/blob/main/models/src_shopify.yml) for the default names. @@ -139,6 +139,8 @@ vars: shopify__identifier: your_table_name ``` +If you are making use of the `shopify_union_schemas` or `shopify_union_databases` variables, the package will assume individual tables to have their default names. + ## (Optional) Step 7: Orchestrate your models with Fivetran Transformations for dbt Core™ diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index 04e9565..ae63f91 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -668,8 +668,7 @@ models: - name: checkout_id description: The ID for the checkout. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: landing_site_base_url description: The URL for the page where the customer entered the shop. - name: location_id @@ -755,13 +754,11 @@ models: - name: collection_id description: ID referencing the `collection` the product belongs to. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: product_id description: ID referencing the `product`. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: source_relation description: "{{ doc('source_relation') }}" @@ -786,8 +783,7 @@ models: - name: collection_id description: The ID for the collection. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: published_at description: The time and date (ISO 8601 format) when the collection was made visible. Returns null for a hidden collection. - name: published_scope @@ -833,13 +829,11 @@ models: - name: index description: Index (starting from 1) representing when the tag was placed on the customer. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: customer_id description: ID of the customer being tagged. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: value description: Value of the tag. - name: source_relation @@ -864,8 +858,7 @@ models: - name: discount_code_id description: The ID for the discount code. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: price_rule_id description: The ID for the price rule that this discount code belongs to. - name: updated_at @@ -892,8 +885,7 @@ models: - name: fulfillment_id description: The ID for the fulfillment. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: location_id description: The unique identifier of the location that the fulfillment was processed at. - name: name @@ -966,8 +958,7 @@ models: - name: inventory_item_id description: The ID of the inventory item. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: province_code_of_origin description: The province code (ISO 3166-2 alpha-2) of where the item came from. The province code is only used if the shipping provider for the inventory item is Canada Post. - name: is_shipping_required @@ -1001,13 +992,11 @@ models: - name: inventory_item_id description: The ID of the inventory item associated with the inventory level. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: location_id description: The ID of the location that the inventory level belongs to. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: updated_at description: The date and time (ISO 8601 format) when the inventory level was last modified. - name: source_relation @@ -1045,8 +1034,7 @@ models: - name: location_id description: The ID of the location. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: is_legacy description: > Boolean representing whether this is a fulfillment service location. If true, then the location is a fulfillment service location. @@ -1089,8 +1077,7 @@ models: - name: metafield_id description: The unique ID of the metafield. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: key description: The key of the metafield. Keys can be up to 64 characters long and can contain alphanumeric characters, hyphens, underscores, and periods. - name: namespace @@ -1126,13 +1113,11 @@ models: - name: name description: Name of the attribute. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: order_id description: ID referencing the order the note attribute belongs to. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: value description: Value of the attribute. - name: source_relation @@ -1161,8 +1146,7 @@ models: - name: order_shipping_line_id description: A globally-unique identifier. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: order_id description: ID of the associated order. - name: phone @@ -1194,13 +1178,11 @@ models: - name: index description: Index (from 1) representing the order of shipping lines per order. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: order_shipping_line_id description: ID of the order shipping line this record is associated with. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: price description: The amount of tax, in shop currency, after discounts and before returns. - name: price_set @@ -1226,13 +1208,11 @@ models: - name: index description: Index (starting from 1) representing when the tag was placed on the order. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: order_id description: ID of the order being tagged. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: value description: Value of the tag. - name: source_relation @@ -1252,13 +1232,11 @@ models: - name: key description: Key of the tag pair. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: order_id description: ID of the order url being tagged. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: value description: Value of the tag. - name: source_relation @@ -1295,8 +1273,7 @@ models: - name: price_rule_id description: The ID for the price rule. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: is_once_per_customer description: Boolean representing whether the generated discount code will be valid only for a single use per customer. This is tracked using customer ID. - name: prereq_min_quantity @@ -1365,8 +1342,7 @@ models: - name: product_image_id description: Unique numeric identifier of the product image. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: position description: The order of the product image in the list. The first product image is at position 1 and is the "main" image for the product. - name: product_id @@ -1396,13 +1372,11 @@ models: - name: index description: Index (starting from 1) representing when the tag was placed on the product. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: product_id description: ID of the product being tagged. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: value description: Value of the tag. - name: source_relation @@ -1471,8 +1445,7 @@ models: - name: shop_id description: The ID for the shop. A 64-bit unsigned integer. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: latitude description: The latitude of the shop's location. - name: longitude @@ -1548,8 +1521,7 @@ models: - name: transaction_id description: The ID of the transaction. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: order_id description: The ID of the order that the tender transaction belongs to. - name: payment_method @@ -1623,13 +1595,11 @@ models: - name: code description: This property returns the discount code that was entered at checkout. Otherwise this property returns the title of the discount that was applied. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: order_id description: Associated order ID. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: type description: The type of discount - `fixed_amount`, `percentage`, or `shipping`. - name: source_relation From 3236f4ff0b21c600b777c452f5c5d28cac75c832 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:06:21 -0800 Subject: [PATCH 09/21] test --- CHANGELOG.md | 3 +- DECISIONLOG.md | 2 +- models/stg_shopify.yml | 180 ++++++++++++++++++++++++++++++++++++++++- 3 files changed, 181 insertions(+), 4 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6e91236..347b571 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,8 @@ # dbt_shopify_source v0.8.2 ## Bug Fixes -- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) with just one string column (`_dbt_source_relation`). ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). +- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) that will work seamlessly with downstream transformations ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). + - A [dbt_expectations](https://github.com/calogica/dbt-expectations#expect_table_row_count_to_be_between) test has been placed on each staging model to raise a **warning** when it is completely empty. ## Contributors - [@dfagnan](https://github.com/dfagnan) (Issue https://github.com/fivetran/dbt_shopify_source/issues/57) diff --git a/DECISIONLOG.md b/DECISIONLOG.md index 249d368..4918063 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -6,7 +6,7 @@ In creating this package, which is meant for a wide range of use cases, we had t Empty source tables are created in the Shopify schema dyanmically if they do not exist in your raw source schema. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. -Thus, the source package will will return **completely** empty staging models (ie `limit 0`) with just one string column (`_dbt_source_relation`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). +Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). > In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index ae63f91..4cc09c6 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -8,6 +8,11 @@ models: combination_of_columns: - customer_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -61,6 +66,11 @@ models: combination_of_columns: - order_line_refund_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -94,6 +104,11 @@ models: combination_of_columns: - order_line_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -193,6 +208,11 @@ models: combination_of_columns: - order_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -372,6 +392,11 @@ models: combination_of_columns: - product_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: is_deleted description: Whether the record has been deleted in the source system. @@ -412,6 +437,11 @@ models: combination_of_columns: - variant_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: barcode description: The barcode, UPC, or ISBN number for the product. @@ -473,6 +503,11 @@ models: combination_of_columns: - transaction_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: transaction_id description: The ID for the transaction. @@ -548,6 +583,11 @@ models: combination_of_columns: - refund_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: refund_id description: The unique numeric identifier for the refund. @@ -577,6 +617,11 @@ models: combination_of_columns: - order_adjustment_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: order_adjustment_id description: The unique numeric identifier for the order adjustment. @@ -608,6 +653,11 @@ models: combination_of_columns: - checkout_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -748,6 +798,11 @@ models: - collection_id - product_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -771,6 +826,11 @@ models: combination_of_columns: - collection_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -823,6 +883,11 @@ models: - customer_id - index - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -848,6 +913,11 @@ models: combination_of_columns: - discount_code_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -876,7 +946,12 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - fulfillment_id - - source_relation + - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -944,6 +1019,11 @@ models: combination_of_columns: - inventory_item_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -984,6 +1064,11 @@ models: - inventory_item_id - location_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1009,6 +1094,11 @@ models: combination_of_columns: - location_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -1067,6 +1157,11 @@ models: combination_of_columns: - metafield_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1107,6 +1202,11 @@ models: - order_id - name - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1130,6 +1230,11 @@ models: combination_of_columns: - order_shipping_line_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1172,6 +1277,11 @@ models: - order_shipping_line_id - index - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1202,6 +1312,11 @@ models: - order_id - index - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1226,6 +1341,11 @@ models: - order_id - key - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1250,7 +1370,12 @@ models: - dbt_utils.unique_combination_of_columns: combination_of_columns: - price_rule_id - - source_relation + - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1332,6 +1457,11 @@ models: combination_of_columns: - product_image_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1366,6 +1496,11 @@ models: - product_id - index - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1391,6 +1526,11 @@ models: combination_of_columns: - shop_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -1511,6 +1651,11 @@ models: combination_of_columns: - transaction_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1556,6 +1701,11 @@ models: - checkout_id - code - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1587,6 +1737,11 @@ models: - order_id - code - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1613,6 +1768,11 @@ models: - checkout_id - index - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1660,6 +1820,11 @@ models: combination_of_columns: - fulfillment_event_id - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1720,6 +1885,17 @@ models: - name: stg_shopify__tax_line description: Represents a single tax applied to the associated line item. + tests: + - dbt_utils.unique_combination_of_columns: + combination_of_columns: + - order_line_id + - index + - source_relation + - dbt_expectations.expect_table_row_count_to_be_between: + min_value: 1 + strictly: false # inclusive + config: + severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" From 91bf4a554e37efcf145ac0aa0e472759e9d77a7c Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:10:39 -0800 Subject: [PATCH 10/21] decision log --- DECISIONLOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/DECISIONLOG.md b/DECISIONLOG.md index 4918063..c4645d2 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -8,6 +8,8 @@ Empty source tables are created in the Shopify schema dyanmically if they do not Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). +A [dbt_expectations](https://github.com/calogica/dbt-expectations#expect_table_row_count_to_be_between) test has been placed on each staging model to raise a **warning** when it is completely empty. + > In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. ## Keeping Deleted Entities From aef65cf4bfb6faef916bb81a5f09e200760e78fa Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 9 Mar 2023 16:20:42 -0800 Subject: [PATCH 11/21] databricks --- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 5e5bbad..92694fe 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index a612eeb..44bc2b4 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,7 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: - shopify_schema: shopify_source_integration_tests_5 + shopify_schema: shopify_source_integration_tests_6 shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" From 58b75911b1824107956a1949153f4b8aa0d745d0 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 10 Mar 2023 12:18:01 -0800 Subject: [PATCH 12/21] work --- CHANGELOG.md | 2 +- DECISIONLOG.md | 2 +- README.md | 15 ++++ models/stg_shopify.yml | 170 ----------------------------------------- 4 files changed, 17 insertions(+), 172 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 347b571..38d5fea 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ ## Bug Fixes - The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) that will work seamlessly with downstream transformations ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). - - A [dbt_expectations](https://github.com/calogica/dbt-expectations#expect_table_row_count_to_be_between) test has been placed on each staging model to raise a **warning** when it is completely empty. + - A compilation **warning** when it is completely empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. ## Contributors - [@dfagnan](https://github.com/dfagnan) (Issue https://github.com/fivetran/dbt_shopify_source/issues/57) diff --git a/DECISIONLOG.md b/DECISIONLOG.md index c4645d2..9696a1e 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -8,7 +8,7 @@ Empty source tables are created in the Shopify schema dyanmically if they do not Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -A [dbt_expectations](https://github.com/calogica/dbt-expectations#expect_table_row_count_to_be_between) test has been placed on each staging model to raise a **warning** when it is completely empty. +The package will raise a compiler warning that the respective staging model is empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. > In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. diff --git a/README.md b/README.md index 97c70d7..c9a10d1 100644 --- a/README.md +++ b/README.md @@ -141,6 +141,21 @@ vars: If you are making use of the `shopify_union_schemas` or `shopify_union_databases` variables, the package will assume individual tables to have their default names. +### Turn off compiler warnings for empty staging models + +Empty source tables are created in the Shopify schema dyanmically if they do not exist in your raw source schema. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. + +Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). + +The package will raise a compiler warning that the respective staging model is empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. + +```yml +# dbt_project.yml + +vars: + remove_empty_table_warnings: true # default = false +``` + ## (Optional) Step 7: Orchestrate your models with Fivetran Transformations for dbt Core™ diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index 4cc09c6..bfff1b4 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -8,11 +8,6 @@ models: combination_of_columns: - customer_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -66,11 +61,6 @@ models: combination_of_columns: - order_line_refund_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -104,11 +94,6 @@ models: combination_of_columns: - order_line_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -208,11 +193,6 @@ models: combination_of_columns: - order_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -392,11 +372,6 @@ models: combination_of_columns: - product_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: is_deleted description: Whether the record has been deleted in the source system. @@ -437,11 +412,6 @@ models: combination_of_columns: - variant_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: barcode description: The barcode, UPC, or ISBN number for the product. @@ -503,11 +473,6 @@ models: combination_of_columns: - transaction_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: transaction_id description: The ID for the transaction. @@ -583,11 +548,6 @@ models: combination_of_columns: - refund_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: refund_id description: The unique numeric identifier for the refund. @@ -617,11 +577,6 @@ models: combination_of_columns: - order_adjustment_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: order_adjustment_id description: The unique numeric identifier for the order adjustment. @@ -653,11 +608,6 @@ models: combination_of_columns: - checkout_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -798,11 +748,6 @@ models: - collection_id - product_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -826,11 +771,6 @@ models: combination_of_columns: - collection_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -883,11 +823,6 @@ models: - customer_id - index - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -913,11 +848,6 @@ models: combination_of_columns: - discount_code_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -947,11 +877,6 @@ models: combination_of_columns: - fulfillment_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1019,11 +944,6 @@ models: combination_of_columns: - inventory_item_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -1064,11 +984,6 @@ models: - inventory_item_id - location_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1094,11 +1009,6 @@ models: combination_of_columns: - location_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -1157,11 +1067,6 @@ models: combination_of_columns: - metafield_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1202,11 +1107,6 @@ models: - order_id - name - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1230,11 +1130,6 @@ models: combination_of_columns: - order_shipping_line_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1277,11 +1172,6 @@ models: - order_shipping_line_id - index - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1312,11 +1202,6 @@ models: - order_id - index - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1341,11 +1226,6 @@ models: - order_id - key - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1371,11 +1251,6 @@ models: combination_of_columns: - price_rule_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1457,11 +1332,6 @@ models: combination_of_columns: - product_image_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1496,11 +1366,6 @@ models: - product_id - index - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1526,11 +1391,6 @@ models: combination_of_columns: - shop_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: is_deleted description: "{{ doc('_fivetran_deleted') }}" @@ -1651,11 +1511,6 @@ models: combination_of_columns: - transaction_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1701,11 +1556,6 @@ models: - checkout_id - code - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1737,11 +1587,6 @@ models: - order_id - code - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1768,11 +1613,6 @@ models: - checkout_id - index - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1820,11 +1660,6 @@ models: combination_of_columns: - fulfillment_event_id - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" @@ -1891,11 +1726,6 @@ models: - order_line_id - index - source_relation - - dbt_expectations.expect_table_row_count_to_be_between: - min_value: 1 - strictly: false # inclusive - config: - severity: warn columns: - name: _fivetran_synced description: "{{ doc('_fivetran_synced') }}" From ce773434231257de65a26bf7bbada9da7a3328bd Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 10 Mar 2023 12:23:08 -0800 Subject: [PATCH 13/21] revert test --- models/stg_shopify.yml | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/stg_shopify.yml b/models/stg_shopify.yml index bfff1b4..bf88b54 100644 --- a/models/stg_shopify.yml +++ b/models/stg_shopify.yml @@ -1564,8 +1564,7 @@ models: - name: checkout_id description: ID of the checkout. tests: - - dbt_expectations.expect_column_values_to_not_be_null: - row_condition: "source_relation is not null" # source_relation is null when the source table doesn't exist (so we made an empty model) + - not_null - name: code description: The discount code. - name: created_at From e9ce54d6b05649c48adbeedaf49bc2cf445b8a06 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Fri, 10 Mar 2023 14:22:58 -0800 Subject: [PATCH 14/21] Docs --- CHANGELOG.md | 6 ++++-- DECISIONLOG.md | 2 +- README.md | 8 ++++---- 3 files changed, 9 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 38d5fea..f1d9e98 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,8 +1,10 @@ # dbt_shopify_source v0.8.2 ## Bug Fixes -- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) that will work seamlessly with downstream transformations ([PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59)). - - A compilation **warning** when it is completely empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. +[PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59) introduces the following changes: +- The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) that will work seamlessly with downstream transformations. + - A compilation message will be raised when a staging model is completely empty. This compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. +- A uniqueness test has been placed on the `order_line_id`, `index`, and `source_relation` columns in `stg_shopify__tax_line`, as it was previously missing a uniqueness test. ## Contributors - [@dfagnan](https://github.com/dfagnan) (Issue https://github.com/fivetran/dbt_shopify_source/issues/57) diff --git a/DECISIONLOG.md b/DECISIONLOG.md index 9696a1e..ea809ac 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -8,7 +8,7 @@ Empty source tables are created in the Shopify schema dyanmically if they do not Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -The package will raise a compiler warning that the respective staging model is empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. +The package will output a compiler message that the respective staging model is empty. This compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. > In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. diff --git a/README.md b/README.md index c9a10d1..16d6524 100644 --- a/README.md +++ b/README.md @@ -141,13 +141,13 @@ vars: If you are making use of the `shopify_union_schemas` or `shopify_union_databases` variables, the package will assume individual tables to have their default names. -### Turn off compiler warnings for empty staging models +### Disable Compiler Warnings for Empty Tables -Empty source tables are created in the Shopify schema dyanmically if they do not exist in your raw source schema. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order. +Empty staging models are created in the Shopify schema dyanmically if the respective source tables do not exist in your raw source schema. For example, if your shop has not incurred any refunds, you will not have a `refund` table yet until you do refund an order, and the package will create an empty `stg_shopify__refund` model. -Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). +The source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -The package will raise a compiler warning that the respective staging model is empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. +The package will raise a compiler warning message that the respective staging model is empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. ```yml # dbt_project.yml From 25a197e23a01ded5d677fe1ebb4eff98799e363f Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 15 Mar 2023 14:10:18 -0700 Subject: [PATCH 15/21] update var --- CHANGELOG.md | 2 +- DECISIONLOG.md | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f1d9e98..249d8ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Bug Fixes [PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59) introduces the following changes: - The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) that will work seamlessly with downstream transformations. - - A compilation message will be raised when a staging model is completely empty. This compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. + - A compilation message will be raised when a staging model is completely empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`. - A uniqueness test has been placed on the `order_line_id`, `index`, and `source_relation` columns in `stg_shopify__tax_line`, as it was previously missing a uniqueness test. ## Contributors diff --git a/DECISIONLOG.md b/DECISIONLOG.md index ea809ac..df2ad49 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -8,7 +8,7 @@ Empty source tables are created in the Shopify schema dyanmically if they do not Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -The package will output a compiler message that the respective staging model is empty. This compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. +The package will output a compiler message that the respective staging model is empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`. > In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. From 2a086e0918f888370b590cac1f421753b6e77edd Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 15 Mar 2023 14:10:46 -0700 Subject: [PATCH 16/21] update var --- README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 16d6524..26819fd 100644 --- a/README.md +++ b/README.md @@ -147,13 +147,13 @@ Empty staging models are created in the Shopify schema dyanmically if the respec The source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -The package will raise a compiler warning message that the respective staging model is empty. The compiler warning can be turned off by the end user by setting the global `remove_empty_table_warnings` variable to `True`. +The package will raise a compiler warning message that the respective staging model is empty. The compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`. ```yml # dbt_project.yml vars: - remove_empty_table_warnings: true # default = false + fivetran__remove_empty_table_warnings: true # default = false ``` From c9b81213dcae52e7b8b045160af503c97403710d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 15 Mar 2023 14:33:43 -0700 Subject: [PATCH 17/21] databricks --- integration_tests/dbt_project.yml | 7 ++----- models/src_shopify.yml | 2 +- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 44bc2b4..aceb273 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -136,6 +136,7 @@ seeds: shopify_tax_line_data: +column_types: price: float + rate: float shopify_price_rule_data: +column_types: prerequisite_subtotal_range: float @@ -166,8 +167,4 @@ seeds: markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" price: float original_shop_markup: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - original_shop_price: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" - shopify_tax_line_data: - +column_types: - price: float - rate: float \ No newline at end of file + original_shop_price: "{{ 'string' if target.type in ('bigquery', 'spark', 'databricks') else 'varchar' }}" \ No newline at end of file diff --git a/models/src_shopify.yml b/models/src_shopify.yml index c587e74..0cab0ce 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -3,7 +3,7 @@ version: 2 sources: - name: shopify # This source will only be used if you are using a single Shopify source connector. If multiple sources are being unioned, their tables will be directly referenced via adatper.get_relation schema: "{{ var('shopify_schema', 'shopify') }}" - database: "{% if target.type != 'spark'%}{{ var('shopify_database', target.database) }}{% endif %}" + database: "{% if target.type not in ['spark', 'databricks'] %}{{ var('shopify_database', target.database) }}{% endif %}" tables: - name: order description: Each record represents an order in Shopify. From 5d86f6546f7333653951c09507bd7994eb67be90 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 15 Mar 2023 15:00:27 -0700 Subject: [PATCH 18/21] databricks --- CHANGELOG.md | 2 +- DECISIONLOG.md | 2 +- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 2 +- models/src_shopify.yml | 2 +- 5 files changed, 9 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 249d8ef..d748279 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,7 +3,7 @@ ## Bug Fixes [PR #59](https://github.com/fivetran/dbt_shopify_source/pull/59) introduces the following changes: - The `fivetan_utils.union_data` [macro](https://github.com/fivetran/dbt_fivetran_utils/pull/100) has been expanded to handle checking if a source table exists. Previously in the Shopify source package, this check happened outside of the macro and depended on the user having a defined shopify `source`. If the package anticipates a table that you do not have in any schema or database, it will return a **completely** empty table (ie `limit 0`) that will work seamlessly with downstream transformations. - - A compilation message will be raised when a staging model is completely empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`. + - A compilation message will be raised when a staging model is completely empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True` (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details). - A uniqueness test has been placed on the `order_line_id`, `index`, and `source_relation` columns in `stg_shopify__tax_line`, as it was previously missing a uniqueness test. ## Contributors diff --git a/DECISIONLOG.md b/DECISIONLOG.md index df2ad49..355194f 100644 --- a/DECISIONLOG.md +++ b/DECISIONLOG.md @@ -8,7 +8,7 @@ Empty source tables are created in the Shopify schema dyanmically if they do not Thus, the source package will will return **completely** empty staging models (ie `limit 0`) if these source tables do not exist in your Shopify schema yet, and the transform package will work seamlessly with these empty models. Once an anticipated source table exists in your schema, the source and transform packages will automatically reference the new populated table(s). ([example](https://github.com/fivetran/dbt_shopify_source/blob/main/models/tmp/stg_shopify__refund_tmp.sql)). -The package will output a compiler message that the respective staging model is empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True`. +The package will output a compiler message that the respective staging model is empty. This compiler warning can be turned off by the end user by setting the `fivetran__remove_empty_table_warnings` variable to `True` (see https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source for details). > In previous versions of the package, you had to manually enable or disable transforms of `refund`, `order_line_refund`, or `order_adjustment` through variables. Because this required you to monitor your Shopify account/schema and update the variable(s) accordingly, we decided to pursue a more automated solution that works for all tables. diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 92694fe..26d8be8 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_source_integration_tests_6 + schema: shopify_source_integration_tests_7 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index aceb273..1c70ccb 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,7 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: - shopify_schema: shopify_source_integration_tests_6 + shopify_schema: shopify_source_integration_tests_7 shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" diff --git a/models/src_shopify.yml b/models/src_shopify.yml index 0cab0ce..3a2a1d3 100644 --- a/models/src_shopify.yml +++ b/models/src_shopify.yml @@ -3,7 +3,7 @@ version: 2 sources: - name: shopify # This source will only be used if you are using a single Shopify source connector. If multiple sources are being unioned, their tables will be directly referenced via adatper.get_relation schema: "{{ var('shopify_schema', 'shopify') }}" - database: "{% if target.type not in ['spark', 'databricks'] %}{{ var('shopify_database', target.database) }}{% endif %}" + database: "{% if target.type not in ('spark', 'databricks') %}{{ var('shopify_database', target.database) }}{% endif %}" tables: - name: order description: Each record represents an order in Shopify. From 1cc20659956fe0491f0547360faee187b3b27ea0 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 15 Mar 2023 15:15:53 -0700 Subject: [PATCH 19/21] int tests --- integration_tests/ci/sample.profiles.yml | 10 +++++----- integration_tests/dbt_project.yml | 2 +- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 26d8be8..5e5bbad 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_source_integration_tests_7 + schema: shopify_source_integration_tests_5 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_source_integration_tests_7 + schema: shopify_source_integration_tests_5 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_source_integration_tests_7 + schema: shopify_source_integration_tests_5 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_source_integration_tests_7 + schema: shopify_source_integration_tests_5 threads: 8 databricks: catalog: null host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_source_integration_tests_7 + schema: shopify_source_integration_tests_5 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 1c70ccb..3d91bfb 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,7 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: - shopify_schema: shopify_source_integration_tests_7 + shopify_schema: shopify_source_integration_tests_5 shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" From 96238e13c572d07b38beb6e2fddd77033094ce78 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 15 Mar 2023 15:52:24 -0700 Subject: [PATCH 20/21] databricks --- .buildkite/hooks/pre-command | 3 ++- .buildkite/pipeline.yml | 1 + integration_tests/ci/sample.profiles.yml | 12 ++++++------ integration_tests/dbt_project.yml | 2 +- 4 files changed, 10 insertions(+), 8 deletions(-) diff --git a/.buildkite/hooks/pre-command b/.buildkite/hooks/pre-command index 04c85c0..81b7b00 100644 --- a/.buildkite/hooks/pre-command +++ b/.buildkite/hooks/pre-command @@ -21,4 +21,5 @@ export CI_SNOWFLAKE_DBT_USER=$(gcloud secrets versions access latest --secret="C export CI_SNOWFLAKE_DBT_WAREHOUSE=$(gcloud secrets versions access latest --secret="CI_SNOWFLAKE_DBT_WAREHOUSE" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HOST=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HOST" --project="dbt-package-testing-363917") export CI_DATABRICKS_DBT_HTTP_PATH=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_HTTP_PATH" --project="dbt-package-testing-363917") -export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") \ No newline at end of file +export CI_DATABRICKS_DBT_TOKEN=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_TOKEN" --project="dbt-package-testing-363917") +export CI_DATABRICKS_DBT_CATALOG=$(gcloud secrets versions access latest --secret="CI_DATABRICKS_DBT_CATALOG" --project="dbt-package-testing-363917") \ No newline at end of file diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml index 6e76ea0..281f1d4 100644 --- a/.buildkite/pipeline.yml +++ b/.buildkite/pipeline.yml @@ -69,5 +69,6 @@ steps: - "CI_DATABRICKS_DBT_HOST" - "CI_DATABRICKS_DBT_HTTP_PATH" - "CI_DATABRICKS_DBT_TOKEN" + - "CI_DATABRICKS_DBT_CATALOG" commands: | bash .buildkite/scripts/run_models.sh databricks \ No newline at end of file diff --git a/integration_tests/ci/sample.profiles.yml b/integration_tests/ci/sample.profiles.yml index 5e5bbad..e76518c 100644 --- a/integration_tests/ci/sample.profiles.yml +++ b/integration_tests/ci/sample.profiles.yml @@ -16,13 +16,13 @@ integration_tests: pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}" dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}" port: 5439 - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 bigquery: type: bigquery method: service-account-json project: 'dbt-package-testing' - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}" snowflake: @@ -33,7 +33,7 @@ integration_tests: role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}" database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}" warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}" - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 postgres: type: postgres @@ -42,13 +42,13 @@ integration_tests: pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}" dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}" port: 5432 - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 8 databricks: - catalog: null + catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}" host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}" http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}" - schema: shopify_source_integration_tests_5 + schema: shopify_source_integration_tests_6 threads: 2 token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}" type: databricks diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 3d91bfb..aceb273 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -4,7 +4,7 @@ profile: 'integration_tests' config-version: 2 vars: - shopify_schema: shopify_source_integration_tests_5 + shopify_schema: shopify_source_integration_tests_6 shopify_source: shopify_customer_identifier: "shopify_customer_data" shopify_order_line_refund_identifier: "shopify_order_line_refund_data" From 7d9d3ff9f8638fc245d3ed81d3ab6abab1dc3a2a Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 20 Mar 2023 14:39:15 -0700 Subject: [PATCH 21/21] update packages.yml --- packages.yml | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/packages.yml b/packages.yml index ba03cb0..c2758dd 100644 --- a/packages.yml +++ b/packages.yml @@ -1,9 +1,6 @@ packages: -# - package: fivetran/fivetran_utils -# version: [">=0.4.0", "<0.5.0"] -# - local: ../dbt_fivetran_utils -- git: https://github.com/fivetran/dbt_fivetran_utils.git - revision: feature/expand-union-data +- package: fivetran/fivetran_utils + version: [">=0.4.0", "<0.5.0"] - package: dbt-labs/spark_utils version: [">=0.3.0", "<0.4.0"] - package: calogica/dbt_expectations