Bugfix/redshift constant expressions

.buildkite/scripts/run_models.sh

@@ -21,15 +21,15 @@ if [ "$db" = "databricks-sql" ]; then
 dbt seed --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true}' --target "$db" --full-refresh
+dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_abandoned_checkout: false}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 
 else
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true}' --target "$db" --full-refresh
+dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_abandoned_checkout: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 fi

CHANGELOG.md

@@ -1,3 +1,9 @@
+# dbt_shopify_source v0.13.0
+
+## Under the Hood
+- Adds enable/disable config for the `metadata` staging model (`stg_shopify__metafield`).
+- Adds enable/disable config for the `abandoned_checkout` staging models (including `stg_shopify__abandoned_checkout`, `stg_shopify__abandoned_checkout_discount_code`, and `stg_shopify__abandoned_checkout_shipping_line`). For more information on how to enable/disable these tables, refer to the [README](https://github.com/fivetran/dbt_shopify_source/blob/main/README.md#step-6-disable-models-for-non-existent-sources).
+
 # dbt_shopify_source v0.12.1
 
 ## 🪲 Bug Fixes 🪛

README.md

@@ -44,7 +44,7 @@ If you  are **not** using the [Shopify transformation package](https://github.co
 ```yml
 packages:
   - package: fivetran/shopify_source
-    version: [">=0.12.0", "<0.13.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.13.0", "<0.14.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ### Step 3: Define database and schema variables
@@ -84,7 +84,18 @@ vars:
     shopify_using_fulfillment_event: true # false by default
 ```
 
-### Step 5: Setting your timezone
+### Step 6: Disable models for non-existent sources
+This package considers that not every Shopify connector uses the `abandoned_checkout` tables (including `abandoned_checkout`, `abandoned_checkout_discount_code`, and `abandoned_checkout_shipping_line`). This package allows you to disable the corresponding functionality. By default, all variables' values are assumed to be `true`. To disable the `abandoned_checkout` tables, add the following variables:
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+    shopify_using_abandoned_checkout: False  #True by default
+```
+
+### Step 7: Setting your timezone
 By default, the data in your Shopify schema is in UTC. However, you may want reporting to reflect a specific timezone for more realistic analysis or data validation.
 
 To convert the timezone of **all** timestamps in the package, update the `shopify_timezone` variable to your target zone in [IANA tz Database format](https://en.wikipedia.org/wiki/List_of_tz_database_time_zones):
@@ -97,7 +108,7 @@ vars:
 
 > **Note**: This will only **numerically** convert timestamps to your target timezone. They will however have a "UTC" appended to them. This is a current limitation of the dbt-date `convert_timezone` [macro](https://github.com/calogica/dbt-date#convert_timezone-column-target_tznone-source_tznone) we leverage.
 
-### (Optional) Step 6: Additional configurations
+### (Optional) Step 8: Additional configurations
 <details open><summary>Expand/Collapse configurations</summary>
 
 #### Passing Through Additional Fields

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_source'
-version: '0.12.1'
+version: '0.13.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
@@ -8,6 +8,8 @@ models:
     +materialized: view
     stg_shopify__metafield:
       +materialized: table
+    stg_shopify__abandoned_checkout_discount_code:
+      +materialized: table
 
 vars:
   shopify_source:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_11
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_11
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_11
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_11
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_11
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,10 @@
 name: 'shopify_source_integration_tests'
-version: '0.12.1'
+version: '0.13.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
-  shopify_schema: "shopify_source_integration_tests_10"
+  shopify_schema: shopify_source_integration_tests_11
   # shopify_using_fulfillment_event: true # enable when regenerating docs
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"

integration_tests/packages.yml

@@ -1,2 +1,6 @@
 packages:
-  - local: ../
+  - local: ../
+
+# packages:
+# - package: fivetran/shopify_source
+#   version: [">=0.12.0", "<0.13.0"]

models/stg_shopify__abandoned_checkout.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
 
 with base as (
 

models/stg_shopify__abandoned_checkout_discount_code.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 with base as (
 
     select * 
@@ -20,6 +22,7 @@ fields as (
         }}
 
     from base
+
 ),
 
 final as (
@@ -34,13 +37,11 @@ final as (
         {{ dbt_date.convert_timezone(column='cast(updated_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as updated_at,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         source_relation, 
-        case when checkout_id is null and code is null and index is null
-            then row_number() over(partition by source_relation order by source_relation)
-            else row_number() over(partition by checkout_id, upper(code), source_relation order by index desc)
-        end as index
+        row_number() over(partition by checkout_id, upper(code), source_relation order by index desc) as index
+
 
     from fields
-    
+
 )
 
 select *

models/stg_shopify__abandoned_checkout_shipping_line.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
 
 with base as (
 

models/stg_shopify__metafield.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False) or var('shopify_using_customer_metafields', False) or var('shopify_using_order_metafields', False) or var('shopify_using_product_metafields', False) or var('shopify_using_product_image_metafields', False) or var('shopify_using_product_variant_metafields', False) or var('shopify_using_shop_metafields', False)) }}
 
 with base as (
 
@@ -38,10 +39,7 @@ final as (
         {{ dbt_date.convert_timezone(column='cast(updated_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as updated_at,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         lower({{ dbt.concat(["namespace","'_'","key"]) }}) as metafield_reference,
-        case when id is null and updated_at is null
-            then row_number() over(partition by source_relation order by source_relation) = 1
-            else row_number() over(partition by id, source_relation order by updated_at desc) = 1
-        end as is_most_recent_record,
+        row_number() over(partition by id, source_relation order by updated_at desc) = 1 as is_most_recent_record,
         source_relation
 
     from fields

models/tmp/stg_shopify__abandoned_checkout_discount_code_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='abandoned_checkout_discount_code', 

models/tmp/stg_shopify__abandoned_checkout_shipping_line_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='abandoned_checkout_shipping_line', 

models/tmp/stg_shopify__abandoned_checkout_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='abandoned_checkout', 

models/tmp/stg_shopify__metafield_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_all_metafields', False) or var('shopify_using_collection_metafields', False) or var('shopify_using_customer_metafields', False) or var('shopify_using_order_metafields', False) or var('shopify_using_product_metafields', False) or var('shopify_using_product_image_metafields', False) or var('shopify_using_product_variant_metafields', False) or var('shopify_using_shop_metafields', False)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='metafield',