Bugfix/redshift constant expressions

.buildkite/scripts/run_models.sh

@@ -21,15 +21,15 @@ if [ "$db" = "databricks-sql" ]; then
 dbt seed --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db"
-dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true}' --target "$db" --full-refresh
+dbt run --vars '{shopify_schema: shopify_source_integrations_tests_sqlw, shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_abandoned_checkout: false, shopify_using_metafield: false}' --target "$db" --full-refresh
 dbt test --vars '{shopify_schema: shopify_source_integrations_tests_sqlw}' --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 
 else
 dbt seed --target "$db" --full-refresh
 dbt run --target "$db" --full-refresh
 dbt test --target "$db"
-dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true}' --target "$db" --full-refresh
+dbt run --vars '{shopify_timezone: "America/New_York", shopify_using_fulfillment_event: true, shopify_using_abandoned_checkout: false, shopify_using_metafield: false}' --target "$db" --full-refresh
 dbt test --target "$db"
 dbt run-operation fivetran_utils.drop_schemas_automation --target "$db"
 fi

CHANGELOG.md

@@ -1,3 +1,15 @@
+# dbt_shopify_source v0.13.0
+
+[PR #91](https://github.com/fivetran/dbt_shopify_source/pull/91) includes the following changes:
+## Under the Hood
+- Adds enable/disable config for the `metadata` staging model using the `shopify_using_metafield` variable (default `true`).
+- Adds enable/disable config for the `abandoned_checkout` staging models using the `shopify_using_abandoned_checkout` variable (default `true`):
+   - `stg_shopify__abandoned_checkout`
+   - `stg_shopify__abandoned_checkout_discount_code`
+   - `stg_shopify__abandoned_checkout_shipping_line`. 
+- For more information on how to enable/disable these tables, refer to the [README](https://github.com/fivetran/dbt_shopify_source/blob/main/README.md#step-4-disable-models-for-non-existent-sources).
+- Updates the `index` calculation in `stg_shopify__abandoned_checkout_discount_code` by removing the conditional logic for null scenarios now that a disable config has been added to the model.
+
 # dbt_shopify_source v0.12.1
 
 ## 🪲 Bug Fixes 🪛

README.md

@@ -44,7 +44,7 @@ If you  are **not** using the [Shopify transformation package](https://github.co
 ```yml
 packages:
   - package: fivetran/shopify_source
-    version: [">=0.12.0", "<0.13.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.13.0", "<0.14.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ### Step 3: Define database and schema variables
@@ -72,18 +72,39 @@ vars:
 
 To connect your multiple schema/database sources to the package models, follow the steps outlined in the [Union Data Defined Sources Configuration](https://github.com/fivetran/dbt_fivetran_utils/tree/releases/v0.4.latest#union_data-source) section of the Fivetran Utils documentation for the union_data macro. This will ensure a proper configuration and correct visualization of connections in the DAG.
 
-### Step 4: Enable `fulfillment_event` data
+### Step 4: Disable models for non-existent sources
 
-The package takes into consideration that not every Shopify connector may have `fulfillment_event` data enabled. However, this table does hold valuable information that is leveraged in the `shopify__daily_shop` model in the transformation package. `fulfillment_event` data is **disabled by default**.
+The package takes into consideration that not every Shopify connector may have the `fulfillment_event`, `metadata`, or `abandoned_checkout` tables (including `abandoned_checkout`, `abandoned_checkout_discount_code`, and `abandoned_checkout_shipping_line`) and allows you to enable or disable the corresponding functionality.
+
+The `fulfillment_event` table is **disabled by default** but does hold valuable information that is leveraged in the `shopify__daily_shop` model in the transformation package. If you would like to enable the modeling of fulfillment events downstream, add the following variable to your `dbt_project.yml` file:
 
-Add the following variable to your `dbt_project.yml` file to enable the modeling of fulfillment events:
 ```yml
 # dbt_project.yml
 
 vars:
     shopify_using_fulfillment_event: true # false by default
 ```
 
+The `metadata` table is **enabled by default**. To disable this tables and prevent metadata fields from persisting downstream, add the following variable to your `dbt_project.yml` file:
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+    shopify_using_metafield: false  #true by default
+```
+
+The `abandoned_checkout` tables (including `abandoned_checkout` in addition to the `abandoned_checkout_discount_code` and `abandoned_checkout_shipping_line` child tables) are **enabled by default**. To disable the `abandoned_checkout` tables, add the following variable to your `dbt_project.yml` file:
+
+```yml
+# dbt_project.yml
+
+...
+vars:
+    shopify_using_abandoned_checkout: false  #true by default
+```
+
 ### Step 5: Setting your timezone
 By default, the data in your Shopify schema is in UTC. However, you may want reporting to reflect a specific timezone for more realistic analysis or data validation.
 

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_source'
-version: '0.12.1'
+version: '0.13.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_12
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_12
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_12
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_12
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: shopify_source_integration_tests_10
+      schema: shopify_source_integration_tests_12
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,10 +1,10 @@
 name: 'shopify_source_integration_tests'
-version: '0.12.1'
+version: '0.13.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
-  shopify_schema: "shopify_source_integration_tests_10"
+  shopify_schema: shopify_source_integration_tests_12
   # shopify_using_fulfillment_event: true # enable when regenerating docs
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"

models/stg_shopify__abandoned_checkout.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
 
 with base as (
 

models/stg_shopify__abandoned_checkout_discount_code.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 with base as (
 
     select * 
@@ -20,6 +22,7 @@ fields as (
         }}
 
     from base
+
 ),
 
 final as (
@@ -34,13 +37,11 @@ final as (
         {{ dbt_date.convert_timezone(column='cast(updated_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as updated_at,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         source_relation, 
-        case when checkout_id is null and code is null and index is null
-            then row_number() over(partition by source_relation order by source_relation)
-            else row_number() over(partition by checkout_id, upper(code), source_relation order by index desc)
-        end as index
+        row_number() over(partition by checkout_id, upper(code), source_relation order by index desc) as index
+
 
     from fields
-    
+
 )
 
 select *

models/stg_shopify__abandoned_checkout_shipping_line.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
 
 with base as (
 

models/stg_shopify__metafield.sql

@@ -1,3 +1,4 @@
+{{ config(enabled=var('shopify_using_metafield', True)) }}
 
 with base as (
 
@@ -38,10 +39,7 @@ final as (
         {{ dbt_date.convert_timezone(column='cast(updated_at as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as updated_at,
         {{ dbt_date.convert_timezone(column='cast(_fivetran_synced as ' ~ dbt.type_timestamp() ~ ')', target_tz=var('shopify_timezone', "UTC"), source_tz="UTC") }} as _fivetran_synced,
         lower({{ dbt.concat(["namespace","'_'","key"]) }}) as metafield_reference,
-        case when id is null and updated_at is null
-            then row_number() over(partition by source_relation order by source_relation) = 1
-            else row_number() over(partition by id, source_relation order by updated_at desc) = 1
-        end as is_most_recent_record,
+        row_number() over(partition by id, source_relation order by updated_at desc) = 1 as is_most_recent_record,
         source_relation
 
     from fields

models/tmp/stg_shopify__abandoned_checkout_discount_code_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='abandoned_checkout_discount_code', 

models/tmp/stg_shopify__abandoned_checkout_shipping_line_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='abandoned_checkout_shipping_line', 

models/tmp/stg_shopify__abandoned_checkout_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_abandoned_checkout', True)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='abandoned_checkout', 

models/tmp/stg_shopify__metafield_tmp.sql

@@ -1,3 +1,5 @@
+{{ config(enabled=var('shopify_using_metafield', True)) }}
+
 {{
     fivetran_utils.union_data(
         table_identifier='metafield',