feature/product-inventory-api-update

CHANGELOG.md

@@ -1,3 +1,68 @@
+# dbt_shopify_source v0.14.0
+[PR #93](https://github.com/fivetran/dbt_shopify_source/pull/93) includes the following changes:
+
+## Breaking Changes
+- Updated for connector changes released on January 6, 2025. See the [release notes](https://fivetran.com/docs/connectors/applications/shopify/changelog#january2025) for more details. Added the following columns to the `stg_shopify__*` staging tables:
+  - `inventory_item`:
+    - `harmonized_system_code`
+    - `inventory_history_url`
+    - `legacy_resource_id`
+    - `measurement_id`
+    - `measurement_weight_value`
+    - `measurement_weight_unit`
+    - `tracked_editable_locked`
+    - `tracked_editable_reason`
+    - `unit_cost_amount`
+    - `unit_cost_currency_code`
+  - `inventory_level`:
+    - `id`
+    - `can_deactivate`
+    - `created_at`
+    - `deactivation_alert`
+  - `product_variant`:
+    - `available_for_sale`
+    - `display_name`
+    - `legacy_resource_id`
+    - `requires_components`
+    - `sellable_online_quantity`
+  - `product_image`:
+    - `media_id`
+    - `status`
+    - `url`
+
+- For backward compatibility, the following columns were coalesced to combine values from the old column name with the new column name, with the resulting column retaining the new name:
+  - `inventory_item`:
+    - The deprecated `cost` column is coalesced with the new column `unit_cost_amount` as `unit_cost_amount`.
+  - `product_image`:
+    - The deprecated `src` column is coalesced with the new column `url` as `url`.
+
+## Documentation
+- Marked the following columns as deprecated in the documentation. These will be removed in a future release:
+  - `inventory_level`:
+    - `available`
+  - `product_variant`:
+    - `fulfillment_service`
+    - `grams`
+    - `inventory_management`
+    - `requires_shipping`
+    - `weight`
+    - `weight_unit`
+    - `option_*`
+  - `product_image`:
+    - `position`
+    - `created_at`
+    - `updated_at`
+    - `variant_ids`
+
+## Under the Hood
+- Updated `shopify_*_data` seed data and `get_*_columns` macros to include new columns for the following tables:
+  - `inventory_item`
+  - `inventory_level`
+  - `product_image`
+  - `product_variant`
+- Standardized boolean casting by updating `get_*_columns` to use `dbt.type_boolean()` for consistency.
+
+
 # dbt_shopify_source v0.13.0
 
 [PR #91](https://github.com/fivetran/dbt_shopify_source/pull/91) includes the following changes:

README.md

@@ -45,7 +45,7 @@ If you  are **not** using the [Shopify transformation package](https://github.co
 ```yml
 packages:
   - package: fivetran/shopify_source
-    version: [">=0.13.0", "<0.14.0"] # we recommend using ranges to capture non-breaking changes automatically
+    version: [">=0.14.0", "<0.15.0"] # we recommend using ranges to capture non-breaking changes automatically
 ```
 
 ### Step 3: Define database and schema variables

dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'shopify_source'
-version: '0.13.0'
+version: '0.14.0'
 config-version: 2
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:

integration_tests/dbt_project.yml

@@ -1,11 +1,13 @@
 name: 'shopify_source_integration_tests'
-version: '0.13.0'
+version: '0.14.0'
 profile: 'integration_tests'
 config-version: 2
 
 vars:
+  # Enable when regenerating docs
+  # shopify_using_fulfillment_event: true
+
   shopify_schema: shopify_source_integration_tests_12
-  # shopify_using_fulfillment_event: true # enable when regenerating docs
   shopify_source:
     shopify_customer_identifier: "shopify_customer_data"
     shopify_order_line_refund_identifier: "shopify_order_line_refund_data"
@@ -97,11 +99,18 @@ seeds:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         order_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         refund_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+    shopify_product_image_data:
+      +column_types:
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        product_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        media_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
     shopify_product_variant_data:
       +column_types:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         product_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         inventory_item_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        image_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        legacy_resource_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
     shopify_transaction_data:
       +column_types:
         id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
@@ -172,8 +181,14 @@ seeds:
     shopify_inventory_level_data:
       +column_types:
         updated_at: timestamp
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        inventory_item_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        location_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
     shopify_inventory_item_data:
       +column_types:
+        id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        measurement_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
+        legacy_resource_id: "{%- if target.type == 'bigquery' -%} INT64 {%- else -%} bigint {%- endif -%}"
         updated_at: timestamp
         created_at: timestamp
     shopify_abandoned_checkout_shipping_line_data:

integration_tests/seeds/shopify_inventory_item_data.csv

@@ -1,4 +1,6 @@
-id,_fivetran_synced,cost,created_at,requires_shipping,sku,tracked,updated_at,country_code_of_origin,province_code_of_origin,_fivetran_deleted
-4555,2021-12-18 06:56:22.877000,,,,,,,,,true
-501419,2022-02-25 06:52:29.767000,,,,,,,,,true
-851179,2022-02-24 06:52:33.361000,,,,,,,,,true
+id,country_code_of_origin,created_at,duplicate_sku_count,harmonized_system_code,inventory_history_url,legacy_resource_id,measurement_id,measurement_weight_value,measurement_weight_unit,province_code_of_origin,requires_shipping,sku,tracked,tracked_editable_locked,tracked_editable_reason,unit_cost_amount,unit_cost_currency_code,updated_at,_fivetran_deleted,_fivetran_synced
+43985459707940,,2024-11-06 07:29:31.000000,1,,shopify.com,43985459707940,42943762858020,0,POUNDS,,TRUE,ITEM15,TRUE,,,,,,FALSE,2024-12-20 00:31:25.048000
+43985392533540,,2024-11-06 06:57:30.000000,1,,shopify.com,43985392533540,42943695683620,0,POUNDS,,TRUE,ITEM24,TRUE,,,,,,FALSE,2024-12-20 00:31:24.899000
+43985392762916,,2024-11-06 06:57:41.000000,1,,shopify.com,43985392762916,42943695912996,0,POUNDS,,TRUE,ITEM31,TRUE,,,,,,FALSE,2024-12-20 00:31:24.910000
+43985403084836,,2024-11-06 07:02:33.000000,1,,shopify.com,43985403084836,42943706234916,0,POUNDS,,TRUE,ITEM45,TRUE,,,,,,FALSE,2024-12-20 00:31:24.927000
+43985467768868,,2024-11-06 07:31:51.000000,1,,shopify.com,43985467768868,42943770918948,0,POUNDS,,TRUE,ITEM99,TRUE,,,,,,FALSE,2024-12-20 00:31:25.052000

integration_tests/seeds/shopify_inventory_level_data.csv

@@ -1,4 +1,6 @@
-inventory_item_id,location_id,_fivetran_synced,available,updated_at
-780939,287748,2021-11-13 08:02:21.760000,,
-6027,287748,2021-11-13 08:02:21.760000,,
-515,28748,2021-11-06 08:04:16.213000,,
+id,inventory_item_id,location_id,can_deactivate,created_at,deactivation_alert,updated_at,_fivetran_synced
+18495311,43985397579812,26191631,FALSE,2024-11-06 06:59:48.000000,Can't unstock from this location.,2024-11-06 07:11:40.000000,2024-12-20 00:31:24.917000
+18495311,43985397448740,26191631,FALSE,2024-11-06 06:59:42.000000,Can't unstock from this location.,2024-11-06 07:11:40.000000,2024-12-06 09:58:30.726000
+18495311,43985397547044,26191631,FALSE,2024-11-06 06:59:46.000000,Can't unstock from this location.,2024-11-06 07:11:40.000000,2024-12-06 09:58:30.728000
+18495311,43985397481508,26191631,FALSE,2024-11-06 06:59:43.000000,Can't unstock from this location.,2024-11-06 07:11:40.000000,2024-12-20 00:31:24.914000
+18495311,43985397612580,26191631,FALSE,2024-11-06 06:59:49.000000,Can't unstock from this location.,2024-11-06 07:11:40.000000,2024-12-10 06:32:37.626000

integration_tests/seeds/shopify_product_image_data.csv

@@ -1,4 +1,6 @@
-id,product_id,_fivetran_deleted,_fivetran_synced,alt,created_at,height,position,src,updated_at,width,is_default,variant_ids
-14180,38804,false,2022-12-01 06:51:36.660000,,2019-06-13 04:06:07.000000,1200,4,https://cdn.shopify.com/s/files/glassess-1784103173.jpg?v=1560398767,2019-06-13 04:06:07.000000,956,false,[]
-748644,34804,false,2022-12-01 06:51:36.660000,,2019-06-13 04:06:07.000000,1200,2,https://cdn.shopify.com/s/files/1/smile.jpg?v=1560398767,2019-06-13 04:06:07.000000,956,false,[]
-679716,34604,false,2022-12-01 06:51:36.660000,,2019-06-13 04:06:07.000000,1200,6,https://cdn.shopify.com/s/files/1/kitten.jpg?v=1560398767,2019-06-13 04:06:07.000000,956,false,"[2755330292,27559733,275597338,275597536,2755931364,2755973,2734989668]"
+product_id,id,media_id,alt_text,height,status,url,width,_fivetran_synced
+7739008745508,25975138287652,25965776863268,,1024,READY,shopify.com/image_1.jpg,687,2024-12-11 12:33:44.438000
+7738981089316,25974954065956,25965593821220,,513,READY,shopify.com/image_2.jpg,570,2024-12-06 09:58:30.973000
+7738956775460,25974778232868,25965419266084,,513,READY,shopify.com/image_3.jpg,570,2024-12-06 09:58:30.669000
+7738983088164,25974962028580,25965601783844,,513,READY,shopify.com/image_4.jpg,570,2024-12-06 09:39:47.803000
+7738956480548,25974777741348,25965418774564,,513,READY,shopify.com/image_5.jpg,570,2024-12-06 09:58:30.664000

integration_tests/seeds/shopify_product_variant_data.csv

@@ -1,6 +1,6 @@
-id,product_id,inventory_item_id,title,price,sku,position,inventory_policy,compare_at_price,fulfillment_service,inventory_management,created_at,updated_at,taxable,barcode,grams,image_id,inventory_quantity,weight,weight_unit,old_inventory_quantity,requires_shipping,_fivetran_synced,option_2,tax_code,option_3,option_1
-39262114414663,6540108431431,41356021661767,my title here,111,,1,deny,,manual,,2021-03-08 16:30:15.000,2021-04-12 19:49:43.000,FALSE,,0,,0,0,lb,0,FALSE,2021-04-16 07:50:32.995,,,,my title here
-39273118957639,6544066379847,41367035936839,my title here,222,,1,deny,,manual,,2021-03-17 16:39:45.000,2021-04-12 19:46:59.000,FALSE,,0,,0,0,lb,0,FALSE,2021-04-16 07:50:29.241,,,,my title here
-39290169262151,6548438188103,41384094924871,my title here,5,,1,deny,,manual,inventory manager,2021-03-30 19:48:15.000,2021-03-30 19:48:15.000,TRUE,,0,,0,0,lb,0,TRUE,2021-04-16 07:50:32.720,,,,my title here
-39262115397703,6540109250631,41356022644807,my title here,333,,1,deny,,manual,,2021-03-08 16:31:31.000,2021-04-12 19:47:26.000,FALSE,,0,,-5,0,lb,-5,FALSE,2021-04-16 07:50:29.822,,,,my title here
-29217058947142,3879735590982,30309980143686,my other title,444,,1,deny,,manual,inventory manager,2019-06-25 18:32:03.000,2019-10-01 23:40:09.000,TRUE,,222,,0,1,lb,0,TRUE,2021-04-16 07:50:25.006,,TR9999,,my other title
+product_id,id,inventory_item_id,image_id,available_for_sale,barcode,compare_at_price,created_at,display_name,inventory_policy,inventory_quantity,legacy_resource_id,metafield,position,price,requires_components,sellable_online_quantity,sku,tax_code,taxable,title,updated_at,_fivetran_synced
+7739017789476,41929063825444,43985575084068,,FALSE,,,2024-11-06 08:35:02.000000,Name 1,DENY,0,41929063825444,,1,169.95,FALSE,0,ITEM15,,TRUE,Default Title,2024-11-06 08:35:02.000000,2024-12-20 00:31:25.083000
+7738992590884,41928969551908,43985483661348,,TRUE,,,2024-11-06 07:37:04.000000,Name 2,DENY,1,41928969551908,,1,109.99,FALSE,1,ITEM24,,TRUE,Default Title,2024-11-06 07:45:11.000000,2024-12-20 00:31:25.070000
+7738992427044,41928969322532,43985483431972,,TRUE,,,2024-11-06 07:36:57.000000,Name 3,DENY,2,41928969322532,,1,14.69,FALSE,2,ITEM31,,TRUE,Default Title,2024-11-06 07:45:12.000000,2024-12-20 00:31:25.068000
+7738959233060,41928883470372,43985397579812,,TRUE,,,2024-11-06 06:59:48.000000,Name 4,DENY,2,41928883470372,,1,64.99,FALSE,2,ITEM45,,TRUE,Default Title,2024-11-06 07:11:40.000000,2024-12-20 00:31:24.917000
+7738991083556,41928967323684,43985481433124,,TRUE,,,2024-11-06 07:36:01.000000,Name 5,DENY,3,41928967323684,,1,9.99,FALSE,3,ITEM99,,TRUE,Default Title,2024-11-06 07:44:50.000000,2024-12-20 00:31:25.064000

macros/get_abandoned_checkout_columns.sql

@@ -1,7 +1,7 @@
 {% macro get_abandoned_checkout_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "abandoned_checkout_url", "datatype": dbt.type_string()},
     {"name": "billing_address_address_1", "datatype": dbt.type_string()},
@@ -19,7 +19,7 @@
     {"name": "billing_address_province", "datatype": dbt.type_string()},
     {"name": "billing_address_province_code", "datatype": dbt.type_string()},
     {"name": "billing_address_zip", "datatype": dbt.type_string()},
-    {"name": "buyer_accepts_marketing", "datatype": "boolean"},
+    {"name": "buyer_accepts_marketing", "datatype": dbt.type_boolean()},
     {"name": "cart_token", "datatype": dbt.type_string()},
     {"name": "closed_at", "datatype": dbt.type_timestamp()},
     {"name": "created_at", "datatype": dbt.type_timestamp()},
@@ -54,7 +54,7 @@
     {"name": "shipping_address_zip", "datatype": dbt.type_string()},
     {"name": "source_name", "datatype": dbt.type_string()},
     {"name": "subtotal_price", "datatype": dbt.type_float()},
-    {"name": "taxes_included", "datatype": "boolean"},
+    {"name": "taxes_included", "datatype": dbt.type_boolean()},
     {"name": "token", "datatype": dbt.type_string()},
     {"name": "total_discounts", "datatype": dbt.type_float()},
     {"name": "total_duties", "datatype": dbt.type_string()},

macros/get_collection_columns.sql

@@ -1,9 +1,9 @@
 {% macro get_collection_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
-    {"name": "disjunctive", "datatype": "boolean"},
+    {"name": "disjunctive", "datatype": dbt.type_boolean()},
     {"name": "handle", "datatype": dbt.type_string()},
     {"name": "id", "datatype": dbt.type_int()},
     {"name": "published_at", "datatype": dbt.type_timestamp()},

macros/get_customer_columns.sql

@@ -2,7 +2,7 @@
 
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
-    {"name": "accepts_marketing", "datatype": "boolean"},
+    {"name": "accepts_marketing", "datatype": dbt.type_boolean()},
     {"name": "created_at", "datatype": dbt.type_timestamp()},
     {"name": "default_address_id", "datatype": dbt.type_numeric()},
     {"name": "email", "datatype": dbt.type_string()},
@@ -12,10 +12,10 @@
     {"name": "orders_count", "datatype": dbt.type_numeric()},
     {"name": "phone", "datatype": dbt.type_string()},
     {"name": "state", "datatype": dbt.type_string()},
-    {"name": "tax_exempt", "datatype": "boolean"},
+    {"name": "tax_exempt", "datatype": dbt.type_boolean()},
     {"name": "total_spent", "datatype": dbt.type_float()},
     {"name": "updated_at", "datatype": dbt.type_timestamp()},
-    {"name": "verified_email", "datatype": "boolean"},
+    {"name": "verified_email", "datatype": dbt.type_boolean()},
     {"name": "email_marketing_consent_consent_updated_at", "datatype": dbt.type_timestamp()},
     {"name": "email_marketing_consent_opt_in_level", "datatype": dbt.type_string()},
     {"name": "email_marketing_consent_state", "datatype": dbt.type_string()},

macros/get_fulfillment_event_columns.sql

@@ -1,7 +1,7 @@
 {% macro get_fulfillment_event_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "address_1", "datatype": dbt.type_string()},
     {"name": "city", "datatype": dbt.type_string()},

macros/get_inventory_item_columns.sql

@@ -1,17 +1,28 @@
 {% macro get_inventory_item_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "cost", "datatype": dbt.type_float()},
     {"name": "country_code_of_origin", "datatype": dbt.type_string()},
     {"name": "created_at", "datatype": dbt.type_timestamp()},
-    {"name": "id", "datatype": dbt.type_int()},
+    {"name": "id", "datatype": dbt.type_bigint()},
     {"name": "province_code_of_origin", "datatype": dbt.type_string()},
-    {"name": "requires_shipping", "datatype": "boolean"},
+    {"name": "requires_shipping", "datatype": dbt.type_boolean()},
     {"name": "sku", "datatype": dbt.type_string()},
-    {"name": "tracked", "datatype": "boolean"},
-    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+    {"name": "tracked", "datatype": dbt.type_boolean()},
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "duplicate_sku_count", "datatype": dbt.type_int()},
+    {"name": "harmonized_system_code", "datatype": dbt.type_string()},
+    {"name": "inventory_history_url", "datatype": dbt.type_string()},
+    {"name": "legacy_resource_id", "datatype": dbt.type_bigint()},
+    {"name": "measurement_id", "datatype": dbt.type_bigint()},
+    {"name": "measurement_weight_value", "datatype": dbt.type_float()},
+    {"name": "measurement_weight_unit", "datatype": dbt.type_string()},
+    {"name": "tracked_editable_locked", "datatype": dbt.type_boolean()},
+    {"name": "tracked_editable_reason", "datatype": dbt.type_string()},
+    {"name": "unit_cost_amount", "datatype": dbt.type_float()},
+    {"name": "unit_cost_currency_code", "datatype": dbt.type_string()}
 ] %}
 
 {{ return(columns) }}

macros/get_inventory_level_columns.sql

@@ -3,9 +3,13 @@
 {% set columns = [
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
     {"name": "available", "datatype": dbt.type_int()},
+    {"name": "id", "datatype": dbt.type_int()},
     {"name": "inventory_item_id", "datatype": dbt.type_int()},
     {"name": "location_id", "datatype": dbt.type_int()},
-    {"name": "updated_at", "datatype": dbt.type_timestamp()}
+    {"name": "updated_at", "datatype": dbt.type_timestamp()},
+    {"name": "can_deactivate", "datatype": dbt.type_boolean()},
+    {"name": "created_at", "datatype": dbt.type_timestamp()},
+    {"name": "deactivation_alert", "datatype": dbt.type_string()}
 ] %}
 
 {{ return(columns) }}

macros/get_location_columns.sql

@@ -1,9 +1,9 @@
 {% macro get_location_columns() %}
 
 {% set columns = [
-    {"name": "_fivetran_deleted", "datatype": "boolean"},
+    {"name": "_fivetran_deleted", "datatype": dbt.type_boolean()},
     {"name": "_fivetran_synced", "datatype": dbt.type_timestamp()},
-    {"name": "active", "datatype": "boolean"},
+    {"name": "active", "datatype": dbt.type_boolean()},
     {"name": "address_1", "datatype": dbt.type_string()},
     {"name": "address_2", "datatype": dbt.type_string()},
     {"name": "city", "datatype": dbt.type_string()},
@@ -12,7 +12,7 @@
     {"name": "country_name", "datatype": dbt.type_string()},
     {"name": "created_at", "datatype": dbt.type_timestamp()},
     {"name": "id", "datatype": dbt.type_int()},
-    {"name": "legacy", "datatype": "boolean"},
+    {"name": "legacy", "datatype": dbt.type_boolean()},
     {"name": "localized_country_name", "datatype": dbt.type_string()},
     {"name": "localized_province_name", "datatype": dbt.type_string()},
     {"name": "name", "datatype": dbt.type_string()},