Feature/normalized billing model

dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 name: 'stripe'
 
-version: '0.13.0'
+version: '0.14.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 models:
   stripe:
@@ -17,6 +17,7 @@ vars:
     charge: "{{ ref('stg_stripe__charge') }}"
     customer: "{{ ref('stg_stripe__customer') }}"
     dispute: "{{ ref('stg_stripe__dispute') }}"
+    discount: "{{ ref('stg_stripe__discount') }}"
     fee: "{{ ref('stg_stripe__fee') }}"
     invoice: "{{ ref('stg_stripe__invoice') }}"
     invoice_line_item: "{{ ref('stg_stripe__invoice_line_item') }}"
@@ -25,6 +26,7 @@ vars:
     payment_method: "{{ ref('stg_stripe__payment_method') }}"
     payout: "{{ ref('stg_stripe__payout') }}"
     price_plan: "{{ ref('stg_stripe__price_plan') }}"
+    product: "{{ ref('stg_stripe__product') }}"
     refund: "{{ ref('stg_stripe__refund') }}"
     subscription: "{{ ref('stg_stripe__subscription') }}"
     transfer: "{{ ref('stg_stripe__transfer') }}"

integration_tests/ci/sample.profiles.yml

@@ -16,13 +16,13 @@ integration_tests:
       pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
       dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
       port: 5439
-      schema: stripe_integrations_tests_11
+      schema: stripe_integrations_tests_14
       threads: 8
     bigquery:
       type: bigquery
       method: service-account-json
       project: 'dbt-package-testing'
-      schema: stripe_integrations_tests_11
+      schema: stripe_integrations_tests_14
       threads: 8
       keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
     snowflake:
@@ -33,7 +33,7 @@ integration_tests:
       role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
       database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
       warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
-      schema: stripe_integrations_tests_11
+      schema: stripe_integrations_tests_14
       threads: 8
     postgres:
       type: postgres
@@ -42,13 +42,13 @@ integration_tests:
       pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
       dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
       port: 5432
-      schema: stripe_integrations_tests_11
+      schema: stripe_integrations_tests_14
       threads: 8
     databricks:
       catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
       host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
       http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
-      schema: stripe_integrations_tests_11
+      schema: stripe_integrations_tests_14
       threads: 8
       token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
       type: databricks

integration_tests/dbt_project.yml

@@ -1,7 +1,7 @@
 config-version: 2
 
 name: 'stripe_integration_tests'
-version: '0.13.0'
+version: '0.14.0'
 
 profile: 'integration_tests'
 
@@ -10,14 +10,15 @@ models:
   +schema: "stripe_{{ var('directed_schema','dev') }}"
 
 vars:
-  stripe_schema: stripe_integrations_tests_11
+  stripe_schema: stripe_integrations_tests_14
   stripe_source:
     stripe_account_identifier:                "account_data"
     stripe_group_identifier:                  "group_data"
     stripe_balance_transaction_identifier:    "balance_transaction_data"
     stripe_card_identifier:                   "card_data"
     stripe_charge_identifier:                 "charge_data"
     stripe_customer_identifier:               "customer_data"
+    stripe_discount_identifier:               "discount_data"
     stripe_dispute_identifier:                "dispute_data"
     stripe_fee_identifier:                    "fee_data"
     stripe_invoice_identifier:                "invoice_data"
@@ -28,6 +29,7 @@ vars:
     stripe_payout_identifier:                 "payout_data"
     stripe_plan_identifier:                   "plan_data"
     stripe_price_identifier:                  "price_data"
+    stripe_product_identifier:                "product_data"
     stripe_refund_identifier:                 "refund_data"
     stripe_subscription_history_identifier:   "subscription_history_data"
     stripe_subscription_identifier:           "subscription_data"
@@ -47,6 +49,7 @@ seeds:
       voided_at: timestamp
       canceled_at: timestamp
       invoice_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
+      end: timestamp
     balance_transaction_data:
       +column_types:
         available_on: timestamp
@@ -61,6 +64,15 @@ seeds:
       +column_types:
         destination_bank_account_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
         destination_card_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
+    discount_data:
+      +enabled: "{{ true if target.type not in ('snowflake', 'postgres') else false }}"
+    discount_data_snowflake:
+      +alias: discount_data
+      +enabled: "{{ true if target.type == 'snowflake' else false }}"
+    discount_data_postgres:
+      +alias: discount_data
+      +enabled: "{{ true if target.type == 'postgres' else false }}"
+
 
 dispatch:
   - macro_namespace: dbt_utils

integration_tests/seeds/discount_data.csv

@@ -0,0 +1,11 @@
+id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,end,invoice_id,invoice_item_id,promotion_code,start,subscription_id
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OvvZQDZlKRVG8GnByl08x0P,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OlPk3DZlKRVG8GnkWTFA0go,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OaAyXDZlKRVG8GnQo19CbEo,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1P7ALFDZlKRVG8GnTUCtI40b,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1PI2dvDZlKRVG8Gn4I34kEqR,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,,,,,2023-08-07 09:37:39.000000,
+di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,2023-08-07 09:38:05.000000,

integration_tests/seeds/discount_data_postgres.csv

@@ -0,0 +1,11 @@
+id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,""end"",invoice_id,invoice_item_id,promotion_code,""start"",subscription_id
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OvvZQDZlKRVG8GnByl08x0P,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OlPk3DZlKRVG8GnkWTFA0go,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OaAyXDZlKRVG8GnQo19CbEo,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1P7ALFDZlKRVG8GnTUCtI40b,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1PI2dvDZlKRVG8Gn4I34kEqR,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,"2022-09-27 05:50:07.000000",,,,"2023-08-07 09:37:39.000000",
+di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,"2023-08-07 09:38:05.000000",

integration_tests/seeds/discount_data_snowflake.csv

@@ -0,0 +1,11 @@
+id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,"""END""",invoice_id,invoice_item_id,promotion_code,"""START""",subscription_id
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OvvZQDZlKRVG8GnByl08x0P,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OlPk3DZlKRVG8GnkWTFA0go,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OaAyXDZlKRVG8GnQo19CbEo,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1P7ALFDZlKRVG8GnTUCtI40b,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1PI2dvDZlKRVG8Gn4I34kEqR,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
+di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,,,,,2023-08-07 09:37:39.000000,
+di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,2023-08-07 09:38:05.000000,

integration_tests/tests/consistency/consistency_line_item_enhanced copy.sql

@@ -0,0 +1,31 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_stripe_prod.stripe__line_item_enhanced
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_stripe_dev.stripe__line_item_enhanced
+), 
+
+final as (
+    -- test will fail if any rows from prod are not found in dev
+    (select * from prod
+    except distinct
+    select * from dev)
+
+    union all -- union since we only care if rows are produced
+
+    -- test will fail if any rows from dev are not found in prod
+    (select * from dev
+    except distinct
+    select * from prod)
+    )
+
+select *
+from final

integration_tests/tests/consistency/consistency_line_item_enhanced.sql

@@ -0,0 +1,21 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure the rows counts are the same between versions
+with prod as (
+    select count(*) as prod_rows
+    from {{ target.schema }}_stripe_prod.stripe__line_item_enhanced
+),
+
+dev as (
+    select count(*) as dev_rows
+    from {{ target.schema }}_stripe_dev.stripe__line_item_enhanced
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from prod
+join dev
+    on prod.prod_rows != dev.dev_rows

integration_tests/tests/integrity/integrity_line_item_enhanced.sql

@@ -0,0 +1,21 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+-- this test is to make sure there is no fanout between the spine and the daily_overview
+with stg_invoice_line_item as (
+    select count(*) as line_item_count
+    from {{ target.schema }}_stripe_dev.stg_stripe__invoice_line_item
+),
+
+line_item_enhanced as (
+    select count(*) as daily_overview_count
+    from {{ target.schema }}_stripe_dev.stripe__line_item_enhanced
+)
+
+-- test will return values and fail if the row counts don't match
+select *
+from stg_invoice_line_item
+join line_item_enhanced
+    on stg_invoice_line_item.line_item_count != line_item_enhanced.daily_overview_count

models/common_data_models/stripe__common_data_models.yml

@@ -0,0 +1,79 @@
+version: 2
+
+models:
+  - name: stripe__line_item_enhanced
+    description: Add description
+    tests:
+      - dbt_utils.unique_combination_of_columns:
+          combination_of_columns:
+            - header_id
+            - line_item_id
+    columns:
+      - name: header_id
+        description: Id of corresponding invoice.
+      - name: line_item_id
+        description: Invoice line item id.
+      - name: line_item_index
+        description: Unique index of each line item id for each invoice.
+      - name: record_type
+        description: header or line_item
+      - name: created_at
+        description: When the invoice was created
+      - name: header_status
+        description: Status of the invoice.
+      - name: billing_type
+        description: Type of invoice line item.
+      - name: currency
+        description: Currency used in the invoice.
+      - name: product_id
+        description: Product ID
+      - name: product_name
+        description: Product name
+      - name: product_type
+        description: Product type
+      - name: transaction_type
+        description: Balance transaction type
+      - name: quantity
+        description: The number of units of this line item.
+      - name: unit_amount
+        description: Amount per unit.
+      - name: discount_amount
+        description: The discount amount associated with this invoice.
+      - name: tax_amount
+        description: The tax amount associated with this invoice.
+      - name: total_amount
+        description: Invoice total
+      - name: payment_id
+        description: Payment intent ID
+      - name: payment_method
+        description: Payment method type
+      - name: payment_method_id
+        description: Payment method ID
+      - name: payment_at
+        description: Charge created date
+      - name: fee_amount
+        description: Balance transaction fee
+      - name: refund_amount
+        description: Refund amount associated with this invoice.
+      - name: subscription_id
+        description: Subscription associated with this invoice.
+      - name: subscription_period_started_at
+        description: Subscription's current period start
+      - name: subscription_period_ended_at
+        description: Subscription's current period end
+      - name: subscription_status
+        description: Subscription status
+      - name: customer_id
+        description: Customer associated with this invoice.
+      - name: customer_level
+        description: Whether 'account' or 'customer'. For Stripe, the level is 'customer'.
+      - name: customer_name
+        description: Customer name
+      - name: customer_company
+        description: Associated account's company if applicable.
+      - name: customer_email
+        description: Customer email
+      - name: customer_city
+        description: Customer city
+      - name: customer_country
+        description: Customer country