Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Feature/normalized billing model #81

Closed
wants to merge 24 commits into from
Closed

Feature/normalized billing model #81

Show file tree
Hide file tree
Changes from 17 commits
Commits
Show all changes
24 commits
Select commit Hold shift + click to select a range
6ac0028
initial
fivetran-reneeli Jun 4, 2024
91fe0c7
updates
fivetran-reneeli Jun 4, 2024
2c60d4f
first pass at common data model ran succesful
fivetran-reneeli Jun 6, 2024
e364cab
updates
fivetran-reneeli Jun 7, 2024
7a8a8af
updates
fivetran-reneeli Jun 10, 2024
ae234b6
fixes
fivetran-reneeli Jun 11, 2024
b5a04c9
ambiguity
fivetran-reneeli Jun 11, 2024
9c9727c
data type cast
fivetran-reneeli Jun 11, 2024
22cfeb7
explicit casts for everythinig
fivetran-reneeli Jun 11, 2024
43d6fc3
fix and add seed data
fivetran-reneeli Jun 11, 2024
db538ad
add integration test config for discount data
fivetran-reneeli Jun 11, 2024
2c94aad
new schema
fivetran-reneeli Jun 11, 2024
b94c97a
try 2
fivetran-reneeli Jun 11, 2024
369a793
add identifiers
fivetran-reneeli Jun 11, 2024
8d2b16c
back to triple
fivetran-reneeli Jun 11, 2024
5565417
guess two double quotes it is
fivetran-reneeli Jun 11, 2024
e2b1f1e
discount table fixes
fivetran-reneeli Jun 12, 2024
9b47735
update line item enhanced
fivetran-reneeli Jun 12, 2024
d072e35
try new schema
fivetran-reneeli Jun 12, 2024
48944ef
this was deleted
fivetran-reneeli Jun 12, 2024
53f58e6
rm product category
fivetran-reneeli Jun 12, 2024
e45d3b9
tests
fivetran-reneeli Jun 13, 2024
f6ea8b0
updates
fivetran-reneeli Jun 13, 2024
b5d29fc
add docs
fivetran-reneeli Jun 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
config-version: 2
name: 'stripe'

version: '0.13.0'
version: '0.13.1'
fivetran-joemarkiewicz marked this conversation as resolved.
Show resolved Hide resolved
require-dbt-version: [">=1.3.0", "<2.0.0"]
models:
stripe:
Expand All @@ -17,6 +17,7 @@ vars:
charge: "{{ ref('stg_stripe__charge') }}"
customer: "{{ ref('stg_stripe__customer') }}"
dispute: "{{ ref('stg_stripe__dispute') }}"
discount: "{{ ref('stg_stripe__discount') }}"
fee: "{{ ref('stg_stripe__fee') }}"
invoice: "{{ ref('stg_stripe__invoice') }}"
invoice_line_item: "{{ ref('stg_stripe__invoice_line_item') }}"
Expand All @@ -25,6 +26,7 @@ vars:
payment_method: "{{ ref('stg_stripe__payment_method') }}"
payout: "{{ ref('stg_stripe__payout') }}"
price_plan: "{{ ref('stg_stripe__price_plan') }}"
product: "{{ ref('stg_stripe__product') }}"
refund: "{{ ref('stg_stripe__refund') }}"
subscription: "{{ ref('stg_stripe__subscription') }}"
transfer: "{{ ref('stg_stripe__transfer') }}"
10 changes: 5 additions & 5 deletions integration_tests/ci/sample.profiles.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -16,13 +16,13 @@ integration_tests:
pass: "{{ env_var('CI_REDSHIFT_DBT_PASS') }}"
dbname: "{{ env_var('CI_REDSHIFT_DBT_DBNAME') }}"
port: 5439
schema: stripe_integrations_tests_11
schema: stripe_integrations_tests_13
threads: 8
bigquery:
type: bigquery
method: service-account-json
project: 'dbt-package-testing'
schema: stripe_integrations_tests_11
schema: stripe_integrations_tests_13
threads: 8
keyfile_json: "{{ env_var('GCLOUD_SERVICE_KEY') | as_native }}"
snowflake:
Expand All @@ -33,7 +33,7 @@ integration_tests:
role: "{{ env_var('CI_SNOWFLAKE_DBT_ROLE') }}"
database: "{{ env_var('CI_SNOWFLAKE_DBT_DATABASE') }}"
warehouse: "{{ env_var('CI_SNOWFLAKE_DBT_WAREHOUSE') }}"
schema: stripe_integrations_tests_11
schema: stripe_integrations_tests_13
threads: 8
postgres:
type: postgres
Expand All @@ -42,13 +42,13 @@ integration_tests:
pass: "{{ env_var('CI_POSTGRES_DBT_PASS') }}"
dbname: "{{ env_var('CI_POSTGRES_DBT_DBNAME') }}"
port: 5432
schema: stripe_integrations_tests_11
schema: stripe_integrations_tests_13
threads: 8
databricks:
catalog: "{{ env_var('CI_DATABRICKS_DBT_CATALOG') }}"
host: "{{ env_var('CI_DATABRICKS_DBT_HOST') }}"
http_path: "{{ env_var('CI_DATABRICKS_DBT_HTTP_PATH') }}"
schema: stripe_integrations_tests_11
schema: stripe_integrations_tests_13
threads: 8
token: "{{ env_var('CI_DATABRICKS_DBT_TOKEN') }}"
type: databricks
16 changes: 14 additions & 2 deletions integration_tests/dbt_project.yml
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
config-version: 2

name: 'stripe_integration_tests'
version: '0.13.0'
version: '0.13.1'
fivetran-joemarkiewicz marked this conversation as resolved.
Show resolved Hide resolved

profile: 'integration_tests'

Expand All @@ -10,14 +10,15 @@ models:
+schema: "stripe_{{ var('directed_schema','dev') }}"

vars:
stripe_schema: stripe_integrations_tests_11
stripe_schema: stripe_integrations_tests_13
stripe_source:
stripe_account_identifier: "account_data"
stripe_group_identifier: "group_data"
stripe_balance_transaction_identifier: "balance_transaction_data"
stripe_card_identifier: "card_data"
stripe_charge_identifier: "charge_data"
stripe_customer_identifier: "customer_data"
stripe_discount_identifier: "discount_data"
stripe_dispute_identifier: "dispute_data"
stripe_fee_identifier: "fee_data"
stripe_invoice_identifier: "invoice_data"
Expand All @@ -28,6 +29,7 @@ vars:
stripe_payout_identifier: "payout_data"
stripe_plan_identifier: "plan_data"
stripe_price_identifier: "price_data"
stripe_product_identifier: "product_data"
stripe_refund_identifier: "refund_data"
stripe_subscription_history_identifier: "subscription_history_data"
stripe_subscription_identifier: "subscription_data"
Expand All @@ -47,6 +49,7 @@ seeds:
voided_at: timestamp
canceled_at: timestamp
invoice_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
end: timestamp
balance_transaction_data:
+column_types:
available_on: timestamp
Expand All @@ -61,6 +64,15 @@ seeds:
+column_types:
destination_bank_account_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
destination_card_id: "{{ 'varchar(500)' if target.type in ('redshift','postgres') else 'string'}}"
discount_data:
+enabled: "{{ true if target.type not in ('snowflake', 'postgres') else false }}"
discount_data_snowflake:
+alias: discount_data
+enabled: "{{ true if target.type == 'snowflake' else false }}"
discount_data_postgres:
+alias: discount_data
+enabled: "{{ true if target.type == 'postgres' else false }}"


dispatch:
- macro_namespace: dbt_utils
Expand Down
11 changes: 11 additions & 0 deletions integration_tests/seeds/discount_data_postgres.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,""end"",invoice_id,invoice_item_id,promotion_code,""start"",subscription_id
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OvvZQDZlKRVG8GnByl08x0P,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OlPk3DZlKRVG8GnkWTFA0go,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1OaAyXDZlKRVG8GnQo19CbEo,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1P7ALFDZlKRVG8GnTUCtI40b,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-27 05:50:07.000000",in_1PI2dvDZlKRVG8Gn4I34kEqR,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,"2022-09-22 05:50:07.000000",,,,"2022-09-19 05:50:07.000000",sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,"2022-09-27 05:50:07.000000",,,,"2023-08-07 09:37:39.000000",
di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,"2023-08-07 09:38:05.000000",
11 changes: 11 additions & 0 deletions integration_tests/seeds/discount_data_snowflake.csv
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
id,type,type_id,_fivetran_synced,amount,checkout_session_id,checkout_session_line_item_id,coupon_id,credit_note_line_item_id,customer_id,"""END""",invoice_id,invoice_item_id,promotion_code,"""START""",subscription_id
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a1b99DZlKRVG8Gn5eb6dced,2024-03-19 12:32:16.476000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OvvZQDZlKRVG8GnByl08x0P,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1cd03bDZlKRVG8Gn694a7916,2024-02-19 12:32:46.371000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OlPk3DZlKRVG8GnkWTFA0go,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_15a605DZlKRVG8Gncf06fd6b,2024-02-16 18:45:08.587000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OaAyXDZlKRVG8GnQo19CbEo,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1d3987DZlKRVG8Gnb404c3ca,2023-11-19 08:18:47.498000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1432ddDZlKRVG8Gna92778f6,2024-04-19 07:45:08.824000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1P7ALFDZlKRVG8GnTUCtI40b,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_1a71eaDZlKRVG8Gn3909c3a4,2024-02-16 18:33:16.340000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1OOwCbDZlKRVG8Gn3Tfhz2RW,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,PARENT_INVOICE_LINE_ITEM,sli_146059DZlKRVG8Gn63a45e70,2024-05-19 12:34:05.168000,101,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,in_1PI2dvDZlKRVG8Gn4I34kEqR,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1LjcrrDZlKRVG8GnP7REhm8t,SUBSCRIPTION,sub_1LjcrrDZlKRVG8Gnv1Bjosun,2024-05-19 06:31:36.758000,,,,ojshw0ZH,,cus_MSXtGSAxxtYWe1,,,,,2022-09-19 05:50:07.000000,sub_1LjcrrDZlKRVG8Gnv1Bjosun
di_1NcPsdDZlKRVG8GnU0gsnD2b,CUSTOMER,cus_OPEJRxhitWa90P,2023-10-17 14:18:42.776000,,,,ioLCPhMr,,cus_OPEJRxhitWa90P,,,,,2023-08-07 09:37:39.000000,
di_1NcPt3DZlKRVG8GnqM2ZJTve,CUSTOMER,cus_OPEKG00sHdsUs8,2024-06-06 12:32:41.114000,,,,Oodskd5H,,cus_OPEKG00sHdsUs8,,,,,2023-08-07 09:38:05.000000,
Loading