From 255c04e5671e5671752203a94db230e19d8bca22 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 17 Jan 2024 10:59:45 +0000 Subject: [PATCH 01/20] Adding the source_relation column to the models and joins in the base model files --- .../base/int_netsuite2__accounting_periods.sql | 4 +++- .../intermediate/base/int_netsuite2__accounts.sql | 4 +++- .../intermediate/base/int_netsuite2__customers.sql | 4 +++- .../intermediate/base/int_netsuite2__locations.sql | 4 +++- .../base/int_netsuite2__transaction_lines.sql | 9 ++++++++- 5 files changed, 20 insertions(+), 5 deletions(-) diff --git a/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql b/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql index c551bbf1..e73909e6 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql @@ -16,11 +16,13 @@ joined as ( select accounting_periods.*, - accounting_period_fiscal_calendars.fiscal_calendar_id + accounting_period_fiscal_calendars.fiscal_calendar_id, + accounting_period_fiscal_calendars.source_relation from accounting_periods left join accounting_period_fiscal_calendars on accounting_periods.accounting_period_id = accounting_period_fiscal_calendars.accounting_period_id + and accounting_periods.source_relation = accounting_period_fiscal_calendars.source_relation ) select * diff --git a/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql b/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql index d67c22a1..dae2794b 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql @@ -18,11 +18,13 @@ joined as ( accounts.*, account_types.type_name, account_types.is_balancesheet, - account_types.is_leftside + account_types.is_leftside, + account_types.source_relation from accounts left join account_types on accounts.account_type_id = account_types.account_type_id + and accounts.source_relation = account_types.source_relation ) select * diff --git a/models/netsuite2/intermediate/base/int_netsuite2__customers.sql b/models/netsuite2/intermediate/base/int_netsuite2__customers.sql index 002d0fb5..3e530271 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__customers.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__customers.sql @@ -19,11 +19,13 @@ joined as ( entity_address.city, entity_address.state, entity_address.zipcode, - entity_address.country + entity_address.country, + entity_address.source_relation from customers left join entity_address on coalesce(customers.default_billing_address_id, customers.default_shipping_address_id) = entity_address.nkey + and customers.source_relation = entity_address.source_relation ) select * diff --git a/models/netsuite2/intermediate/base/int_netsuite2__locations.sql b/models/netsuite2/intermediate/base/int_netsuite2__locations.sql index 0b3d1341..fe67b1ca 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__locations.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__locations.sql @@ -19,11 +19,13 @@ joined as ( location_main_address.city, location_main_address.state, location_main_address.zipcode, - location_main_address.country + location_main_address.country, + location_main_address.source_relation from locations left join location_main_address on locations.main_address_id = location_main_address.nkey + and locations.source_relation = location_main_address.source_relation ) select * diff --git a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql index 3c709d38..3c33c508 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql @@ -29,6 +29,7 @@ joined as ( {% if var('netsuite2__multibook_accounting_enabled', false) %} transaction_accounting_lines.accounting_book_id, accounting_books.accounting_book_name, + accounting_books.source_relation {% endif %} transaction_accounting_lines.amount, @@ -36,16 +37,19 @@ joined as ( transaction_accounting_lines.debit_amount, transaction_accounting_lines.paid_amount, transaction_accounting_lines.unpaid_amount, - transaction_accounting_lines.is_posting + transaction_accounting_lines.is_posting, + transaction_accounting_lines.source_relation from transaction_lines left join transaction_accounting_lines on transaction_lines.transaction_line_id = transaction_accounting_lines.transaction_line_id and transaction_lines.transaction_id = transaction_accounting_lines.transaction_id + and transaction_lines.source_relation = transaction_accounting_lines.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} left join accounting_books on accounting_books.accounting_book_id = transaction_accounting_lines.accounting_book_id + and accounting_books.source_relation = transaction_accounting_lines.source_relation union all @@ -60,13 +64,16 @@ joined as ( transaction_accounting_lines.paid_amount, transaction_accounting_lines.unpaid_amount, transaction_accounting_lines.is_posting + transaction_accounting_lines.source_relation from transaction_lines left join transaction_accounting_lines on transaction_lines.transaction_line_id = transaction_accounting_lines.transaction_line_id and transaction_lines.transaction_id = transaction_accounting_lines.transaction_id + and transaction_lines.source_relation = transaction_accounting_lines.source_relation left join accounting_books on accounting_books.base_book_id = transaction_accounting_lines.accounting_book_id + and accounting_books.source_relation = transaction_accounting_lines.source_relation where accounting_books.base_book_id is not null {% endif %} From 60d4a699a57c1d64dc26c37d0e8840e32ebac9b8 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 17 Jan 2024 11:03:33 +0000 Subject: [PATCH 02/20] Updates --- .../intermediate/base/int_netsuite2__transaction_lines.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql index 3c33c508..da7b3bec 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql @@ -58,6 +58,7 @@ joined as ( transaction_accounting_lines.account_id, accounting_books.accounting_book_id, accounting_books.accounting_book_name, + accounting_books.source_relation transaction_accounting_lines.amount, transaction_accounting_lines.credit_amount, transaction_accounting_lines.debit_amount, From dbf957f027a8e8900eccf22979d48c1213b5314a Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 17 Jan 2024 11:46:36 +0000 Subject: [PATCH 03/20] Adding the source_relation column to the models and joins in the intermediate model files --- ...nt_netsuite2__acctxperiod_exchange_rate_map.sql | 9 ++++++++- .../int_netsuite2__tran_and_reporting_periods.sql | 5 ++++- ...t_netsuite2__tran_lines_w_accounting_period.sql | 6 +++++- .../int_netsuite2__tran_with_converted_amounts.sql | 14 +++++++++++--- 4 files changed, 28 insertions(+), 6 deletions(-) diff --git a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql index 723dff7d..55cf851f 100644 --- a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql +++ b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql @@ -35,20 +35,25 @@ period_exchange_rate_map as ( -- exchange rates used, by accounting period, to c consolidated_exchange_rates.accounting_book_id, {% endif %} + consolidated_exchange_rates.source_relation, consolidated_exchange_rates.average_rate, consolidated_exchange_rates.current_rate, consolidated_exchange_rates.historical_rate, consolidated_exchange_rates.from_subsidiary_id, consolidated_exchange_rates.to_subsidiary_id, to_subsidiaries.name as to_subsidiary_name, - currencies.symbol as to_subsidiary_currency_symbol + to_subsidiaries.source_relation + currencies.symbol as to_subsidiary_currency_symbol, + currencies.source_relation from consolidated_exchange_rates left join subsidiaries as to_subsidiaries on consolidated_exchange_rates.to_subsidiary_id = to_subsidiaries.subsidiary_id + and consolidated_exchange_rates.source_relation = to_subsidiaries.source_relation left join currencies on currencies.currency_id = to_subsidiaries.currency_id + and currencies.source_relation = to_subsidiaries.source_relation {% if not var('netsuite2__using_to_subsidiary', false) %} where consolidated_exchange_rates.to_subsidiary_id in (select subsidiary_id from subsidiaries where parent_id is null) -- constraint - only the primary subsidiary has no parent @@ -63,11 +68,13 @@ accountxperiod_exchange_rate_map as ( -- account table with exchange rate detail period_exchange_rate_map.accounting_book_id, {% endif %} + period_exchange_rate_map.source_relation, period_exchange_rate_map.from_subsidiary_id, period_exchange_rate_map.to_subsidiary_id, period_exchange_rate_map.to_subsidiary_name, period_exchange_rate_map.to_subsidiary_currency_symbol, accounts.account_id, + accounts.source_relation case when lower(accounts.general_rate_type) = 'historical' then period_exchange_rate_map.historical_rate when lower(accounts.general_rate_type) = 'current' then period_exchange_rate_map.current_rate diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql index 046a9142..0ca72330 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql @@ -13,7 +13,9 @@ subsidiaries as ( transaction_and_reporting_periods as ( select base.accounting_period_id as accounting_period_id, - multiplier.accounting_period_id as reporting_accounting_period_id + base.source_relation, + multiplier.accounting_period_id as reporting_accounting_period_id, + multiplier.source_relation from accounting_periods as base join accounting_periods as multiplier @@ -21,6 +23,7 @@ transaction_and_reporting_periods as ( and multiplier.is_quarter = base.is_quarter and multiplier.is_year = base.is_year -- this was year_0 in netsuite1 and multiplier.fiscal_calendar_id = base.fiscal_calendar_id + and multiplier.source_relation = base.source_relation and cast(multiplier.starting_at as {{ dbt.type_timestamp() }}) <= {{ current_timestamp() }} where not base.is_quarter diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql b/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql index 0fbc134c..df498a81 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql @@ -20,13 +20,17 @@ transaction_lines_w_accounting_period as ( -- transaction line totals, by accoun {% if var('netsuite2__multibook_accounting_enabled', false) %} transaction_lines.accounting_book_id, transaction_lines.accounting_book_name, + transaction_lines.source_relation, {% endif %} transactions.accounting_period_id as transaction_accounting_period_id, + transactions.source_relation, coalesce(transaction_lines.amount, 0) as unconverted_amount from transaction_lines - join transactions on transactions.transaction_id = transaction_lines.transaction_id + join transactions + on transactions.transaction_id = transaction_lines.transaction_id + and transactions.source_relation = transaction_lines.source_relation where lower(transactions.transaction_type) != 'revenue arrangement' and transaction_lines.is_posting diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql index 28e35906..183ca923 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql @@ -25,32 +25,37 @@ accounts as ( transactions_in_every_calculation_period_w_exchange_rates as ( select transaction_lines_w_accounting_period.*, - reporting_accounting_period_id + transaction_and_reporting_periods.reporting_accounting_period_id {% if var('netsuite2__using_exchange_rate', true) %} , exchange_reporting_period.exchange_rate as exchange_rate_reporting_period , exchange_transaction_period.exchange_rate as exchange_rate_transaction_period + , exchange_transaction_period.source_relation {% endif %} {% if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} , exchange_reporting_period.to_subsidiary_id , exchange_reporting_period.to_subsidiary_name , exchange_reporting_period.to_subsidiary_currency_symbol + , exchange_reporting_period.source_relation {% endif %} from transaction_lines_w_accounting_period left join transaction_and_reporting_periods - on transaction_and_reporting_periods.accounting_period_id = transaction_lines_w_accounting_period.transaction_accounting_period_id + on transaction_and_reporting_periods.accounting_period_id = transaction_lines_w_accounting_period.transaction_accounting_period_ida + and transaction_and_reporting_periods.source_relation = transaction_lines_w_accounting_period.source_relation {% if var('netsuite2__using_exchange_rate', true) %} left join accountxperiod_exchange_rate_map as exchange_reporting_period on exchange_reporting_period.accounting_period_id = transaction_and_reporting_periods.reporting_accounting_period_id and exchange_reporting_period.account_id = transaction_lines_w_accounting_period.account_id and exchange_reporting_period.from_subsidiary_id = transaction_lines_w_accounting_period.subsidiary_id + and exchange_reporting_period.source_relation = transaction_lines_w_accounting_period.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and exchange_reporting_period.accounting_book_id = transaction_lines_w_accounting_period.accounting_book_id + and exchange_reporting_period.source_relation = transaction_lines_w_accounting_period.source_relation {% endif %} left join accountxperiod_exchange_rate_map as exchange_transaction_period @@ -64,6 +69,7 @@ transactions_in_every_calculation_period_w_exchange_rates as ( {% if var('netsuite2__using_to_subsidiary', false) %} and exchange_transaction_period.to_subsidiary_id = exchange_reporting_period.to_subsidiary_id + and exchange_transaction_period.source_relation = transaction_lines_w_accounting_period.source_relation {% endif %} {% endif %} ), @@ -78,6 +84,7 @@ transactions_with_converted_amounts as ( unconverted_amount as converted_amount_using_transaction_accounting_period, unconverted_amount as converted_amount_using_reporting_month, {% endif %} + accounts.source_relation, case when lower(accounts.account_type_id) in ('income','othincome','expense','othexpense','cogs') then true else false @@ -94,7 +101,8 @@ transactions_with_converted_amounts as ( from transactions_in_every_calculation_period_w_exchange_rates left join accounts - on accounts.account_id = transactions_in_every_calculation_period_w_exchange_rates.account_id + on accounts.account_id = transactions_in_every_calculation_period_w_exchange_rates.account_id + and accounts.source_relation = transactions_in_every_calculation_period_w_exchange_rates.source_relation ) select * From 0aedb3592ed57fe87631ebf0f58e0e6ec0b188fd Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 17 Jan 2024 15:22:59 +0000 Subject: [PATCH 04/20] Adding the source_relation column to the models and joins in the main transform model files --- models/netsuite2/netsuite2__balance_sheet.sql | 24 ++++++++++++-- .../netsuite2/netsuite2__income_statement.sql | 26 ++++++++++++--- .../netsuite2__transaction_details.sql | 32 +++++++++++++++++-- 3 files changed, 71 insertions(+), 11 deletions(-) diff --git a/models/netsuite2/netsuite2__balance_sheet.sql b/models/netsuite2/netsuite2__balance_sheet.sql index 8245254c..779285eb 100644 --- a/models/netsuite2/netsuite2__balance_sheet.sql +++ b/models/netsuite2/netsuite2__balance_sheet.sql @@ -32,8 +32,10 @@ balance_sheet as ( select transactions_with_converted_amounts.transaction_id, transactions_with_converted_amounts.transaction_line_id, + transactions_with_converted_amounts.source_relation, transactions_with_converted_amounts.subsidiary_id, subsidiaries.name as subsidiary_name, + subsidiaries.name as source_relation, {% if var('netsuite2__multibook_accounting_enabled', false) %} transactions_with_converted_amounts.accounting_book_id, @@ -47,6 +49,7 @@ balance_sheet as ( {% endif %} reporting_accounting_periods.accounting_period_id as accounting_period_id, + reporting_accounting_periods.source_relation, reporting_accounting_periods.ending_at as accounting_period_ending, reporting_accounting_periods.name as accounting_period_name, reporting_accounting_periods.is_adjustment as is_accounting_period_adjustment, @@ -120,12 +123,14 @@ balance_sheet as ( and reporting_accounting_periods.fiscal_calendar_id = transaction_accounting_periods.fiscal_calendar_id) then 15 when not accounts.is_balancesheet then 14 else null - end as balance_sheet_sort_helper + end as balance_sheet_sort_helper, + accounts.source_relation --Below is only used if balance sheet transaction detail columns are specified dbt_project.yml file. {% if var('balance_sheet_transaction_detail_columns') %} , transaction_details.{{ var('balance_sheet_transaction_detail_columns') | join (", transaction_details.")}} + , transaction_details.source_relation {% endif %} @@ -136,6 +141,7 @@ balance_sheet as ( left join transaction_details on transaction_details.transaction_id = transactions_with_converted_amounts.transaction_id and transaction_details.transaction_line_id = transactions_with_converted_amounts.transaction_line_id + and transaction_details.source_relation = transactions_with_converted_amounts.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and transaction_details.accounting_book_id = transactions_with_converted_amounts.accounting_book_id @@ -149,15 +155,18 @@ balance_sheet as ( left join accounts on accounts.account_id = transactions_with_converted_amounts.account_id + on accounts.source_relation = transactions_with_converted_amounts.source_relation left join accounting_periods as reporting_accounting_periods on reporting_accounting_periods.accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id + and reporting_accounting_periods.source_relation = transactions_with_converted_amounts.source_relation left join accounting_periods as transaction_accounting_periods on transaction_accounting_periods.accounting_period_id = transactions_with_converted_amounts.transaction_accounting_period_id left join subsidiaries on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id + and subsidiaries.source_relation = transactions_with_converted_amounts.source_relation where reporting_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) and transaction_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) @@ -169,8 +178,10 @@ balance_sheet as ( select transactions_with_converted_amounts.transaction_id, transactions_with_converted_amounts.transaction_line_id, + transactions_with_converted_amounts.source_relation, transactions_with_converted_amounts.subsidiary_id, subsidiaries.name as subsidiary_name, + subsidiaries.name as source_relation, {% if var('netsuite2__multibook_accounting_enabled', false) %} transactions_with_converted_amounts.accounting_book_id, @@ -184,6 +195,7 @@ balance_sheet as ( {% endif %} reporting_accounting_periods.accounting_period_id as accounting_period_id, + reporting_accounting_periods.source_relation, reporting_accounting_periods.ending_at as accounting_period_ending, reporting_accounting_periods.name as accounting_period_name, reporting_accounting_periods.is_adjustment as is_accounting_period_adjustment, @@ -205,12 +217,14 @@ balance_sheet as ( when lower(accounts.general_rate_type) in ('historical', 'average') then converted_amount_using_transaction_accounting_period else converted_amount_using_reporting_month end as converted_amount, - 16 as balance_sheet_sort_helper + 16 as balance_sheet_sort_helper, + accounts.source_relation --Below is only used if balance sheet transaction detail columns are specified dbt_project.yml file. {% if var('balance_sheet_transaction_detail_columns') %} , transaction_details.{{ var('balance_sheet_transaction_detail_columns') | join (", transaction_details.")}} + , transaction_details.source_relation {% endif %} @@ -221,6 +235,7 @@ balance_sheet as ( left join transaction_details on transaction_details.transaction_id = transactions_with_converted_amounts.transaction_id and transaction_details.transaction_line_id = transactions_with_converted_amounts.transaction_line_id + and transaction_details.source_relation = transactions_with_converted_amounts.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and transaction_details.accounting_book_id = transactions_with_converted_amounts.accounting_book_id @@ -233,12 +248,15 @@ balance_sheet as ( left join accounts on accounts.account_id = transactions_with_converted_amounts.account_id + on accounts.source_relation = transactions_with_converted_amounts.source_relation left join accounting_periods as reporting_accounting_periods on reporting_accounting_periods.accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id + on reporting_accounting_periods.source_relation = transactions_with_converted_amounts.source_relation left join subsidiaries on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id + and subsidiaries.source_relation = transactions_with_converted_amounts.source_relation where reporting_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) and (accounts.is_balancesheet @@ -246,7 +264,7 @@ balance_sheet as ( ), surrogate_key as ( - {% set surrogate_key_fields = ['transaction_line_id', 'transaction_id', 'accounting_period_id', 'account_name', 'account_id'] %} + {% set surrogate_key_fields = ['transaction_line_id', 'transaction_id', 'accounting_period_id', 'account_name', 'account_id', 'source_relation'] %} {% do surrogate_key_fields.append('to_subsidiary_id') if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} {% do surrogate_key_fields.append('accounting_book_id') if var('netsuite2__multibook_accounting_enabled', false) %} diff --git a/models/netsuite2/netsuite2__income_statement.sql b/models/netsuite2/netsuite2__income_statement.sql index 66ef8d13..9bfaf673 100644 --- a/models/netsuite2/netsuite2__income_statement.sql +++ b/models/netsuite2/netsuite2__income_statement.sql @@ -52,6 +52,7 @@ income_statement as ( select transactions_with_converted_amounts.transaction_id, transactions_with_converted_amounts.transaction_line_id, + transactions_with_converted_amounts.source_relation, {% if var('netsuite2__multibook_accounting_enabled', false) %} transactions_with_converted_amounts.accounting_book_id, @@ -69,26 +70,32 @@ income_statement as ( reporting_accounting_periods.name as accounting_period_name, reporting_accounting_periods.is_adjustment as is_accounting_period_adjustment, reporting_accounting_periods.is_closed as is_accounting_period_closed, + reporting_accounting_periods.source_relation, accounts.name as account_name, accounts.type_name as account_type_name, accounts.account_type_id, accounts.account_id as account_id, accounts.account_number, + accounts.source_relation, subsidiaries.subsidiary_id, subsidiaries.full_name as subsidiary_full_name, - subsidiaries.name as subsidiary_name + subsidiaries.name as subsidiary_name, + subsidiaries.source_relation, --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, {{ dbt.concat(['accounts.account_number',"'-'", 'accounts.name']) }} as account_number_and_name, - classes.full_name as class_full_name + classes.full_name as class_full_name, + classes.source_relation, --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('classes_pass_through_columns', identifier='classes') }}, locations.full_name as location_full_name, - departments.full_name as department_full_name + locations.source_relation, + departments.full_name as department_full_name, + departments.source_relation, --The below script allows for departments table pass through columns. {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, @@ -106,6 +113,7 @@ income_statement as ( {% if var('income_statement_transaction_detail_columns') %} , transaction_details.{{ var('income_statement_transaction_detail_columns') | join (", transaction_details.")}} + , transaction_details.source_relation {% endif %} @@ -116,6 +124,7 @@ income_statement as ( join transaction_lines as transaction_lines on transaction_lines.transaction_line_id = transactions_with_converted_amounts.transaction_line_id and transaction_lines.transaction_id = transactions_with_converted_amounts.transaction_id + and transaction_lines.source_relation = transactions_with_converted_amounts.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and transaction_lines.accounting_book_id = transactions_with_converted_amounts.accounting_book_id @@ -123,21 +132,27 @@ income_statement as ( left join departments on departments.department_id = transaction_lines.department_id + and departments.source_relation = transaction_lines.source_relation left join accounts on accounts.account_id = transactions_with_converted_amounts.account_id + and accounts.source_relation = transactions_with_converted_amounts.source_relation left join locations on locations.location_id = transaction_lines.location_id + on locations.source_relation = transaction_lines.source_relation left join classes on classes.class_id = transaction_lines.class_id + and classes.source_relation = transaction_lines.source_relation left join accounting_periods as reporting_accounting_periods on reporting_accounting_periods.accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id + and reporting_accounting_periods.source_relation = transactions_with_converted_amounts.source_relation left join subsidiaries on transactions_with_converted_amounts.subsidiary_id = subsidiaries.subsidiary_id + on transactions_with_converted_amounts.source_relation = subsidiaries.source_relation --Below is only used if income statement transaction detail columns are specified dbt_project.yml file. {% if var('income_statement_transaction_detail_columns') != []%} @@ -150,6 +165,7 @@ income_statement as ( {% if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} and transaction_details.to_subsidiary_id = transactions_with_converted_amounts.to_subsidiary_id + and transaction_details.source_relation = transactions_with_converted_amounts.source_relation {% endif %} {% endif %} @@ -159,7 +175,7 @@ income_statement as ( ), surrogate_key as ( - {% set surrogate_key_fields = ['transaction_line_id', 'transaction_id', 'accounting_period_id', 'account_name'] %} + {% set surrogate_key_fields = ['transaction_line_id', 'transaction_id', 'accounting_period_id', 'account_name', 'source_relation'] %} {% do surrogate_key_fields.append('to_subsidiary_id') if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} {% do surrogate_key_fields.append('accounting_book_id') if var('netsuite2__multibook_accounting_enabled', false) %} @@ -171,4 +187,4 @@ surrogate_key as ( ) select * -from surrogate_key +from surrogate_key \ No newline at end of file diff --git a/models/netsuite2/netsuite2__transaction_details.sql b/models/netsuite2/netsuite2__transaction_details.sql index 4e836b24..84646cd5 100644 --- a/models/netsuite2/netsuite2__transaction_details.sql +++ b/models/netsuite2/netsuite2__transaction_details.sql @@ -83,18 +83,21 @@ transaction_details as ( {% if var('netsuite2__multibook_accounting_enabled', false) %} transaction_lines.accounting_book_id, transaction_lines.accounting_book_name, + transaction_lines.source_relation, {% endif %} {% if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} transactions_with_converted_amounts.to_subsidiary_id, transactions_with_converted_amounts.to_subsidiary_name, transactions_with_converted_amounts.to_subsidiary_currency_symbol, + transactions_with_converted_amounts.source_relation, {% endif %} transaction_lines.transaction_line_id, transaction_lines.memo as transaction_memo, not transaction_lines.is_posting as is_transaction_non_posting, transactions.transaction_id, + transactions.source_relation, transactions.status as transaction_status, transactions.transaction_date, transactions.due_date_at as transaction_due_date, @@ -112,11 +115,13 @@ transaction_details as ( accounting_periods.accounting_period_id as accounting_period_id, accounting_periods.is_adjustment as is_accounting_period_adjustment, accounting_periods.is_closed as is_accounting_period_closed, + accounting_periods.source_relation, accounts.name as account_name, accounts.type_name as account_type_name, accounts.account_type_id, accounts.account_id as account_id, - accounts.account_number + accounts.account_number, + accounts.source_relation --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, @@ -135,27 +140,35 @@ transaction_details as ( customers.country as customer_country, customers.date_first_order_at as customer_date_first_order, customers.customer_external_id, + customers.source_relation, classes.full_name as class_full_name, + classes.source_relation, items.name as item_name, items.type_name as item_type_name, items.sales_description, + items.source_relation, locations.name as location_name, locations.city as location_city, locations.country as location_country, + locations.source_relation, {% if var('netsuite2__using_vendor_categories', true) %} vendor_categories.name as vendor_category_name, {% endif %} vendors.company_name as vendor_name, vendors.create_date_at as vendor_create_date, + vendors.source_relation, currencies.name as currency_name, currencies.symbol as currency_symbol, - departments.name as department_name + currencies.source_relation, + departments.name as department_name, + departments.source_relation --The below script allows for departments table pass through columns. {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, subsidiaries.subsidiary_id, subsidiaries.name as subsidiary_name, + subsidiaries.source_relation, case when lower(accounts.account_type_id) in ('income', 'othincome') then -converted_amount_using_transaction_accounting_period else converted_amount_using_transaction_accounting_period @@ -168,11 +181,13 @@ transaction_details as ( join transactions on transactions.transaction_id = transaction_lines.transaction_id + and transactions.source_relation = transaction_lines.source_relation left join transactions_with_converted_amounts as transactions_with_converted_amounts on transactions_with_converted_amounts.transaction_line_id = transaction_lines.transaction_line_id and transactions_with_converted_amounts.transaction_id = transaction_lines.transaction_id and transactions_with_converted_amounts.transaction_accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id + and transactions_with_converted_amounts.source_relation = transactions_with_converted_amounts.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and transactions_with_converted_amounts.accounting_book_id = transaction_lines.accounting_book_id @@ -180,27 +195,35 @@ transaction_details as ( left join accounts on accounts.account_id = transaction_lines.account_id + and accounts.source_relation = transaction_lines.source_relation left join accounts as parent_account on parent_account.account_id = accounts.parent_id + on parent_account.source_relation = accounts.source_relation left join accounting_periods on accounting_periods.accounting_period_id = transactions.accounting_period_id + and accounting_periods.source_relation = transactions.source_relation left join customers on customers.customer_id = coalesce(transaction_lines.entity_id, transactions.entity_id) + on customers.source_relation = coalesce(transaction_lines.source_relation, transactions.source_relation) left join classes on classes.class_id = transaction_lines.class_id + and classes.source_relation = transaction_lines.source_relation left join items on items.item_id = transaction_lines.item_id + and items.source_relation = transaction_lines.source_relation left join locations on locations.location_id = transaction_lines.location_id + on locations.source_relation = transaction_lines.source_relation left join vendors on vendors.vendor_id = coalesce(transaction_lines.entity_id, transactions.entity_id) + on vendors.source_relation = coalesce(transaction_lines.source_relation, transactions.source_relation) {% if var('netsuite2__using_vendor_categories', true) %} left join vendor_categories @@ -209,19 +232,22 @@ transaction_details as ( left join currencies on currencies.currency_id = transactions.currency_id + and currencies.source_relation = transactions.source_relation left join departments on departments.department_id = transaction_lines.department_id + on departments.source_relation = transaction_lines.source_relation join subsidiaries on subsidiaries.subsidiary_id = transaction_lines.subsidiary_id + and subsidiaries.source_relation = transaction_lines.source_relation where (accounting_periods.fiscal_calendar_id is null or accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null)) ), surrogate_key as ( - {% set surrogate_key_fields = ['transaction_line_id', 'transaction_id'] %} + {% set surrogate_key_fields = ['transaction_line_id', 'transaction_id', 'source_relation'] %} {% do surrogate_key_fields.append('to_subsidiary_id') if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} {% do surrogate_key_fields.append('accounting_book_id') if var('netsuite2__multibook_accounting_enabled', false) %} From 7fb08a3de8c697be40f996405af3bc610387bb77 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 17 Jan 2024 15:31:59 +0000 Subject: [PATCH 05/20] Addin source_relation to the description of the uniqueness tests --- models/netsuite2.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/models/netsuite2.yml b/models/netsuite2.yml index 124760f8..2a120f3c 100644 --- a/models/netsuite2.yml +++ b/models/netsuite2.yml @@ -102,7 +102,7 @@ models: - name: to_subsidiary_currency_symbol description: Currency symbol for the consolidated exchange rate. - name: balance_sheet_id - description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, `account_name`, and `account_id`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. + description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, `account_name`, `account_id` and `source_relation`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. tests: - unique - not_null @@ -172,7 +172,7 @@ models: - name: to_subsidiary_currency_symbol description: Currency symbol for the consolidated exchange rate. - name: income_statement_id - description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, and `account_name`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. + description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, `account_name` and `source_relation`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. tests: - unique - not_null @@ -300,7 +300,7 @@ models: - name: to_subsidiary_currency_symbol description: Currency symbol for the consolidated exchange rate. - name: transaction_details_id - description: Surrogate key hashed on `transaction_line_id` and `transaction_id`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. + description: Surrogate key hashed on `transaction_line_id` and `transaction_id`, `source_relation`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. tests: - unique - not_null \ No newline at end of file From 588441d6c02c838d760969de8a6f5269f339fb13 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 17 Jan 2024 15:45:44 +0000 Subject: [PATCH 06/20] Updates to the CHANGELOG, README, dbt_project.yml file (integration_tests) and packages.yml file --- CHANGELOG.md | 6 +- README.md | 621 +++++++++++++++++++++++++++++- integration_tests/dbt_project.yml | 2 +- packages.yml | 7 +- 4 files changed, 630 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ffb2807b..6c050dbe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,4 +1,8 @@ -# dbt_netsuite v0.12.0 +# dbt_netsuite v0.13.0 + +## πŸŽ‰ Feature Update πŸŽ‰ +This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-3-define-netsuitecom-or-netsuite2-source) for details on how to leverage this feature ([PR #xxx](Placeholder)). + ## 🎁 Official release for Netsuite2! 🎁 [PR #98](https://github.com/fivetran/dbt_netsuite/pull/98) is the official supported release of [dbt_netsuite v0.12.0-b1](https://github.com/fivetran/dbt_netsuite/releases/tag/v0.12.0-b1). diff --git a/README.md b/README.md index e7effb3f..a3cec0db 100644 --- a/README.md +++ b/README.md @@ -102,7 +102,7 @@ Include the following netsuite package version in your `packages.yml` file: ```yaml packages: - package: fivetran/netsuite - version: [">=0.12.0", "<0.13.0"] + version: [">=0.13.0", "<0.14.0"] ``` ## Step 3: Define Netsuite.com or Netsuite2 Source As of April 2022 Fivetran made available a new Netsuite connector which leverages the Netsuite2 endpoint opposed to the original Netsuite.com endpoint. This package is designed to run for either or, not both. By default the `netsuite_data_model` variable for this package is set to the original `netsuite` value which runs the netsuite.com version of the package. If you would like to run the package on Netsuite2 data, you may adjust the `netsuite_data_model` variable to run the `netsuite2` version of the package. @@ -112,6 +112,7 @@ vars: ``` ## Step 4: Define database and schema variables +### Option 1: Single connector By default, this package runs using your destination and the `netsuite` schema. If this is not where your Netsuite data is (for example, if your netsuite schema is named `netsuite_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -120,6 +121,622 @@ vars: netsuite_schema: your_schema_name ``` +> **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. + +### Option 2: Union multiple connectors +If you have multiple Netsuite connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `netsuite_union_schemas` OR `netsuite_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: + +```yml +# dbt_project.yml + +vars: + netsuite_union_schemas: ['netsuite_usa','netsuite_canada'] # use this if the data is in different schemas/datasets of the same database/project + netsuite_union_databases: ['netsuite_usa','netsuite_canada'] # use this if the data is in different databases/projects but uses the same schema name +``` + +#### Recommended: Incorporate unioned sources into DAG +By default, this package defines one single-connector source, called `netsuite`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Netsuite sources, though the package will run successfully. + +To properly incorporate all of your Netsuite connectors into your project's DAG: +1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. + +
Expand for source configuration template

+ +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. + +```yml +sources: + - name: + schema: + database: + loader: fivetran + loaded_at_field: _fivetran_synced + + freshness: + warn_after: {count: 72, period: hour} + error_after: {count: 168, period: hour} + + tables: &netsuite2_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ + - name: account_type + description: A table containing the various account types within Netsuite. + columns: + - name: _fivetran_deleted + description: Unique ID used by Fivetran to sync and dedupe data. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: Unique identifier of thea account type. + - name: balancesheet + description: Boolean indicating if the account type is a balance sheet account. Represented as "T" or "F" for true and false respectively. + - name: left + description: Boolean indicating if the account type is leftside. Represented as "T" or "F" for true and false respectively. + - name: longname + description: The name of the account type. + + - name: accounting_book_subsidiaries + description: A table containing the various account books and the respective subsidiaries. + config: + enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" + columns: + - name: _fivetran_id + description: Unique ID used by Fivetran to sync and dedupe data. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: accountingbook + description: Unique identifier of the accounting book. + - name: status + description: The status of the accounting book subsidiary. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + + - name: accounting_book + description: Table detailing all accounting books set up in Netsuite. + config: + enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: Unique identifier of the accounting book. + - name: name + description: Name of the accounting book. + - name: basebook + description: Reference to the base book. + - name: effectiveperiod + description: Reference to the effective period of the accounting book. + - name: isadjustmentonly + description: Boolean indicating if the accounting book is an adjustment only. Represented as "T" or "F" for true and false respectively. + - name: isconsolidated + description: Boolean indicating if the accounting book is a consolidated entry. Represented as "T" or "F" for true and false respectively. + - name: contingentrevenuehandling + description: Boolean indicating if the accounting book is contingent revenue handling. Represented as "T" or "F" for true and false respectively. + - name: isprimary + description: Boolean indicating if the accounting book is a primary entry. Represented as "T" or "F" for true and false respectively. + - name: twosteprevenueallocation + description: Boolean indicating if the accounting book is a two step revenue allocation entry. Represented as "T" or "F" for true and false respectively. + - name: unbilledreceivablegrouping + description: Boolean indicating if the accounting book is an unbilled receivable grouping. Represented as "T" or "F" for true and false respectively. + + - name: accounting_period_fiscal_calendars + description: A table containing the accounting fiscal calendar periods. + columns: + - name: _fivetran_id + description: Unique ID used by Fivetran to sync and dedupe data. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: accountingperiod + description: The accounting period id of the accounting period which the transaction took place in. + - name: fiscalcalendar + description: Reference to the fiscal calendar used for the record. + - name: parent + description: Reference to the parent fiscal calendar accounting period. + + - name: accounting_period + description: Table detailing all accounting periods, including monthly, quarterly and yearly. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The accounting period id of the accounting period which the transaction took place in. + - name: parent + description: Reference to the parent accounting period. + - name: periodname + description: Name of the accounting period. + - name: startdate + description: Timestamp of when the accounting period starts. + - name: enddate + description: Timestamp if when the accounting period ends. + - name: closedondate + description: Timestamp of when the accounting period is closed. + - name: isquarter + description: Boolean indicating if the accounting period is the initial quarter. Represented as "T" or "F" for true and false respectively. + - name: isyear + description: Boolean indicating if the accounting period is the initial period. Represented as "T" or "F" for true and false respectively. + - name: isadjust + description: Boolean indicating if the accounting period is an adjustment. Represented as "T" or "F" for true and false respectively. + - name: isposting + description: Boolean indicating if the accounting period is posting. Represented as "T" or "F" for true and false respectively. + - name: closed + description: Boolean indicating if the accounting period is closed. Represented as "T" or "F" for true and false respectively. + - name: alllocked + description: Boolean indicating if all the accounting periods are locked. Represented as "T" or "F" for true and false respectively. + - name: arlocked + description: Boolean indicating if the ar accounting period is locked. Represented as "T" or "F" for true and false respectively. + - name: aplocked + description: Boolean indicating if the ap accounting period is locked. Represented as "T" or "F" for true and false respectively. + + - name: account + description: Table detailing all accounts set up in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier associated with the account. + - name: externalid + description: Reference to the external account, + - name: parent + description: Reference to the parent account. + - name: acctnumber + description: Netsuite generated account number. + - name: accttype + description: Reference to the account type. + - name: sspecacct + description: Special account type. + - name: fullname + description: Name of the account. + - name: description + description: Description of the account. + - name: deferralacct + description: Reference to the deferral account. + - name: cashflowrate + description: The cash flow rate type of the account. + - name: generalrate + description: The general rate type of the account (Current, Historical, Average). + - name: currency + description: The currency id of the currency used within the record. + - name: class + description: The unique identifier of the class used for the record. + - name: department + description: The unique identifier of the department used for the record. + - name: location + description: The unique identifier of the location used for the record. + - name: includechildren + description: Boolean indicating if the account includes sub accounts. Represented as "T" or "F" for true and false respectively. + - name: isinactive + description: Boolean indicating if the account is inactive. Represented as "T" or "F" for true and false respectively. + - name: issummary + description: Boolean indicating if the account is a summary account. Represented as "T" or "F" for true and false respectively. + - name: eliminate + description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period. Represented as "T" or "F" for true and false respectively. + - name: _fivetran_deleted + description: Timestamp of when a record was deleted. + + - name: classification + description: Table detailing all classes set up in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the class used for the record. + - name: externalid + description: Reference to the external class. + - name: name + description: Name of the class. + - name: fullname + description: Full name of the class. + - name: isinactive + description: Boolean indicating if the class is active. Represented as "T" or "F" for true and false respectively. + - name: _fivetran_deleted + description: Timestamp of when a record was deleted. + + - name: consolidated_exchange_rate + description: Table detailing average, historical and current exchange rates for all accounting periods. + columns: + - name: id + description: Unique identifier for the consolidated exchange rate. + - name: postingperiod + description: The accounting period id of the accounting period which the transaction took place in. + - name: fromcurrency + description: The currency id which the consolidated exchange rate is from. + - name: fromsubsidiary + description: The subsidiary id which the consolidated exchange rate is from. + - name: tocurrency + description: The subsidiary id which the consolidated exchange rate is for. + - name: tosubsidiary + description: The subsidiary id which the consolidated exchange rate is for. + - name: currentrate + description: The current rate associated with the exchange rate. + - name: averagerate + description: The consolidated exchange rates average rate. + - name: accountingbook + description: Unique identifier of the accounting book. + - name: historicalrate + description: The historical rate of the exchange rate. + + - name: currency + description: Table detailing all currency information. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The currency id of the currency used within the record. + - name: name + description: Name of the currency. + - name: symbol + description: Currency symbol. + + - name: customer + description: Table detailing all customer information. + columns: + - name: id + description: Unique identifier of the customer. + - name: entityid + description: The entity id of the entity used for the record. + - name: externalid + description: Reference to the associated external customer. + - name: parent + description: Reference to the parent customer. + - name: isperson + description: Boolean indicating if the customer is an individual person. Represented as "T" or "F" for true and false respectively. + - name: companyname + description: Name of the company. + - name: firstname + description: First name of the customer. + - name: lastname + description: Last name of the customer. + - name: email + description: Customers email address. + - name: phone + description: Phone number of the customer. + - name: defaultbillingaddress + description: Reference to the associated billing address. + - name: defaultshippingaddress + description: Reference to the associated default shipping address. + - name: receivablesaccount + description: Reference to the associated receivables account. + - name: currency + description: The currency id of the currency used within the record. + - name: firstorderdate + description: Timestamp of when the first order was created. + + - name: department + description: Table detailing all departments set up in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the department used for the record. + - name: parent + description: Reference to the parent department. + - name: name + description: Name of the department. + - name: fullname + description: Full name of the department. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + - name: isinactive + description: Boolean indicating if the department is active. Represented as "T" or "F" for true and false respectively. + - name: _fivetran_deleted + description: Timestamp of when a record was deleted. + + - name: entity + description: Table detailing all entities in Netsuite. + columns: + - name: id + description: The entity id of the entity used for the record. + - name: contact + description: The unique identifier of the contact associated with the entity. + - name: customer + description: The unique identifier of the customer associated with the entity. + - name: employee + description: The unique identifier of the employee associated with the entity. + - name: entitytitle + description: The entity name. + - name: isperson + description: Value indicating whether the entity is a person (either yes or no). + - name: parent + description: The unique identifier of the parent entity. + - name: project + description: The unique identifier of the project (job) associated with the entity. + - name: type + description: The entity type (Contact, CustJob, Job, etc). + - name: vendor + description: The unique identifier of the vendor associated with the entity. + + - name: entity_address + description: A table containing addresses and the various entities which they map. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: addr1 + description: The associated address 1. + - name: addr2 + description: The associated address 2. + - name: addr3 + description: The associated address 3. + - name: addressee + description: The individual associated with the address. + - name: addrtext + description: The full address associated. + - name: city + description: The associated city. + - name: country + description: The associated country. + - name: state + description: The associated state. + - name: nkey + description: The associated Netsuite key. + - name: zip + description: The associated zipcode. + + - name: item + description: Table detailing information about the items created in Netsuite. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the item used within the record. + - name: fullname + description: Name of the item. + - name: itemtype + description: Item type name. + - name: description + description: Sales description associated with the item. + - name: department + description: The unique identifier of the department used for the record. + - name: class + description: The unique identifier of the class used for the record. + - name: location + description: The unique identifier of the location used for the record. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + - name: assetaccount + description: Reference to the asset account. + - name: expenseaccount + description: Reference to the expense account. + - name: gainlossaccount + description: Reference to the gain or loss account. + - name: incomeaccount + description: Reference to the income account. + - name: intercoexpenseaccount + description: Reference to the intercompany expense account. + - name: intercoincomeaccount + description: Reference to the intercompany income account. + - name: deferralaccount + description: Reference to the deferred expense account. + - name: deferredrevenueaccount + description: Reference to the deferred revenue account. + - name: parent + description: Reference to the parent item. + + - name: job + description: Table detailing all jobs. + config: + enabled: "{{ var('netsuite2__using_jobs', true) }}" + columns: + - name: id + description: The unique identifier of the job. + - name: externalid + description: The unique identifier of the external job reference. + - name: customer + description: The unique identifier of the customer associated with the job. + - name: entityid + description: Reference the the entity. + - name: defaultbillingaddress + description: Default billing address. + - name: defaultshippingaddress + description: Default shipping address. + - name: parent + description: Reference to the parent job. + + - name: location_main_address + description: A table containing the location main addresses. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: addr1 + description: The associated address 1. + - name: addr2 + description: The associated address 2. + - name: addr3 + description: The associated address 3. + - name: addressee + description: The individual associated with the address. + - name: addrtext + description: The full address associated. + - name: city + description: The associated city. + - name: country + description: The associated country. + - name: state + description: The associated state. + - name: nkey + description: The associated Netsuite key. + - name: zip + description: The associated zipcode. + + - name: location + description: Table detailing all locations, including store, warehouse and office locations. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the location used for the record. + - name: name + description: Name of the location. + - name: fullname + description: Full name of the location. + - name: mainaddress + description: Reference to the main address used for the record. + - name: parent + description: Reference to the parent location. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + + - name: subsidiary + description: Table detailing all subsidiaries. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the subsidiary used for the record. + - name: name + description: Name of the subsidiary. + - name: fullname + description: Full name of the subsidiary. + - name: email + description: Email address associated with the subsidiary. + - name: mainaddress + description: Reference to the main address used for the record. + - name: country + description: The country which the subsidiary is located. + - name: state + description: The state which the subsidiary is located. + - name: fiscalcalendar + description: Reference to the fiscal calendar used for the record. + - name: parent + description: Reference to the parent subsidiary. + - name: currency + description: The currency id of the currency used within the record. + + - name: transaction_accounting_line + description: A table detailing all transaction lines for all transactions. + columns: + - name: transaction + description: The transaction id which the transaction line is associated with. + - name: transactionline + description: The unique identifier of the transaction line. + - name: amount + description: The amount of the transaction line. + - name: netamount + description: The net amount of the transaction line. + - name: accountingbook + description: Unique identifier of the accounting book. + - name: account + description: Reference to the account associated with the entry. + - name: posting + description: Boolean indicating if the entry is posting. Represented as "T" or "F" for true and false respectively. + - name: credit + description: Amount associated as a credit. + - name: debit + description: Amount associated as a debit. + - name: amountpaid + description: Total amount paid. + - name: amountunpaid + description: Total amount unpaid. + + - name: transaction_line + description: A table detailing all transaction lines for all transactions. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: Unique identifier of the transaction line. + - name: transaction + description: The transaction id of referenced for the record. + - name: linesequencenumber + description: Netsuite generated number associated with the transaction line. + - name: memo + description: The memo attached to the transaction line. + - name: entity + description: The entity id of the entity used for the record. + - name: item + description: The unique identifier of the item used within the record. + - name: class + description: The unique identifier of the class used for the record. + - name: location + description: The unique identifier of the location used for the record. + - name: subsidiary + description: The unique identifier of the subsidiary used for the record. + - name: department + description: The unique identifier of the department used for the record. + - name: isclosed + description: Boolean indicating if the transaction line is closed. Represented as "T" or "F" for true and false respectively. + - name: isbillable + description: Boolean indicating if the transaction line is billable. Represented as "T" or "F" for true and false respectively. + - name: iscogs + description: Boolean indicating if the transaction line is a cost of goods sold entry. Represented as "T" or "F" for true and false respectively. + - name: cleared + description: Boolean indicating if the transaction line is cleared. Represented as "T" or "F" for true and false respectively. + - name: commitmentfirm + description: Boolean indicating if the transaction line is a commitment firm. Represented as "T" or "F" for true and false respectively. + - name: mainline + description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively. + - name: taxline + description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. + + - name: transaction + description: A table detailing all transactions. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The transaction id of referenced for the record. + - name: transactionnumber + description: The Netsuite generated number of the transaction. + - name: type + description: The type of the transaction. + - name: memo + description: Memo attached to the transaction. + - name: trandate + description: The timestamp of the transaction date. + - name: status + description: Status of the transaction. + - name: createddate + description: Timestamp of when the record was created. + - name: duedate + description: Timestamp of the transactions due date. + - name: closedate + description: Timestamp of when the transaction was closed. + - name: currency + description: The currency id of the currency used within the record. + - name: entity + description: The entity id of the entity used for the record. + - name: postingperiod + description: The accounting period id of the accounting period which the transaction took place in. + - name: posting + description: Boolean indicating if the transaction is a posting event. Represented as "T" or "F" for true and false respectively. + - name: intercoadj + description: Boolean indicating if the transaction is an intercompany adjustment. Represented as "T" or "F" for true and false respectively. + - name: isreversal + description: Boolean indicating if the transaction is a reversal entry. Represented as "T" or "F" for true and false respectively. + + - name: vendor_category + description: A table containing categories and how they map to vendors. + config: + enabled: "{{ var('netsuite2__using_vendor_categories', true) }}" + columns: + - name: id + description: Unique identifier of the vendor category. + - name: name + description: Name of the vendor category. + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + + - name: vendor + description: A table detailing all vendor information. + columns: + - name: _fivetran_synced + description: Timestamp of when a record was last synced. + - name: id + description: The unique identifier of the vendor. + - name: companyname + description: Name of the company. + - name: datecreated + description: Timestamp of the record creation. + - name: category + description: Unique identifier of the vendor category +``` +

+ +2. Set the `has_defined_sources` variable (scoped to the `netsuite_source` package) to true, like such: +```yml +# dbt_project.yml +vars: + netsuite_source: + has_defined_sources: true +``` + ## Step 5: Disable models for non-existent sources (Netsuite2 only) It's possible that your Netsuite connector does not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that feature in Netsuite or actively excluded some tables from your syncs. To disable the corresponding functionality in the package, you must add the relevant variables. By default, all variables are assumed to be true. Add variables for only the tables you would like to disable: ```yml @@ -253,7 +870,7 @@ This dbt package is dependent on the following dbt packages. Please be aware tha ```yml packages: - package: fivetran/netsuite_source - version: [">=0.9.0", "<0.10.0"] + version: [">=0.10.0", "<0.11.0"] - package: fivetran/fivetran_utils version: [">=0.4.0", "<0.5.0"] diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 36f400c7..bc7c3613 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -1,5 +1,5 @@ name: 'netsuite_integration_tests' -version: '0.12.0' +version: '0.13.0' profile: 'integration_tests' config-version: 2 models: diff --git a/packages.yml b/packages.yml index 4a671962..71bad159 100644 --- a/packages.yml +++ b/packages.yml @@ -1,3 +1,6 @@ packages: - - package: fivetran/netsuite_source - version: [">=0.9.0", "<0.10.0"] + # - package: fivetran/netsuite_source + # version: [">=0.9.0", "<0.10.0"] + - git: https://github.com/fivetran/dbt_netsuite_source.git + revision: feature/add-union-data + warn-unpinned: false \ No newline at end of file From 8785203fa82e3b19f4a1a114d73b8a0bd4c640c0 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Wed, 17 Jan 2024 15:49:20 +0000 Subject: [PATCH 07/20] Updates --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 6c050dbe..9838334b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,7 @@ # dbt_netsuite v0.13.0 ## πŸŽ‰ Feature Update πŸŽ‰ -This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-3-define-netsuitecom-or-netsuite2-source) for details on how to leverage this feature ([PR #xxx](Placeholder)). +This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-3-define-netsuitecom-or-netsuite2-source) for details on how to leverage this feature ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)). ## 🎁 Official release for Netsuite2! 🎁 [PR #98](https://github.com/fivetran/dbt_netsuite/pull/98) is the official supported release of [dbt_netsuite v0.12.0-b1](https://github.com/fivetran/dbt_netsuite/releases/tag/v0.12.0-b1). From 368c3bd4167aeae30190a2d29581d233a30fd209 Mon Sep 17 00:00:00 2001 From: Poonam Agate Date: Thu, 18 Jan 2024 22:39:03 +0000 Subject: [PATCH 08/20] Removing the duplicate source_relation column from the models --- .../base/int_netsuite2__transaction_lines.sql | 4 ---- ...nt_netsuite2__acctxperiod_exchange_rate_map.sql | 5 +---- .../int_netsuite2__tran_and_reporting_periods.sql | 1 - ...t_netsuite2__tran_lines_w_accounting_period.sql | 1 - .../int_netsuite2__tran_with_converted_amounts.sql | 3 --- models/netsuite2/netsuite2__balance_sheet.sql | 12 ++---------- models/netsuite2/netsuite2__income_statement.sql | 7 ------- .../netsuite2/netsuite2__transaction_details.sql | 14 +------------- 8 files changed, 4 insertions(+), 43 deletions(-) diff --git a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql index da7b3bec..2d16c82b 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql @@ -29,7 +29,6 @@ joined as ( {% if var('netsuite2__multibook_accounting_enabled', false) %} transaction_accounting_lines.accounting_book_id, accounting_books.accounting_book_name, - accounting_books.source_relation {% endif %} transaction_accounting_lines.amount, @@ -38,7 +37,6 @@ joined as ( transaction_accounting_lines.paid_amount, transaction_accounting_lines.unpaid_amount, transaction_accounting_lines.is_posting, - transaction_accounting_lines.source_relation from transaction_lines left join transaction_accounting_lines @@ -58,14 +56,12 @@ joined as ( transaction_accounting_lines.account_id, accounting_books.accounting_book_id, accounting_books.accounting_book_name, - accounting_books.source_relation transaction_accounting_lines.amount, transaction_accounting_lines.credit_amount, transaction_accounting_lines.debit_amount, transaction_accounting_lines.paid_amount, transaction_accounting_lines.unpaid_amount, transaction_accounting_lines.is_posting - transaction_accounting_lines.source_relation from transaction_lines left join transaction_accounting_lines diff --git a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql index 55cf851f..456a74e5 100644 --- a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql +++ b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql @@ -42,9 +42,7 @@ period_exchange_rate_map as ( -- exchange rates used, by accounting period, to c consolidated_exchange_rates.from_subsidiary_id, consolidated_exchange_rates.to_subsidiary_id, to_subsidiaries.name as to_subsidiary_name, - to_subsidiaries.source_relation - currencies.symbol as to_subsidiary_currency_symbol, - currencies.source_relation + currencies.symbol as to_subsidiary_currency_symbol from consolidated_exchange_rates left join subsidiaries as to_subsidiaries @@ -68,7 +66,6 @@ accountxperiod_exchange_rate_map as ( -- account table with exchange rate detail period_exchange_rate_map.accounting_book_id, {% endif %} - period_exchange_rate_map.source_relation, period_exchange_rate_map.from_subsidiary_id, period_exchange_rate_map.to_subsidiary_id, period_exchange_rate_map.to_subsidiary_name, diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql index 0ca72330..0de06e20 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql @@ -15,7 +15,6 @@ transaction_and_reporting_periods as ( base.accounting_period_id as accounting_period_id, base.source_relation, multiplier.accounting_period_id as reporting_accounting_period_id, - multiplier.source_relation from accounting_periods as base join accounting_periods as multiplier diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql b/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql index df498a81..ba5c374d 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql @@ -24,7 +24,6 @@ transaction_lines_w_accounting_period as ( -- transaction line totals, by accoun {% endif %} transactions.accounting_period_id as transaction_accounting_period_id, - transactions.source_relation, coalesce(transaction_lines.amount, 0) as unconverted_amount from transaction_lines diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql index 183ca923..662db0c2 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql @@ -30,14 +30,12 @@ transactions_in_every_calculation_period_w_exchange_rates as ( {% if var('netsuite2__using_exchange_rate', true) %} , exchange_reporting_period.exchange_rate as exchange_rate_reporting_period , exchange_transaction_period.exchange_rate as exchange_rate_transaction_period - , exchange_transaction_period.source_relation {% endif %} {% if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} , exchange_reporting_period.to_subsidiary_id , exchange_reporting_period.to_subsidiary_name , exchange_reporting_period.to_subsidiary_currency_symbol - , exchange_reporting_period.source_relation {% endif %} from transaction_lines_w_accounting_period @@ -84,7 +82,6 @@ transactions_with_converted_amounts as ( unconverted_amount as converted_amount_using_transaction_accounting_period, unconverted_amount as converted_amount_using_reporting_month, {% endif %} - accounts.source_relation, case when lower(accounts.account_type_id) in ('income','othincome','expense','othexpense','cogs') then true else false diff --git a/models/netsuite2/netsuite2__balance_sheet.sql b/models/netsuite2/netsuite2__balance_sheet.sql index 779285eb..ef7e3009 100644 --- a/models/netsuite2/netsuite2__balance_sheet.sql +++ b/models/netsuite2/netsuite2__balance_sheet.sql @@ -35,7 +35,6 @@ balance_sheet as ( transactions_with_converted_amounts.source_relation, transactions_with_converted_amounts.subsidiary_id, subsidiaries.name as subsidiary_name, - subsidiaries.name as source_relation, {% if var('netsuite2__multibook_accounting_enabled', false) %} transactions_with_converted_amounts.accounting_book_id, @@ -49,7 +48,6 @@ balance_sheet as ( {% endif %} reporting_accounting_periods.accounting_period_id as accounting_period_id, - reporting_accounting_periods.source_relation, reporting_accounting_periods.ending_at as accounting_period_ending, reporting_accounting_periods.name as accounting_period_name, reporting_accounting_periods.is_adjustment as is_accounting_period_adjustment, @@ -123,14 +121,12 @@ balance_sheet as ( and reporting_accounting_periods.fiscal_calendar_id = transaction_accounting_periods.fiscal_calendar_id) then 15 when not accounts.is_balancesheet then 14 else null - end as balance_sheet_sort_helper, - accounts.source_relation + end as balance_sheet_sort_helper --Below is only used if balance sheet transaction detail columns are specified dbt_project.yml file. {% if var('balance_sheet_transaction_detail_columns') %} , transaction_details.{{ var('balance_sheet_transaction_detail_columns') | join (", transaction_details.")}} - , transaction_details.source_relation {% endif %} @@ -181,7 +177,6 @@ balance_sheet as ( transactions_with_converted_amounts.source_relation, transactions_with_converted_amounts.subsidiary_id, subsidiaries.name as subsidiary_name, - subsidiaries.name as source_relation, {% if var('netsuite2__multibook_accounting_enabled', false) %} transactions_with_converted_amounts.accounting_book_id, @@ -195,7 +190,6 @@ balance_sheet as ( {% endif %} reporting_accounting_periods.accounting_period_id as accounting_period_id, - reporting_accounting_periods.source_relation, reporting_accounting_periods.ending_at as accounting_period_ending, reporting_accounting_periods.name as accounting_period_name, reporting_accounting_periods.is_adjustment as is_accounting_period_adjustment, @@ -217,14 +211,12 @@ balance_sheet as ( when lower(accounts.general_rate_type) in ('historical', 'average') then converted_amount_using_transaction_accounting_period else converted_amount_using_reporting_month end as converted_amount, - 16 as balance_sheet_sort_helper, - accounts.source_relation + 16 as balance_sheet_sort_helper --Below is only used if balance sheet transaction detail columns are specified dbt_project.yml file. {% if var('balance_sheet_transaction_detail_columns') %} , transaction_details.{{ var('balance_sheet_transaction_detail_columns') | join (", transaction_details.")}} - , transaction_details.source_relation {% endif %} diff --git a/models/netsuite2/netsuite2__income_statement.sql b/models/netsuite2/netsuite2__income_statement.sql index 9bfaf673..8bf82d39 100644 --- a/models/netsuite2/netsuite2__income_statement.sql +++ b/models/netsuite2/netsuite2__income_statement.sql @@ -70,32 +70,26 @@ income_statement as ( reporting_accounting_periods.name as accounting_period_name, reporting_accounting_periods.is_adjustment as is_accounting_period_adjustment, reporting_accounting_periods.is_closed as is_accounting_period_closed, - reporting_accounting_periods.source_relation, accounts.name as account_name, accounts.type_name as account_type_name, accounts.account_type_id, accounts.account_id as account_id, accounts.account_number, - accounts.source_relation, subsidiaries.subsidiary_id, subsidiaries.full_name as subsidiary_full_name, subsidiaries.name as subsidiary_name, - subsidiaries.source_relation, --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, {{ dbt.concat(['accounts.account_number',"'-'", 'accounts.name']) }} as account_number_and_name, classes.full_name as class_full_name, - classes.source_relation, --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('classes_pass_through_columns', identifier='classes') }}, locations.full_name as location_full_name, - locations.source_relation, departments.full_name as department_full_name, - departments.source_relation, --The below script allows for departments table pass through columns. {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, @@ -113,7 +107,6 @@ income_statement as ( {% if var('income_statement_transaction_detail_columns') %} , transaction_details.{{ var('income_statement_transaction_detail_columns') | join (", transaction_details.")}} - , transaction_details.source_relation {% endif %} diff --git a/models/netsuite2/netsuite2__transaction_details.sql b/models/netsuite2/netsuite2__transaction_details.sql index 84646cd5..106faa7c 100644 --- a/models/netsuite2/netsuite2__transaction_details.sql +++ b/models/netsuite2/netsuite2__transaction_details.sql @@ -90,14 +90,12 @@ transaction_details as ( transactions_with_converted_amounts.to_subsidiary_id, transactions_with_converted_amounts.to_subsidiary_name, transactions_with_converted_amounts.to_subsidiary_currency_symbol, - transactions_with_converted_amounts.source_relation, {% endif %} transaction_lines.transaction_line_id, transaction_lines.memo as transaction_memo, not transaction_lines.is_posting as is_transaction_non_posting, transactions.transaction_id, - transactions.source_relation, transactions.status as transaction_status, transactions.transaction_date, transactions.due_date_at as transaction_due_date, @@ -115,13 +113,11 @@ transaction_details as ( accounting_periods.accounting_period_id as accounting_period_id, accounting_periods.is_adjustment as is_accounting_period_adjustment, accounting_periods.is_closed as is_accounting_period_closed, - accounting_periods.source_relation, accounts.name as account_name, accounts.type_name as account_type_name, accounts.account_type_id, accounts.account_id as account_id, - accounts.account_number, - accounts.source_relation + accounts.account_number --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, @@ -140,35 +136,27 @@ transaction_details as ( customers.country as customer_country, customers.date_first_order_at as customer_date_first_order, customers.customer_external_id, - customers.source_relation, classes.full_name as class_full_name, - classes.source_relation, items.name as item_name, items.type_name as item_type_name, items.sales_description, - items.source_relation, locations.name as location_name, locations.city as location_city, locations.country as location_country, - locations.source_relation, {% if var('netsuite2__using_vendor_categories', true) %} vendor_categories.name as vendor_category_name, {% endif %} vendors.company_name as vendor_name, vendors.create_date_at as vendor_create_date, - vendors.source_relation, currencies.name as currency_name, currencies.symbol as currency_symbol, - currencies.source_relation, departments.name as department_name, - departments.source_relation --The below script allows for departments table pass through columns. {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, subsidiaries.subsidiary_id, subsidiaries.name as subsidiary_name, - subsidiaries.source_relation, case when lower(accounts.account_type_id) in ('income', 'othincome') then -converted_amount_using_transaction_accounting_period else converted_amount_using_transaction_accounting_period From 45b4fd6857efb96f6338751970d93e01bb7df2f5 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 25 Jan 2024 16:17:45 -0300 Subject: [PATCH 09/20] remove duplicative source_relation fields --- .../intermediate/base/int_netsuite2__accounting_periods.sql | 3 +-- models/netsuite2/intermediate/base/int_netsuite2__accounts.sql | 3 +-- .../netsuite2/intermediate/base/int_netsuite2__customers.sql | 3 +-- .../netsuite2/intermediate/base/int_netsuite2__locations.sql | 3 +-- .../intermediate/base/int_netsuite2__transaction_lines.sql | 2 +- 5 files changed, 5 insertions(+), 9 deletions(-) diff --git a/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql b/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql index e73909e6..e3f68cbd 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql @@ -16,8 +16,7 @@ joined as ( select accounting_periods.*, - accounting_period_fiscal_calendars.fiscal_calendar_id, - accounting_period_fiscal_calendars.source_relation + accounting_period_fiscal_calendars.fiscal_calendar_id from accounting_periods left join accounting_period_fiscal_calendars diff --git a/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql b/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql index dae2794b..73ed6392 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__accounts.sql @@ -18,8 +18,7 @@ joined as ( accounts.*, account_types.type_name, account_types.is_balancesheet, - account_types.is_leftside, - account_types.source_relation + account_types.is_leftside from accounts left join account_types diff --git a/models/netsuite2/intermediate/base/int_netsuite2__customers.sql b/models/netsuite2/intermediate/base/int_netsuite2__customers.sql index 3e530271..4f7c8854 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__customers.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__customers.sql @@ -19,8 +19,7 @@ joined as ( entity_address.city, entity_address.state, entity_address.zipcode, - entity_address.country, - entity_address.source_relation + entity_address.country from customers left join entity_address diff --git a/models/netsuite2/intermediate/base/int_netsuite2__locations.sql b/models/netsuite2/intermediate/base/int_netsuite2__locations.sql index fe67b1ca..b9a72b76 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__locations.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__locations.sql @@ -19,8 +19,7 @@ joined as ( location_main_address.city, location_main_address.state, location_main_address.zipcode, - location_main_address.country, - location_main_address.source_relation + location_main_address.country from locations left join location_main_address diff --git a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql index 2d16c82b..96a963d2 100644 --- a/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql +++ b/models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql @@ -36,7 +36,7 @@ joined as ( transaction_accounting_lines.debit_amount, transaction_accounting_lines.paid_amount, transaction_accounting_lines.unpaid_amount, - transaction_accounting_lines.is_posting, + transaction_accounting_lines.is_posting from transaction_lines left join transaction_accounting_lines From eb8e1220fe6c20f638c436f00dbd388ef331292d Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 25 Jan 2024 21:34:19 -0300 Subject: [PATCH 10/20] getting weird 'Scalar subquery produced more than one element' error --- ..._netsuite2__acctxperiod_exchange_rate_map.sql | 4 ++-- ...netsuite2__tran_lines_w_accounting_period.sql | 2 +- ...nt_netsuite2__tran_with_converted_amounts.sql | 2 +- models/netsuite2/netsuite2__balance_sheet.sql | 7 ++++--- models/netsuite2/netsuite2__income_statement.sql | 12 ++++++------ .../netsuite2/netsuite2__transaction_details.sql | 16 ++++++++-------- 6 files changed, 22 insertions(+), 21 deletions(-) diff --git a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql index 456a74e5..491af78d 100644 --- a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql +++ b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql @@ -71,13 +71,13 @@ accountxperiod_exchange_rate_map as ( -- account table with exchange rate detail period_exchange_rate_map.to_subsidiary_name, period_exchange_rate_map.to_subsidiary_currency_symbol, accounts.account_id, - accounts.source_relation case when lower(accounts.general_rate_type) = 'historical' then period_exchange_rate_map.historical_rate when lower(accounts.general_rate_type) = 'current' then period_exchange_rate_map.current_rate when lower(accounts.general_rate_type) = 'average' then period_exchange_rate_map.average_rate else null - end as exchange_rate + end as exchange_rate, + accounts.source_relation from accounts cross join period_exchange_rate_map diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql b/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql index ba5c374d..ab02a2aa 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql @@ -16,11 +16,11 @@ transaction_lines_w_accounting_period as ( -- transaction line totals, by accoun transaction_lines.transaction_line_id, transaction_lines.subsidiary_id, transaction_lines.account_id, + transaction_lines.source_relation, {% if var('netsuite2__multibook_accounting_enabled', false) %} transaction_lines.accounting_book_id, transaction_lines.accounting_book_name, - transaction_lines.source_relation, {% endif %} transactions.accounting_period_id as transaction_accounting_period_id, diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql index 662db0c2..eb15f6a6 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql @@ -41,7 +41,7 @@ transactions_in_every_calculation_period_w_exchange_rates as ( from transaction_lines_w_accounting_period left join transaction_and_reporting_periods - on transaction_and_reporting_periods.accounting_period_id = transaction_lines_w_accounting_period.transaction_accounting_period_ida + on transaction_and_reporting_periods.accounting_period_id = transaction_lines_w_accounting_period.transaction_accounting_period_id and transaction_and_reporting_periods.source_relation = transaction_lines_w_accounting_period.source_relation {% if var('netsuite2__using_exchange_rate', true) %} diff --git a/models/netsuite2/netsuite2__balance_sheet.sql b/models/netsuite2/netsuite2__balance_sheet.sql index ef7e3009..fae35383 100644 --- a/models/netsuite2/netsuite2__balance_sheet.sql +++ b/models/netsuite2/netsuite2__balance_sheet.sql @@ -151,7 +151,7 @@ balance_sheet as ( left join accounts on accounts.account_id = transactions_with_converted_amounts.account_id - on accounts.source_relation = transactions_with_converted_amounts.source_relation + and accounts.source_relation = transactions_with_converted_amounts.source_relation left join accounting_periods as reporting_accounting_periods on reporting_accounting_periods.accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id @@ -159,6 +159,7 @@ balance_sheet as ( left join accounting_periods as transaction_accounting_periods on transaction_accounting_periods.accounting_period_id = transactions_with_converted_amounts.transaction_accounting_period_id + and transaction_accounting_periods.source_relation = transactions_with_converted_amounts.source_relation left join subsidiaries on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id @@ -240,11 +241,11 @@ balance_sheet as ( left join accounts on accounts.account_id = transactions_with_converted_amounts.account_id - on accounts.source_relation = transactions_with_converted_amounts.source_relation + and accounts.source_relation = transactions_with_converted_amounts.source_relation left join accounting_periods as reporting_accounting_periods on reporting_accounting_periods.accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id - on reporting_accounting_periods.source_relation = transactions_with_converted_amounts.source_relation + and reporting_accounting_periods.source_relation = transactions_with_converted_amounts.source_relation left join subsidiaries on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id diff --git a/models/netsuite2/netsuite2__income_statement.sql b/models/netsuite2/netsuite2__income_statement.sql index 8bf82d39..9ed3e559 100644 --- a/models/netsuite2/netsuite2__income_statement.sql +++ b/models/netsuite2/netsuite2__income_statement.sql @@ -77,19 +77,19 @@ income_statement as ( accounts.account_number, subsidiaries.subsidiary_id, subsidiaries.full_name as subsidiary_full_name, - subsidiaries.name as subsidiary_name, + subsidiaries.name as subsidiary_name --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }}, {{ dbt.concat(['accounts.account_number',"'-'", 'accounts.name']) }} as account_number_and_name, - classes.full_name as class_full_name, + classes.full_name as class_full_name --The below script allows for accounts table pass through columns. {{ fivetran_utils.persist_pass_through_columns('classes_pass_through_columns', identifier='classes') }}, locations.full_name as location_full_name, - departments.full_name as department_full_name, + departments.full_name as department_full_name --The below script allows for departments table pass through columns. {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, @@ -133,7 +133,7 @@ income_statement as ( left join locations on locations.location_id = transaction_lines.location_id - on locations.source_relation = transaction_lines.source_relation + and locations.source_relation = transaction_lines.source_relation left join classes on classes.class_id = transaction_lines.class_id @@ -145,20 +145,20 @@ income_statement as ( left join subsidiaries on transactions_with_converted_amounts.subsidiary_id = subsidiaries.subsidiary_id - on transactions_with_converted_amounts.source_relation = subsidiaries.source_relation + and transactions_with_converted_amounts.source_relation = subsidiaries.source_relation --Below is only used if income statement transaction detail columns are specified dbt_project.yml file. {% if var('income_statement_transaction_detail_columns') != []%} join transaction_details on transaction_details.transaction_id = transactions_with_converted_amounts.transaction_id and transaction_details.transaction_line_id = transactions_with_converted_amounts.transaction_line_id + and transaction_details.source_relation = transactions_with_converted_amounts.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and transaction_details.accounting_book_id = transactions_with_converted_amounts.accounting_book_id {% endif %} {% if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} and transaction_details.to_subsidiary_id = transactions_with_converted_amounts.to_subsidiary_id - and transaction_details.source_relation = transactions_with_converted_amounts.source_relation {% endif %} {% endif %} diff --git a/models/netsuite2/netsuite2__transaction_details.sql b/models/netsuite2/netsuite2__transaction_details.sql index 106faa7c..5fadbb6f 100644 --- a/models/netsuite2/netsuite2__transaction_details.sql +++ b/models/netsuite2/netsuite2__transaction_details.sql @@ -79,11 +79,10 @@ entities as ( transaction_details as ( select - + transaction_lines.source_relation, {% if var('netsuite2__multibook_accounting_enabled', false) %} transaction_lines.accounting_book_id, transaction_lines.accounting_book_name, - transaction_lines.source_relation, {% endif %} {% if var('netsuite2__using_to_subsidiary', false) and var('netsuite2__using_exchange_rate', true) %} @@ -150,7 +149,7 @@ transaction_details as ( vendors.create_date_at as vendor_create_date, currencies.name as currency_name, currencies.symbol as currency_symbol, - departments.name as department_name, + departments.name as department_name --The below script allows for departments table pass through columns. {{ fivetran_utils.persist_pass_through_columns('departments_pass_through_columns', identifier='departments') }}, @@ -187,7 +186,7 @@ transaction_details as ( left join accounts as parent_account on parent_account.account_id = accounts.parent_id - on parent_account.source_relation = accounts.source_relation + and parent_account.source_relation = accounts.source_relation left join accounting_periods on accounting_periods.accounting_period_id = transactions.accounting_period_id @@ -195,7 +194,7 @@ transaction_details as ( left join customers on customers.customer_id = coalesce(transaction_lines.entity_id, transactions.entity_id) - on customers.source_relation = coalesce(transaction_lines.source_relation, transactions.source_relation) + and customers.source_relation = coalesce(transaction_lines.source_relation, transactions.source_relation) left join classes on classes.class_id = transaction_lines.class_id @@ -207,15 +206,16 @@ transaction_details as ( left join locations on locations.location_id = transaction_lines.location_id - on locations.source_relation = transaction_lines.source_relation + and locations.source_relation = transaction_lines.source_relation left join vendors on vendors.vendor_id = coalesce(transaction_lines.entity_id, transactions.entity_id) - on vendors.source_relation = coalesce(transaction_lines.source_relation, transactions.source_relation) + and vendors.source_relation = coalesce(transaction_lines.source_relation, transactions.source_relation) {% if var('netsuite2__using_vendor_categories', true) %} left join vendor_categories on vendor_categories.vendor_category_id = vendors.vendor_category_id + and vendor_categories.source_relation = vendors.source_relation {% endif %} left join currencies @@ -224,7 +224,7 @@ transaction_details as ( left join departments on departments.department_id = transaction_lines.department_id - on departments.source_relation = transaction_lines.source_relation + and departments.source_relation = transaction_lines.source_relation join subsidiaries on subsidiaries.subsidiary_id = transaction_lines.subsidiary_id From 97c0cef24676c1a594e7a8357c1f4c0a31f98df6 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:03:57 -0400 Subject: [PATCH 11/20] working on bq --- ...tsuite2__acctxperiod_exchange_rate_map.sql | 15 +++++++++- ..._netsuite2__tran_and_reporting_periods.sql | 14 ++++++++- models/netsuite2/netsuite2__balance_sheet.sql | 29 ++++++++++++++----- .../netsuite2/netsuite2__income_statement.sql | 15 ++++++++-- .../netsuite2__transaction_details.sql | 15 ++++++++-- 5 files changed, 75 insertions(+), 13 deletions(-) diff --git a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql index 491af78d..c428ce10 100644 --- a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql +++ b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql @@ -27,6 +27,15 @@ currencies as ( from {{ var('netsuite2_currencies') }} ), +{% if not var('netsuite2__using_to_subsidiary', false) %} +primary_subsidiaries as ( + select + subsidiary_id, + source_relation + from subsidiaries where parent_id is null +), +{% endif %} + period_exchange_rate_map as ( -- exchange rates used, by accounting period, to convert to parent subsidiary select consolidated_exchange_rates.accounting_period_id, @@ -54,7 +63,11 @@ period_exchange_rate_map as ( -- exchange rates used, by accounting period, to c and currencies.source_relation = to_subsidiaries.source_relation {% if not var('netsuite2__using_to_subsidiary', false) %} - where consolidated_exchange_rates.to_subsidiary_id in (select subsidiary_id from subsidiaries where parent_id is null) -- constraint - only the primary subsidiary has no parent + join primary_subsidiaries + on consolidated_exchange_rates.to_subsidiary_id = primary_subsidiaries.subsidiary_id + and consolidated_exchange_rates.source_relation = primary_subsidiaries.source_relation + + {# where consolidated_exchange_rates.to_subsidiary_id in (select subsidiary_id from subsidiaries where parent_id is null) -- constraint - only the primary subsidiary has no parent #} {% endif %} ), diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql index 0de06e20..e0b16567 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql @@ -10,6 +10,14 @@ subsidiaries as ( from {{ var('netsuite2_subsidiaries') }} ), +primary_subsidiary_calendar as ( + select + fiscal_calendar_id, + source_relation + from subsidiaries + where parent_id is null +), + transaction_and_reporting_periods as ( select base.accounting_period_id as accounting_period_id, @@ -25,9 +33,13 @@ transaction_and_reporting_periods as ( and multiplier.source_relation = base.source_relation and cast(multiplier.starting_at as {{ dbt.type_timestamp() }}) <= {{ current_timestamp() }} + join primary_subsidiary_calendar + on base.fiscal_calendar_id = primary_subsidiary_calendar.fiscal_calendar_id + and base.source_relation = primary_subsidiary_calendar.source_relation + where not base.is_quarter and not base.is_year - and base.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) -- fiscal calendar will align with parent subsidiary's default calendar + {# and base.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) -- fiscal calendar will align with parent subsidiary's default calendar #} ) select * diff --git a/models/netsuite2/netsuite2__balance_sheet.sql b/models/netsuite2/netsuite2__balance_sheet.sql index fae35383..a6eed193 100644 --- a/models/netsuite2/netsuite2__balance_sheet.sql +++ b/models/netsuite2/netsuite2__balance_sheet.sql @@ -28,6 +28,14 @@ subsidiaries as ( from {{ var('netsuite2_subsidiaries') }} ), +primary_subsidiary_calendar as ( + select + fiscal_calendar_id, + source_relation + from subsidiaries + where parent_id is null +), + balance_sheet as ( select transactions_with_converted_amounts.transaction_id, @@ -165,10 +173,14 @@ balance_sheet as ( on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id and subsidiaries.source_relation = transactions_with_converted_amounts.source_relation - where reporting_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) - and transaction_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) - and (accounts.is_balancesheet - or transactions_with_converted_amounts.is_income_statement) + join primary_subsidiary_calendar + on reporting_accounting_periods.fiscal_calendar_id = primary_subsidiary_calendar.fiscal_calendar_id + and reporting_accounting_periods.source_relation = primary_subsidiary_calendar.source_relation + and transaction_accounting_periods.fiscal_calendar_id = primary_subsidiary_calendar.fiscal_calendar_id + and transaction_accounting_periods.source_relation = primary_subsidiary_calendar.source_relation + + where accounts.is_balancesheet + or transactions_with_converted_amounts.is_income_statement union all @@ -251,9 +263,12 @@ balance_sheet as ( on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id and subsidiaries.source_relation = transactions_with_converted_amounts.source_relation - where reporting_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) - and (accounts.is_balancesheet - or transactions_with_converted_amounts.is_income_statement) + join primary_subsidiary_calendar + on reporting_accounting_periods.fiscal_calendar_id = primary_subsidiary_calendar.fiscal_calendar_id + and reporting_accounting_periods.source_relation = primary_subsidiary_calendar.source_relation + + where accounts.is_balancesheet + or transactions_with_converted_amounts.is_income_statement ), surrogate_key as ( diff --git a/models/netsuite2/netsuite2__income_statement.sql b/models/netsuite2/netsuite2__income_statement.sql index 9ed3e559..fe5cb522 100644 --- a/models/netsuite2/netsuite2__income_statement.sql +++ b/models/netsuite2/netsuite2__income_statement.sql @@ -48,6 +48,14 @@ departments as ( from {{ var('netsuite2_departments') }} ), +primary_subsidiary_calendar as ( + select + fiscal_calendar_id, + source_relation + from subsidiaries + where parent_id is null +), + income_statement as ( select transactions_with_converted_amounts.transaction_id, @@ -162,8 +170,11 @@ income_statement as ( {% endif %} {% endif %} - where reporting_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) - and transactions_with_converted_amounts.transaction_accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id + join primary_subsidiary_calendar + on reporting_accounting_periods.fiscal_calendar_id = primary_subsidiary_calendar.fiscal_calendar_id + and reporting_accounting_periods.source_relation = primary_subsidiary_calendar.source_relation + + where transactions_with_converted_amounts.transaction_accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id and transactions_with_converted_amounts.is_income_statement ), diff --git a/models/netsuite2/netsuite2__transaction_details.sql b/models/netsuite2/netsuite2__transaction_details.sql index 5fadbb6f..20884cbe 100644 --- a/models/netsuite2/netsuite2__transaction_details.sql +++ b/models/netsuite2/netsuite2__transaction_details.sql @@ -77,6 +77,14 @@ entities as ( from {{ var('netsuite2_entities') }} ), +primary_subsidiary_calendar as ( + select + fiscal_calendar_id, + source_relation + from subsidiaries + where parent_id is null +), + transaction_details as ( select transaction_lines.source_relation, @@ -229,9 +237,12 @@ transaction_details as ( join subsidiaries on subsidiaries.subsidiary_id = transaction_lines.subsidiary_id and subsidiaries.source_relation = transaction_lines.source_relation + + left join primary_subsidiary_calendar + on accounting_periods.fiscal_calendar_id = primary_subsidiary_calendar.fiscal_calendar_id + and accounting_periods.source_relation = primary_subsidiary_calendar.source_relation - where (accounting_periods.fiscal_calendar_id is null - or accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null)) + where accounting_periods.fiscal_calendar_id is null or primary_subsidiary_calendar.fiscal_calendar_id is not null ), surrogate_key as ( From 21cb74c64de8e7322bf8dfeec9c96c8df0afcd43 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:27:29 -0400 Subject: [PATCH 12/20] remove trailing comma and include source_relation in missing place --- .../intermediate/int_netsuite2__tran_and_reporting_periods.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql index e0b16567..6ba2a3a8 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql @@ -23,6 +23,7 @@ transaction_and_reporting_periods as ( base.accounting_period_id as accounting_period_id, base.source_relation, multiplier.accounting_period_id as reporting_accounting_period_id, + base.source_relation from accounting_periods as base join accounting_periods as multiplier @@ -39,7 +40,6 @@ transaction_and_reporting_periods as ( where not base.is_quarter and not base.is_year - {# and base.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) -- fiscal calendar will align with parent subsidiary's default calendar #} ) select * From 06aa0ff774a525ee42bf6fdb84d3d52f08545a45 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 29 Jan 2024 13:46:19 -0400 Subject: [PATCH 13/20] remove duplicate source_relation --- .../intermediate/int_netsuite2__tran_and_reporting_periods.sql | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql index 6ba2a3a8..042b9868 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql @@ -22,8 +22,7 @@ transaction_and_reporting_periods as ( select base.accounting_period_id as accounting_period_id, base.source_relation, - multiplier.accounting_period_id as reporting_accounting_period_id, - base.source_relation + multiplier.accounting_period_id as reporting_accounting_period_id from accounting_periods as base join accounting_periods as multiplier From 9d6b89c775795d439a74f26f098ba7a2966473d5 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Tue, 30 Jan 2024 10:35:13 -0400 Subject: [PATCH 14/20] fixed fanout --- README.md | 33 ++++++++++++++++--- integration_tests/dbt_project.yml | 1 + ...tsuite2__acctxperiod_exchange_rate_map.sql | 4 +-- ...netsuite2__tran_with_converted_amounts.sql | 2 +- .../netsuite2__transaction_details.sql | 2 +- 5 files changed, 33 insertions(+), 9 deletions(-) diff --git a/README.md b/README.md index a3cec0db..ec5116a0 100644 --- a/README.md +++ b/README.md @@ -153,11 +153,12 @@ sources: loaded_at_field: _fivetran_synced freshness: - warn_after: {count: 72, period: hour} - error_after: {count: 168, period: hour} + warn_after: {count: 72, period: hour} + error_after: {count: 168, period: hour} tables: &netsuite2_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ - name: account_type + identifier: accounttype description: A table containing the various account types within Netsuite. columns: - name: _fivetran_deleted @@ -174,6 +175,7 @@ sources: description: The name of the account type. - name: accounting_book_subsidiaries + identifier: accountingbooksubsidiaries description: A table containing the various account books and the respective subsidiaries. config: enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" @@ -188,8 +190,9 @@ sources: description: The status of the accounting book subsidiary. - name: subsidiary description: The unique identifier of the subsidiary used for the record. - + - name: accounting_book + identifier: accountingbook description: Table detailing all accounting books set up in Netsuite. config: enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" @@ -216,8 +219,9 @@ sources: description: Boolean indicating if the accounting book is a two step revenue allocation entry. Represented as "T" or "F" for true and false respectively. - name: unbilledreceivablegrouping description: Boolean indicating if the accounting book is an unbilled receivable grouping. Represented as "T" or "F" for true and false respectively. - + - name: accounting_period_fiscal_calendars + identifier: accountingperiodfiscalcalendars description: A table containing the accounting fiscal calendar periods. columns: - name: _fivetran_id @@ -232,6 +236,7 @@ sources: description: Reference to the parent fiscal calendar accounting period. - name: accounting_period + identifier: accountingperiod description: Table detailing all accounting periods, including monthly, quarterly and yearly. columns: - name: _fivetran_synced @@ -264,8 +269,9 @@ sources: description: Boolean indicating if the ar accounting period is locked. Represented as "T" or "F" for true and false respectively. - name: aplocked description: Boolean indicating if the ap accounting period is locked. Represented as "T" or "F" for true and false respectively. - + - name: account + identifier: account description: Table detailing all accounts set up in Netsuite. columns: - name: _fivetran_synced @@ -312,6 +318,7 @@ sources: description: Timestamp of when a record was deleted. - name: classification + identifier: classification description: Table detailing all classes set up in Netsuite. columns: - name: _fivetran_synced @@ -330,6 +337,7 @@ sources: description: Timestamp of when a record was deleted. - name: consolidated_exchange_rate + identifier: consolidatedexchangerate description: Table detailing average, historical and current exchange rates for all accounting periods. columns: - name: id @@ -354,6 +362,7 @@ sources: description: The historical rate of the exchange rate. - name: currency + identifier: currency description: Table detailing all currency information. columns: - name: _fivetran_synced @@ -366,6 +375,7 @@ sources: description: Currency symbol. - name: customer + identifier: customer description: Table detailing all customer information. columns: - name: id @@ -400,6 +410,7 @@ sources: description: Timestamp of when the first order was created. - name: department + identifier: department description: Table detailing all departments set up in Netsuite. columns: - name: _fivetran_synced @@ -420,6 +431,7 @@ sources: description: Timestamp of when a record was deleted. - name: entity + identifier: entity description: Table detailing all entities in Netsuite. columns: - name: id @@ -444,6 +456,7 @@ sources: description: The unique identifier of the vendor associated with the entity. - name: entity_address + identifier: entityaddress description: A table containing addresses and the various entities which they map. columns: - name: _fivetran_synced @@ -470,6 +483,7 @@ sources: description: The associated zipcode. - name: item + identifier: item description: Table detailing information about the items created in Netsuite. columns: - name: _fivetran_synced @@ -510,6 +524,7 @@ sources: description: Reference to the parent item. - name: job + identifier: job description: Table detailing all jobs. config: enabled: "{{ var('netsuite2__using_jobs', true) }}" @@ -530,6 +545,7 @@ sources: description: Reference to the parent job. - name: location_main_address + identifier: locationmainaddress description: A table containing the location main addresses. columns: - name: _fivetran_synced @@ -556,6 +572,7 @@ sources: description: The associated zipcode. - name: location + identifier: location description: Table detailing all locations, including store, warehouse and office locations. columns: - name: _fivetran_synced @@ -574,6 +591,7 @@ sources: description: The unique identifier of the subsidiary used for the record. - name: subsidiary + identifier: subsidiary description: Table detailing all subsidiaries. columns: - name: _fivetran_synced @@ -600,6 +618,7 @@ sources: description: The currency id of the currency used within the record. - name: transaction_accounting_line + identifier: transactionaccountingline description: A table detailing all transaction lines for all transactions. columns: - name: transaction @@ -626,6 +645,7 @@ sources: description: Total amount unpaid. - name: transaction_line + identifier: transactionline description: A table detailing all transaction lines for all transactions. columns: - name: _fivetran_synced @@ -666,6 +686,7 @@ sources: description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. - name: transaction + identifier: transaction description: A table detailing all transactions. columns: - name: _fivetran_synced @@ -702,6 +723,7 @@ sources: description: Boolean indicating if the transaction is a reversal entry. Represented as "T" or "F" for true and false respectively. - name: vendor_category + identifier: vendorcategory description: A table containing categories and how they map to vendors. config: enabled: "{{ var('netsuite2__using_vendor_categories', true) }}" @@ -714,6 +736,7 @@ sources: description: Timestamp of when a record was last synced. - name: vendor + identifier: vendor description: A table detailing all vendor information. columns: - name: _fivetran_synced diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index bc7c3613..87e7823c 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,6 +7,7 @@ models: vars: netsuite_schema: netsuite_integration_tests_5 netsuite_data_model_override: netsuite + integration_tests_seed_identifer_override: true # to use seed identifiers in integration tests # Enable below when generating docs # netsuite2__multibook_accounting_enabled: true diff --git a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql index c428ce10..3575e237 100644 --- a/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql +++ b/models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql @@ -67,7 +67,6 @@ period_exchange_rate_map as ( -- exchange rates used, by accounting period, to c on consolidated_exchange_rates.to_subsidiary_id = primary_subsidiaries.subsidiary_id and consolidated_exchange_rates.source_relation = primary_subsidiaries.source_relation - {# where consolidated_exchange_rates.to_subsidiary_id in (select subsidiary_id from subsidiaries where parent_id is null) -- constraint - only the primary subsidiary has no parent #} {% endif %} ), @@ -93,7 +92,8 @@ accountxperiod_exchange_rate_map as ( -- account table with exchange rate detail accounts.source_relation from accounts - cross join period_exchange_rate_map + join period_exchange_rate_map + on accounts.source_relation = period_exchange_rate_map.source_relation ) select * diff --git a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql index eb15f6a6..63b5cce6 100644 --- a/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql +++ b/models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql @@ -60,6 +60,7 @@ transactions_in_every_calculation_period_w_exchange_rates as ( on exchange_transaction_period.accounting_period_id = transaction_and_reporting_periods.accounting_period_id and exchange_transaction_period.account_id = transaction_lines_w_accounting_period.account_id and exchange_transaction_period.from_subsidiary_id = transaction_lines_w_accounting_period.subsidiary_id + and exchange_transaction_period.source_relation = transaction_lines_w_accounting_period.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and exchange_transaction_period.accounting_book_id = transaction_lines_w_accounting_period.accounting_book_id @@ -67,7 +68,6 @@ transactions_in_every_calculation_period_w_exchange_rates as ( {% if var('netsuite2__using_to_subsidiary', false) %} and exchange_transaction_period.to_subsidiary_id = exchange_reporting_period.to_subsidiary_id - and exchange_transaction_period.source_relation = transaction_lines_w_accounting_period.source_relation {% endif %} {% endif %} ), diff --git a/models/netsuite2/netsuite2__transaction_details.sql b/models/netsuite2/netsuite2__transaction_details.sql index 20884cbe..a7174e40 100644 --- a/models/netsuite2/netsuite2__transaction_details.sql +++ b/models/netsuite2/netsuite2__transaction_details.sql @@ -182,7 +182,7 @@ transaction_details as ( on transactions_with_converted_amounts.transaction_line_id = transaction_lines.transaction_line_id and transactions_with_converted_amounts.transaction_id = transaction_lines.transaction_id and transactions_with_converted_amounts.transaction_accounting_period_id = transactions_with_converted_amounts.reporting_accounting_period_id - and transactions_with_converted_amounts.source_relation = transactions_with_converted_amounts.source_relation + and transactions_with_converted_amounts.source_relation = transaction_lines.source_relation {% if var('netsuite2__multibook_accounting_enabled', false) %} and transactions_with_converted_amounts.accounting_book_id = transaction_lines.accounting_book_id From 4c50cc64f7949a88584568e12d11913dc510c90b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Wed, 31 Jan 2024 15:56:35 -0400 Subject: [PATCH 15/20] polish and try out --- CHANGELOG.md | 5 ++++- README.md | 25 ++++++++++++++++--------- dbt_project.yml | 2 +- integration_tests/dbt_project.yml | 2 +- 4 files changed, 22 insertions(+), 12 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 9838334b..12f578fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,7 +1,10 @@ # dbt_netsuite v0.13.0 ## πŸŽ‰ Feature Update πŸŽ‰ -This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-3-define-netsuitecom-or-netsuite2-source) for details on how to leverage this feature ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)). +This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite/tree/main?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)). + +## Contributors: +- [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)) ## 🎁 Official release for Netsuite2! 🎁 [PR #98](https://github.com/fivetran/dbt_netsuite/pull/98) is the official supported release of [dbt_netsuite v0.12.0-b1](https://github.com/fivetran/dbt_netsuite/releases/tag/v0.12.0-b1). diff --git a/README.md b/README.md index ec5116a0..864692f0 100644 --- a/README.md +++ b/README.md @@ -112,7 +112,7 @@ vars: ``` ## Step 4: Define database and schema variables -### Option 1: Single connector +### Option 1: Single connector πŸ’ƒ By default, this package runs using your destination and the `netsuite` schema. If this is not where your Netsuite data is (for example, if your netsuite schema is named `netsuite_fivetran`), add the following configuration to your root `dbt_project.yml` file: ```yml @@ -123,15 +123,15 @@ vars: > **Note**: If you are running the package on one source connector, each model will have a `source_relation` column that is just an empty string. -### Option 2: Union multiple connectors +### Option 2: Union multiple connectors (Netsuite2 only) πŸ‘― If you have multiple Netsuite connectors in Fivetran and would like to use this package on all of them simultaneously, we have provided functionality to do so. The package will union all of the data together and pass the unioned table into the transformations. You will be able to see which source it came from in the `source_relation` column of each model. To use this functionality, you will need to set either the `netsuite_union_schemas` OR `netsuite_union_databases` variables (cannot do both, though a more flexible approach is in the works...) in your root `dbt_project.yml` file: ```yml # dbt_project.yml vars: - netsuite_union_schemas: ['netsuite_usa','netsuite_canada'] # use this if the data is in different schemas/datasets of the same database/project - netsuite_union_databases: ['netsuite_usa','netsuite_canada'] # use this if the data is in different databases/projects but uses the same schema name + netsuite2_union_schemas: ['netsuite2_usa','netsuite2_canada'] # use this if the data is in different schemas/datasets of the same database/project + netsuite2_union_databases: ['netsuite2_usa','netsuite2_canada'] # use this if the data is in different databases/projects but uses the same schema name ``` #### Recommended: Incorporate unioned sources into DAG @@ -142,7 +142,7 @@ To properly incorporate all of your Netsuite connectors into your project's DAG:
Expand for source configuration template

-> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right variables to `False`. Otherwise, you may remove them from your source definitions. +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right [variables](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only) to `False`. Otherwise, you may remove them from your source definitions. ```yml sources: @@ -761,6 +761,8 @@ vars: ``` ## Step 5: Disable models for non-existent sources (Netsuite2 only) +> _This step is unnecessary (but still available for use) if you are unioning multiple connectors together in the previous step. That is, the `union_data` macro we use will create completely empty staging models for sources that are not found in any of your Netsuite2 schemas/databases. However, you can still leverage the below variables if you would like to avoid this behavior._ + It's possible that your Netsuite connector does not sync every table that this package expects. If your syncs exclude certain tables, it is because you either don't use that feature in Netsuite or actively excluded some tables from your syncs. To disable the corresponding functionality in the package, you must add the relevant variables. By default, all variables are assumed to be true. Add variables for only the tables you would like to disable: ```yml vars: @@ -774,10 +776,11 @@ vars: > To determine if a table or field is activated by a feature, access the [Records Catalog](https://docs.oracle.com/en/cloud/saas/netsuite/ns-online-help/article_159367781370.html). ## (Optional) Step 6: Additional configurations +

Expand/collapse configurations ### Enable additional features -### Multi-Book (Netsuite2 only) +#### Multi-Book (Netsuite2 only) To include `accounting_book_id` and `accounting_book_name` columns in the end models, set the below variable to `true` in your `dbt_project.yml`. This feature is disabled by default. >❗Notes: > - If you choose to enable this feature, this will add rows for transactions for any non-primary `accounting_book_id`, and your downstream use cases may need to be adjusted. @@ -788,7 +791,7 @@ vars: netsuite2__multibook_accounting_enabled: true # False by default. ``` -### To Subsidiary (Netsuite2 only) +#### To Subsidiary (Netsuite2 only) To include `to_subsidiary_id` and `to_subsidiary_name` columns in the end models, set the below variable to `true` in your `dbt_project.yml`. This feature is disabled by default. You will also need to be using exchange rates, which is enabled by default. >❗Notes: @@ -856,13 +859,15 @@ models: +schema: my_new_schema_name # leave blank for just the target_schema ``` -### Change the source table references -If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable: +### Change the source table references (only if using a single connector) +If an individual source table has a different name than the package expects, add the table name as it appears in your destination to the respective variable (and set `use_table_name_identifer_override` to `true` if using Netsuite2): > IMPORTANT: See this project's [`dbt_project.yml`](https://github.com/fivetran/dbt_netsuite_source/blob/main/dbt_project.yml) variable declarations to see the expected names. ```yml vars: + use_table_name_identifer_override: true # Netsuite2 users must set this to TRUE. default = false + # For all Netsuite source tables netsuite__identifier: your_table_name @@ -873,6 +878,8 @@ vars: ### Override the data models variable This package is designed to run **either** the Netsuite.com or Netsuite2 data models. However, for documentation purposes, an additional variable `netsuite_data_model_override` was created to allow for both data model types to be run at the same time by setting the variable value to `netsuite`. This is only to ensure the [dbt docs](https://fivetran.github.io/dbt_netsuite/) (which is hosted on this repository) is generated for both model types. While this variable is provided, we recommend you do not adjust the variable and instead change the `netsuite_data_model` variable to fit your configuration needs. +
+ ## (Optional) Step 7: Produce Analytics-Ready Reports with Streamlit App (Bigquery and Snowflake users only) For those who want to take their reports a step further, our team has created the [Fivetran Netsuite Streamlit App](https://fivetran-netsuite.streamlit.app/) to generate end model visualizations based off of the reports we created in this package. This way you can replicate much of the reporting you see internally in Netsuite and automate a lot of the work needed to report on your core metrics. diff --git a/dbt_project.yml b/dbt_project.yml index 61379e38..d74644d7 100644 --- a/dbt_project.yml +++ b/dbt_project.yml @@ -1,6 +1,6 @@ config-version: 2 name: 'netsuite' -version: '0.12.0' +version: '0.13.0' require-dbt-version: [">=1.3.0", "<2.0.0"] models: diff --git a/integration_tests/dbt_project.yml b/integration_tests/dbt_project.yml index 87e7823c..6b2e4908 100644 --- a/integration_tests/dbt_project.yml +++ b/integration_tests/dbt_project.yml @@ -7,7 +7,7 @@ models: vars: netsuite_schema: netsuite_integration_tests_5 netsuite_data_model_override: netsuite - integration_tests_seed_identifer_override: true # to use seed identifiers in integration tests + use_table_name_identifer_override: true # to use seed identifiers in integration tests # Enable below when generating docs # netsuite2__multibook_accounting_enabled: true From 0f470e9fccfea8e370a5b54463070d70d08ead6c Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 1 Feb 2024 09:48:58 -0400 Subject: [PATCH 16/20] changelog typo --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 12f578fd..91504f8c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,8 @@ This release supports running the package on multiple Netsuite sources at once! ## Contributors: - [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)) +# dbt_netsuite v0.12.0 + ## 🎁 Official release for Netsuite2! 🎁 [PR #98](https://github.com/fivetran/dbt_netsuite/pull/98) is the official supported release of [dbt_netsuite v0.12.0-b1](https://github.com/fivetran/dbt_netsuite/releases/tag/v0.12.0-b1). From 7bfc4f42cc0b2eb2aaf86ffc14f3d66bd8b37f0f Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:11:41 -0400 Subject: [PATCH 17/20] github stuff --- .../maintainer_pull_request_template.md | 7 +++- .github/workflows/auto-release.yml | 13 +++++++ .github/workflows/check_docs.yml | 35 +++++++++++++++++++ CHANGELOG.md | 5 +++ 4 files changed, 59 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/auto-release.yml create mode 100644 .github/workflows/check_docs.yml diff --git a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md index fdc71e6c..1e22b099 100644 --- a/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md +++ b/.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md @@ -11,11 +11,12 @@ ### Basic Validation Please acknowledge that you have successfully performed the following commands locally: - [ ] dbt run –full-refresh && dbt test -- [ ] dbt run (if incremental models are present) +- [ ] dbt run (if incremental models are present) && dbt test Before marking this PR as "ready for review" the following have been applied: - [ ] The appropriate issue has been linked, tagged, and properly assigned - [ ] All necessary documentation and version upgrades have been applied + - [ ] docs were regenerated (unless this PR does not include any code or yml updates) - [ ] BuildKite integration tests are passing - [ ] Detailed validation steps have been provided below @@ -23,3 +24,7 @@ Before marking this PR as "ready for review" the following have been applied: ### Detailed Validation Please share any and all of your validation steps: + +### If you had to summarize this PR in an emoji, which would it be? + +:dancer: \ No newline at end of file diff --git a/.github/workflows/auto-release.yml b/.github/workflows/auto-release.yml new file mode 100644 index 00000000..8ed58532 --- /dev/null +++ b/.github/workflows/auto-release.yml @@ -0,0 +1,13 @@ +name: 'auto release' +on: + pull_request: + types: + - closed + branches: + - main + +jobs: + call-workflow-passing-data: + if: github.event.pull_request.merged + uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main + secrets: inherit \ No newline at end of file diff --git a/.github/workflows/check_docs.yml b/.github/workflows/check_docs.yml new file mode 100644 index 00000000..c1e2a89a --- /dev/null +++ b/.github/workflows/check_docs.yml @@ -0,0 +1,35 @@ +name: 'check docs' +on: + push: + branches: + - main + pull_request: + branches: + - main + +jobs: + changed-files: + runs-on: ubuntu-latest + name: test changed-files + steps: + - uses: actions/checkout@v3 + with: + fetch-depth: 0 + + - name: Get changed files + id: changed-files + uses: tj-actions/changed-files@v41.0.0 + with: + files: docs/** + + - name: Check to see if docs folder hasn't changed + if: steps.changed-files.outputs.any_changed == 'false' + run: | + echo "Docs have not been regenerated." + exit 1 + + - name: Check if docs folder has changed + if: steps.changed-files.outputs.any_changed == 'true' + run: | + echo "Docs have been regenerated!" + exit 0 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 91504f8c..2d447330 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,11 @@ ## πŸŽ‰ Feature Update πŸŽ‰ This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite/tree/main?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)). +## πŸ› οΈ Under the Hood πŸ› οΈ +- Included auto-releaser GitHub Actions workflow to automate future releases. +- Included Github Actions workflow to check for docs updates. +- Updated the maintainer PR template to resemble the most up to date format. + ## Contributors: - [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)) From 9b0e7587a05e2b8960536b381b0c8bf1027689ed Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 10:11:22 -0500 Subject: [PATCH 18/20] joe feedback --- .github/workflows/check_docs.yml | 35 -------------------------------- CHANGELOG.md | 1 - 2 files changed, 36 deletions(-) delete mode 100644 .github/workflows/check_docs.yml diff --git a/.github/workflows/check_docs.yml b/.github/workflows/check_docs.yml deleted file mode 100644 index c1e2a89a..00000000 --- a/.github/workflows/check_docs.yml +++ /dev/null @@ -1,35 +0,0 @@ -name: 'check docs' -on: - push: - branches: - - main - pull_request: - branches: - - main - -jobs: - changed-files: - runs-on: ubuntu-latest - name: test changed-files - steps: - - uses: actions/checkout@v3 - with: - fetch-depth: 0 - - - name: Get changed files - id: changed-files - uses: tj-actions/changed-files@v41.0.0 - with: - files: docs/** - - - name: Check to see if docs folder hasn't changed - if: steps.changed-files.outputs.any_changed == 'false' - run: | - echo "Docs have not been regenerated." - exit 1 - - - name: Check if docs folder has changed - if: steps.changed-files.outputs.any_changed == 'true' - run: | - echo "Docs have been regenerated!" - exit 0 \ No newline at end of file diff --git a/CHANGELOG.md b/CHANGELOG.md index 2d447330..39c7b684 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,6 @@ This release supports running the package on multiple Netsuite sources at once! ## πŸ› οΈ Under the Hood πŸ› οΈ - Included auto-releaser GitHub Actions workflow to automate future releases. -- Included Github Actions workflow to check for docs updates. - Updated the maintainer PR template to resemble the most up to date format. ## Contributors: From 3ce838c27737b45ad9870bf2cedcb79125c44512 Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 11:34:55 -0500 Subject: [PATCH 19/20] add source_relation note to changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 39c7b684..81b43e65 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,7 @@ ## πŸŽ‰ Feature Update πŸŽ‰ This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite/tree/main?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)). +> Please note: For each end model, we have a added a new field, `source_relation`, that points to the source connector from which the record originated. ## πŸ› οΈ Under the Hood πŸ› οΈ - Included auto-releaser GitHub Actions workflow to automate future releases. From 8defe67391c500dcb22117f9c214cfdf67b6028b Mon Sep 17 00:00:00 2001 From: Jamie Rodriguez <65564846+fivetran-jamie@users.noreply.github.com> Date: Mon, 5 Feb 2024 18:34:52 -0500 Subject: [PATCH 20/20] swap source definition reference --- README.md | 618 ++---------------------------------------------------- 1 file changed, 12 insertions(+), 606 deletions(-) diff --git a/README.md b/README.md index 864692f0..f48cdda2 100644 --- a/README.md +++ b/README.md @@ -135,624 +135,30 @@ vars: ``` #### Recommended: Incorporate unioned sources into DAG -By default, this package defines one single-connector source, called `netsuite`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Netsuite sources, though the package will run successfully. +By default, this package defines one single-connector source, called `netsuite2`, which will be disabled if you are unioning multiple connectors. This means that your DAG will not include your Netsuite sources, though the package will run successfully. -To properly incorporate all of your Netsuite connectors into your project's DAG: -1. Define each of your sources in a `.yml` file in your project. Utilize the following template to leverage our table and column documentation. - -
Expand for source configuration template

- -> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right [variables](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only) to `False`. Otherwise, you may remove them from your source definitions. +To properly incorporate all of your Netsuite2 connectors into your project's DAG: +1. Define each of your sources in a `.yml` file in your project. Utilize the following template for the `source`-level configurations, and, **most importantly**, copy and paste the table and column-level definitions from the package's `src_netsuite2.yml` [file](https://github.com/fivetran/dbt_netsuite_source/blob/main/models/netsuite2/src_netsuite2.yml#L15-L607). ```yml +# a .yml file in your root project sources: - - name: - schema: - database: + - name: # ex: netsuite2_usa + schema: # one of var('netsuite2_union_schemas') if unioning schemas, otherwise just 'netsuite2' + database: # one of var('netsuite2_union_databases') if unioning databases, otherwise whatever DB your netsuite2 schemas all live in loader: fivetran loaded_at_field: _fivetran_synced - freshness: + freshness: # feel free to adjust to your liking warn_after: {count: 72, period: hour} error_after: {count: 168, period: hour} - tables: &netsuite2_table_defs # <- see https://support.atlassian.com/bitbucket-cloud/docs/yaml-anchors/ - - name: account_type - identifier: accounttype - description: A table containing the various account types within Netsuite. - columns: - - name: _fivetran_deleted - description: Unique ID used by Fivetran to sync and dedupe data. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: Unique identifier of thea account type. - - name: balancesheet - description: Boolean indicating if the account type is a balance sheet account. Represented as "T" or "F" for true and false respectively. - - name: left - description: Boolean indicating if the account type is leftside. Represented as "T" or "F" for true and false respectively. - - name: longname - description: The name of the account type. - - - name: accounting_book_subsidiaries - identifier: accountingbooksubsidiaries - description: A table containing the various account books and the respective subsidiaries. - config: - enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" - columns: - - name: _fivetran_id - description: Unique ID used by Fivetran to sync and dedupe data. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: accountingbook - description: Unique identifier of the accounting book. - - name: status - description: The status of the accounting book subsidiary. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - - name: accounting_book - identifier: accountingbook - description: Table detailing all accounting books set up in Netsuite. - config: - enabled: "{{ var('netsuite2__multibook_accounting_enabled', true) }}" - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: Unique identifier of the accounting book. - - name: name - description: Name of the accounting book. - - name: basebook - description: Reference to the base book. - - name: effectiveperiod - description: Reference to the effective period of the accounting book. - - name: isadjustmentonly - description: Boolean indicating if the accounting book is an adjustment only. Represented as "T" or "F" for true and false respectively. - - name: isconsolidated - description: Boolean indicating if the accounting book is a consolidated entry. Represented as "T" or "F" for true and false respectively. - - name: contingentrevenuehandling - description: Boolean indicating if the accounting book is contingent revenue handling. Represented as "T" or "F" for true and false respectively. - - name: isprimary - description: Boolean indicating if the accounting book is a primary entry. Represented as "T" or "F" for true and false respectively. - - name: twosteprevenueallocation - description: Boolean indicating if the accounting book is a two step revenue allocation entry. Represented as "T" or "F" for true and false respectively. - - name: unbilledreceivablegrouping - description: Boolean indicating if the accounting book is an unbilled receivable grouping. Represented as "T" or "F" for true and false respectively. - - - name: accounting_period_fiscal_calendars - identifier: accountingperiodfiscalcalendars - description: A table containing the accounting fiscal calendar periods. - columns: - - name: _fivetran_id - description: Unique ID used by Fivetran to sync and dedupe data. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: accountingperiod - description: The accounting period id of the accounting period which the transaction took place in. - - name: fiscalcalendar - description: Reference to the fiscal calendar used for the record. - - name: parent - description: Reference to the parent fiscal calendar accounting period. - - - name: accounting_period - identifier: accountingperiod - description: Table detailing all accounting periods, including monthly, quarterly and yearly. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The accounting period id of the accounting period which the transaction took place in. - - name: parent - description: Reference to the parent accounting period. - - name: periodname - description: Name of the accounting period. - - name: startdate - description: Timestamp of when the accounting period starts. - - name: enddate - description: Timestamp if when the accounting period ends. - - name: closedondate - description: Timestamp of when the accounting period is closed. - - name: isquarter - description: Boolean indicating if the accounting period is the initial quarter. Represented as "T" or "F" for true and false respectively. - - name: isyear - description: Boolean indicating if the accounting period is the initial period. Represented as "T" or "F" for true and false respectively. - - name: isadjust - description: Boolean indicating if the accounting period is an adjustment. Represented as "T" or "F" for true and false respectively. - - name: isposting - description: Boolean indicating if the accounting period is posting. Represented as "T" or "F" for true and false respectively. - - name: closed - description: Boolean indicating if the accounting period is closed. Represented as "T" or "F" for true and false respectively. - - name: alllocked - description: Boolean indicating if all the accounting periods are locked. Represented as "T" or "F" for true and false respectively. - - name: arlocked - description: Boolean indicating if the ar accounting period is locked. Represented as "T" or "F" for true and false respectively. - - name: aplocked - description: Boolean indicating if the ap accounting period is locked. Represented as "T" or "F" for true and false respectively. - - - name: account - identifier: account - description: Table detailing all accounts set up in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier associated with the account. - - name: externalid - description: Reference to the external account, - - name: parent - description: Reference to the parent account. - - name: acctnumber - description: Netsuite generated account number. - - name: accttype - description: Reference to the account type. - - name: sspecacct - description: Special account type. - - name: fullname - description: Name of the account. - - name: description - description: Description of the account. - - name: deferralacct - description: Reference to the deferral account. - - name: cashflowrate - description: The cash flow rate type of the account. - - name: generalrate - description: The general rate type of the account (Current, Historical, Average). - - name: currency - description: The currency id of the currency used within the record. - - name: class - description: The unique identifier of the class used for the record. - - name: department - description: The unique identifier of the department used for the record. - - name: location - description: The unique identifier of the location used for the record. - - name: includechildren - description: Boolean indicating if the account includes sub accounts. Represented as "T" or "F" for true and false respectively. - - name: isinactive - description: Boolean indicating if the account is inactive. Represented as "T" or "F" for true and false respectively. - - name: issummary - description: Boolean indicating if the account is a summary account. Represented as "T" or "F" for true and false respectively. - - name: eliminate - description: Indicates this is an intercompany account used only to record transactions between subsidiaries. Amounts posted to intercompany accounts are eliminated when you run the intercompany elimination process at the end of an accounting period. Represented as "T" or "F" for true and false respectively. - - name: _fivetran_deleted - description: Timestamp of when a record was deleted. - - - name: classification - identifier: classification - description: Table detailing all classes set up in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the class used for the record. - - name: externalid - description: Reference to the external class. - - name: name - description: Name of the class. - - name: fullname - description: Full name of the class. - - name: isinactive - description: Boolean indicating if the class is active. Represented as "T" or "F" for true and false respectively. - - name: _fivetran_deleted - description: Timestamp of when a record was deleted. - - - name: consolidated_exchange_rate - identifier: consolidatedexchangerate - description: Table detailing average, historical and current exchange rates for all accounting periods. - columns: - - name: id - description: Unique identifier for the consolidated exchange rate. - - name: postingperiod - description: The accounting period id of the accounting period which the transaction took place in. - - name: fromcurrency - description: The currency id which the consolidated exchange rate is from. - - name: fromsubsidiary - description: The subsidiary id which the consolidated exchange rate is from. - - name: tocurrency - description: The subsidiary id which the consolidated exchange rate is for. - - name: tosubsidiary - description: The subsidiary id which the consolidated exchange rate is for. - - name: currentrate - description: The current rate associated with the exchange rate. - - name: averagerate - description: The consolidated exchange rates average rate. - - name: accountingbook - description: Unique identifier of the accounting book. - - name: historicalrate - description: The historical rate of the exchange rate. - - - name: currency - identifier: currency - description: Table detailing all currency information. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The currency id of the currency used within the record. - - name: name - description: Name of the currency. - - name: symbol - description: Currency symbol. - - - name: customer - identifier: customer - description: Table detailing all customer information. - columns: - - name: id - description: Unique identifier of the customer. - - name: entityid - description: The entity id of the entity used for the record. - - name: externalid - description: Reference to the associated external customer. - - name: parent - description: Reference to the parent customer. - - name: isperson - description: Boolean indicating if the customer is an individual person. Represented as "T" or "F" for true and false respectively. - - name: companyname - description: Name of the company. - - name: firstname - description: First name of the customer. - - name: lastname - description: Last name of the customer. - - name: email - description: Customers email address. - - name: phone - description: Phone number of the customer. - - name: defaultbillingaddress - description: Reference to the associated billing address. - - name: defaultshippingaddress - description: Reference to the associated default shipping address. - - name: receivablesaccount - description: Reference to the associated receivables account. - - name: currency - description: The currency id of the currency used within the record. - - name: firstorderdate - description: Timestamp of when the first order was created. - - - name: department - identifier: department - description: Table detailing all departments set up in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the department used for the record. - - name: parent - description: Reference to the parent department. - - name: name - description: Name of the department. - - name: fullname - description: Full name of the department. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - name: isinactive - description: Boolean indicating if the department is active. Represented as "T" or "F" for true and false respectively. - - name: _fivetran_deleted - description: Timestamp of when a record was deleted. - - - name: entity - identifier: entity - description: Table detailing all entities in Netsuite. - columns: - - name: id - description: The entity id of the entity used for the record. - - name: contact - description: The unique identifier of the contact associated with the entity. - - name: customer - description: The unique identifier of the customer associated with the entity. - - name: employee - description: The unique identifier of the employee associated with the entity. - - name: entitytitle - description: The entity name. - - name: isperson - description: Value indicating whether the entity is a person (either yes or no). - - name: parent - description: The unique identifier of the parent entity. - - name: project - description: The unique identifier of the project (job) associated with the entity. - - name: type - description: The entity type (Contact, CustJob, Job, etc). - - name: vendor - description: The unique identifier of the vendor associated with the entity. - - - name: entity_address - identifier: entityaddress - description: A table containing addresses and the various entities which they map. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: addr1 - description: The associated address 1. - - name: addr2 - description: The associated address 2. - - name: addr3 - description: The associated address 3. - - name: addressee - description: The individual associated with the address. - - name: addrtext - description: The full address associated. - - name: city - description: The associated city. - - name: country - description: The associated country. - - name: state - description: The associated state. - - name: nkey - description: The associated Netsuite key. - - name: zip - description: The associated zipcode. - - - name: item - identifier: item - description: Table detailing information about the items created in Netsuite. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the item used within the record. - - name: fullname - description: Name of the item. - - name: itemtype - description: Item type name. - - name: description - description: Sales description associated with the item. - - name: department - description: The unique identifier of the department used for the record. - - name: class - description: The unique identifier of the class used for the record. - - name: location - description: The unique identifier of the location used for the record. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - name: assetaccount - description: Reference to the asset account. - - name: expenseaccount - description: Reference to the expense account. - - name: gainlossaccount - description: Reference to the gain or loss account. - - name: incomeaccount - description: Reference to the income account. - - name: intercoexpenseaccount - description: Reference to the intercompany expense account. - - name: intercoincomeaccount - description: Reference to the intercompany income account. - - name: deferralaccount - description: Reference to the deferred expense account. - - name: deferredrevenueaccount - description: Reference to the deferred revenue account. - - name: parent - description: Reference to the parent item. - - - name: job - identifier: job - description: Table detailing all jobs. - config: - enabled: "{{ var('netsuite2__using_jobs', true) }}" - columns: - - name: id - description: The unique identifier of the job. - - name: externalid - description: The unique identifier of the external job reference. - - name: customer - description: The unique identifier of the customer associated with the job. - - name: entityid - description: Reference the the entity. - - name: defaultbillingaddress - description: Default billing address. - - name: defaultshippingaddress - description: Default shipping address. - - name: parent - description: Reference to the parent job. - - - name: location_main_address - identifier: locationmainaddress - description: A table containing the location main addresses. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: addr1 - description: The associated address 1. - - name: addr2 - description: The associated address 2. - - name: addr3 - description: The associated address 3. - - name: addressee - description: The individual associated with the address. - - name: addrtext - description: The full address associated. - - name: city - description: The associated city. - - name: country - description: The associated country. - - name: state - description: The associated state. - - name: nkey - description: The associated Netsuite key. - - name: zip - description: The associated zipcode. - - - name: location - identifier: location - description: Table detailing all locations, including store, warehouse and office locations. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the location used for the record. - - name: name - description: Name of the location. - - name: fullname - description: Full name of the location. - - name: mainaddress - description: Reference to the main address used for the record. - - name: parent - description: Reference to the parent location. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - - name: subsidiary - identifier: subsidiary - description: Table detailing all subsidiaries. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the subsidiary used for the record. - - name: name - description: Name of the subsidiary. - - name: fullname - description: Full name of the subsidiary. - - name: email - description: Email address associated with the subsidiary. - - name: mainaddress - description: Reference to the main address used for the record. - - name: country - description: The country which the subsidiary is located. - - name: state - description: The state which the subsidiary is located. - - name: fiscalcalendar - description: Reference to the fiscal calendar used for the record. - - name: parent - description: Reference to the parent subsidiary. - - name: currency - description: The currency id of the currency used within the record. - - - name: transaction_accounting_line - identifier: transactionaccountingline - description: A table detailing all transaction lines for all transactions. - columns: - - name: transaction - description: The transaction id which the transaction line is associated with. - - name: transactionline - description: The unique identifier of the transaction line. - - name: amount - description: The amount of the transaction line. - - name: netamount - description: The net amount of the transaction line. - - name: accountingbook - description: Unique identifier of the accounting book. - - name: account - description: Reference to the account associated with the entry. - - name: posting - description: Boolean indicating if the entry is posting. Represented as "T" or "F" for true and false respectively. - - name: credit - description: Amount associated as a credit. - - name: debit - description: Amount associated as a debit. - - name: amountpaid - description: Total amount paid. - - name: amountunpaid - description: Total amount unpaid. - - - name: transaction_line - identifier: transactionline - description: A table detailing all transaction lines for all transactions. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: Unique identifier of the transaction line. - - name: transaction - description: The transaction id of referenced for the record. - - name: linesequencenumber - description: Netsuite generated number associated with the transaction line. - - name: memo - description: The memo attached to the transaction line. - - name: entity - description: The entity id of the entity used for the record. - - name: item - description: The unique identifier of the item used within the record. - - name: class - description: The unique identifier of the class used for the record. - - name: location - description: The unique identifier of the location used for the record. - - name: subsidiary - description: The unique identifier of the subsidiary used for the record. - - name: department - description: The unique identifier of the department used for the record. - - name: isclosed - description: Boolean indicating if the transaction line is closed. Represented as "T" or "F" for true and false respectively. - - name: isbillable - description: Boolean indicating if the transaction line is billable. Represented as "T" or "F" for true and false respectively. - - name: iscogs - description: Boolean indicating if the transaction line is a cost of goods sold entry. Represented as "T" or "F" for true and false respectively. - - name: cleared - description: Boolean indicating if the transaction line is cleared. Represented as "T" or "F" for true and false respectively. - - name: commitmentfirm - description: Boolean indicating if the transaction line is a commitment firm. Represented as "T" or "F" for true and false respectively. - - name: mainline - description: Boolean indicating if the transaction line is a main line entry. Represented as "T" or "F" for true and false respectively. - - name: taxline - description: Boolean indicating if the transaction line is a tax line. Represented as "T" or "F" for true and false respectively. - - - name: transaction - identifier: transaction - description: A table detailing all transactions. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The transaction id of referenced for the record. - - name: transactionnumber - description: The Netsuite generated number of the transaction. - - name: type - description: The type of the transaction. - - name: memo - description: Memo attached to the transaction. - - name: trandate - description: The timestamp of the transaction date. - - name: status - description: Status of the transaction. - - name: createddate - description: Timestamp of when the record was created. - - name: duedate - description: Timestamp of the transactions due date. - - name: closedate - description: Timestamp of when the transaction was closed. - - name: currency - description: The currency id of the currency used within the record. - - name: entity - description: The entity id of the entity used for the record. - - name: postingperiod - description: The accounting period id of the accounting period which the transaction took place in. - - name: posting - description: Boolean indicating if the transaction is a posting event. Represented as "T" or "F" for true and false respectively. - - name: intercoadj - description: Boolean indicating if the transaction is an intercompany adjustment. Represented as "T" or "F" for true and false respectively. - - name: isreversal - description: Boolean indicating if the transaction is a reversal entry. Represented as "T" or "F" for true and false respectively. - - - name: vendor_category - identifier: vendorcategory - description: A table containing categories and how they map to vendors. - config: - enabled: "{{ var('netsuite2__using_vendor_categories', true) }}" - columns: - - name: id - description: Unique identifier of the vendor category. - - name: name - description: Name of the vendor category. - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - - name: vendor - identifier: vendor - description: A table detailing all vendor information. - columns: - - name: _fivetran_synced - description: Timestamp of when a record was last synced. - - name: id - description: The unique identifier of the vendor. - - name: companyname - description: Name of the company. - - name: datecreated - description: Timestamp of the record creation. - - name: category - description: Unique identifier of the vendor category + tables: # copy and paste from netsuite_source/models/netsuite2/src_netsuite2.yml ``` -

-2. Set the `has_defined_sources` variable (scoped to the `netsuite_source` package) to true, like such: +> **Note**: If there are source tables you do not have (see [Step 4](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only)), you may still include them, as long as you have set the right [variables](https://github.com/fivetran/dbt_netsuite?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only) to `False`. Otherwise, you may remove them from your source definitions. + +2. Set the `has_defined_sources` variable (scoped to the `netsuite_source` package) to `True`, like such: ```yml # dbt_project.yml vars: