Feature/add union data

.github/PULL_REQUEST_TEMPLATE/maintainer_pull_request_template.md

@@ -11,15 +11,20 @@
 ### Basic Validation
 Please acknowledge that you have successfully performed the following commands locally:
 - [ ] dbt run –full-refresh && dbt test
-- [ ] dbt run (if incremental models are present)
+- [ ] dbt run (if incremental models are present) && dbt test
 
 Before marking this PR as "ready for review" the following have been applied:
 - [ ] The appropriate issue has been linked, tagged, and properly assigned
 - [ ] All necessary documentation and version upgrades have been applied
+    <!--- Be sure to update the package version in the dbt_project.yml, integration_tests/dbt_project.yml, and README if necessary. -->
 - [ ] docs were regenerated (unless this PR does not include any code or yml updates)
 - [ ] BuildKite integration tests are passing
 - [ ] Detailed validation steps have been provided below
 
 ### Detailed Validation
 Please share any and all of your validation steps:
 <!--- Provide the steps you took to validate your changes below. -->
+
+### If you had to summarize this PR in an emoji, which would it be?
+<!--- For a complete list of markdown compatible emojis check our this git repo (https://gist.github.com/rxaviers/7360908)  --> 
+:dancer:

.github/workflows/auto-release.yml

@@ -0,0 +1,13 @@
+name: 'auto release'
+on:
+  pull_request:
+    types:
+      - closed
+    branches:
+      - main
+
+jobs:
+  call-workflow-passing-data:
+    if: github.event.pull_request.merged
+    uses: fivetran/dbt_package_automations/.github/workflows/auto-release.yml@main
+    secrets: inherit

.github/workflows/check_docs.yml

@@ -0,0 +1,35 @@
+name: 'check docs'
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  changed-files:
+    runs-on: ubuntu-latest
+    name: test changed-files
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          fetch-depth: 0
+
+      - name: Get changed files
+        id: changed-files
+        uses: tj-actions/[email protected]
+        with:
+          files: docs/**
+
+      - name: Check to see if docs folder hasn't changed
+        if: steps.changed-files.outputs.any_changed == 'false'
+        run: |
+          echo "Docs have not been regenerated."
+          exit 1
+
+      - name: Check if docs folder has changed
+        if: steps.changed-files.outputs.any_changed == 'true'
+        run: |
+          echo "Docs have been regenerated!"
+          exit 0 

CHANGELOG.md

@@ -1,4 +1,18 @@
+# dbt_netsuite v0.13.0
+
+## 🎉 Feature Update 🎉 
+This release supports running the package on multiple Netsuite sources at once! See the [README](https://github.com/fivetran/dbt_netsuite/tree/main?tab=readme-ov-file#step-4-define-database-and-schema-variables) for details on how to leverage this feature ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104)).
+
+## 🛠️ Under the Hood 🛠️
+- Included auto-releaser GitHub Actions workflow to automate future releases.
+- Included Github Actions workflow to check for docs updates.
+- Updated the maintainer PR template to resemble the most up to date format.
+
+## Contributors:
+- [@fivetran-poonamagate](https://github.com/fivetran-poonamagate) ([PR #104](https://github.com/fivetran/dbt_netsuite/pull/104))
+
 # dbt_netsuite v0.12.0
+
 ## 🎁 Official release for Netsuite2! 🎁
 [PR #98](https://github.com/fivetran/dbt_netsuite/pull/98) is the official supported release of [dbt_netsuite v0.12.0-b1](https://github.com/fivetran/dbt_netsuite/releases/tag/v0.12.0-b1). 
 

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '0.12.0'
+version: '0.13.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/dbt_project.yml

@@ -1,12 +1,13 @@
 name: 'netsuite_integration_tests'
-version: '0.12.0'
+version: '0.13.0'
 profile: 'integration_tests'
 config-version: 2
 models:
   +materialized: table
 vars:
   netsuite_schema: netsuite_integration_tests_5
   netsuite_data_model_override: netsuite
+  use_table_name_identifer_override: true # to use seed identifiers in integration tests
 
   # Enable below when generating docs
   # netsuite2__multibook_accounting_enabled: true

models/netsuite2.yml

@@ -102,7 +102,7 @@ models:
     - name: to_subsidiary_currency_symbol
       description: Currency symbol for the consolidated exchange rate.
     - name: balance_sheet_id
-      description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, `account_name`, and `account_id`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. 
+      description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, `account_name`, `account_id` and `source_relation`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. 
       tests:
           - unique
           - not_null
@@ -172,7 +172,7 @@ models:
     - name: to_subsidiary_currency_symbol
       description: Currency symbol for the consolidated exchange rate.
     - name: income_statement_id
-      description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, and `account_name`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. 
+      description: Surrogate key hashed on `transaction_line_id`, `transaction_id`, `accounting_period_id`, `account_name` and `source_relation`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. 
       tests:
           - unique
           - not_null
@@ -300,7 +300,7 @@ models:
     - name: to_subsidiary_currency_symbol
       description: Currency symbol for the consolidated exchange rate.
     - name: transaction_details_id
-      description: Surrogate key hashed on `transaction_line_id` and `transaction_id`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. 
+      description: Surrogate key hashed on `transaction_line_id` and `transaction_id`, `source_relation`. Adds `to_subsidiary_id` if using subsidiaries and `accounting_book_id` if using multibook accounting. 
       tests:
           - unique
           - not_null

models/netsuite2/intermediate/base/int_netsuite2__accounting_periods.sql

@@ -21,6 +21,7 @@ joined as (
     from accounting_periods
     left join accounting_period_fiscal_calendars
         on accounting_periods.accounting_period_id = accounting_period_fiscal_calendars.accounting_period_id
+        and accounting_periods.source_relation = accounting_period_fiscal_calendars.source_relation
 )
 
 select *

models/netsuite2/intermediate/base/int_netsuite2__accounts.sql

@@ -23,6 +23,7 @@ joined as (
     from accounts
     left join account_types
         on accounts.account_type_id = account_types.account_type_id
+        and accounts.source_relation = account_types.source_relation
 )
 
 select *

models/netsuite2/intermediate/base/int_netsuite2__customers.sql

@@ -24,6 +24,7 @@ joined as (
     from customers
     left join entity_address
         on coalesce(customers.default_billing_address_id, customers.default_shipping_address_id) = entity_address.nkey
+        and customers.source_relation = entity_address.source_relation
 )
 
 select *

models/netsuite2/intermediate/base/int_netsuite2__locations.sql

@@ -24,6 +24,7 @@ joined as (
     from locations
     left join location_main_address
         on locations.main_address_id = location_main_address.nkey
+        and locations.source_relation = location_main_address.source_relation
 )
 
 select *

models/netsuite2/intermediate/base/int_netsuite2__transaction_lines.sql

@@ -42,10 +42,12 @@ joined as (
     left join transaction_accounting_lines
         on transaction_lines.transaction_line_id = transaction_accounting_lines.transaction_line_id
         and transaction_lines.transaction_id = transaction_accounting_lines.transaction_id
+        and transaction_lines.source_relation = transaction_accounting_lines.source_relation
 
     {% if var('netsuite2__multibook_accounting_enabled', false) %}
     left join accounting_books
         on accounting_books.accounting_book_id = transaction_accounting_lines.accounting_book_id
+        and accounting_books.source_relation = transaction_accounting_lines.source_relation
 
     union all
 
@@ -65,8 +67,10 @@ joined as (
     left join transaction_accounting_lines
         on transaction_lines.transaction_line_id = transaction_accounting_lines.transaction_line_id
         and transaction_lines.transaction_id = transaction_accounting_lines.transaction_id
+        and transaction_lines.source_relation = transaction_accounting_lines.source_relation
     left join accounting_books
         on accounting_books.base_book_id = transaction_accounting_lines.accounting_book_id
+        and accounting_books.source_relation = transaction_accounting_lines.source_relation
     where accounting_books.base_book_id is not null
     {% endif %}
 

models/netsuite2/intermediate/int_netsuite2__acctxperiod_exchange_rate_map.sql

@@ -27,6 +27,15 @@ currencies as (
     from {{ var('netsuite2_currencies') }}
 ),
 
+{% if not var('netsuite2__using_to_subsidiary', false) %}
+primary_subsidiaries as (
+  select 
+    subsidiary_id,
+    source_relation
+  from subsidiaries where parent_id is null
+),
+{% endif %}
+
 period_exchange_rate_map as ( -- exchange rates used, by accounting period, to convert to parent subsidiary
   select
     consolidated_exchange_rates.accounting_period_id,
@@ -35,6 +44,7 @@ period_exchange_rate_map as ( -- exchange rates used, by accounting period, to c
     consolidated_exchange_rates.accounting_book_id,
     {% endif %}
 
+    consolidated_exchange_rates.source_relation,
     consolidated_exchange_rates.average_rate,
     consolidated_exchange_rates.current_rate,
     consolidated_exchange_rates.historical_rate,
@@ -46,12 +56,17 @@ period_exchange_rate_map as ( -- exchange rates used, by accounting period, to c
 
   left join subsidiaries as to_subsidiaries
     on consolidated_exchange_rates.to_subsidiary_id = to_subsidiaries.subsidiary_id
+    and consolidated_exchange_rates.source_relation = to_subsidiaries.source_relation
 
   left join currencies
     on currencies.currency_id = to_subsidiaries.currency_id
+    and currencies.source_relation = to_subsidiaries.source_relation
 
   {% if not var('netsuite2__using_to_subsidiary', false) %}
-  where consolidated_exchange_rates.to_subsidiary_id in (select subsidiary_id from subsidiaries where parent_id is null)  -- constraint - only the primary subsidiary has no parent
+  join primary_subsidiaries
+    on consolidated_exchange_rates.to_subsidiary_id = primary_subsidiaries.subsidiary_id
+    and consolidated_exchange_rates.source_relation = primary_subsidiaries.source_relation
+
   {% endif %}
 ), 
 
@@ -73,10 +88,12 @@ accountxperiod_exchange_rate_map as ( -- account table with exchange rate detail
       when lower(accounts.general_rate_type) = 'current' then period_exchange_rate_map.current_rate
       when lower(accounts.general_rate_type) = 'average' then period_exchange_rate_map.average_rate
       else null
-        end as exchange_rate
+        end as exchange_rate,
+    accounts.source_relation
   from accounts
 
-  cross join period_exchange_rate_map
+  join period_exchange_rate_map
+    on accounts.source_relation = period_exchange_rate_map.source_relation
 )
 
 select *

models/netsuite2/intermediate/int_netsuite2__tran_and_reporting_periods.sql

@@ -10,9 +10,18 @@ subsidiaries as (
     from {{ var('netsuite2_subsidiaries') }}
 ),
 
+primary_subsidiary_calendar as (
+    select 
+      fiscal_calendar_id, 
+      source_relation 
+    from subsidiaries 
+    where parent_id is null
+),
+
 transaction_and_reporting_periods as ( 
   select
     base.accounting_period_id as accounting_period_id,
+    base.source_relation,
     multiplier.accounting_period_id as reporting_accounting_period_id
   from accounting_periods as base
 
@@ -21,11 +30,15 @@ transaction_and_reporting_periods as (
       and multiplier.is_quarter = base.is_quarter
       and multiplier.is_year = base.is_year -- this was year_0 in netsuite1
       and multiplier.fiscal_calendar_id = base.fiscal_calendar_id
+      and multiplier.source_relation = base.source_relation
       and cast(multiplier.starting_at as {{ dbt.type_timestamp() }}) <= {{ current_timestamp() }} 
 
+  join primary_subsidiary_calendar
+    on base.fiscal_calendar_id = primary_subsidiary_calendar.fiscal_calendar_id
+    and base.source_relation = primary_subsidiary_calendar.source_relation
+
   where not base.is_quarter
     and not base.is_year
-    and base.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null) -- fiscal calendar will align with parent subsidiary's default calendar
 )
 
 select * 

models/netsuite2/intermediate/int_netsuite2__tran_lines_w_accounting_period.sql

@@ -16,6 +16,7 @@ transaction_lines_w_accounting_period as ( -- transaction line totals, by accoun
     transaction_lines.transaction_line_id,
     transaction_lines.subsidiary_id,
     transaction_lines.account_id,
+    transaction_lines.source_relation,
 
     {% if var('netsuite2__multibook_accounting_enabled', false) %}
     transaction_lines.accounting_book_id,
@@ -26,7 +27,9 @@ transaction_lines_w_accounting_period as ( -- transaction line totals, by accoun
     coalesce(transaction_lines.amount, 0) as unconverted_amount
   from transaction_lines
 
-  join transactions on transactions.transaction_id = transaction_lines.transaction_id
+  join transactions 
+    on transactions.transaction_id = transaction_lines.transaction_id
+    and transactions.source_relation = transaction_lines.source_relation
 
   where lower(transactions.transaction_type) != 'revenue arrangement'
     and transaction_lines.is_posting

models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql

@@ -25,7 +25,7 @@ accounts as (
 transactions_in_every_calculation_period_w_exchange_rates as (
   select
     transaction_lines_w_accounting_period.*,
-    reporting_accounting_period_id
+    transaction_and_reporting_periods.reporting_accounting_period_id
 
     {% if var('netsuite2__using_exchange_rate', true) %}
     , exchange_reporting_period.exchange_rate as exchange_rate_reporting_period
@@ -41,22 +41,26 @@ transactions_in_every_calculation_period_w_exchange_rates as (
   from transaction_lines_w_accounting_period
 
   left join transaction_and_reporting_periods 
-    on transaction_and_reporting_periods.accounting_period_id = transaction_lines_w_accounting_period.transaction_accounting_period_id 
+    on transaction_and_reporting_periods.accounting_period_id = transaction_lines_w_accounting_period.transaction_accounting_period_id
+    and transaction_and_reporting_periods.source_relation = transaction_lines_w_accounting_period.source_relation
 
   {% if var('netsuite2__using_exchange_rate', true) %}
   left join accountxperiod_exchange_rate_map as exchange_reporting_period
     on exchange_reporting_period.accounting_period_id = transaction_and_reporting_periods.reporting_accounting_period_id
       and exchange_reporting_period.account_id = transaction_lines_w_accounting_period.account_id
       and exchange_reporting_period.from_subsidiary_id = transaction_lines_w_accounting_period.subsidiary_id
+      and exchange_reporting_period.source_relation = transaction_lines_w_accounting_period.source_relation
 
       {% if var('netsuite2__multibook_accounting_enabled', false) %}
       and exchange_reporting_period.accounting_book_id = transaction_lines_w_accounting_period.accounting_book_id
+      and exchange_reporting_period.source_relation = transaction_lines_w_accounting_period.source_relation
       {% endif %}
 
   left join accountxperiod_exchange_rate_map as exchange_transaction_period
     on exchange_transaction_period.accounting_period_id = transaction_and_reporting_periods.accounting_period_id
       and exchange_transaction_period.account_id = transaction_lines_w_accounting_period.account_id
       and exchange_transaction_period.from_subsidiary_id = transaction_lines_w_accounting_period.subsidiary_id
+      and exchange_transaction_period.source_relation = transaction_lines_w_accounting_period.source_relation
 
       {% if var('netsuite2__multibook_accounting_enabled', false) %}
       and exchange_transaction_period.accounting_book_id = transaction_lines_w_accounting_period.accounting_book_id
@@ -94,7 +98,8 @@ transactions_with_converted_amounts as (
   from transactions_in_every_calculation_period_w_exchange_rates
 
   left join accounts
-    on accounts.account_id = transactions_in_every_calculation_period_w_exchange_rates.account_id 
+    on accounts.account_id = transactions_in_every_calculation_period_w_exchange_rates.account_id
+    and accounts.source_relation = transactions_in_every_calculation_period_w_exchange_rates.source_relation
 )
 
 select *