Releases/v0.17.latest

CHANGELOG.md

@@ -1,3 +1,21 @@
+# dbt_netsuite v0.17.0
+
+This release involves **breaking changes** and will require running a **full refresh**.
+
+## Bug Fixes
+- Adjusted the materialization of the `int_netsuite2__tran_with_converted_amounts` model **from incremental to [ephemeral](https://docs.getdbt.com/docs/build/materializations#ephemeral)**. There was a previous issue with the incremental strategy that could potentially produce duplicate records in certain situations ([PR #153](https://github.com/fivetran/dbt_netsuite/pull/153)).
+   - Our team has determined that the most sensible path forward is to remove the incremental logic for this model entirely, as its performance benefits have proved marginal and an ephemeral materialization will avoid any duplication issues.
+> This is a **Breaking Change**, as `int_netsuite2__tran_with_converted_amounts` will no longer materialize in the warehouse.
+
+## Feature Updates
+- Added two fields to the `netsuite2__balance_sheet` and `netsuite2__income_statement` models ([PR #151](https://github.com/fivetran/dbt_netsuite/pull/151)):
+  - `transaction_amount`
+  - `subsidiary_currency_symbol`
+> This change **will require running a full refresh**, as we are adding new fields to incrementally materialized models.
+
+## Contribtors
+- [@jmongerlyra](https://github.com/jmongerlyra) ([PR #151](https://github.com/fivetran/dbt_netsuite/pull/151))
+
 # dbt_netsuite v0.16.0
 For Netsuite2, [PR #149](https://github.com/fivetran/dbt_netsuite/pull/149) includes the following updates: 
 

README.md

@@ -102,7 +102,7 @@ Include the following netsuite package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/netsuite
-    version: [">=0.16.0", "<0.17.0"]
+    version: [">=0.17.0", "<0.18.0"]
 
 ```
 ### Step 3: Define Netsuite.com or Netsuite2 Source
@@ -263,9 +263,6 @@ models:
         +materialized: incremental # default is table for Bigquery and Databricks
       netsuite2__balance_sheet:
         +materialized: incremental # default is table for Bigquery and Databricks
-      intermediate:
-        int_netsuite2__tran_with_converted_amounts:
-          +materialized: incremental # default is ephemeral for Bigquery and Databricks
 ```
 
 ### (Optional) Step 7: Produce Analytics-Ready Reports with Streamlit App (Bigquery and Snowflake users only)

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '0.16.0'
+version: '0.17.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/dbt_project.yml

@@ -1,5 +1,5 @@
 name: 'netsuite_integration_tests'
-version: '0.16.0'
+version: '0.17.0'
 profile: 'integration_tests'
 config-version: 2
 

integration_tests/tests/consistency/consistency_balance_sheet.sql

@@ -9,7 +9,7 @@ with prod as (
 ),
 
 dev as (
-    select *
+    select * except (transaction_amount, subsidiary_currency_symbol) -- remove after v0.17.0
     from {{ target.schema }}_netsuite_dev.netsuite2__balance_sheet
 ),
 

integration_tests/tests/consistency/consistency_income_statement.sql

@@ -9,7 +9,7 @@ with prod as (
 ),
 
 dev as (
-    select *
+    select * except (transaction_amount, subsidiary_currency_symbol) -- remove after v0.17.0
     from {{ target.schema }}_netsuite_dev.netsuite2__income_statement
 ),
 

models/netsuite2.yml

@@ -116,6 +116,10 @@ models:
           - not_null
     - name: _fivetran_synced_date
       description: _fivetran_synced formatted as a date for incremental logic use.
+    - name: transaction_amount
+      description: Total amount of the transaction line.
+    - name: subsidiary_currency_symbol
+      description: Currency of the subsidiary.
 
   - name: netsuite2__income_statement
     description: >
@@ -196,6 +200,10 @@ models:
           - not_null
     - name: _fivetran_synced_date
       description: _fivetran_synced formatted as a date for incremental logic use.
+    - name: transaction_amount
+      description: Total amount of the transaction line.
+    - name: subsidiary_currency_symbol
+      description: Currency of the subsidiary.
 
   - name: netsuite2__transaction_details
     description: >

models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql

@@ -1,23 +1,12 @@
 {{
   config(
-    enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
-    materialized='ephemeral' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-    partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
-      if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],
-    cluster_by = ['transaction_id'],
-    unique_key='tran_with_converted_amounts_id',
-    incremental_strategy = 'merge' if target.type in ('bigquery', 'databricks', 'spark') else 'delete+insert',
-    file_format='delta'
+    enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2')
   )
 }}
 
 with transaction_lines_w_accounting_period as (
   select * 
   from {{ ref('int_netsuite2__tran_lines_w_accounting_period') }}
-
-  {% if is_incremental() %}
-  where _fivetran_synced_date >= {{ netsuite.netsuite_lookback(from_date='max(_fivetran_synced_date)', datepart='day', interval=var('lookback_window', 3)) }}
-  {% endif %}
 ), 
 
 {% if var('netsuite2__using_exchange_rate', true) %}

models/netsuite2/netsuite2__balance_sheet.sql

@@ -43,6 +43,11 @@ subsidiaries as (
     from {{ var('netsuite2_subsidiaries') }}
 ),
 
+currencies as (
+    select *
+    from {{ var('netsuite2_currencies') }}
+),
+
 balance_sheet as ( 
     select
         transactions_with_converted_amounts.transaction_id,
@@ -51,6 +56,7 @@ balance_sheet as (
         transactions_with_converted_amounts._fivetran_synced_date,
         subsidiaries.full_name as subsidiary_full_name,
         subsidiaries.name as subsidiary_name,
+        subsidiaries_currencies.symbol as subsidiary_currency_symbol,
 
         {% if var('netsuite2__multibook_accounting_enabled', false) %}
         transactions_with_converted_amounts.accounting_book_id,
@@ -130,6 +136,13 @@ balance_sheet as (
         else 0
             end as converted_amount,
 
+        case
+        when not accounts.is_balancesheet then -unconverted_amount
+        when accounts.is_balancesheet and not accounts.is_leftside then -unconverted_amount
+        when accounts.is_balancesheet and accounts.is_leftside then unconverted_amount
+        else 0
+            end as transaction_amount,
+
         case
         when lower(accounts.account_type_id) = 'bank' then 1
         when lower(accounts.account_type_id) = 'acctrec' then 2
@@ -190,6 +203,9 @@ balance_sheet as (
     left join subsidiaries
         on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id
 
+    left join currencies subsidiaries_currencies
+        on subsidiaries_currencies.currency_id = subsidiaries.currency_id
+
     where reporting_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null)
         and transaction_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null)
         and (accounts.is_balancesheet
@@ -204,6 +220,7 @@ balance_sheet as (
         transactions_with_converted_amounts._fivetran_synced_date,
         subsidiaries.full_name as subsidiary_full_name,
         subsidiaries.name as subsidiary_name,
+        subsidiaries_currencies.symbol as subsidiary_currency_symbol,
 
         {% if var('netsuite2__multibook_accounting_enabled', false) %}
         transactions_with_converted_amounts.accounting_book_id,
@@ -241,6 +258,9 @@ balance_sheet as (
         when lower(accounts.general_rate_type) in ('historical', 'average') then converted_amount_using_transaction_accounting_period
         else converted_amount_using_reporting_month
             end as converted_amount,
+
+        unconverted_amount as transaction_amount,
+
         16 as balance_sheet_sort_helper
 
         --Below is only used if balance sheet transaction detail columns are specified dbt_project.yml file.
@@ -276,6 +296,9 @@ balance_sheet as (
     left join subsidiaries
         on subsidiaries.subsidiary_id = transactions_with_converted_amounts.subsidiary_id
 
+    left join currencies subsidiaries_currencies
+        on subsidiaries_currencies.currency_id = subsidiaries.currency_id
+
     where reporting_accounting_periods.fiscal_calendar_id = (select fiscal_calendar_id from subsidiaries where parent_id is null)
         and (accounts.is_balancesheet
         or transactions_with_converted_amounts.is_income_statement)

models/netsuite2/netsuite2__income_statement.sql

@@ -43,6 +43,11 @@ subsidiaries as (
     from {{ var('netsuite2_subsidiaries') }}
 ),
 
+currencies as (
+    select *
+    from {{ var('netsuite2_currencies') }}
+),
+
 transaction_lines as (
     select * 
     from {{ ref('int_netsuite2__transaction_lines') }}
@@ -93,7 +98,8 @@ income_statement as (
         accounts.account_number,
         subsidiaries.subsidiary_id,
         subsidiaries.full_name as subsidiary_full_name,
-        subsidiaries.name as subsidiary_name
+        subsidiaries.name as subsidiary_name,
+        subsidiaries_currencies.symbol as subsidiary_currency_symbol
 
         --The below script allows for accounts table pass through columns.
         {{ fivetran_utils.persist_pass_through_columns('accounts_pass_through_columns', identifier='accounts') }},
@@ -129,7 +135,9 @@ income_statement as (
 
         {% endif %}
 
-        , -converted_amount_using_transaction_accounting_period as converted_amount
+        , -converted_amount_using_transaction_accounting_period as converted_amount,
+
+        -unconverted_amount as transaction_amount
 
     from transactions_with_converted_amounts
 
@@ -159,6 +167,9 @@ income_statement as (
     left join subsidiaries
         on transactions_with_converted_amounts.subsidiary_id = subsidiaries.subsidiary_id
 
+    left join currencies subsidiaries_currencies
+        on subsidiaries_currencies.currency_id = subsidiaries.currency_id
+
     --Below is only used if income statement transaction detail columns are specified dbt_project.yml file.
     {% if var('income_statement_transaction_detail_columns') != []%}
     join transaction_details