Bugfix/too many partitions

.gitignore

@@ -4,4 +4,5 @@ dbt_modules/
 logs/
 .DS_Store
 dbt_packages/
-env/
+env/
+package-lock.yml

CHANGELOG.md

@@ -1,3 +1,17 @@
+# dbt_netsuite v0.14.0
+
+[PR #132](https://github.com/fivetran/dbt_netsuite/pull/132) includes the following updates:
+
+## Breaking Changes (Full refresh required after upgrading)
+- Partitioned models have had the `partition_by` logic adjusted to include a granularity of a month. This change should only impact BigQuery warehouses and was applied to avoid the common `too many partitions` error some users have experienced do to over partitioning by day. Therefore, adjusting the partition to a month granularity will increase the partition windows and allow for more performant querying. This change was applied to the following models:
+  - `int_netsuite2__tran_with_converted_amounts`
+  - `netsuite2__balance_sheet`
+  - `netsuite2__income_statement`
+  - `netsuite2__transaction_details`
+
+## Under the Hood
+- Consistency tests add for each Netsuite2 end model to be used during integration test validations.
+
 # dbt_netsuite v0.13.0
 
 For Netsuite2, [PR #116](https://github.com/fivetran/dbt_netsuite/pull/116) includes the following updates: 

README.md

@@ -101,7 +101,7 @@ Include the following netsuite package version in your `packages.yml` file:
 ```yaml
 packages:
   - package: fivetran/netsuite
-    version: [">=0.13.0", "<0.14.0"]
+    version: [">=0.14.0", "<0.15.0"]
 ```
 ## Step 3: Define Netsuite.com or Netsuite2 Source
 As of April 2022 Fivetran made available a new Netsuite connector which leverages the Netsuite2 endpoint opposed to the original Netsuite.com endpoint. This package is designed to run for either or, not both. By default the `netsuite_data_model` variable for this package is set to the original `netsuite` value which runs the netsuite.com version of the package. If you would like to run the package on Netsuite2 data, you may adjust the `netsuite_data_model` variable to run the `netsuite2` version of the package.

dbt_project.yml

@@ -1,6 +1,6 @@
 config-version: 2
 name: 'netsuite'
-version: '0.13.0'
+version: '0.14.0'
 require-dbt-version: [">=1.3.0", "<2.0.0"]
 
 models:

integration_tests/dbt_project.yml

@@ -1,11 +1,11 @@
 name: 'netsuite_integration_tests'
-version: '0.13.0'
+version: '0.14.0'
 profile: 'integration_tests'
 config-version: 2
 
 models:
   +schema: "{{ 'netsuite_integrations_tests_sqlw' if target.name == 'databricks-sql' else 'netsuite' }}"
-
+  # +schema: "netsuite_{{ var('directed_schema','dev') }}"  ## To be used for validation tests
 vars:
   netsuite_schema: netsuite_integration_tests_5
   netsuite_data_model_override: netsuite
@@ -56,6 +56,7 @@ vars:
     netsuite2_account_type_identifier: "netsuite2_account_type_data"
     netsuite2_entity_address_identifier: "netsuite2_entity_address_data"
     netsuite2_location_main_address_identifier: "netsuite2_location_main_address_data"
+
 seeds:
   +quote_columns: "{{ true if target.type in ('redshift','postgres') else false }}"
   netsuite_integration_tests:

integration_tests/packages.yml

@@ -1,2 +1,2 @@
 packages:
-  - local: ../
+  - local: ../

integration_tests/tests/consistency/comparison__balance_sheet.sql

@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_netsuite_prod.netsuite2__balance_sheet
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_netsuite_dev.netsuite2__balance_sheet
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/comparison__income_statement.sql

@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_netsuite_prod.netsuite2__income_statement
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_netsuite_dev.netsuite2__income_statement
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/comparison__transaction_details.sql

@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_netsuite_prod.netsuite2__transaction_details
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_netsuite_dev.netsuite2__transaction_details
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql

@@ -2,7 +2,7 @@
   config(
     enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
     materialized='ephemeral' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-    partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+    partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
       if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],
     cluster_by = ['transaction_id'],
     unique_key='tran_with_converted_amounts_id',

models/netsuite2/netsuite2__balance_sheet.sql

@@ -2,7 +2,7 @@
     config(
         enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
         materialized='table' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
             if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],       
         cluster_by = ['transaction_id'],
         unique_key='balance_sheet_id',

models/netsuite2/netsuite2__income_statement.sql

@@ -2,7 +2,7 @@
     config(
         enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
         materialized='table' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
             if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],
         cluster_by = ['transaction_id'],
         unique_key='income_statement_id',

models/netsuite2/netsuite2__transaction_details.sql

@@ -2,7 +2,7 @@
     config(
         enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
         materialized='table' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
             if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],
         cluster_by = ['transaction_id'],
         unique_key='transaction_details_id',