Bugfix/too many partitions

CHANGELOG.md

@@ -1,5 +1,14 @@
 # dbt_netsuite v0.14.0
-For Netsuite2, [PR #138](https://github.com/fivetran/dbt_netsuite/pull/138) includes the following updates: 
+
+For Netsuite2, [PR #138](https://github.com/fivetran/dbt_netsuite/pull/138) and [PR #132](https://github.com/fivetran/dbt_netsuite/pull/132) include the following updates: 
+
+## Breaking Changes (Full refresh required after upgrading)
+- Partitioned models have had the `partition_by` logic adjusted to include a granularity of a month. This change should only impact BigQuery warehouses and was applied to avoid the common `too many partitions` error users have experienced due to over-partitioning by day. Therefore, adjusting the partition to a monthly granularity will increase the partition windows and allow for more performant querying. 
+- This change was applied to the following models:
+  - `int_netsuite2__tran_with_converted_amounts`
+  - `netsuite2__balance_sheet`
+  - `netsuite2__income_statement`
+  - `netsuite2__transaction_details`
 
 ## Upstream Netsuite Source Breaking Changes (Full refresh required after upgrading)
 - Casted specific timestamp fields across all staging models as dates where the Netsuite UI does not perform timezone conversion. Keeping these fields as type timestamp causes issues in reporting tools that perform automatic timezone conversion.   
@@ -17,7 +26,7 @@ For Netsuite2, [PR #138](https://github.com/fivetran/dbt_netsuite/pull/138) incl
   - Since this model is only used by a subset of customers, we've introduced the variable `netsuite2__using_employees` to allow users who don't utilize the `employee` table in Netsuite2 the ability to disable that functionality within your `dbt_project.yml`. [Instructions are available in the README](https://github.com/fivetran/dbt_netsuite/?tab=readme-ov-file#step-5-disable-models-for-non-existent-sources-netsuite2-only).
 
 ## Under the Hood
-- Generated consistency test for `netsuite2__transaction_details` to make sure amounts match for integration test validations. 
+- Consistency tests added for each Netsuite2 end model in order to be used during integration test validations.
 
 ## Contributors
 - [@jmongerlyra](https://github.com/jmongerlyra) ([PR #131](https://github.com/fivetran/dbt_netsuite/pull/131))

integration_tests/tests/consistency/comparison__balance_sheet.sql

@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_netsuite_prod.netsuite2__balance_sheet
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_netsuite_dev.netsuite2__balance_sheet
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/comparison__income_statement.sql

@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_netsuite_prod.netsuite2__income_statement
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_netsuite_dev.netsuite2__income_statement
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

integration_tests/tests/consistency/comparison__transaction_details.sql

@@ -0,0 +1,45 @@
+{{ config(
+    tags="fivetran_validations",
+    enabled=var('fivetran_validation_tests_enabled', false)
+) }}
+
+with prod as (
+    select *
+    from {{ target.schema }}_netsuite_prod.netsuite2__transaction_details
+),
+
+dev as (
+    select *
+    from {{ target.schema }}_netsuite_dev.netsuite2__transaction_details
+),
+
+prod_not_in_dev as (
+    -- rows from prod not found in dev
+    select * from prod
+    except distinct
+    select * from dev
+),
+
+dev_not_in_prod as (
+    -- rows from dev not found in prod
+    select * from dev
+    except distinct
+    select * from prod
+),
+
+final as (
+    select
+        *,
+        'from prod' as source
+    from prod_not_in_dev
+
+    union all -- union since we only care if rows are produced
+
+    select
+        *,
+        'from dev' as source
+    from dev_not_in_prod
+)
+
+select *
+from final

models/netsuite2/intermediate/int_netsuite2__tran_with_converted_amounts.sql

@@ -2,7 +2,7 @@
   config(
     enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
     materialized='ephemeral' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-    partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+    partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
       if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],
     cluster_by = ['transaction_id'],
     unique_key='tran_with_converted_amounts_id',

models/netsuite2/netsuite2__balance_sheet.sql

@@ -2,7 +2,7 @@
     config(
         enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
         materialized='table' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
             if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],       
         cluster_by = ['transaction_id'],
         unique_key='balance_sheet_id',

models/netsuite2/netsuite2__income_statement.sql

@@ -2,7 +2,7 @@
     config(
         enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
         materialized='table' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
             if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],
         cluster_by = ['transaction_id'],
         unique_key='income_statement_id',

models/netsuite2/netsuite2__transaction_details.sql

@@ -2,7 +2,7 @@
     config(
         enabled=var('netsuite_data_model', 'netsuite') == var('netsuite_data_model_override','netsuite2'),
         materialized='table' if target.type in ('bigquery', 'databricks', 'spark') else 'incremental',
-        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date'}
+        partition_by = {'field': '_fivetran_synced_date', 'data_type': 'date', 'granularity': 'month'}
             if target.type not in ['spark', 'databricks'] else ['_fivetran_synced_date'],
         cluster_by = ['transaction_id'],
         unique_key='transaction_details_id',