Merge pull request #43 from elementary-data/schema_snapshots_fix

Schema snapshots fix
elementary-data · Jul 6, 2022 · 24f6ed1 · 24f6ed1
2 parents db1a95d + 0c6da15
commit 24f6ed1
Show file tree

Hide file tree

Showing 2 changed files with 12 additions and 8 deletions.
diff --git a/models/edr/data_monitoring/schema_changes/schema_columns_snapshot.sql b/models/edr/data_monitoring/schema_changes/schema_columns_snapshot.sql
@@ -25,20 +25,22 @@ columns_snapshot as (
 
         {% if is_incremental() %}
             {%- set known_columns_query %}
+                (
                 select full_column_name from {{ this }}
                 where detected_at = (select max(detected_at) from {{ this }})
+                )
             {% endset %}
-            {%- set known_columns = elementary.result_column_to_list(known_columns_query) %}
 
             {%- set known_tables_query %}
+                (
                 select distinct full_table_name from {{ this }}
                 where detected_at = (select max(detected_at) from {{ this }})
+                )
             {% endset %}
-            {%- set known_tables = elementary.result_column_to_list(known_tables_query) %}
 
             case when
-                {{ elementary.full_column_name() }} not in {{ elementary.strings_list_to_tuple(known_columns) }}
-                and full_table_name in {{ elementary.strings_list_to_tuple(known_tables) }}
+                {{ elementary.full_column_name() }} not in {{ known_columns_query }}
+                and full_table_name in {{ known_tables_query }}
             then true
             else false end
             as is_new

diff --git a/models/edr/data_monitoring/schema_changes/schema_tables_snapshot.sql b/models/edr/data_monitoring/schema_changes/schema_tables_snapshot.sql
@@ -21,20 +21,22 @@ schema_tables as (
 
         {% if is_incremental() %}
             {%- set known_tables_query %}
+                (
                 select full_table_name from {{ this }}
                 where detected_at = (select max(detected_at) from {{ this }})
+                )
             {% endset %}
-            {%- set known_tables = elementary.result_column_to_list(known_tables_query) %}
 
             {%- set known_schemas_query %}
+                (
                 select distinct full_schema_name from {{ this }}
                 where detected_at = (select max(detected_at) from {{ this }})
+                )
             {% endset %}
-            {%- set known_schemas = elementary.result_column_to_list(known_schemas_query) %}
 
             case when
-                full_table_name not in {{ elementary.strings_list_to_tuple(known_tables) }}
-                and full_schema_name in {{ elementary.strings_list_to_tuple(known_schemas) }}
+                full_table_name not in {{ known_tables_query }}
+                and full_schema_name in {{ known_schemas_query }}
             then true
             else false end
             as is_new