fixed ambiguous columns (#224)

GoogleCloudPlatform · Feb 7, 2023 · dc91f6e · dc91f6e
1 parent d5249d1
commit dc91f6e
Show file tree

Hide file tree

Showing 16 changed files with 1,097 additions and 1,097 deletions.
diff --git a/clouddq/templates/dbt/macros/create_rule_binding_view.sql b/clouddq/templates/dbt/macros/create_rule_binding_view.sql
@@ -51,12 +51,12 @@ WITH
 {% endif %}
 zero_record AS (
     SELECT
-        '{{ rule_binding_id }}' AS rule_binding_id,
+        '{{ rule_binding_id }}' AS _internal_rule_binding_id,
 ),
 data AS (
     SELECT
       *,
-      '{{ rule_binding_id }}' AS rule_binding_id,
+      '{{ rule_binding_id }}' AS _internal_rule_binding_id,
     FROM
       `{{- fully_qualified_table_name -}}` d
 {%- if configs.get('incremental_time_filter_column') %}
@@ -77,7 +77,7 @@ data AS (
 ),
 last_mod AS (
     SELECT
-        project_id || '.' || dataset_id || '.' || table_id AS table_id,
+        project_id || '.' || dataset_id || '.' || table_id AS _internal_table_id,
         TIMESTAMP_MILLIS(last_modified_time) AS last_modified
     FROM `{{ instance_name }}.{{ database_name }}.__TABLES__`
 ),
@@ -96,17 +96,17 @@ validation_results AS (
 ),
 all_validation_results AS (
   SELECT
-    r.execution_ts AS execution_ts,
-    r.rule_binding_id AS rule_binding_id,
-    r.rule_id AS rule_id,
-    r.table_id AS table_id,
-    r.column_id AS column_id,
-    r.column_value AS column_value,
-    CAST(r.dimension AS STRING) AS dimension,
-    r.skip_null_count AS skip_null_count,
-    r.simple_rule_row_is_valid AS simple_rule_row_is_valid,
-    r.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
-    r.complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
+    r._internal_execution_ts AS execution_ts,
+    r._internal_rule_binding_id AS rule_binding_id,
+    r._internal_rule_id AS rule_id,
+    r._internal_table_id AS table_id,
+    r._internal_column_id AS column_id,
+    r._internal_column_value AS column_value,
+    CAST(r._internal_dimension AS STRING) AS dimension,
+    r._internal_skip_null_count AS skip_null_count,
+    r._internal_simple_rule_row_is_valid AS simple_rule_row_is_valid,
+    r._internal_complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
+    r._internal_complex_rule_validation_success_flag AS complex_rule_validation_success_flag,
     (SELECT COUNT(*) FROM data) AS rows_validated,
     last_mod.last_modified,
     '{{ metadata|tojson }}' AS metadata_json_string,
@@ -126,13 +126,13 @@ all_validation_results AS (
 {%- else %}
     CAST(NULL AS STRING) AS dataplex_asset_id,
 {%- endif %}
-    CONCAT(r.rule_binding_id, '_', r.rule_id, '_', r.execution_ts, '_', {{ progress_watermark }}) AS dq_run_id,
+    CONCAT(r._internal_rule_binding_id, '_', r._internal_rule_id, '_', r._internal_execution_ts, '_', {{ progress_watermark }}) AS dq_run_id,
     {{ progress_watermark|upper }} AS progress_watermark,
-    failed_records_query AS failed_records_query,
+    _internal_failed_records_query AS failed_records_query,
   r.include_reference_columns_json as include_reference_columns_json,
   FROM
     validation_results r
-  JOIN last_mod USING(table_id)
+  JOIN last_mod USING(_internal_table_id)
 )
 SELECT
   *

diff --git a/clouddq/templates/dbt/macros/failed_records_query.sql b/clouddq/templates/dbt/macros/failed_records_query.sql
@@ -65,7 +65,7 @@ data AS (
 ),
 last_mod AS (
     SELECT
-        project_id || '.' || dataset_id || '.' || table_id AS table_id,
+        project_id || '.' || dataset_id || '.' || table_id AS _internal_table_id,
         TIMESTAMP_MILLIS(last_modified_time) AS last_modified
     FROM `{{ instance_name }}.{{ database_name }}.__TABLES__`
 ),
@@ -81,17 +81,17 @@ all_validation_results AS (
   SELECT
     '{{ invocation_id }}' as _dq_validation_invocation_id,
     r.rule_binding_id AS _dq_validation_rule_binding_id,
-    r.rule_id AS _dq_validation_rule_id,
-    r.column_id AS _dq_validation_column_id,
-    r.column_value AS _dq_validation_column_value,
-    CAST(r.dimension AS STRING) AS _dq_validation_dimension,
-    r.simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
+    r._internal_rule_id AS _dq_validation_rule_id,
+    r._internal_column_id AS _dq_validation_column_id,
+    r._internal_column_value AS _dq_validation_column_value,
+    CAST(r._internal_dimension AS STRING) AS _dq_validation_dimension,
+    r._internal_simple_rule_row_is_valid AS _dq_validation_simple_rule_row_is_valid,
     r.complex_rule_validation_errors_count AS _dq_validation_complex_rule_validation_errors_count,
-    r.complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
+    r._internal_complex_rule_validation_success_flag AS _dq_validation_complex_rule_validation_success_flag,
     {{ '\n' }}
     {%- if rule_type != "CUSTOM_SQL_STATEMENT" -%}
         {% for ref_column_name in include_reference_columns %}
-            r.{{ ref_column_name }} AS {{ ref_column_name }},
+            r._internal_{{ ref_column_name }} AS {{ ref_column_name }},
         {%- endfor -%}
     {%- else -%}
         r.custom_sql_statement_validation_errors,

diff --git a/clouddq/templates/dbt/macros/macros.sql b/clouddq/templates/dbt/macros/macros.sql
@@ -14,16 +14,16 @@
 
 {% macro validate_simple_rule(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns, configs) -%}
   SELECT
-    CURRENT_TIMESTAMP() AS execution_ts,
-    '{{ rule_binding_id }}' AS rule_binding_id,
-    '{{ rule_id }}' AS rule_id,
-    '{{ fully_qualified_table_name }}' AS table_id,
-    '{{ column_name }}' AS column_id,
-    data.{{ column_name }} AS column_value,
+    CURRENT_TIMESTAMP() AS _internal_execution_ts,
+    '{{ rule_binding_id }}' AS _internal_rule_binding_id,
+    '{{ rule_id }}' AS _internal_rule_id,
+    '{{ fully_qualified_table_name }}' AS _internal_table_id,
+    '{{ column_name }}' AS _internal_column_id,
+    data.{{ column_name }} AS _internal_column_value,
 {% if rule_configs.get("dimension") %}
-    '{{ rule_configs.get("dimension") }}' AS dimension,
+    '{{ rule_configs.get("dimension") }}' AS _internal_dimension,
 {% else %}
-    CAST(NULL AS STRING) AS dimension,
+    CAST(NULL AS STRING) AS _internal_dimension,
 {% endif %}
     CASE
 {% if rule_configs.get("rule_type") == "NOT_NULL" %}
@@ -34,16 +34,16 @@
 {% endif %}
     ELSE
       FALSE
-    END AS simple_rule_row_is_valid,
+    END AS _internal_simple_rule_row_is_valid,
 {% if rule_configs.get("rule_type") == "NOT_NULL" %}
-    TRUE AS skip_null_count,
+    TRUE AS _internal_skip_null_count,
 {% else %}
-    FALSE AS skip_null_count,
+    FALSE AS _internal_skip_null_count,
 {% endif %}
-    CAST(NULL AS INT64) AS complex_rule_validation_errors_count,
-    CAST(NULL AS BOOLEAN) AS complex_rule_validation_success_flag,
+    CAST(NULL AS INT64) AS _internal_complex_rule_validation_errors_count,
+    CAST(NULL AS BOOLEAN) AS _internal_complex_rule_validation_success_flag,
     r"""{{ configs.get(rule_binding_id ~ '_' ~ rule_id ~ '_failed_records_sql_string') }}"""
-    AS failed_records_query,
+    AS _internal_failed_records_query,
   {% if include_reference_columns|length %}
     {% set include_reference_columns_string = "data."~include_reference_columns|join(",data.") %}
     to_json(struct({{include_reference_columns_string}})) AS include_reference_columns_json,
@@ -55,32 +55,32 @@
   LEFT JOIN
     data
   ON
-    zero_record.rule_binding_id = data.rule_binding_id
+    zero_record._internal_rule_binding_id = data._internal_rule_binding_id
 {% endmacro -%}
 
 {% macro validate_complex_rule(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, configs) -%}
   SELECT
-    CURRENT_TIMESTAMP() AS execution_ts,
-    '{{ rule_binding_id }}' AS rule_binding_id,
-    '{{ rule_id }}' AS rule_id,
-    '{{ fully_qualified_table_name }}' AS table_id,
-    CAST(NULL AS STRING) AS column_id,
-    NULL AS column_value,
+    CURRENT_TIMESTAMP() AS _internal_execution_ts,
+    '{{ rule_binding_id }}' AS _internal_rule_binding_id,
+    '{{ rule_id }}' AS _internal_rule_id,
+    '{{ fully_qualified_table_name }}' AS _internal_table_id,
+    CAST(NULL AS STRING) AS _internal_column_id,
+    NULL AS _internal_column_value,
 {% if rule_configs.get("dimension") %}
-    '{{ rule_configs.get("dimension") }}' AS dimension,
+    '{{ rule_configs.get("dimension") }}' AS _internal_dimension,
 {% else %}
-    CAST(NULL AS STRING) AS dimension,
+    CAST(NULL AS STRING) AS _internal_dimension,
 {% endif %}
-    CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
-    TRUE AS skip_null_count,
-    custom_sql_statement_validation_errors.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
+    CAST(NULL AS BOOLEAN) AS _internal_simple_rule_row_is_valid,
+    TRUE AS _internal_skip_null_count,
+    custom_sql_statement_validation_errors._internal_complex_rule_validation_errors_count AS _internal_complex_rule_validation_errors_count,
     CASE
-      WHEN custom_sql_statement_validation_errors.complex_rule_validation_errors_count IS NULL THEN TRUE
-      WHEN custom_sql_statement_validation_errors.complex_rule_validation_errors_count = 0 THEN TRUE
+      WHEN custom_sql_statement_validation_errors._internal_complex_rule_validation_errors_count IS NULL THEN TRUE
+      WHEN custom_sql_statement_validation_errors._internal_complex_rule_validation_errors_count = 0 THEN TRUE
       ELSE FALSE
-    END AS complex_rule_validation_success_flag,
+    END AS _internal_complex_rule_validation_success_flag,
     r"""{{ configs.get(rule_binding_id ~ '_' ~ rule_id ~ '_failed_records_sql_string') }}"""
-    AS failed_records_query,
+    AS _internal_failed_records_query,
     CAST(NULL AS JSON) AS include_reference_columns_json,
   FROM
     zero_record
@@ -89,30 +89,30 @@
       SELECT
          *,
         '{{ rule_binding_id }}' AS _rule_binding_id,
-        COUNT(*) OVER() AS complex_rule_validation_errors_count,
+        COUNT(*) OVER() AS _internal_complex_rule_validation_errors_count,
       FROM (
       {{ rule_configs.get("rule_sql_expr") }}
       ) custom_sql
     ) custom_sql_statement_validation_errors
   ON
-    zero_record.rule_binding_id = custom_sql_statement_validation_errors._rule_binding_id
+    zero_record._internal_rule_binding_id = custom_sql_statement_validation_errors._rule_binding_id
 {% endmacro -%}
 
 
 {% macro validate_simple_rule_failed_records_query(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name, include_reference_columns) -%}
   SELECT
     '{{ rule_binding_id }}' AS rule_binding_id,
-    '{{ rule_id }}' AS rule_id,
-    '{{ fully_qualified_table_name }}' AS table_id,
-    '{{ column_name }}' AS column_id,
-    data.{{ column_name }} AS column_value,
+    '{{ rule_id }}' AS _internal_rule_id,
+    '{{ fully_qualified_table_name }}' AS _internal_table_id,
+    '{{ column_name }}' AS _internal_column_id,
+    data.{{ column_name }} AS _internal_column_value,
     {% for ref_column_name in include_reference_columns %}
-        data.{{ ref_column_name }} AS {{ ref_column_name }},
+        data.{{ ref_column_name }} AS _internal_{{ ref_column_name }},
     {%- endfor -%}
 {% if rule_configs.get("dimension") %}
-    '{{ rule_configs.get("dimension") }}' AS dimension,
+    '{{ rule_configs.get("dimension") }}' AS _internal_dimension,
 {% else %}
-    CAST(NULL AS STRING) AS dimension,
+    CAST(NULL AS STRING) AS _internal_dimension,
 {% endif %}
     CASE
 {% if rule_configs.get("rule_type") == "NOT_NULL" %}
@@ -123,14 +123,14 @@
 {% endif %}
     ELSE
       FALSE
-    END AS simple_rule_row_is_valid,
+    END AS _internal_simple_rule_row_is_valid,
 {% if rule_configs.get("rule_type") == "NOT_NULL" %}
-    TRUE AS skip_null_count,
+    TRUE AS _internal_skip_null_count,
 {% else %}
-    FALSE AS skip_null_count,
+    FALSE AS _internal_skip_null_count,
 {% endif %}
     CAST(NULL AS INT64) AS complex_rule_validation_errors_count,
-    CAST(NULL AS BOOLEAN) AS complex_rule_validation_success_flag,
+    CAST(NULL AS BOOLEAN) AS _internal_complex_rule_validation_success_flag,
   FROM
     zero_record
   LEFT JOIN
@@ -142,24 +142,24 @@
 {% macro validate_complex_rule_failed_records_query(rule_id, rule_configs, rule_binding_id, column_name, fully_qualified_table_name) -%}
   SELECT
     '{{ rule_binding_id }}' AS rule_binding_id,
-    '{{ rule_id }}' AS rule_id,
-    '{{ fully_qualified_table_name }}' AS table_id,
-    CAST(NULL AS STRING) AS column_id,
-    NULL AS column_value,
+    '{{ rule_id }}' AS _internal_rule_id,
+    '{{ fully_qualified_table_name }}' AS _internal_table_id,
+    CAST(NULL AS STRING) AS _internal_column_id,
+    NULL AS _internal_column_value,
     custom_sql_statement_validation_errors,
 {% if rule_configs.get("dimension") %}
-    '{{ rule_configs.get("dimension") }}' AS dimension,
+    '{{ rule_configs.get("dimension") }}' AS _internal_dimension,
 {% else %}
-    CAST(NULL AS STRING) AS dimension,
+    CAST(NULL AS STRING) AS _internal_dimension,
 {% endif %}
-    CAST(NULL AS BOOLEAN) AS simple_rule_row_is_valid,
-    TRUE AS skip_null_count,
+    CAST(NULL AS BOOLEAN) AS _internal_simple_rule_row_is_valid,
+    TRUE AS _internal_skip_null_count,
     custom_sql_statement_validation_errors.complex_rule_validation_errors_count AS complex_rule_validation_errors_count,
     CASE
       WHEN custom_sql_statement_validation_errors.complex_rule_validation_errors_count IS NULL THEN TRUE
       WHEN custom_sql_statement_validation_errors.complex_rule_validation_errors_count = 0 THEN TRUE
       ELSE FALSE
-    END AS complex_rule_validation_success_flag,
+    END AS _internal_complex_rule_validation_success_flag,
   FROM
     zero_record
   LEFT JOIN