From 5715e734b46ad36dcf21a944b76b734c5d103b00 Mon Sep 17 00:00:00 2001 From: sp-tkerlavage <90415765+sp-tkerlavage@users.noreply.github.com> Date: Wed, 28 Feb 2024 21:24:57 -0500 Subject: [PATCH 1/3] * Implementation of get_test_executions_dml_sql has code that references `model.timing`. Local loop variable is `test`, not `model`. Revised code to use `test` instead of `model`. * Implemented snowflake_get_test_executions_dml_sql to support rows_affected. --- .../upload_test_executions.sql | 61 +++++++++++++++++-- 1 file changed, 57 insertions(+), 4 deletions(-) diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index ea3553ae..74223ce5 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -35,9 +35,9 @@ '{{ test.thread_id }}', {# thread_id #} '{{ test.status }}', {# status #} - {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% set compile_started_at = (test.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} - {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% set query_completed_at = (test.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ test.execution_time }}, {# total_node_runtime #} @@ -73,9 +73,9 @@ '{{ test.thread_id }}', {# thread_id #} '{{ test.status }}', {# status #} - {% set compile_started_at = (model.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% set compile_started_at = (test.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} - {% set query_completed_at = (model.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% set query_completed_at = (test.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ test.execution_time }}, {# total_node_runtime #} @@ -148,3 +148,56 @@ {{ return("") }} {% endif %} {% endmacro -%} + +{% macro snowflake_get_test_executions_dml_sql(tests) -%} + {% if tests != [] %} + {% set test_execution_values %} + select + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(1) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(2) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(3) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(4) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(5) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(6) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(7) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(8) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(9) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(10) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(11) }}, + {{ adapter.dispatch('column_identifier', 'dbt_artifacts')(12) }}, + {{ adapter.dispatch('parse_json', 'dbt_artifacts')(adapter.dispatch('column_identifier', 'dbt_artifacts')(13)) }} + from values + {% for test in tests -%} + ( + '{{ invocation_id }}', {# command_invocation_id #} + '{{ test.node.unique_id }}', {# node_id #} + '{{ run_started_at }}', {# run_started_at #} + + {% set config_full_refresh = test.node.config.full_refresh %} + {% if config_full_refresh is none %} + {% set config_full_refresh = flags.FULL_REFRESH %} + {% endif %} + '{{ config_full_refresh }}', {# was_full_refresh #} + + '{{ test.thread_id }}', {# thread_id #} + '{{ test.status }}', {# status #} + + {% set compile_started_at = (test.timing | selectattr("name", "eq", "compile") | first | default({}))["started_at"] %} + {% if compile_started_at %}'{{ compile_started_at }}'{% else %}null{% endif %}, {# compile_started_at #} + {% set query_completed_at = (test.timing | selectattr("name", "eq", "execute") | first | default({}))["completed_at"] %} + {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} + + {{ test.execution_time }}, {# total_node_runtime #} + try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} + {{ 'null' if test.failures is none else test.failures }}, {# failures #} + '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} + '{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #} + ) + {%- if not loop.last %},{%- endif %} + {%- endfor %} + {% endset %} + {{ test_execution_values }} + {% else %} + {{ return("") }} + {% endif %} +{% endmacro -%} \ No newline at end of file From 17060c0b776c23b35be3996f514f33ed6439e328 Mon Sep 17 00:00:00 2001 From: michelley-an <101600153+michelley-an@users.noreply.github.com> Date: Mon, 12 Aug 2024 06:42:36 -0700 Subject: [PATCH 2/3] Update macros/upload_individual_datasets/upload_test_executions.sql Co-authored-by: Laura Li Foa Wing <100870296+llifoawing@users.noreply.github.com> --- macros/upload_individual_datasets/upload_test_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index 74223ce5..5401d4e7 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -149,7 +149,7 @@ {% endif %} {% endmacro -%} -{% macro snowflake_get_test_executions_dml_sql(tests) -%} +{% macro snowflake__get_test_executions_dml_sql(tests) -%} {% if tests != [] %} {% set test_execution_values %} select From a0b041759db7642e70f400f172d2a69aea2db3f3 Mon Sep 17 00:00:00 2001 From: michelley-an <101600153+michelley-an@users.noreply.github.com> Date: Mon, 12 Aug 2024 06:42:45 -0700 Subject: [PATCH 3/3] Update macros/upload_individual_datasets/upload_test_executions.sql Co-authored-by: Laura Li Foa Wing <100870296+llifoawing@users.noreply.github.com> --- macros/upload_individual_datasets/upload_test_executions.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/macros/upload_individual_datasets/upload_test_executions.sql b/macros/upload_individual_datasets/upload_test_executions.sql index 5401d4e7..cb13288b 100644 --- a/macros/upload_individual_datasets/upload_test_executions.sql +++ b/macros/upload_individual_datasets/upload_test_executions.sql @@ -188,7 +188,7 @@ {% if query_completed_at %}'{{ query_completed_at }}'{% else %}null{% endif %}, {# query_completed_at #} {{ test.execution_time }}, {# total_node_runtime #} - try_cast('{{ model.adapter_response.rows_affected }}' as int), {# rows_affected #} + try_cast('{{ test.adapter_response.rows_affected }}' as int), {# rows_affected #} {{ 'null' if test.failures is none else test.failures }}, {# failures #} '{{ test.message | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}', {# message #} '{{ tojson(test.adapter_response) | replace("\\", "\\\\") | replace("'", "\\'") | replace('"', '\\"') }}' {# adapter_response #}