From 7e1be1bc756e2aef4e1f4b97be84b9ef81deabcf Mon Sep 17 00:00:00 2001 From: niccoloalexander Date: Tue, 31 Mar 2026 22:04:42 +0800 Subject: [PATCH 1/2] feat: add timestamp normalization for BigQuery artifacts - Introduced a new macro `normalize_artifact_timestamp_precision` to ensure timestamp precision for BigQuery. - Updated existing macros to utilize this new function for `execute_started_at`, `execute_completed_at`, `compile_started_at`, and `compile_completed_at` fields in `upload_run_results.sql` and `upload_source_freshness.sql`. - Enhanced schema existence checks for BigQuery in `create_elementary_tests_schema.sql` and `get_elementary_tests_schema.sql` to improve compatibility. --- .../edr/dbt_artifacts/upload_run_results.sql | 35 ++++++++++++++++--- .../dbt_artifacts/upload_source_freshness.sql | 16 ++++++--- .../create_elementary_tests_schema.sql | 19 ++++++++-- .../get_elementary_tests_schema.sql | 21 +++++++++-- 4 files changed, 77 insertions(+), 14 deletions(-) diff --git a/macros/edr/dbt_artifacts/upload_run_results.sql b/macros/edr/dbt_artifacts/upload_run_results.sql index 251f9737b..7a5b925d0 100644 --- a/macros/edr/dbt_artifacts/upload_run_results.sql +++ b/macros/edr/dbt_artifacts/upload_run_results.sql @@ -16,6 +16,25 @@ {% endmacro %} +{% macro normalize_artifact_timestamp_precision(timestamp_value) %} + {% if target.type != "bigquery" %} + {{ return(timestamp_value) }} + {% endif %} + + {% if timestamp_value is string and timestamp_value.endswith("Z") and "." in timestamp_value %} + {% set ts_no_z = timestamp_value[:-1] %} + {% set ts_parts = ts_no_z.split(".") %} + {% if ts_parts | length == 2 %} + {% set fractional_part = ts_parts[1] %} + {% if fractional_part | length > 6 %} + {{ return(ts_parts[0] ~ "." ~ fractional_part[:6] ~ "Z") }} + {% endif %} + {% endif %} + {% endif %} + {{ return(timestamp_value) }} +{% endmacro %} + + {% macro get_dbt_run_results_empty_table_query() %} {% set dbt_run_results_empty_table_query = elementary.empty_table( [ @@ -102,15 +121,23 @@ {% if timing.get("name") == "execute" %} {% do flatten_run_result_dict.update( { - "execute_started_at": timing.get("started_at"), - "execute_completed_at": timing.get("completed_at"), + "execute_started_at": elementary.normalize_artifact_timestamp_precision( + timing.get("started_at") + ), + "execute_completed_at": elementary.normalize_artifact_timestamp_precision( + timing.get("completed_at") + ), } ) %} {% elif timing.get("name") == "compile" %} {% do flatten_run_result_dict.update( { - "compile_started_at": timing.get("started_at"), - "compile_completed_at": timing.get("completed_at"), + "compile_started_at": elementary.normalize_artifact_timestamp_precision( + timing.get("started_at") + ), + "compile_completed_at": elementary.normalize_artifact_timestamp_precision( + timing.get("completed_at") + ), } ) %} {% endif %} diff --git a/macros/edr/dbt_artifacts/upload_source_freshness.sql b/macros/edr/dbt_artifacts/upload_source_freshness.sql index 3926eedd5..0f592c2f5 100644 --- a/macros/edr/dbt_artifacts/upload_source_freshness.sql +++ b/macros/edr/dbt_artifacts/upload_source_freshness.sql @@ -80,10 +80,18 @@ "filter": criteria_dict.get("filter"), "generated_at": elementary.datetime_now_utc_as_string(), "invocation_id": source_freshness_invocation_id, - "compile_started_at": compile_timing.get("started_at"), - "compile_completed_at": compile_timing.get("completed_at"), - "execute_started_at": execute_timing.get("started_at"), - "execute_completed_at": execute_timing.get("completed_at"), + "compile_started_at": elementary.normalize_artifact_timestamp_precision( + compile_timing.get("started_at") + ), + "compile_completed_at": elementary.normalize_artifact_timestamp_precision( + compile_timing.get("completed_at") + ), + "execute_started_at": elementary.normalize_artifact_timestamp_precision( + execute_timing.get("started_at") + ), + "execute_completed_at": elementary.normalize_artifact_timestamp_precision( + execute_timing.get("completed_at") + ), } %} {{ return(flatten_source_freshness_dict) }} {% endmacro %} diff --git a/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql b/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql index 5b69c3d3e..725a0f4bf 100644 --- a/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql +++ b/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql @@ -6,9 +6,22 @@ {% set tests_schema_name = elementary.get_elementary_tests_schema( database_name, schema_name ) %} - {%- if tests_schema_name != schema_name and not adapter.check_schema_exists( - database_name, tests_schema_name - ) %} + {% if target.type == "bigquery" %} + {% set schema_exists_sql %} + select count(*) as schema_count + from `{{ database_name }}`.INFORMATION_SCHEMA.SCHEMATA + where upper(schema_name) = upper('{{ tests_schema_name }}') + {% endset %} + {% set schema_exists_result = elementary.run_query(schema_exists_sql) %} + {% set schema_exists = ( + schema_exists_result is not none + and schema_exists_result.rows | length > 0 + and schema_exists_result.rows[0][0] | int > 0 + ) %} + {% else %} + {% set schema_exists = adapter.check_schema_exists(database_name, tests_schema_name) %} + {% endif %} + {%- if tests_schema_name != schema_name and not schema_exists %} {{ elementary.edr_log("Creating Elementary's tests schema.") }} {% set schema_relation = api.Relation.create( database=database_name, schema=tests_schema_name diff --git a/macros/edr/tests/test_utils/get_elementary_tests_schema.sql b/macros/edr/tests/test_utils/get_elementary_tests_schema.sql index 24611e878..81dd0c479 100644 --- a/macros/edr/tests/test_utils/get_elementary_tests_schema.sql +++ b/macros/edr/tests/test_utils/get_elementary_tests_schema.sql @@ -15,9 +15,24 @@ {% set legacy_tests_schema_name = ( elementary_schema ~ LEGACY_TESTS_SCHEMA_SUFFIX ) %} - {% if adapter.check_schema_exists( - elementary_database, legacy_tests_schema_name - ) %} + {% if target.type == "bigquery" %} + {% set legacy_schema_exists_sql %} + select count(*) as schema_count + from `{{ elementary_database }}`.INFORMATION_SCHEMA.SCHEMATA + where upper(schema_name) = upper('{{ legacy_tests_schema_name }}') + {% endset %} + {% set legacy_schema_exists_result = elementary.run_query(legacy_schema_exists_sql) %} + {% set legacy_schema_exists = ( + legacy_schema_exists_result is not none + and legacy_schema_exists_result.rows | length > 0 + and legacy_schema_exists_result.rows[0][0] | int > 0 + ) %} + {% else %} + {% set legacy_schema_exists = adapter.check_schema_exists( + elementary_database, legacy_tests_schema_name + ) %} + {% endif %} + {% if legacy_schema_exists %} {% set tests_schema_name = legacy_tests_schema_name %} {% endif %} {% endif %} From 8a7967c0ab81d580505a2f4094b766c1aff6ad91 Mon Sep 17 00:00:00 2001 From: niccoloalexander Date: Tue, 7 Apr 2026 12:10:51 +0200 Subject: [PATCH 2/2] refactor: enhance schema existence checks in test macros following PR review comments - Updated `create_elementary_tests_schema.sql` and `get_elementary_tests_schema.sql` to utilize a more robust method for checking schema existence by converting query results to dictionaries. - Improved readability and maintainability of the schema existence logic. --- .../on_run_start/create_elementary_tests_schema.sql | 11 +++++++---- .../tests/test_utils/get_elementary_tests_schema.sql | 11 +++++++---- 2 files changed, 14 insertions(+), 8 deletions(-) diff --git a/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql b/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql index 725a0f4bf..ca3913f4a 100644 --- a/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql +++ b/macros/edr/tests/on_run_start/create_elementary_tests_schema.sql @@ -13,10 +13,13 @@ where upper(schema_name) = upper('{{ tests_schema_name }}') {% endset %} {% set schema_exists_result = elementary.run_query(schema_exists_sql) %} + {% set schema_count_rows = [] %} + {% if schema_exists_result is not none %} + {% set schema_count_rows = elementary.agate_to_dicts(schema_exists_result) %} + {% endif %} {% set schema_exists = ( - schema_exists_result is not none - and schema_exists_result.rows | length > 0 - and schema_exists_result.rows[0][0] | int > 0 + schema_count_rows | length > 0 + and schema_count_rows[0]["schema_count"] | int > 0 ) %} {% else %} {% set schema_exists = adapter.check_schema_exists(database_name, tests_schema_name) %} @@ -31,4 +34,4 @@ {%- endif %} {% endif %} {{ return("") }} -{% endmacro %} +{% endmacro %} \ No newline at end of file diff --git a/macros/edr/tests/test_utils/get_elementary_tests_schema.sql b/macros/edr/tests/test_utils/get_elementary_tests_schema.sql index 81dd0c479..02f1983c3 100644 --- a/macros/edr/tests/test_utils/get_elementary_tests_schema.sql +++ b/macros/edr/tests/test_utils/get_elementary_tests_schema.sql @@ -22,10 +22,13 @@ where upper(schema_name) = upper('{{ legacy_tests_schema_name }}') {% endset %} {% set legacy_schema_exists_result = elementary.run_query(legacy_schema_exists_sql) %} + {% set legacy_schema_count_rows = [] %} + {% if legacy_schema_exists_result is not none %} + {% set legacy_schema_count_rows = elementary.agate_to_dicts(legacy_schema_exists_result) %} + {% endif %} {% set legacy_schema_exists = ( - legacy_schema_exists_result is not none - and legacy_schema_exists_result.rows | length > 0 - and legacy_schema_exists_result.rows[0][0] | int > 0 + legacy_schema_count_rows | length > 0 + and legacy_schema_count_rows[0]["schema_count"] | int > 0 ) %} {% else %} {% set legacy_schema_exists = adapter.check_schema_exists( @@ -40,4 +43,4 @@ {% do elementary.set_cache("tests_schema_name", tests_schema_name) %} {{ return(tests_schema_name) }} -{% endmacro %} +{% endmacro %} \ No newline at end of file