diff --git a/integration_tests/models/schema_tests/schema.yml b/integration_tests/models/schema_tests/schema.yml index 00b9f2e..d2b7521 100644 --- a/integration_tests/models/schema_tests/schema.yml +++ b/integration_tests/models/schema_tests/schema.yml @@ -44,15 +44,15 @@ models: - dbt_expectations.expect_column_values_to_be_within_n_stdevs: sigma_threshold: 6 - dbt_expectations.expect_column_values_to_be_within_n_moving_stdevs: - group_by: date_day + date_column_name: date_day sigma_threshold: 6 take_logs: true - - name: timeseries_data_extended tests: - dbt_expectations.expect_table_columns_to_match_ordered_list: column_list: ["date_day", "row_value", "row_value_log"] + columns: - name: date_day tests: @@ -77,10 +77,34 @@ models: - name: row_value_log tests: - dbt_expectations.expect_column_values_to_be_within_n_moving_stdevs: - group_by: date_day + date_column_name: cast(date_day as datetime) sigma_threshold: 6 take_logs: false + - name: timeseries_hourly_data_extended + columns: + - name: date_hour + tests: + - dbt_expectations.expect_row_values_to_have_recent_data: + datepart: hour + interval: 12 + - dbt_expectations.expect_column_values_to_be_of_type: + column_type: datetime + - dbt_expectations.expect_column_values_to_be_in_type_list: + column_type_list: [date, datetime] + + + - name: row_value_log + tests: + - dbt_expectations.expect_column_values_to_be_within_n_moving_stdevs: + date_column_name: cast(date_hour as datetime) + period: hour + trend_periods: 48 + test_periods: 12 + sigma_threshold: 6 + take_logs: true + + - name: data_test tests: - dbt_expectations.expect_compound_columns_to_be_unique: diff --git a/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql b/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql new file mode 100644 index 0000000..2f09ba2 --- /dev/null +++ b/integration_tests/models/schema_tests/timeseries_hourly_data_extended.sql @@ -0,0 +1,31 @@ +with dates as ( + + {{ dbt_utils.date_spine('hour', + start_date=dbt_date.n_days_ago(10), + end_date=dbt_date.today() + ) }} + +), +add_row_values as ( + + select + d.date_hour, + cast(floor(100 * rnd) as {{ dbt_utils.type_int() }}) as row_value + from + dates d + cross join + unnest(generate_array(1, 10)) as rnd + +), +add_logs as ( + + select + *, + {{ dbt_expectations.log_natural('nullif(row_value, 0)') }} as row_value_log + from + add_row_values +) +select + * +from + add_logs diff --git a/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql b/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql index 751fd50..b2bef96 100644 --- a/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql +++ b/macros/schema_tests/distributional/expect_column_values_to_be_within_n_moving_stdevs.sql @@ -11,10 +11,11 @@ coalesce({{ metric_column }}, 0) {% macro test_expect_column_values_to_be_within_n_moving_stdevs(model, column_name, - group_by, - lookback_days=1, - trend_days=7, - test_days=14, + date_column_name, + period='day', + lookback_periods=1, + trend_periods=7, + test_periods=14, sigma_threshold=3, sigma_threshold_upper=None, sigma_threshold_lower=None, @@ -30,7 +31,7 @@ with metric_values as ( with grouped_metric_values as ( select - {{ group_by }} as metric_date, + {{ dbt_utils.date_trunc(period, date_column_name) }} as metric_period, sum({{ column_name }}) as agg_metric_value from {{ model }} @@ -43,7 +44,7 @@ with metric_values as ( select *, - lag(agg_metric_value, {{ lookback_days }}) over(order by metric_date) as prior_agg_metric_value + lag(agg_metric_value, {{ lookback_periods }}) over(order by metric_period) as prior_agg_metric_value from grouped_metric_values d @@ -73,11 +74,11 @@ metric_moving_calcs as ( select *, avg(metric_test_value) - over(order by metric_date rows - between {{ trend_days }} preceding and 1 preceding) as metric_test_rolling_average, + over(order by metric_period rows + between {{ trend_periods }} preceding and 1 preceding) as metric_test_rolling_average, stddev(metric_test_value) - over(order by metric_date rows - between {{ trend_days }} preceding and 1 preceding) as metric_test_rolling_stddev + over(order by metric_period rows + between {{ trend_periods }} preceding and 1 preceding) as metric_test_rolling_stddev from metric_values @@ -87,7 +88,7 @@ metric_sigma as ( select *, (metric_test_value - metric_test_rolling_average) as metric_test_delta, - (metric_test_value - metric_test_rolling_average)/metric_test_rolling_stddev as metric_test_sigma + (metric_test_value - metric_test_rolling_average)/nullif(metric_test_rolling_stddev, 0) as metric_test_sigma from metric_moving_calcs @@ -97,8 +98,14 @@ select from metric_sigma where - metric_date >= date({{ dbt_date.n_days_ago(test_days) }}) and - metric_date < {{ dbt_date.today() }} and + + metric_period >= cast( + {{ dbt_utils.dateadd(period, -test_periods, dbt_utils.date_trunc(period, dbt_date.now())) }} + as {{ dbt_utils.type_timestamp() }}) + and + metric_period < {{ dbt_utils.date_trunc(period, dbt_date.now()) }} + and + not ( metric_test_sigma >= {{ sigma_threshold_lower }} and metric_test_sigma <= {{ sigma_threshold_upper }}