From f94bc9a24472218b216efd406b9ede89c22f3ca0 Mon Sep 17 00:00:00 2001 From: Piotr Czarnas Date: Wed, 31 Jan 2024 21:19:34 +0100 Subject: [PATCH] Documentation update for the top text values check. --- ...-to-validate-accepted-values-in-columns.md | 39 +++++++ .../expected-numbers-in-use-count.md | 60 +++++++---- .../expected-text-values-in-use-count.md | 60 +++++++---- .../expected-texts-in-top-values-count.md | 60 +++++++---- docs/reference/rules/Averages.md | 100 +++++++++--------- docs/reference/rules/Stdev.md | 72 ++++++------- .../MaxMissingRule0ErrorParametersSpec.java | 1 + 7 files changed, 246 insertions(+), 146 deletions(-) diff --git a/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md b/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md index ae9dbb8ccc..e29d66d653 100644 --- a/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md +++ b/docs/categories-of-data-quality-checks/how-to-validate-accepted-values-in-columns.md @@ -59,6 +59,26 @@ spec: ![Asserting that all expected text values are present in a column](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/all-expected-column-values-are-in-use-data-quality-check-min.png){ loading=lazy } +``` { .yaml linenums="1" hl_lines="12-15" } +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + columns: + monitoring_checks: + daily: + accepted_values: + daily_expected_text_values_in_use_count: + parameters: + expected_values: + - TRAVIS + - WILLIAMSON + - HAYS + - BASTROP + error: + max_missing: 0 +``` + ![Detecting expected values that are missing in a column](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-values-found-vs-text-values-in-use-min.png){ loading=lazy } ![Top values in a column to assert in a data quality check](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/column-profiling-result-top-two-column-values-min.png){ width="619px"; loading=lazy } @@ -66,6 +86,25 @@ spec: ![Asserting that expected values are in the top of most popular values in a column](https://dqops.com/docs/images/concepts/categories-of-data-quality-checks/text-values-in-top-most-popular-min.png){ loading=lazy } +``` { .yaml linenums="1" hl_lines="12-15" } +# yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json +apiVersion: dqo/v1 +kind: table +spec: + columns: + monitoring_checks: + daily: + accepted_values: + daily_expected_texts_in_top_values_count: + parameters: + expected_values: + - TRAVIS + - WILLIAMSON + top: 3 + error: + max_missing: 0 +``` + ## Detecting accepted values issues How to detect accepted values data quality issues. diff --git a/docs/checks/column/accepted_values/expected-numbers-in-use-count.md b/docs/checks/column/accepted_values/expected-numbers-in-use-count.md index 51fabbe9a4..799fb0b550 100644 --- a/docs/checks/column/accepted_values/expected-numbers-in-use-count.md +++ b/docs/checks/column/accepted_values/expected-numbers-in-use-count.md @@ -101,7 +101,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-17" +```yaml hl_lines="7-19" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -115,8 +115,10 @@ spec: expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -793,7 +795,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 29-34" + ```yaml hl_lines="5-15 31-36" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -816,8 +818,10 @@ Expand the *Configure with data grouping* section to see additional examples for expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -1604,7 +1608,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-18" +```yaml hl_lines="7-20" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -1619,8 +1623,10 @@ spec: expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -2297,7 +2303,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 30-35" + ```yaml hl_lines="5-15 32-37" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -2321,8 +2327,10 @@ Expand the *Configure with data grouping* section to see additional examples for expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -3109,7 +3117,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-18" +```yaml hl_lines="7-20" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -3124,8 +3132,10 @@ spec: expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -3802,7 +3812,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 30-35" + ```yaml hl_lines="5-15 32-37" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -3826,8 +3836,10 @@ Expand the *Configure with data grouping* section to see additional examples for expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -4614,7 +4626,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="12-23" +```yaml hl_lines="12-25" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -4634,8 +4646,10 @@ spec: expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -5321,7 +5335,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="10-20 40-45" + ```yaml hl_lines="10-20 42-47" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -5350,8 +5364,10 @@ Expand the *Configure with data grouping* section to see additional examples for expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -6141,7 +6157,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="12-23" +```yaml hl_lines="12-25" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -6161,8 +6177,10 @@ spec: expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -6848,7 +6866,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="10-20 40-45" + ```yaml hl_lines="10-20 42-47" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -6877,8 +6895,10 @@ Expand the *Configure with data grouping* section to see additional examples for expected_values: - 2 - 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: diff --git a/docs/checks/column/accepted_values/expected-text-values-in-use-count.md b/docs/checks/column/accepted_values/expected-text-values-in-use-count.md index 27b72e328d..49f75b56ee 100644 --- a/docs/checks/column/accepted_values/expected-text-values-in-use-count.md +++ b/docs/checks/column/accepted_values/expected-text-values-in-use-count.md @@ -101,7 +101,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-18" +```yaml hl_lines="7-20" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -116,8 +116,10 @@ spec: - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -870,7 +872,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 30-35" + ```yaml hl_lines="5-15 32-37" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -894,8 +896,10 @@ Expand the *Configure with data grouping* section to see additional examples for - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -1758,7 +1762,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-19" +```yaml hl_lines="7-21" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -1774,8 +1778,10 @@ spec: - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -2528,7 +2534,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 31-36" + ```yaml hl_lines="5-15 33-38" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -2553,8 +2559,10 @@ Expand the *Configure with data grouping* section to see additional examples for - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -3417,7 +3425,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-19" +```yaml hl_lines="7-21" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -3433,8 +3441,10 @@ spec: - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -4187,7 +4197,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 31-36" + ```yaml hl_lines="5-15 33-38" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -4212,8 +4222,10 @@ Expand the *Configure with data grouping* section to see additional examples for - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -5076,7 +5088,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="12-24" +```yaml hl_lines="12-26" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -5097,8 +5109,10 @@ spec: - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -5860,7 +5874,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="10-20 41-46" + ```yaml hl_lines="10-20 43-48" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -5890,8 +5904,10 @@ Expand the *Configure with data grouping* section to see additional examples for - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -6757,7 +6773,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="12-24" +```yaml hl_lines="12-26" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -6778,8 +6794,10 @@ spec: - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -7541,7 +7559,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="10-20 41-46" + ```yaml hl_lines="10-20 43-48" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -7571,8 +7589,10 @@ Expand the *Configure with data grouping* section to see additional examples for - USD - GBP - EUR - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: diff --git a/docs/checks/column/accepted_values/expected-texts-in-top-values-count.md b/docs/checks/column/accepted_values/expected-texts-in-top-values-count.md index c7c97f54d3..e61afbb570 100644 --- a/docs/checks/column/accepted_values/expected-texts-in-top-values-count.md +++ b/docs/checks/column/accepted_values/expected-texts-in-top-values-count.md @@ -103,7 +103,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-19" +```yaml hl_lines="7-21" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -119,8 +119,10 @@ spec: - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -1513,7 +1515,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 31-36" + ```yaml hl_lines="5-15 33-38" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -1538,8 +1540,10 @@ Expand the *Configure with data grouping* section to see additional examples for - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -3056,7 +3060,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-20" +```yaml hl_lines="7-22" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -3073,8 +3077,10 @@ spec: - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -4467,7 +4473,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 32-37" + ```yaml hl_lines="5-15 34-39" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -4493,8 +4499,10 @@ Expand the *Configure with data grouping* section to see additional examples for - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -6011,7 +6019,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="7-20" +```yaml hl_lines="7-22" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -6028,8 +6036,10 @@ spec: - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -7422,7 +7432,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="5-15 32-37" + ```yaml hl_lines="5-15 34-39" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -7448,8 +7458,10 @@ Expand the *Configure with data grouping* section to see additional examples for - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -8966,7 +8978,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="12-25" +```yaml hl_lines="12-27" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -8988,8 +9000,10 @@ spec: - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -10387,7 +10401,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="10-20 42-47" + ```yaml hl_lines="10-20 44-49" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -10418,8 +10432,10 @@ Expand the *Configure with data grouping* section to see additional examples for - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -11941,7 +11957,7 @@ Please expand the section below to see the [DQOps command-line](../../../dqo-con The sample *schema_name.table_name.dqotable.yaml* file with the check configured is shown below. -```yaml hl_lines="12-25" +```yaml hl_lines="12-27" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -11963,8 +11979,10 @@ spec: - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: @@ -13362,7 +13380,7 @@ Expand the *Configure with data grouping* section to see additional examples for **Sample configuration with data grouping enabled (YAML)** The sample below shows how to configure the data grouping and how it affects the generated SQL query. - ```yaml hl_lines="10-20 42-47" + ```yaml hl_lines="10-20 44-49" # yaml-language-server: $schema=https://cloud.dqops.com/dqo-yaml-schema/TableYaml-schema.json apiVersion: dqo/v1 kind: table @@ -13393,8 +13411,10 @@ Expand the *Configure with data grouping* section to see additional examples for - GBP - EUR top: 3 - error: + warning: max_missing: 0 + error: + max_missing: 1 fatal: max_missing: 2 labels: diff --git a/docs/reference/rules/Averages.md b/docs/reference/rules/Averages.md index f746130bb3..3c5bcec103 100644 --- a/docs/reference/rules/Averages.md +++ b/docs/reference/rules/Averages.md @@ -49,20 +49,20 @@ The rule definition YAML file *averages/between_percent_moving_average_30_days.d fields: - field_name: max_percent_above display_name: max_percent_above - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ above a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be above a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double default_value: 10.0 - field_name: max_percent_below display_name: max_percent_below - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ below a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be below a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double default_value: 10.0 ``` @@ -214,20 +214,20 @@ The rule definition YAML file *averages/between_percent_moving_average_60_days.d fields: - field_name: max_percent_above display_name: max_percent_above - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ above a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be above a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double default_value: 10.0 - field_name: max_percent_below display_name: max_percent_below - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ below a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be below a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double default_value: 10.0 ``` @@ -379,20 +379,20 @@ The rule definition YAML file *averages/between_percent_moving_average_7_days.dq fields: - field_name: max_percent_above display_name: max_percent_above - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ above a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be above a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double default_value: 10.0 - field_name: max_percent_below display_name: max_percent_below - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ below a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be below a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double default_value: 10.0 ``` @@ -544,19 +544,19 @@ The rule definition YAML file *averages/percent_moving_average.dqorule.yaml* wit fields: - field_name: max_percent_above display_name: max_percent_above - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ above a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (warning, error, fatal) at once. The\ - \ default is a 14 time periods (days, etc.) time window, but at least 7 readouts\ - \ must exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be above a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (warning, error, fatal) at\ + \ once. The default is a 14 time periods (days, etc.) time window, but at least\ + \ 7 readouts must exist to run the calculation." data_type: double - field_name: max_percent_below display_name: max_percent_below - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ below a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (warning, error, fatal) at once. The\ - \ default is a 14 time periods (days, etc.) time window, but at least 7 readouts\ - \ must exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be below a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (warning, error, fatal) at\ + \ once. The default is a 14 time periods (days, etc.) time window, but at least\ + \ 7 readouts must exist to run the calculation." data_type: double ``` @@ -706,11 +706,11 @@ The rule definition YAML file *averages/within_percent_moving_average_30_days.dq fields: - field_name: max_percent_within display_name: max_percent_within - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ within a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be within a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double required: true default_value: 10.0 @@ -1016,11 +1016,11 @@ The rule definition YAML file *averages/within_percent_moving_average_7_days.dqo fields: - field_name: max_percent_within display_name: max_percent_within - help_text: "Maximum percent (e.q. 3%) that the current sensor readout could be\ - \ within a moving average within the time window. Set the time window at the\ - \ threshold level for all severity levels (low, medium, high) at once. The default\ - \ is a 14 time periods (days, etc.) time window, but at least 7 readouts must\ - \ exist to run the calculation." + help_text: "The maximum percentage (e.g., 3%) by which the current sensor readout\ + \ can be within a moving average within the time window. Set the time window\ + \ at the threshold level for all severity levels (low, medium, high) at once.\ + \ The default is a 14 time periods (days, etc.) time window, but at least 7\ + \ readouts must exist to run the calculation." data_type: double required: true default_value: 10.0 diff --git a/docs/reference/rules/Stdev.md b/docs/reference/rules/Stdev.md index 206ce4041c..1963b7ea20 100644 --- a/docs/reference/rules/Stdev.md +++ b/docs/reference/rules/Stdev.md @@ -50,17 +50,17 @@ The rule definition YAML file *stdev/change_multiply_moving_stdev_30_days.dqorul fields: - field_name: multiply_stdev_above display_name: multiply_stdev_above - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be above the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be above the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 30 time periods (days, etc.) time window,\ \ but at least 10 readouts must exist to run the calculation." data_type: double - field_name: multiply_stdev_below display_name: multiply_stdev_below - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be below the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be below the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 30 time periods (days, etc.) time window,\ \ but at least 10 readouts must exist to run the calculation." data_type: double @@ -234,17 +234,17 @@ The rule definition YAML file *stdev/change_multiply_moving_stdev_60_days.dqorul fields: - field_name: multiply_stdev_above display_name: multiply_stdev_above - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be above the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be above the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 60 time periods (days, etc.) time window,\ \ but at least 20 readouts must exist to run the calculation." data_type: double - field_name: multiply_stdev_below display_name: multiply_stdev_below - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be below the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be below the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 60 time periods (days, etc.) time window,\ \ but at least 20 readouts must exist to run the calculation." data_type: double @@ -418,17 +418,17 @@ The rule definition YAML file *stdev/change_multiply_moving_stdev_7_days.dqorule fields: - field_name: multiply_stdev_above display_name: multiply_stdev_above - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be above the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be above the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 7 time periods (days, etc.) time window,\ \ but at least 3 readouts must exist to run the calculation." data_type: double - field_name: multiply_stdev_below display_name: multiply_stdev_below - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be below the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be below the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 7 time periods (days, etc.) time window,\ \ but at least 3 readouts must exist to run the calculation." data_type: double @@ -1103,17 +1103,17 @@ The rule definition YAML file *stdev/multiply_moving_stdev_30_days.dqorule.yaml* fields: - field_name: multiply_stdev_above display_name: multiply_stdev_above - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be above the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be above the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 30 time periods (days, etc.) time window,\ \ but at least 10 readouts must exist to run the calculation." data_type: double - field_name: multiply_stdev_below display_name: multiply_stdev_below - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be below the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be below the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 30 time periods (days, etc.) time window,\ \ but at least 10 readouts must exist to run the calculation." data_type: double @@ -1283,17 +1283,17 @@ The rule definition YAML file *stdev/multiply_moving_stdev_60_days.dqorule.yaml* fields: - field_name: multiply_stdev_above display_name: multiply_stdev_above - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be above the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be above the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 60 time periods (days, etc.) time window,\ \ but at least 20 readouts must exist to run the calculation." data_type: double - field_name: multiply_stdev_below display_name: multiply_stdev_below - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be below the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be below the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 60 time periods (days, etc.) time window,\ \ but at least 20 readouts must exist to run the calculation." data_type: double @@ -1463,17 +1463,17 @@ The rule definition YAML file *stdev/multiply_moving_stdev_7_days.dqorule.yaml* fields: - field_name: multiply_stdev_above display_name: multiply_stdev_above - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be above the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be above the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 7 time periods (days, etc.) time window,\ \ but at least 3 readouts must exist to run the calculation." data_type: double - field_name: multiply_stdev_below display_name: multiply_stdev_below - help_text: "How many multiples of the estimated standard deviation the current\ - \ sensor readout could be below the moving average within the time window. Set\ - \ the time window at the threshold level for all severity levels (warning, error,\ + help_text: "How many multiples of the estimated standard deviation can the current\ + \ sensor readout be below the moving average within the time window? Set the\ + \ time window at the threshold level for all severity levels (warning, error,\ \ fatal) at once. The default is a 7 time periods (days, etc.) time window,\ \ but at least 3 readouts must exist to run the calculation." data_type: double diff --git a/dqops/src/main/java/com/dqops/rules/comparison/MaxMissingRule0ErrorParametersSpec.java b/dqops/src/main/java/com/dqops/rules/comparison/MaxMissingRule0ErrorParametersSpec.java index b55b017be9..1963489554 100644 --- a/dqops/src/main/java/com/dqops/rules/comparison/MaxMissingRule0ErrorParametersSpec.java +++ b/dqops/src/main/java/com/dqops/rules/comparison/MaxMissingRule0ErrorParametersSpec.java @@ -46,6 +46,7 @@ public class MaxMissingRule0ErrorParametersSpec extends AbstractRuleParametersSp @JsonPropertyDescription("The maximum number of values from the expected_values list that were not found in the column (inclusive).") @RequiredField + @SampleValues(values = "1") private Long maxMissing = 0L; /**