Custom identity: support in Reference and Group Evolution (#2208)

sodadata · Feb 4, 2025 · 53686db · 53686db
1 parent 6f169d4
commit 53686db
Show file tree

Hide file tree

Showing 3 changed files with 45 additions and 2 deletions.
diff --git a/soda/core/soda/sodacl/sodacl_parser.py b/soda/core/soda/sodacl/sodacl_parser.py
@@ -1007,7 +1007,7 @@ def __parse_group_evolution_check(
         if isinstance(check_configurations, dict):
             self._push_path_element(check_str, check_configurations)
             for configuration_key in check_configurations:
-                if configuration_key not in [NAME, WARN, FAIL, ATTRIBUTES, QUERY]:
+                if configuration_key not in [NAME, WARN, FAIL, ATTRIBUTES, QUERY, IDENTITY]:
                     self.logs.error(
                         f'Invalid group evolution check configuration key "{configuration_key}"', location=self.location
                     )
@@ -1375,7 +1375,7 @@ def __parse_reference_check(
             samples_limit = self._get_optional(SAMPLES_LIMIT, int)
 
             for configuration_key in check_configurations:
-                if configuration_key not in [NAME, SAMPLES_LIMIT, ATTRIBUTES]:
+                if configuration_key not in [NAME, SAMPLES_LIMIT, ATTRIBUTES, IDENTITY]:
                     self.logs.error(
                         f"Invalid reference check configuration key {configuration_key}", location=self.location
                     )

diff --git a/soda/core/tests/data_source/test_group_evolution.py b/soda/core/tests/data_source/test_group_evolution.py
@@ -1,5 +1,6 @@
 from helpers.common_test_tables import customers_test_table
 from helpers.data_source_fixture import DataSourceFixture
+from helpers.utils import execute_scan_and_get_scan_result
 
 
 def test_group_evolution(data_source_fixture: DataSourceFixture):
@@ -25,6 +26,30 @@ def test_group_evolution(data_source_fixture: DataSourceFixture):
     scan.assert_all_checks_pass()
 
 
+def test_group_evolution_identity(data_source_fixture: DataSourceFixture):
+    table_name = data_source_fixture.ensure_test_table(customers_test_table)
+    qualified_table_name = data_source_fixture.data_source.qualified_table_name(table_name)
+    casify = data_source_fixture.data_source.default_casify_column_name
+
+    identity = "test_identity"
+    scan_result = execute_scan_and_get_scan_result(
+        data_source_fixture,
+        f"""
+            checks for {table_name}:
+              - group evolution:
+                  identity: {identity}
+                  query: |
+                    SELECT distinct({casify('country')})
+                    FROM {qualified_table_name}
+                  fail:
+                    when required group missing: ["BE"]
+                    when forbidden group present: ["US"]
+    """,
+    )
+    assert "v4" in scan_result["checks"][0]["identities"]
+    assert scan_result["checks"][0]["identities"]["v4"] == identity
+
+
 def test_group_evolution_query_multiline(data_source_fixture: DataSourceFixture):
     table_name = data_source_fixture.ensure_test_table(customers_test_table)
     qualified_table_name = data_source_fixture.data_source.qualified_table_name(table_name)

diff --git a/soda/core/tests/data_source/test_reference_check.py b/soda/core/tests/data_source/test_reference_check.py
@@ -1,5 +1,6 @@
 from helpers.common_test_tables import customers_test_table, orders_test_table
 from helpers.data_source_fixture import DataSourceFixture
+from helpers.utils import execute_scan_and_get_scan_result
 
 
 def test_reference_check_fail(data_source_fixture: DataSourceFixture):
@@ -34,6 +35,23 @@ def test_reference_check_pass(data_source_fixture: DataSourceFixture):
     scan.assert_all_checks_pass()
 
 
+def test_reference_check_pass_identity(data_source_fixture: DataSourceFixture):
+    customers_table_name = data_source_fixture.ensure_test_table(customers_test_table)
+    orders_table_name = data_source_fixture.ensure_test_table(orders_test_table)
+
+    identity = "test_identity"
+    scan_result = execute_scan_and_get_scan_result(
+        data_source_fixture,
+        f"""
+          checks for {orders_table_name}:
+            - values in (customer_id_ok) must exist in {customers_table_name} (id):
+                identity: {identity}
+        """,
+    )
+    assert "v4" in scan_result["checks"][0]["identities"]
+    assert scan_result["checks"][0]["identities"]["v4"] == identity
+
+
 def test_multi_column_reference_check(data_source_fixture: DataSourceFixture):
     customers_table_name = data_source_fixture.ensure_test_table(customers_test_table)
     orders_table_name = data_source_fixture.ensure_test_table(orders_test_table)