Addition of fix for are complete (#112)

cuallee/pyspark_validation.py

@@ -71,15 +71,11 @@ def is_complete(self, rule: Rule):
 
     def are_complete(self, rule: Rule):
         """Validation for non-null values in a group of columns"""
-        predicate = [F.col(f"`{c}`").isNotNull() for c in rule.column]
+        predicate = (reduce(operator.add, [F.col(f"`{c}`").isNotNull().cast("integer") for c in rule.column]) == len(rule.column)).cast("integer")
         self.compute_instruction = ComputeInstruction(
             predicate,
-            reduce(
-                operator.add,
-                [self._sum_predicate_to_integer(p) for p in predicate],
-            )
-            / len(rule.column),
-            ComputeMethod.OBSERVE,
+            F.sum(predicate),
+            ComputeMethod.SELECT,
         )
         return self.compute_instruction
 

pyproject.toml

@@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cuallee"
-version = "0.4.8"
+version = "0.4.9"
 authors = [
   { name="Virginie Grosboillot", email="[email protected]" },
   { name="Herminio Vazquez", email="[email protected]"}

setup.cfg

@@ -1,5 +1,5 @@
 [metadata]
 name = cuallee
-version = 0.4.8
+version = 0.4.9
 [options]
 packages = find:

test/unit/pyspark_dataframe/test_are_complete.py

@@ -22,7 +22,7 @@ def test_negative(spark):
     assert rs.first().status == "FAIL"
     assert rs.first().violations == 1
     assert rs.first().pass_threshold == 1.0
-    assert rs.first().pass_rate == 7 / 8
+    assert rs.first().pass_rate == 3 / 4
 
 
 @pytest.mark.parametrize(
@@ -41,9 +41,9 @@ def test_coverage(spark):
         [[0, "zero"], [1, None], [2, "deux"], [3, "trois"]], ["id", "desc"]
     )
     check = Check(CheckLevel.WARNING, "pytest")
-    check.are_complete(("id", "desc"), 0.7)
+    check.are_complete(("id", "desc"), 0.75)
     rs = check.validate(df)
     assert rs.first().status == "PASS"
     assert rs.first().violations == 1
-    assert rs.first().pass_threshold == 0.7
-    assert rs.first().pass_rate == 7 / 8
+    assert rs.first().pass_threshold == 0.75
+    assert rs.first().pass_rate == 3 / 4