Skip to content

Commit

Permalink
Mark filtered rows as true
Browse files Browse the repository at this point in the history
We recently fixed the outcome of filtered rows and made them default to true instead of false, which was a bug earlier. This change maintains that behavior.
  • Loading branch information
rdsharma26 committed Mar 7, 2024
1 parent 8fb010c commit 1195dcf
Show file tree
Hide file tree
Showing 2 changed files with 6 additions and 3 deletions.
5 changes: 4 additions & 1 deletion src/main/scala/com/amazon/deequ/constraints/Constraint.scala
Original file line number Diff line number Diff line change
Expand Up @@ -960,7 +960,10 @@ object Constraint {
case FilteredRowOutcome.TRUE => true
case FilteredRowOutcome.NULL => null
}
case None => null
// https://github.com/awslabs/deequ/issues/530
// Filtered rows should be marked as true by default.
// They can be set to null using the FilteredRowOutcome option.
case None => true
}
}

Expand Down
4 changes: 2 additions & 2 deletions src/test/scala/com/amazon/deequ/VerificationSuiteTest.scala
Original file line number Diff line number Diff line change
Expand Up @@ -371,11 +371,11 @@ class VerificationSuiteTest extends WordSpec with Matchers with SparkContextSpec

// filtered rows 1, 2, 3 (where item > 3)
val minRowLevel = resultData.select(expectedColumn4).collect().map(r => r.getAs[Any](0))
assert(Seq(null, null, null, true, true, true).sameElements(minRowLevel))
assert(Seq(true, true, true, true, true, true).sameElements(minRowLevel))

// filtered rows 4, 5, 6 (where item < 4)
val maxRowLevel = resultData.select(expectedColumn5).collect().map(r => r.getAs[Any](0))
assert(Seq(true, true, true, null, null, null).sameElements(maxRowLevel))
assert(Seq(true, true, true, true, true, true).sameElements(maxRowLevel))

// filtered rows 4, 5, 6 (where item < 4)
val rowLevel6 = resultData.select(expectedColumn6).collect().map(r => r.getAs[Any](0))
Expand Down

0 comments on commit 1195dcf

Please sign in to comment.