Skip to content

Commit

Permalink
Fix code-style
Browse files Browse the repository at this point in the history
  • Loading branch information
SemyonSinchenko committed Sep 6, 2024
1 parent d4ad85e commit a7a34b0
Show file tree
Hide file tree
Showing 3 changed files with 28 additions and 18 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,12 @@ import com.amazon.deequ.constraints.Constraint
import com.amazon.deequ.repository.ResultKey
import com.amazon.deequ.repository.fs.FileSystemMetricsRepository
import com.amazon.deequ.repository.sparktable.SparkTableMetricsRepository
import com.amazon.deequ.{AnomalyCheckConfig, VerificationRunBuilder, VerificationRunBuilderWithRepository, VerificationSuite}
import com.amazon.deequ.{
AnomalyCheckConfig,
VerificationRunBuilder,
VerificationRunBuilderWithRepository,
VerificationSuite
}
import com.ssinchenko.proto
import org.apache.spark.sql.DataFrame

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,12 @@ class DeequConnectPlugin extends RelationPlugin {

// TODO: pass returnRows here
val resultDf =
DeequUtils.runAndCollectResults(verificationSuiteBuilder, Option(spark), returnRows = false, dataFrame = Option(data))
DeequUtils.runAndCollectResults(
verificationSuiteBuilder,
Option(spark),
returnRows = false,
dataFrame = Option(data)
)
Option(resultDf.logicalPlan)
} else {
Option.empty
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -143,22 +143,22 @@ class DeequSuiteBuilderTest extends ConfTest {
.run()
.metrics

metrics.foreach(p =>
p._1 match {
case _: analyzers.ApproxCountDistinct => assert(p._2.value.get == 5.0)
case _: analyzers.ApproxQuantile => assert(p._2.value.get == 5.0)
case _: analyzers.ColumnCount => assert(p._2.value.get == 5.0)
case _: analyzers.Completeness => assert(p._2.value.get == 1.0)
case _: analyzers.Compliance => assert(p._2.value.get == 0.2)
case _: analyzers.Correlation => assert(p._2.value.get.asInstanceOf[Double] > 0.9)
case _: analyzers.CountDistinct => assert(p._2.value.get == 5.0)
case _: analyzers.Distinctness => assert(p._2.value.get == 1.0)
case _: analyzers.Entropy => assert(p._2.value.get.asInstanceOf[Double] > 0.85)
case _: analyzers.Size => assert(p._2.value.get == 5.0)
case _: analyzers.Sum => assert(p._2.value.get == 27.0)
case _: analyzers.MaxLength => assert(p._2.value.get == 31.0)
case _: analyzers.UniqueValueRatio => assert(p._2.value.get == 1.0)
}
metrics.foreach(p =>
p._1 match {
case _: analyzers.ApproxCountDistinct => assert(p._2.value.get == 5.0)
case _: analyzers.ApproxQuantile => assert(p._2.value.get == 5.0)
case _: analyzers.ColumnCount => assert(p._2.value.get == 5.0)
case _: analyzers.Completeness => assert(p._2.value.get == 1.0)
case _: analyzers.Compliance => assert(p._2.value.get == 0.2)
case _: analyzers.Correlation => assert(p._2.value.get.asInstanceOf[Double] > 0.9)
case _: analyzers.CountDistinct => assert(p._2.value.get == 5.0)
case _: analyzers.Distinctness => assert(p._2.value.get == 1.0)
case _: analyzers.Entropy => assert(p._2.value.get.asInstanceOf[Double] > 0.85)
case _: analyzers.Size => assert(p._2.value.get == 5.0)
case _: analyzers.Sum => assert(p._2.value.get == 27.0)
case _: analyzers.MaxLength => assert(p._2.value.get == 31.0)
case _: analyzers.UniqueValueRatio => assert(p._2.value.get == 1.0)
}
)
}

Expand Down

0 comments on commit a7a34b0

Please sign in to comment.