Skip to content

Commit

Permalink
#116: measuredCol -> measuredColumn,, for the sake of consistency
Browse files Browse the repository at this point in the history
  • Loading branch information
lsulak committed Nov 12, 2023
1 parent 6ab4763 commit 8bdcc2e
Show file tree
Hide file tree
Showing 3 changed files with 16 additions and 16 deletions.
4 changes: 2 additions & 2 deletions agent/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -12,10 +12,10 @@ Create multiple `AtumContext` with different control measures to be applied
### Option 1
```scala
val atumContextInstanceWithRecordCount = AtumContext(processor = processor)
.withMeasureAdded(RecordCount(MockMeasureNames.recordCount1, measuredCol = "id"))
.withMeasureAdded(RecordCount(MockMeasureNames.recordCount1, measuredColumn = "id"))

val atumContextWithSalaryAbsMeasure = atumContextInstanceWithRecordCount
.withMeasureAdded(AbsSumOfValuesOfColumn(measuredCol = "salary"))
.withMeasureAdded(AbsSumOfValuesOfColumn(measuredColumn = "salary"))
```

### Option 2
Expand Down
20 changes: 10 additions & 10 deletions agent/src/main/scala/za/co/absa/atum/agent/model/Measure.scala
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,7 @@ object Measure {
}
}
object RecordCount extends MeasureType {
def apply(measuredCol: String): RecordCount = RecordCount(measuredCol, measureName, resultValueType)
def apply(measuredColumn: String): RecordCount = RecordCount(measuredColumn, measureName, resultValueType)

override val measureName: String = "count"
override val resultValueType: ResultValueType.ResultValueType = ResultValueType.Long
Expand All @@ -81,8 +81,8 @@ object Measure {
}
}
object DistinctRecordCount extends MeasureType {
def apply(measuredCol: String): DistinctRecordCount = {
DistinctRecordCount(measuredCol, measureName, resultValueType)
def apply(measuredColumn: String): DistinctRecordCount = {
DistinctRecordCount(measuredColumn, measureName, resultValueType)
}

override val measureName: String = "distinctCount"
Expand All @@ -102,8 +102,8 @@ object Measure {
}
}
object SumOfValuesOfColumn extends MeasureType {
def apply(measuredCol: String): SumOfValuesOfColumn = {
SumOfValuesOfColumn(measuredCol, measureName, resultValueType)
def apply(measuredColumn: String): SumOfValuesOfColumn = {
SumOfValuesOfColumn(measuredColumn, measureName, resultValueType)
}

override val measureName: String = "aggregatedTotal"
Expand All @@ -123,8 +123,8 @@ object Measure {
}
}
object AbsSumOfValuesOfColumn extends MeasureType {
def apply(measuredCol: String): AbsSumOfValuesOfColumn = {
AbsSumOfValuesOfColumn(measuredCol, measureName, resultValueType)
def apply(measuredColumn: String): AbsSumOfValuesOfColumn = {
AbsSumOfValuesOfColumn(measuredColumn, measureName, resultValueType)
}

override val measureName: String = "absAggregatedTotal"
Expand All @@ -149,8 +149,8 @@ object Measure {
}
}
object SumOfHashesOfColumn extends MeasureType {
def apply(measuredCol: String): SumOfHashesOfColumn = {
SumOfHashesOfColumn(measuredCol, measureName, resultValueType)
def apply(measuredColumn: String): SumOfHashesOfColumn = {
SumOfHashesOfColumn(measuredColumn, measureName, resultValueType)
}

override val measureName: String = "hashCrc32"
Expand All @@ -169,7 +169,7 @@ object Measure {
// scala> sc.parallelize(List(Long.MaxValue, 1)).toDF.agg(sum("value")).take(1)(0)(0)
// res11: Any = -9223372036854775808
// Converting to BigDecimal fixes the issue
// val ds2 = ds.select(col(measurement.measuredCol).cast(DecimalType(38, 0)).as("value"))
// val ds2 = ds.select(col(measurement.measuredColumn).cast(DecimalType(38, 0)).as("value"))
// ds2.agg(sum(abs($"value"))).collect()(0)(0)
val ds2 = ds.select(
col(measureColumn).cast(DecimalType(38, 0)).as(valueColumnName)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -29,11 +29,11 @@ class MeasureTest extends AnyFlatSpec with Matchers with SparkTestBase { self =>
"Measure" should "be based on the dataframe" in {

// Measures
val measureIds: Measure = RecordCount(measuredCol = "id")
val salaryAbsSum: Measure = AbsSumOfValuesOfColumn(measuredCol = "salary")
val measureIds: Measure = RecordCount(measuredColumn = "id")
val salaryAbsSum: Measure = AbsSumOfValuesOfColumn(measuredColumn = "salary")

val salarySum = SumOfValuesOfColumn(measuredCol = "salary")
val sumOfHashes: Measure = SumOfHashesOfColumn(measuredCol = "id")
val salarySum = SumOfValuesOfColumn(measuredColumn = "salary")
val sumOfHashes: Measure = SumOfHashesOfColumn(measuredColumn = "id")

// AtumContext contains `Measurement`
val atumContextInstanceWithRecordCount = AtumAgent
Expand Down

0 comments on commit 8bdcc2e

Please sign in to comment.