Skip to content

Commit

Permalink
#65 Add tests
Browse files Browse the repository at this point in the history
  • Loading branch information
Zejnilovic committed Oct 22, 2024
1 parent e928bdf commit 3618340
Showing 1 changed file with 4 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -23,7 +23,7 @@ import za.co.absa.spark.commons.implicits.DataFrameImplicits.DataFrameEnhancemen
import za.co.absa.spark.commons.test.SparkTestBase
import za.co.absa.spark.commons.utils.JsonUtils
import za.co.absa.standardization.RecordIdGeneration.IdType.NoId
import za.co.absa.standardization.config.{BasicMetadataColumnsConfig, BasicStandardizationConfig, ErrorCodesConfig}
import za.co.absa.standardization.config.{BasicMetadataColumnsConfig, BasicStandardizationConfig, DefaultStandardizationConfig, ErrorCodesConfig}
import za.co.absa.standardization.types.{CommonTypeDefaults, TypeDefaults}
import za.co.absa.standardization.udf.UDFLibrary
import za.co.absa.standardization._
Expand Down Expand Up @@ -135,6 +135,7 @@ class StandardizationInterpreterSuite extends AnyFunSuite with SparkTestBase wit

val standardizedDF = Standardization.standardize(orig, schema)

assert(standardizedDF.schema.treeString.contains("standardization_record_id"))
assertResult(exp)(standardizedDF.as[MyWrapperStd].collect().toList)
}

Expand Down Expand Up @@ -170,8 +171,9 @@ class StandardizationInterpreterSuite extends AnyFunSuite with SparkTestBase wit
StructField("errCol",
ArrayType(
ErrorMessage.errorColSchema, containsNull = false)))
.add(StructField("standardization_record_id", StringType, nullable = false))

val standardizedDF = Standardization.standardize(sourceDF, stdExpectedSchema, stdConfig)
val standardizedDF = Standardization.standardize(sourceDF, stdExpectedSchema)

logger.debug(standardizedDF.schema.treeString)
logger.debug(expectedSchema.treeString)
Expand Down

0 comments on commit 3618340

Please sign in to comment.