From 8369ce5b890009ee518dd38427087747a6155505 Mon Sep 17 00:00:00 2001 From: Tuan Pham Date: Sat, 5 Oct 2024 17:30:20 +1000 Subject: [PATCH] Move native function to "unsafe" project --- .github/workflows/{ci.yml => core-ci.yml} | 6 ++-- .github/workflows/unsafe-ci.yml | 22 ++++++++++++ build.sbt | 34 +++++++++++++++---- .../spark/daria/delta/DeltaLogHelpers.scala | 0 .../mrpowers/spark/daria/elt/Parser.scala | 0 .../spark/daria/hadoop/FsHelpers.scala | 0 .../mrpowers/spark/daria/sql/ColumnExt.scala | 0 .../spark/daria/sql/CustomTransform.scala | 0 .../spark/daria/sql/DariaValidator.scala | 0 .../spark/daria/sql/DariaWriters.scala | 0 .../daria/sql/DataFrameColumnsAbsence.scala | 0 .../daria/sql/DataFrameColumnsChecker.scala | 0 .../spark/daria/sql/DataFrameExt.scala | 0 .../spark/daria/sql/DataFrameHelpers.scala | 0 .../daria/sql/DataFrameSchemaChecker.scala | 0 .../spark/daria/sql/DataFrameValidator.scala | 0 .../spark/daria/sql/EtlDefinition.scala | 0 .../daria/sql/FunctionsAsColumnExt.scala | 0 .../spark/daria/sql/ParquetCompactor.scala | 0 .../spark/daria/sql/SparkSessionExt.scala | 0 .../mrpowers/spark/daria/sql/functions.scala | 0 .../spark/daria/sql/transformations.scala | 0 .../daria/sql/types/StructFieldHelpers.scala | 0 .../daria/sql/types/StructTypeHelpers.scala | 0 .../spark/daria/sql/udafs/ArrayConcat.scala | 0 .../spark/daria/utils/ArrayHelpers.scala | 0 .../spark/daria/utils/DirHelpers.scala | 0 .../mrpowers/spark/daria/utils/NioUtils.scala | 0 .../spark/daria/utils/RowHelpers.scala | 0 .../spark/daria/utils/SchemaSafeWriter.scala | 0 .../spark/daria/utils/StringHelpers.scala | 0 .../src}/test/resources/csvs/file1.csv | 0 .../src}/test/resources/csvs/file2.csv | 0 .../src}/test/resources/csvs/file3.csv | 0 .../src}/test/resources/log4j.properties | 0 {src => core/src}/test/resources/people.csv | 0 .../src}/test/resources/some_data.csv | 0 {src => core/src}/test/resources/zipcodes.txt | 0 .../spark/daria/CustomFramework.scala | 0 .../daria/delta/DeltaLogHelpersTest.scala | 0 .../spark/daria/elt/StagingParser.scala | 0 .../spark/daria/elt/StagingParserTest.scala | 0 .../spark/daria/hadoop/FsHelpersTest.scala | 0 .../spark/daria/sql/ColumnExtTest.scala | 0 .../spark/daria/sql/DariaValidatorTest.scala | 0 .../spark/daria/sql/DariaWritersTest.scala | 0 .../sql/DataFrameColumnsAbsenceTest.scala | 0 .../sql/DataFrameColumnsCheckerTest.scala | 0 .../spark/daria/sql/DataFrameExtTest.scala | 0 .../daria/sql/DataFrameHelpersTest.scala | 0 .../sql/DataFrameSchemaCheckerTest.scala | 0 .../daria/sql/DataFrameValidatorTest.scala | 0 .../spark/daria/sql/EtlDefinitionTest.scala | 0 .../mrpowers/spark/daria/sql/EtlHelpers.scala | 0 .../spark/daria/sql/ExampleTransforms.scala | 0 .../daria/sql/FunctionsAsColumnExtTest.scala | 0 .../spark/daria/sql/FunctionsTest.scala | 0 .../daria/sql/ParquetCompactorTest.scala | 0 .../spark/daria/sql/SparkSessionExtTest.scala | 0 .../daria/sql/SparkSessionTestWrapper.scala | 0 .../spark/daria/sql/TransformationsTest.scala | 0 .../sql/types/StructFieldHelpersTest.scala | 0 .../sql/types/StructTypeHelpersTest.scala | 0 .../daria/sql/udafs/ArrayConcatTest.scala | 0 .../spark/daria/utils/ArrayHelpersTest.scala | 0 .../spark/daria/utils/RowHelpersTest.scala | 0 .../daria/utils/SchemaSafeWriterTest.scala | 0 .../spark/daria/utils/StringHelpersTest.scala | 0 .../sql/catalyst/expressions/RandGamma.scala | 0 .../apache/spark/sql/daria/functions.scala | 0 .../util/random/XORShiftRandomAdapted.scala | 0 .../spark/sql/daria/functionsTests.scala | 0 72 files changed, 52 insertions(+), 10 deletions(-) rename .github/workflows/{ci.yml => core-ci.yml} (70%) create mode 100644 .github/workflows/unsafe-ci.yml rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/elt/Parser.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/ColumnExt.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/CustomTransform.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DariaValidator.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DariaWriters.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsence.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsChecker.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaChecker.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidator.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/EtlDefinition.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExt.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactor.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExt.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/functions.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/transformations.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcat.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/utils/DirHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/utils/NioUtils.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/utils/RowHelpers.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriter.scala (100%) rename {src => core/src}/main/scala/com/github/mrpowers/spark/daria/utils/StringHelpers.scala (100%) rename {src => core/src}/test/resources/csvs/file1.csv (100%) rename {src => core/src}/test/resources/csvs/file2.csv (100%) rename {src => core/src}/test/resources/csvs/file3.csv (100%) rename {src => core/src}/test/resources/log4j.properties (100%) rename {src => core/src}/test/resources/people.csv (100%) rename {src => core/src}/test/resources/some_data.csv (100%) rename {src => core/src}/test/resources/zipcodes.txt (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/CustomFramework.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/elt/StagingParser.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/elt/StagingParserTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/ColumnExtTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DariaValidatorTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DariaWritersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsenceTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsCheckerTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameExtTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaCheckerTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidatorTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/EtlDefinitionTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/EtlHelpers.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/ExampleTransforms.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExtTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactorTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExtTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionTestWrapper.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/TransformationsTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcatTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/utils/RowHelpersTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriterTest.scala (100%) rename {src => core/src}/test/scala/com/github/mrpowers/spark/daria/utils/StringHelpersTest.scala (100%) rename {src => unsafe/src}/main/scala/org/apache/spark/sql/catalyst/expressions/RandGamma.scala (100%) rename {src => unsafe/src}/main/scala/org/apache/spark/sql/daria/functions.scala (100%) rename {src => unsafe/src}/main/scala/org/apache/spark/util/random/XORShiftRandomAdapted.scala (100%) rename {src => unsafe/src}/test/scala/org/apache/spark/sql/daria/functionsTests.scala (100%) diff --git a/.github/workflows/ci.yml b/.github/workflows/core-ci.yml similarity index 70% rename from .github/workflows/ci.yml rename to .github/workflows/core-ci.yml index 73f80792..3ef8368c 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/core-ci.yml @@ -1,4 +1,4 @@ -name: ci +name: core-ci on: push: @@ -17,6 +17,6 @@ jobs: - uses: actions/checkout@v1 - uses: olafurpg/setup-scala@v10 - name: Test - run: sbt -Dspark.testVersion=${{ matrix.spark }} +test + run: sbt -Dspark.testVersion=${{ matrix.spark }} +"project core" test - name: Code Quality - run: sbt scalafmtCheckAll + run: sbt "project core" scalafmtCheckAll diff --git a/.github/workflows/unsafe-ci.yml b/.github/workflows/unsafe-ci.yml new file mode 100644 index 00000000..9b25c316 --- /dev/null +++ b/.github/workflows/unsafe-ci.yml @@ -0,0 +1,22 @@ +name: unsafe-ci + +on: + push: + branches: + - main + pull_request: + +jobs: + build: + strategy: + fail-fast: false + matrix: + spark: ["3.2.4", "3.3.4"] + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v1 + - uses: olafurpg/setup-scala@v10 + - name: Test + run: sbt -Dspark.testVersion=${{ matrix.spark }} +"project unsafe" test + - name: Code Quality + run: sbt "project unsafe" scalafmtCheckAll diff --git a/build.sbt b/build.sbt index 9cfc127a..511d34cd 100644 --- a/build.sbt +++ b/build.sbt @@ -20,22 +20,42 @@ crossScalaVersions := { scalaVersion := crossScalaVersions.value.head -libraryDependencies += "org.apache.spark" %% "spark-sql" % sparkVersion % "provided" -libraryDependencies += "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided" -libraryDependencies += "com.github.mrpowers" %% "spark-fast-tests" % "1.1.0" % "test" -libraryDependencies += "com.lihaoyi" %% "utest" % "0.7.11" % "test" -libraryDependencies += "com.lihaoyi" %% "os-lib" % "0.8.0" % "test" +lazy val commonSettings = Seq( + javaOptions ++= { + Seq("-Xms512M", "-Xmx2048M", "-Duser.timezone=GMT") ++ (if (System.getProperty("java.version").startsWith("1.8.0")) + Seq("-XX:+CMSClassUnloadingEnabled") + else Seq.empty) + }, + libraryDependencies ++= Seq( + "org.apache.spark" %% "spark-sql" % sparkVersion % "provided", + "org.apache.spark" %% "spark-mllib" % sparkVersion % "provided", + "com.lihaoyi" %% "utest" % "0.7.11" % "test", + "com.lihaoyi" %% "os-lib" % "0.8.0" % "test" + ), +) + +lazy val core = (project in file("core")) + .settings( + commonSettings, + name := "core", + ) + +lazy val unsafe = (project in file("unsafe")) + .settings( + commonSettings, + name := "unsafe", + ) + testFrameworks += new TestFramework("com.github.mrpowers.spark.daria.CustomFramework") credentials += Credentials(Path.userHome / ".sbt" / "sonatype_credentials") fork in Test := true -javaOptions ++= Seq("-Xms512M", "-Xmx2048M", "-XX:+CMSClassUnloadingEnabled", "-Duser.timezone=GMT") - licenses := Seq("MIT" -> url("http://opensource.org/licenses/MIT")) homepage := Some(url("https://github.com/MrPowers/spark-daria")) + developers ++= List( Developer("MrPowers", "Matthew Powers", "@MrPowers", url("https://github.com/MrPowers")) ) diff --git a/src/main/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/elt/Parser.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/elt/Parser.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/elt/Parser.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/elt/Parser.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/ColumnExt.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/ColumnExt.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/ColumnExt.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/ColumnExt.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/CustomTransform.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/CustomTransform.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/CustomTransform.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/CustomTransform.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DariaValidator.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DariaValidator.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DariaValidator.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DariaValidator.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DariaWriters.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DariaWriters.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DariaWriters.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DariaWriters.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsence.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsence.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsence.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsence.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsChecker.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsChecker.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsChecker.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsChecker.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameExt.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaChecker.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaChecker.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaChecker.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaChecker.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidator.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidator.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidator.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidator.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/EtlDefinition.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/EtlDefinition.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/EtlDefinition.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/EtlDefinition.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExt.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExt.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExt.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExt.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactor.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactor.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactor.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactor.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExt.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExt.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExt.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExt.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/functions.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/functions.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/functions.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/functions.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/transformations.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/transformations.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/transformations.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/transformations.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcat.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcat.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcat.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcat.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/utils/DirHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/utils/DirHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/utils/DirHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/utils/DirHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/utils/NioUtils.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/utils/NioUtils.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/utils/NioUtils.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/utils/NioUtils.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/utils/RowHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/utils/RowHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/utils/RowHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/utils/RowHelpers.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriter.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriter.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriter.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriter.scala diff --git a/src/main/scala/com/github/mrpowers/spark/daria/utils/StringHelpers.scala b/core/src/main/scala/com/github/mrpowers/spark/daria/utils/StringHelpers.scala similarity index 100% rename from src/main/scala/com/github/mrpowers/spark/daria/utils/StringHelpers.scala rename to core/src/main/scala/com/github/mrpowers/spark/daria/utils/StringHelpers.scala diff --git a/src/test/resources/csvs/file1.csv b/core/src/test/resources/csvs/file1.csv similarity index 100% rename from src/test/resources/csvs/file1.csv rename to core/src/test/resources/csvs/file1.csv diff --git a/src/test/resources/csvs/file2.csv b/core/src/test/resources/csvs/file2.csv similarity index 100% rename from src/test/resources/csvs/file2.csv rename to core/src/test/resources/csvs/file2.csv diff --git a/src/test/resources/csvs/file3.csv b/core/src/test/resources/csvs/file3.csv similarity index 100% rename from src/test/resources/csvs/file3.csv rename to core/src/test/resources/csvs/file3.csv diff --git a/src/test/resources/log4j.properties b/core/src/test/resources/log4j.properties similarity index 100% rename from src/test/resources/log4j.properties rename to core/src/test/resources/log4j.properties diff --git a/src/test/resources/people.csv b/core/src/test/resources/people.csv similarity index 100% rename from src/test/resources/people.csv rename to core/src/test/resources/people.csv diff --git a/src/test/resources/some_data.csv b/core/src/test/resources/some_data.csv similarity index 100% rename from src/test/resources/some_data.csv rename to core/src/test/resources/some_data.csv diff --git a/src/test/resources/zipcodes.txt b/core/src/test/resources/zipcodes.txt similarity index 100% rename from src/test/resources/zipcodes.txt rename to core/src/test/resources/zipcodes.txt diff --git a/src/test/scala/com/github/mrpowers/spark/daria/CustomFramework.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/CustomFramework.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/CustomFramework.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/CustomFramework.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/delta/DeltaLogHelpersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParser.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParser.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParser.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParser.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParserTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParserTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParserTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/elt/StagingParserTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/hadoop/FsHelpersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/ColumnExtTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/ColumnExtTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/ColumnExtTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/ColumnExtTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DariaValidatorTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DariaValidatorTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DariaValidatorTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DariaValidatorTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DariaWritersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DariaWritersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DariaWritersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DariaWritersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsenceTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsenceTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsenceTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsAbsenceTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsCheckerTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsCheckerTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsCheckerTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameColumnsCheckerTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameExtTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameExtTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameExtTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameExtTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameHelpersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaCheckerTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaCheckerTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaCheckerTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameSchemaCheckerTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidatorTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidatorTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidatorTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/DataFrameValidatorTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/EtlDefinitionTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/EtlDefinitionTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/EtlDefinitionTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/EtlDefinitionTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/EtlHelpers.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/EtlHelpers.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/EtlHelpers.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/EtlHelpers.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/ExampleTransforms.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/ExampleTransforms.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/ExampleTransforms.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/ExampleTransforms.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExtTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExtTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExtTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsAsColumnExtTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/FunctionsTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactorTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactorTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactorTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/ParquetCompactorTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExtTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExtTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExtTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionExtTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionTestWrapper.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionTestWrapper.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionTestWrapper.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/SparkSessionTestWrapper.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/TransformationsTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/TransformationsTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/TransformationsTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/TransformationsTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructFieldHelpersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/types/StructTypeHelpersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcatTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcatTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcatTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/sql/udafs/ArrayConcatTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/utils/ArrayHelpersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/utils/RowHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/utils/RowHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/utils/RowHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/utils/RowHelpersTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriterTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriterTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriterTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/utils/SchemaSafeWriterTest.scala diff --git a/src/test/scala/com/github/mrpowers/spark/daria/utils/StringHelpersTest.scala b/core/src/test/scala/com/github/mrpowers/spark/daria/utils/StringHelpersTest.scala similarity index 100% rename from src/test/scala/com/github/mrpowers/spark/daria/utils/StringHelpersTest.scala rename to core/src/test/scala/com/github/mrpowers/spark/daria/utils/StringHelpersTest.scala diff --git a/src/main/scala/org/apache/spark/sql/catalyst/expressions/RandGamma.scala b/unsafe/src/main/scala/org/apache/spark/sql/catalyst/expressions/RandGamma.scala similarity index 100% rename from src/main/scala/org/apache/spark/sql/catalyst/expressions/RandGamma.scala rename to unsafe/src/main/scala/org/apache/spark/sql/catalyst/expressions/RandGamma.scala diff --git a/src/main/scala/org/apache/spark/sql/daria/functions.scala b/unsafe/src/main/scala/org/apache/spark/sql/daria/functions.scala similarity index 100% rename from src/main/scala/org/apache/spark/sql/daria/functions.scala rename to unsafe/src/main/scala/org/apache/spark/sql/daria/functions.scala diff --git a/src/main/scala/org/apache/spark/util/random/XORShiftRandomAdapted.scala b/unsafe/src/main/scala/org/apache/spark/util/random/XORShiftRandomAdapted.scala similarity index 100% rename from src/main/scala/org/apache/spark/util/random/XORShiftRandomAdapted.scala rename to unsafe/src/main/scala/org/apache/spark/util/random/XORShiftRandomAdapted.scala diff --git a/src/test/scala/org/apache/spark/sql/daria/functionsTests.scala b/unsafe/src/test/scala/org/apache/spark/sql/daria/functionsTests.scala similarity index 100% rename from src/test/scala/org/apache/spark/sql/daria/functionsTests.scala rename to unsafe/src/test/scala/org/apache/spark/sql/daria/functionsTests.scala