Merge pull request #43 from sauliusvl/bump-versions

Bump versions, refactor integration test configuration in sbt
adform · Apr 2, 2024 · b299365 · b299365
2 parents 1852081 + 2efea7d
commit b299365
Show file tree

Hide file tree

Showing 27 changed files with 98 additions and 75 deletions.
diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml
@@ -16,25 +16,25 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Cache Coursier
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/coursier
         key: ${{ runner.os }}-coursier-${{ hashFiles('**/*.sbt') }}
 
     - name: Cache SBT
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.sbt
         key: ${{ runner.os }}-sbt-${{ hashFiles('**/*.sbt') }}
 
     - name: Setup JDK
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
-        java-version: '11'
+        java-version: '21'
 
     - name: Run Tests
       run: sbt -Dsbt.color=always -Dsbt.supershell=false scalafmtCheck scalafmtSbtCheck headerCheckAll test IntegrationTest/test
diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml
@@ -12,32 +12,32 @@ jobs:
 
     steps:
     - name: Checkout
-      uses: actions/checkout@v3
+      uses: actions/checkout@v4
 
     - name: Cache Coursier
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.cache/coursier
         key: ${{ runner.os }}-coursier-${{ hashFiles('**/*.sbt') }}
 
     - name: Cache SBT
-      uses: actions/cache@v3
+      uses: actions/cache@v4
       with:
         path: ~/.sbt
         key: ${{ runner.os }}-sbt-${{ hashFiles('**/*.sbt') }}
 
     - name: Setup JDK
-      uses: actions/setup-java@v3
+      uses: actions/setup-java@v4
       with:
         distribution: 'temurin'
-        java-version: '11'
+        java-version: '21'
 
     - name: Setup Graphviz
-      uses: ts-graphviz/setup-graphviz@v1
+      uses: ts-graphviz/setup-graphviz@v2
 
     - name: Import Sonatype GPG key
       id: import_gpg
-      uses: crazy-max/ghaction-import-gpg@v5
+      uses: crazy-max/ghaction-import-gpg@v6
       with:
         gpg_private_key: ${{ secrets.PGP_SECRET }}
         passphrase: ${{ secrets.PGP_PASSPHRASE }}

diff --git a/build.sbt b/build.sbt
@@ -2,11 +2,20 @@ name := "stream-loader"
 
 ThisBuild / organization := "com.adform"
 ThisBuild / organizationName := "Adform"
-ThisBuild / scalaVersion := "2.13.11"
-ThisBuild / scalacOptions := Seq("-unchecked", "-deprecation", "-feature", "-encoding", "utf8", "-release", "11")
+ThisBuild / scalaVersion := "2.13.13"
+ThisBuild / scalacOptions := Seq(
+  "-unchecked",
+  "-deprecation",
+  "-feature",
+  "-encoding",
+  "utf8",
+  "-release",
+  "11",
+  "-Wconf:msg=While parsing annotations in:silent"
+)
 
 ThisBuild / startYear := Some(2020)
-ThisBuild / licenses += ("MPL-2.0", new URL("http://mozilla.org/MPL/2.0/"))
+ThisBuild / licenses += ("MPL-2.0", url("http://mozilla.org/MPL/2.0/"))
 
 ThisBuild / developers := List(
   Developer("sauliusvl", "Saulius Valatka", "[email protected]", url("https://github.com/sauliusvl"))
@@ -25,9 +34,9 @@ ThisBuild / git.remoteRepo := {
   }
 }
 
-val scalaTestVersion = "3.2.16"
+val scalaTestVersion = "3.2.18"
 val scalaCheckVersion = "1.17.0"
-val scalaCheckTestVersion = "3.2.16.0"
+val scalaCheckTestVersion = "3.2.18.0"
 
 lazy val `stream-loader-core` = project
   .in(file("stream-loader-core"))
@@ -38,24 +47,19 @@ lazy val `stream-loader-core` = project
     buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, git.gitHeadCommit),
     libraryDependencies ++= Seq(
       "org.scala-lang"     % "scala-reflect"     % scalaVersion.value,
-      "org.apache.kafka"   % "kafka-clients"     % "3.5.1",
+      "org.apache.kafka"   % "kafka-clients"     % "3.7.0",
       "org.log4s"         %% "log4s"             % "1.10.0",
-      "org.apache.commons" % "commons-compress"  % "1.21",
-      "org.xerial.snappy"  % "snappy-java"       % "1.1.10.3",
+      "org.apache.commons" % "commons-compress"  % "1.26.0",
+      "org.xerial.snappy"  % "snappy-java"       % "1.1.10.5",
       "org.lz4"            % "lz4-java"          % "1.8.0",
-      "com.github.luben"   % "zstd-jni"          % "1.5.5-5",
+      "com.github.luben"   % "zstd-jni"          % "1.5.5-6",
       "com.univocity"      % "univocity-parsers" % "2.9.1",
-      "org.json4s"        %% "json4s-native"     % "4.0.6",
-      "io.micrometer"      % "micrometer-core"   % "1.11.2",
+      "org.json4s"        %% "json4s-native"     % "4.0.7",
+      "io.micrometer"      % "micrometer-core"   % "1.12.3",
       "org.scalatest"     %% "scalatest"         % scalaTestVersion      % "test",
       "org.scalatestplus" %% "scalacheck-1-17"   % scalaCheckTestVersion % "test",
       "org.scalacheck"    %% "scalacheck"        % scalaCheckVersion     % "test",
-      "ch.qos.logback"     % "logback-classic"   % "1.4.8"               % "test"
-    ),
-    testOptions += sbt.Tests.Setup(cl =>
-      cl.loadClass("org.slf4j.LoggerFactory")
-        .getMethod("getLogger", cl.loadClass("java.lang.String"))
-        .invoke(null, "ROOT") // Prevents slf4j replay warnings during tests
+      "ch.qos.logback"     % "logback-classic"   % "1.5.3"               % "test"
     )
   )
 
@@ -66,10 +70,11 @@ lazy val `stream-loader-clickhouse` = project
   .settings(
     resolvers += "jitpack" at "https://jitpack.io",
     libraryDependencies ++= Seq(
-      "com.clickhouse"     % "clickhouse-jdbc" % "0.4.6",
-      "org.scalatest"     %% "scalatest"       % scalaTestVersion      % "test",
-      "org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test",
-      "org.scalacheck"    %% "scalacheck"      % scalaCheckVersion     % "test"
+      "org.apache.httpcomponents.client5" % "httpclient5"     % "5.3.1",
+      "com.clickhouse"                    % "clickhouse-jdbc" % "0.6.0",
+      "org.scalatest"                    %% "scalatest"       % scalaTestVersion      % "test",
+      "org.scalatestplus"                %% "scalacheck-1-17" % scalaCheckTestVersion % "test",
+      "org.scalacheck"                   %% "scalacheck"      % scalaCheckVersion     % "test"
     )
   )
 
@@ -81,7 +86,7 @@ lazy val `stream-loader-hadoop` = project
   .settings(commonSettings)
   .settings(
     libraryDependencies ++= Seq(
-      "com.sksamuel.avro4s" %% "avro4s-core"      % "4.1.1",
+      "com.sksamuel.avro4s" %% "avro4s-core"      % "4.1.2",
       "org.apache.parquet"   % "parquet-avro"     % parquetVersion,
       "org.apache.parquet"   % "parquet-protobuf" % parquetVersion,
       "org.apache.hadoop"    % "hadoop-client"    % "3.3.6" exclude ("log4j", "log4j"),
@@ -95,14 +100,14 @@ lazy val `stream-loader-s3` = project
   .settings(commonSettings)
   .settings(
     libraryDependencies ++= Seq(
-      "software.amazon.awssdk" % "s3"              % "2.20.119",
+      "software.amazon.awssdk" % "s3"              % "2.25.1",
       "org.scalatest"         %% "scalatest"       % scalaTestVersion % "test",
-      "com.amazonaws"          % "aws-java-sdk-s3" % "1.12.522"       % "test",
-      "org.gaul"               % "s3proxy"         % "2.0.0"          % "test"
+      "com.amazonaws"          % "aws-java-sdk-s3" % "1.12.671"       % "test",
+      "org.gaul"               % "s3proxy"         % "2.1.0"          % "test"
     )
   )
 
-val verticaVersion = "12.0.4-0"
+val verticaVersion = "24.1.0-2"
 
 lazy val `stream-loader-vertica` = project
   .in(file("stream-loader-vertica"))
@@ -121,32 +126,32 @@ lazy val packAndSplitJars =
   taskKey[(File, File)]("Runs pack and splits out the application jars from the external dependency jars")
 lazy val dockerImage = settingKey[String]("Full docker image name")
 
+val IntegrationTest = config("it").extend(Test)
+
 lazy val `stream-loader-tests` = project
   .in(file("stream-loader-tests"))
   .dependsOn(`stream-loader-clickhouse`)
   .dependsOn(`stream-loader-hadoop`)
   .dependsOn(`stream-loader-s3`)
   .dependsOn(`stream-loader-vertica`)
-  .configs(IntegrationTest)
   .enablePlugins(PackPlugin)
   .enablePlugins(DockerPlugin)
   .enablePlugins(BuildInfoPlugin)
-  .settings(Defaults.itSettings: _*)
-  .settings(headerSettings(IntegrationTest))
+  .configs(IntegrationTest)
   .settings(commonSettings)
   .settings(
     libraryDependencies ++= Seq(
-      "com.typesafe"       % "config"           % "1.4.2",
-      "ch.qos.logback"     % "logback-classic"  % "1.4.8",
-      "com.zaxxer"         % "HikariCP"         % "5.0.1",
+      "com.typesafe"       % "config"           % "1.4.3",
+      "ch.qos.logback"     % "logback-classic"  % "1.5.3",
+      "com.zaxxer"         % "HikariCP"         % "5.1.0",
       "com.vertica.jdbc"   % "vertica-jdbc"     % verticaVersion,
       "org.scalacheck"    %% "scalacheck"       % scalaCheckVersion,
-      "org.scalatest"     %% "scalatest"        % scalaTestVersion              % "test,it",
-      "org.scalatestplus" %% "scalacheck-1-17"  % scalaCheckTestVersion         % "test,it",
-      ("com.spotify"       % "docker-client"    % "8.16.0" classifier "shaded") % "it",
-      "org.slf4j"          % "log4j-over-slf4j" % "2.0.7"                       % "it"
+      "org.scalatest"     %% "scalatest"        % scalaTestVersion              % "test",
+      "org.scalatestplus" %% "scalacheck-1-17"  % scalaCheckTestVersion         % "test",
+      ("com.spotify"       % "docker-client"    % "8.16.0" classifier "shaded") % "test",
+      "org.slf4j"          % "log4j-over-slf4j" % "2.0.12"                      % "test"
     ),
-    test := {}, // only integration tests present
+    inConfig(IntegrationTest)(Defaults.testTasks),
     publish := {},
     publishLocal := {},
     publish / skip := true,
@@ -176,7 +181,7 @@ lazy val `stream-loader-tests` = project
       val bin = s"/opt/${name.value}/bin/"
 
       new Dockerfile {
-        from("eclipse-temurin:11.0.17_8-jre")
+        from("eclipse-temurin:21.0.2_13-jre")
 
         env("APP_CLASS_PATH" -> s"$lib/*")
 
@@ -209,18 +214,20 @@ lazy val `stream-loader-tests` = project
         )
       )
     },
+    Test / testOptions ++= Seq(
+      Tests.Argument(TestFrameworks.ScalaTest, "-l", "org.scalatest.tags.Slow")
+    ),
     IntegrationTest / test := (IntegrationTest / test).dependsOn(docker).value,
     IntegrationTest / testOnly := (IntegrationTest / testOnly).dependsOn(docker).evaluated,
-    // Prevents slf4j replay warnings during tests
-    IntegrationTest / testOptions ++= Seq(
+    IntegrationTest / testOptions := Seq(
       sbt.Tests.Setup(cl =>
         cl.loadClass("org.slf4j.LoggerFactory")
           .getMethod("getLogger", cl.loadClass("java.lang.String"))
-          .invoke(null, "ROOT")
+          .invoke(null, "ROOT") // Prevents slf4j replay warnings during tests
       ),
+      Tests.Argument(TestFrameworks.ScalaTest, "-n", "org.scalatest.tags.Slow"),
       Tests.Argument(TestFrameworks.ScalaCheck, "-verbosity", "3", "-minSuccessfulTests", "10")
-    ),
-    inConfig(IntegrationTest)(org.scalafmt.sbt.ScalafmtPlugin.scalafmtConfigSettings)
+    )
   )
 
 lazy val generateDiagrams = taskKey[Seq[File]]("Renders UML diagrams to SVG images")
@@ -250,6 +257,13 @@ lazy val commonSettings = Seq(
   versionScheme := Some("early-semver"),
   publishMavenStyle := true,
   Test / publishArtifact := false,
+  Test / testOptions ++= Seq(
+    sbt.Tests.Setup(cl =>
+      cl.loadClass("org.slf4j.LoggerFactory")
+        .getMethod("getLogger", cl.loadClass("java.lang.String"))
+        .invoke(null, "ROOT") // Prevents slf4j replay warnings during tests
+    )
+  ),
   publishTo := sonatypePublishToBundle.value,
   homepage := Some(url(gitRepoUrl)),
   scmInfo := Some(ScmInfo(url(gitRepoUrl), s"scm:git:[email protected]:$gitRepo.git"))

diff --git a/project/build.properties b/project/build.properties
@@ -1 +1 @@
-sbt.version=1.9.2
+sbt.version=1.9.9
diff --git a/project/plugins.sbt b/project/plugins.sbt
@@ -1,25 +1,25 @@
 ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always
 
-addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.10.0")
+addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.11.0")
 
 addSbtPlugin("com.github.sbt" % "sbt-git" % "2.0.1")
 
-addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.17")
+addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.19")
 
 addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0")
 
-addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0")
+addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")
 
 addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2")
 
 addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0")
 
 addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0")
 
-libraryDependencies += "net.sourceforge.plantuml" % "plantuml" % "1.2023.10"
+libraryDependencies += "net.sourceforge.plantuml" % "plantuml" % "1.2024.3"
 
 addSbtPlugin("com.github.sbt" % "sbt-ghpages" % "0.8.0")
 
 addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1")
 
-addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.21")
+addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0")
diff --git a/...re/src/main/scala/com/adform/streamloader/sink/batch/storage/TwoPhaseCommitMetadata.scala b/...re/src/main/scala/com/adform/streamloader/sink/batch/storage/TwoPhaseCommitMetadata.scala
@@ -71,7 +71,9 @@ object TwoPhaseCommitMetadata extends Logging {
         (metadata.getBytes("UTF-8"), false)
     }
     if (isCompressed) {
-      Try(new String(Zstd.decompress(bytes, Zstd.decompressedSize(bytes).toInt), "UTF-8")) match {
+      Try(
+        new String(Zstd.decompress(bytes, Zstd.getFrameContentSize(bytes, 0, bytes.length, false).toInt), "UTF-8")
+      ) match {
         case Success(decompressed) => TwoPhaseCommitMetadata.tryParseJson[S](decompressed)
         case Failure(ex) =>
           log.error(ex)(s"Failed decompressing base64 encoded metadata '$metadata'")

diff --git a/...r-tests/src/it/resources/logback-test.xml → ...tests/src/test/resources/logback-test.xml b/...r-tests/src/it/resources/logback-test.xml → ...tests/src/test/resources/logback-test.xml
diff --git a/...adform/streamloader/ClickHouseTests.scala → ...amloader/ClickHouseIntegrationTests.scala b/...adform/streamloader/ClickHouseTests.scala → ...amloader/ClickHouseIntegrationTests.scala
@@ -15,12 +15,14 @@ import com.adform.streamloader.storage.ClickHouseStorageBackend
 import com.zaxxer.hikari.{HikariConfig, HikariDataSource}
 import org.scalatest.concurrent.Eventually
 import org.scalatest.funspec.AnyFunSpec
+import org.scalatest.tags.Slow
 import org.scalatest.matchers.should.Matchers
 import org.scalatestplus.scalacheck.Checkers
 
 import scala.concurrent.ExecutionContext
 
-class ClickHouseTests
+@Slow
+class ClickHouseIntegrationTests
     extends AnyFunSpec
     with Matchers
     with Eventually

diff --git a/...a/com/adform/streamloader/HdfsTests.scala → ...m/streamloader/HdfsIntegrationTests.scala b/...a/com/adform/streamloader/HdfsTests.scala → ...m/streamloader/HdfsIntegrationTests.scala
@@ -15,11 +15,13 @@ import com.adform.streamloader.storage.HdfsStorageBackend
 import org.scalatest.concurrent.Eventually
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
+import org.scalatest.tags.Slow
 import org.scalatestplus.scalacheck.Checkers
 
 import scala.concurrent.ExecutionContext
 
-class HdfsTests
+@Slow
+class HdfsIntegrationTests
     extends AnyFunSpec
     with Checkers
     with Matchers

diff --git a/...ala/com/adform/streamloader/S3Tests.scala → ...orm/streamloader/S3IntegrationTests.scala b/...ala/com/adform/streamloader/S3Tests.scala → ...orm/streamloader/S3IntegrationTests.scala
@@ -15,11 +15,13 @@ import com.adform.streamloader.storage.{LoaderS3Config, S3StorageBackend}
 import org.scalatest.concurrent.Eventually
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
+import org.scalatest.tags.Slow
 import org.scalatestplus.scalacheck.Checkers
 
 import scala.concurrent.ExecutionContext
 
-class S3Tests
+@Slow
+class S3IntegrationTests
     extends AnyFunSpec
     with Matchers
     with Eventually

diff --git a/...om/adform/streamloader/VerticaTests.scala → ...treamloader/VerticaIntegrationTests.scala b/...om/adform/streamloader/VerticaTests.scala → ...treamloader/VerticaIntegrationTests.scala
@@ -15,11 +15,13 @@ import com.zaxxer.hikari.{HikariConfig, HikariDataSource}
 import org.scalatest.concurrent.Eventually
 import org.scalatest.funspec.AnyFunSpec
 import org.scalatest.matchers.should.Matchers
+import org.scalatest.tags.Slow
 import org.scalatestplus.scalacheck.Checkers
 
 import scala.concurrent.ExecutionContext
 
-class VerticaTests
+@Slow
+class VerticaIntegrationTests
     extends AnyFunSpec
     with Matchers
     with Eventually

diff --git a/...ader/behaviors/BasicLoaderBehaviors.scala → ...ader/behaviors/BasicLoaderBehaviors.scala b/...ader/behaviors/BasicLoaderBehaviors.scala → ...ader/behaviors/BasicLoaderBehaviors.scala
diff --git a/...der/behaviors/KafkaRestartBehaviors.scala → ...der/behaviors/KafkaRestartBehaviors.scala b/...der/behaviors/KafkaRestartBehaviors.scala → ...der/behaviors/KafkaRestartBehaviors.scala
diff --git a/...loader/behaviors/RebalanceBehaviors.scala → ...loader/behaviors/RebalanceBehaviors.scala b/...loader/behaviors/RebalanceBehaviors.scala → ...loader/behaviors/RebalanceBehaviors.scala
diff --git a/...rm/streamloader/fixtures/ClickHouse.scala → ...rm/streamloader/fixtures/ClickHouse.scala b/...rm/streamloader/fixtures/ClickHouse.scala → ...rm/streamloader/fixtures/ClickHouse.scala
@@ -17,7 +17,7 @@ import org.scalatest.{BeforeAndAfterAll, Suite}
 
 import scala.jdk.CollectionConverters._
 
-case class ClickHouseConfig(dbName: String = "default", image: String = "clickhouse/clickhouse-server:23.1.3.5")
+case class ClickHouseConfig(dbName: String = "default", image: String = "clickhouse/clickhouse-server:24.2.1.2248")
 
 trait ClickHouseTestFixture extends ClickHouse with BeforeAndAfterAll { this: Suite with DockerTestFixture =>
   override def beforeAll(): Unit = {

diff --git a/...adform/streamloader/fixtures/Docker.scala → ...adform/streamloader/fixtures/Docker.scala b/...adform/streamloader/fixtures/Docker.scala → ...adform/streamloader/fixtures/Docker.scala
diff --git a/...m/adform/streamloader/fixtures/Hdfs.scala → ...m/adform/streamloader/fixtures/Hdfs.scala b/...m/adform/streamloader/fixtures/Hdfs.scala → ...m/adform/streamloader/fixtures/Hdfs.scala