diff --git a/.github/workflows/build.yml b/.github/workflows/build.yml index 6fe2086..d5aeb01 100644 --- a/.github/workflows/build.yml +++ b/.github/workflows/build.yml @@ -16,25 +16,25 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache Coursier - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/coursier key: ${{ runner.os }}-coursier-${{ hashFiles('**/*.sbt') }} - name: Cache SBT - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.sbt key: ${{ runner.os }}-sbt-${{ hashFiles('**/*.sbt') }} - name: Setup JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: '11' + java-version: '21' - name: Run Tests run: sbt -Dsbt.color=always -Dsbt.supershell=false scalafmtCheck scalafmtSbtCheck headerCheckAll test IntegrationTest/test diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index 573a71f..e6e38a2 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -12,32 +12,32 @@ jobs: steps: - name: Checkout - uses: actions/checkout@v3 + uses: actions/checkout@v4 - name: Cache Coursier - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.cache/coursier key: ${{ runner.os }}-coursier-${{ hashFiles('**/*.sbt') }} - name: Cache SBT - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ~/.sbt key: ${{ runner.os }}-sbt-${{ hashFiles('**/*.sbt') }} - name: Setup JDK - uses: actions/setup-java@v3 + uses: actions/setup-java@v4 with: distribution: 'temurin' - java-version: '11' + java-version: '21' - name: Setup Graphviz - uses: ts-graphviz/setup-graphviz@v1 + uses: ts-graphviz/setup-graphviz@v2 - name: Import Sonatype GPG key id: import_gpg - uses: crazy-max/ghaction-import-gpg@v5 + uses: crazy-max/ghaction-import-gpg@v6 with: gpg_private_key: ${{ secrets.PGP_SECRET }} passphrase: ${{ secrets.PGP_PASSPHRASE }} diff --git a/build.sbt b/build.sbt index 591fdd5..fc7b746 100644 --- a/build.sbt +++ b/build.sbt @@ -2,11 +2,20 @@ name := "stream-loader" ThisBuild / organization := "com.adform" ThisBuild / organizationName := "Adform" -ThisBuild / scalaVersion := "2.13.11" -ThisBuild / scalacOptions := Seq("-unchecked", "-deprecation", "-feature", "-encoding", "utf8", "-release", "11") +ThisBuild / scalaVersion := "2.13.13" +ThisBuild / scalacOptions := Seq( + "-unchecked", + "-deprecation", + "-feature", + "-encoding", + "utf8", + "-release", + "11", + "-Wconf:msg=While parsing annotations in:silent" +) ThisBuild / startYear := Some(2020) -ThisBuild / licenses += ("MPL-2.0", new URL("http://mozilla.org/MPL/2.0/")) +ThisBuild / licenses += ("MPL-2.0", url("http://mozilla.org/MPL/2.0/")) ThisBuild / developers := List( Developer("sauliusvl", "Saulius Valatka", "saulius.vl@gmail.com", url("https://github.com/sauliusvl")) @@ -25,9 +34,9 @@ ThisBuild / git.remoteRepo := { } } -val scalaTestVersion = "3.2.16" +val scalaTestVersion = "3.2.18" val scalaCheckVersion = "1.17.0" -val scalaCheckTestVersion = "3.2.16.0" +val scalaCheckTestVersion = "3.2.18.0" lazy val `stream-loader-core` = project .in(file("stream-loader-core")) @@ -38,24 +47,19 @@ lazy val `stream-loader-core` = project buildInfoKeys := Seq[BuildInfoKey](name, version, scalaVersion, git.gitHeadCommit), libraryDependencies ++= Seq( "org.scala-lang" % "scala-reflect" % scalaVersion.value, - "org.apache.kafka" % "kafka-clients" % "3.5.1", + "org.apache.kafka" % "kafka-clients" % "3.7.0", "org.log4s" %% "log4s" % "1.10.0", - "org.apache.commons" % "commons-compress" % "1.21", - "org.xerial.snappy" % "snappy-java" % "1.1.10.3", + "org.apache.commons" % "commons-compress" % "1.26.0", + "org.xerial.snappy" % "snappy-java" % "1.1.10.5", "org.lz4" % "lz4-java" % "1.8.0", - "com.github.luben" % "zstd-jni" % "1.5.5-5", + "com.github.luben" % "zstd-jni" % "1.5.5-6", "com.univocity" % "univocity-parsers" % "2.9.1", - "org.json4s" %% "json4s-native" % "4.0.6", - "io.micrometer" % "micrometer-core" % "1.11.2", + "org.json4s" %% "json4s-native" % "4.0.7", + "io.micrometer" % "micrometer-core" % "1.12.3", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", "org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test", "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test", - "ch.qos.logback" % "logback-classic" % "1.4.8" % "test" - ), - testOptions += sbt.Tests.Setup(cl => - cl.loadClass("org.slf4j.LoggerFactory") - .getMethod("getLogger", cl.loadClass("java.lang.String")) - .invoke(null, "ROOT") // Prevents slf4j replay warnings during tests + "ch.qos.logback" % "logback-classic" % "1.5.3" % "test" ) ) @@ -66,10 +70,11 @@ lazy val `stream-loader-clickhouse` = project .settings( resolvers += "jitpack" at "https://jitpack.io", libraryDependencies ++= Seq( - "com.clickhouse" % "clickhouse-jdbc" % "0.4.6", - "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test", - "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test" + "org.apache.httpcomponents.client5" % "httpclient5" % "5.3.1", + "com.clickhouse" % "clickhouse-jdbc" % "0.6.0", + "org.scalatest" %% "scalatest" % scalaTestVersion % "test", + "org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test", + "org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test" ) ) @@ -81,7 +86,7 @@ lazy val `stream-loader-hadoop` = project .settings(commonSettings) .settings( libraryDependencies ++= Seq( - "com.sksamuel.avro4s" %% "avro4s-core" % "4.1.1", + "com.sksamuel.avro4s" %% "avro4s-core" % "4.1.2", "org.apache.parquet" % "parquet-avro" % parquetVersion, "org.apache.parquet" % "parquet-protobuf" % parquetVersion, "org.apache.hadoop" % "hadoop-client" % "3.3.6" exclude ("log4j", "log4j"), @@ -95,14 +100,14 @@ lazy val `stream-loader-s3` = project .settings(commonSettings) .settings( libraryDependencies ++= Seq( - "software.amazon.awssdk" % "s3" % "2.20.119", + "software.amazon.awssdk" % "s3" % "2.25.1", "org.scalatest" %% "scalatest" % scalaTestVersion % "test", - "com.amazonaws" % "aws-java-sdk-s3" % "1.12.522" % "test", - "org.gaul" % "s3proxy" % "2.0.0" % "test" + "com.amazonaws" % "aws-java-sdk-s3" % "1.12.671" % "test", + "org.gaul" % "s3proxy" % "2.1.0" % "test" ) ) -val verticaVersion = "12.0.4-0" +val verticaVersion = "24.1.0-2" lazy val `stream-loader-vertica` = project .in(file("stream-loader-vertica")) @@ -121,32 +126,32 @@ lazy val packAndSplitJars = taskKey[(File, File)]("Runs pack and splits out the application jars from the external dependency jars") lazy val dockerImage = settingKey[String]("Full docker image name") +val IntegrationTest = config("it").extend(Test) + lazy val `stream-loader-tests` = project .in(file("stream-loader-tests")) .dependsOn(`stream-loader-clickhouse`) .dependsOn(`stream-loader-hadoop`) .dependsOn(`stream-loader-s3`) .dependsOn(`stream-loader-vertica`) - .configs(IntegrationTest) .enablePlugins(PackPlugin) .enablePlugins(DockerPlugin) .enablePlugins(BuildInfoPlugin) - .settings(Defaults.itSettings: _*) - .settings(headerSettings(IntegrationTest)) + .configs(IntegrationTest) .settings(commonSettings) .settings( libraryDependencies ++= Seq( - "com.typesafe" % "config" % "1.4.2", - "ch.qos.logback" % "logback-classic" % "1.4.8", - "com.zaxxer" % "HikariCP" % "5.0.1", + "com.typesafe" % "config" % "1.4.3", + "ch.qos.logback" % "logback-classic" % "1.5.3", + "com.zaxxer" % "HikariCP" % "5.1.0", "com.vertica.jdbc" % "vertica-jdbc" % verticaVersion, "org.scalacheck" %% "scalacheck" % scalaCheckVersion, - "org.scalatest" %% "scalatest" % scalaTestVersion % "test,it", - "org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test,it", - ("com.spotify" % "docker-client" % "8.16.0" classifier "shaded") % "it", - "org.slf4j" % "log4j-over-slf4j" % "2.0.7" % "it" + "org.scalatest" %% "scalatest" % scalaTestVersion % "test", + "org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test", + ("com.spotify" % "docker-client" % "8.16.0" classifier "shaded") % "test", + "org.slf4j" % "log4j-over-slf4j" % "2.0.12" % "test" ), - test := {}, // only integration tests present + inConfig(IntegrationTest)(Defaults.testTasks), publish := {}, publishLocal := {}, publish / skip := true, @@ -176,7 +181,7 @@ lazy val `stream-loader-tests` = project val bin = s"/opt/${name.value}/bin/" new Dockerfile { - from("eclipse-temurin:11.0.17_8-jre") + from("eclipse-temurin:21.0.2_13-jre") env("APP_CLASS_PATH" -> s"$lib/*") @@ -209,18 +214,20 @@ lazy val `stream-loader-tests` = project ) ) }, + Test / testOptions ++= Seq( + Tests.Argument(TestFrameworks.ScalaTest, "-l", "org.scalatest.tags.Slow") + ), IntegrationTest / test := (IntegrationTest / test).dependsOn(docker).value, IntegrationTest / testOnly := (IntegrationTest / testOnly).dependsOn(docker).evaluated, - // Prevents slf4j replay warnings during tests - IntegrationTest / testOptions ++= Seq( + IntegrationTest / testOptions := Seq( sbt.Tests.Setup(cl => cl.loadClass("org.slf4j.LoggerFactory") .getMethod("getLogger", cl.loadClass("java.lang.String")) - .invoke(null, "ROOT") + .invoke(null, "ROOT") // Prevents slf4j replay warnings during tests ), + Tests.Argument(TestFrameworks.ScalaTest, "-n", "org.scalatest.tags.Slow"), Tests.Argument(TestFrameworks.ScalaCheck, "-verbosity", "3", "-minSuccessfulTests", "10") - ), - inConfig(IntegrationTest)(org.scalafmt.sbt.ScalafmtPlugin.scalafmtConfigSettings) + ) ) lazy val generateDiagrams = taskKey[Seq[File]]("Renders UML diagrams to SVG images") @@ -250,6 +257,13 @@ lazy val commonSettings = Seq( versionScheme := Some("early-semver"), publishMavenStyle := true, Test / publishArtifact := false, + Test / testOptions ++= Seq( + sbt.Tests.Setup(cl => + cl.loadClass("org.slf4j.LoggerFactory") + .getMethod("getLogger", cl.loadClass("java.lang.String")) + .invoke(null, "ROOT") // Prevents slf4j replay warnings during tests + ) + ), publishTo := sonatypePublishToBundle.value, homepage := Some(url(gitRepoUrl)), scmInfo := Some(ScmInfo(url(gitRepoUrl), s"scm:git:git@github.com:$gitRepo.git")) diff --git a/project/build.properties b/project/build.properties index 875b706..04267b1 100644 --- a/project/build.properties +++ b/project/build.properties @@ -1 +1 @@ -sbt.version=1.9.2 +sbt.version=1.9.9 diff --git a/project/plugins.sbt b/project/plugins.sbt index 0f113b3..2690694 100644 --- a/project/plugins.sbt +++ b/project/plugins.sbt @@ -1,14 +1,14 @@ ThisBuild / libraryDependencySchemes += "org.scala-lang.modules" %% "scala-xml" % VersionScheme.Always -addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.10.0") +addSbtPlugin("se.marcuslonnberg" % "sbt-docker" % "1.11.0") addSbtPlugin("com.github.sbt" % "sbt-git" % "2.0.1") -addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.17") +addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.19") addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0") -addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.0") +addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2") addSbtPlugin("com.thoughtworks.sbt-api-mappings" % "sbt-api-mappings" % "3.0.2") @@ -16,10 +16,10 @@ addSbtPlugin("com.github.sbt" % "sbt-unidoc" % "0.5.0") addSbtPlugin("de.heikoseeberger" % "sbt-header" % "5.10.0") -libraryDependencies += "net.sourceforge.plantuml" % "plantuml" % "1.2023.10" +libraryDependencies += "net.sourceforge.plantuml" % "plantuml" % "1.2024.3" addSbtPlugin("com.github.sbt" % "sbt-ghpages" % "0.8.0") addSbtPlugin("com.github.sbt" % "sbt-pgp" % "2.2.1") -addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.9.21") +addSbtPlugin("org.xerial.sbt" % "sbt-sonatype" % "3.10.0") diff --git a/stream-loader-core/src/main/scala/com/adform/streamloader/sink/batch/storage/TwoPhaseCommitMetadata.scala b/stream-loader-core/src/main/scala/com/adform/streamloader/sink/batch/storage/TwoPhaseCommitMetadata.scala index a30c788..f0b6dbc 100644 --- a/stream-loader-core/src/main/scala/com/adform/streamloader/sink/batch/storage/TwoPhaseCommitMetadata.scala +++ b/stream-loader-core/src/main/scala/com/adform/streamloader/sink/batch/storage/TwoPhaseCommitMetadata.scala @@ -71,7 +71,9 @@ object TwoPhaseCommitMetadata extends Logging { (metadata.getBytes("UTF-8"), false) } if (isCompressed) { - Try(new String(Zstd.decompress(bytes, Zstd.decompressedSize(bytes).toInt), "UTF-8")) match { + Try( + new String(Zstd.decompress(bytes, Zstd.getFrameContentSize(bytes, 0, bytes.length, false).toInt), "UTF-8") + ) match { case Success(decompressed) => TwoPhaseCommitMetadata.tryParseJson[S](decompressed) case Failure(ex) => log.error(ex)(s"Failed decompressing base64 encoded metadata '$metadata'") diff --git a/stream-loader-tests/src/it/resources/logback-test.xml b/stream-loader-tests/src/test/resources/logback-test.xml similarity index 100% rename from stream-loader-tests/src/it/resources/logback-test.xml rename to stream-loader-tests/src/test/resources/logback-test.xml diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/ClickHouseTests.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/ClickHouseIntegrationTests.scala similarity index 97% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/ClickHouseTests.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/ClickHouseIntegrationTests.scala index e6d8b00..6e2c929 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/ClickHouseTests.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/ClickHouseIntegrationTests.scala @@ -15,12 +15,14 @@ import com.adform.streamloader.storage.ClickHouseStorageBackend import com.zaxxer.hikari.{HikariConfig, HikariDataSource} import org.scalatest.concurrent.Eventually import org.scalatest.funspec.AnyFunSpec +import org.scalatest.tags.Slow import org.scalatest.matchers.should.Matchers import org.scalatestplus.scalacheck.Checkers import scala.concurrent.ExecutionContext -class ClickHouseTests +@Slow +class ClickHouseIntegrationTests extends AnyFunSpec with Matchers with Eventually diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/HdfsTests.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/HdfsIntegrationTests.scala similarity index 95% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/HdfsTests.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/HdfsIntegrationTests.scala index 7ef8210..a49875b 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/HdfsTests.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/HdfsIntegrationTests.scala @@ -15,11 +15,13 @@ import com.adform.streamloader.storage.HdfsStorageBackend import org.scalatest.concurrent.Eventually import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers +import org.scalatest.tags.Slow import org.scalatestplus.scalacheck.Checkers import scala.concurrent.ExecutionContext -class HdfsTests +@Slow +class HdfsIntegrationTests extends AnyFunSpec with Checkers with Matchers diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/S3Tests.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/S3IntegrationTests.scala similarity index 96% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/S3Tests.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/S3IntegrationTests.scala index 2c2fc1b..bc4fb89 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/S3Tests.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/S3IntegrationTests.scala @@ -15,11 +15,13 @@ import com.adform.streamloader.storage.{LoaderS3Config, S3StorageBackend} import org.scalatest.concurrent.Eventually import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers +import org.scalatest.tags.Slow import org.scalatestplus.scalacheck.Checkers import scala.concurrent.ExecutionContext -class S3Tests +@Slow +class S3IntegrationTests extends AnyFunSpec with Matchers with Eventually diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/VerticaTests.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/VerticaIntegrationTests.scala similarity index 97% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/VerticaTests.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/VerticaIntegrationTests.scala index 9a3bf89..fc830c7 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/VerticaTests.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/VerticaIntegrationTests.scala @@ -15,11 +15,13 @@ import com.zaxxer.hikari.{HikariConfig, HikariDataSource} import org.scalatest.concurrent.Eventually import org.scalatest.funspec.AnyFunSpec import org.scalatest.matchers.should.Matchers +import org.scalatest.tags.Slow import org.scalatestplus.scalacheck.Checkers import scala.concurrent.ExecutionContext -class VerticaTests +@Slow +class VerticaIntegrationTests extends AnyFunSpec with Matchers with Eventually diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/BasicLoaderBehaviors.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/KafkaRestartBehaviors.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/behaviors/RebalanceBehaviors.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/ClickHouse.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/ClickHouse.scala similarity index 98% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/ClickHouse.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/ClickHouse.scala index 79af660..6c70025 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/ClickHouse.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/ClickHouse.scala @@ -17,7 +17,7 @@ import org.scalatest.{BeforeAndAfterAll, Suite} import scala.jdk.CollectionConverters._ -case class ClickHouseConfig(dbName: String = "default", image: String = "clickhouse/clickhouse-server:23.1.3.5") +case class ClickHouseConfig(dbName: String = "default", image: String = "clickhouse/clickhouse-server:24.2.1.2248") trait ClickHouseTestFixture extends ClickHouse with BeforeAndAfterAll { this: Suite with DockerTestFixture => override def beforeAll(): Unit = { diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Docker.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Docker.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Docker.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Docker.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Hdfs.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Hdfs.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Hdfs.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Hdfs.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Kafka.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Kafka.scala similarity index 98% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Kafka.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Kafka.scala index b88e164..16f58db 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Kafka.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Kafka.scala @@ -26,7 +26,7 @@ import scala.collection.mutable.ArrayBuffer import scala.jdk.CollectionConverters._ import scala.util.Using -case class KafkaConfig(image: String = "bitnami/kafka:3.4.0-debian-11-r21") +case class KafkaConfig(image: String = "bitnami/kafka:3.7.0-debian-12-r0") trait KafkaTestFixture extends Kafka with BeforeAndAfterAll with BeforeAndAfterEach { this: Suite with DockerTestFixture => @@ -154,8 +154,7 @@ trait Kafka { this: Docker => ) .exposedPorts(kafkaPort.toString) .env( - "KAFKA_BROKER_ID=1", - "KAFKA_ENABLE_KRAFT=yes", + "KAFKA_CFG_NODE_ID=1", "KAFKA_CFG_PROCESS_ROLES=broker,controller", "KAFKA_CFG_CONTROLLER_LISTENER_NAMES=CONTROLLER", s"KAFKA_CFG_LISTENERS=PLAINTEXT://:$kafkaPort,CONTROLLER://:$kafkaControllerPort", diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Loaders.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Loaders.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Loaders.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Loaders.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/S3.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/S3.scala similarity index 92% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/S3.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/S3.scala index 700fc44..ac2708d 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/S3.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/S3.scala @@ -20,7 +20,7 @@ import software.amazon.awssdk.services.s3.S3Client import scala.jdk.CollectionConverters._ -case class S3Config(image: String = "minio/minio:RELEASE.2023-02-10T18-48-39Z") +case class S3Config(image: String = "minio/minio:RELEASE.2024-03-05T04-48-44Z") trait S3TestFixture extends S3 with BeforeAndAfterAll { this: Suite with DockerTestFixture => override def beforeAll(): Unit = { @@ -81,7 +81,7 @@ trait S3 { this: Docker => .healthcheck( Healthcheck .builder() - .test(List("CMD-SHELL", s"curl localhost:$s3Port/minio/health/live").asJava) + .test(List("CMD-SHELL", "mc ready local").asJava) .retries(6) .interval(10000000000L) .timeout(1000000000L) @@ -89,8 +89,8 @@ trait S3 { this: Docker => ) .exposedPorts(s3Port.toString) .env( - s"MINIO_ACCESS_KEY=$accessKey", - s"MINIO_SECRET_KEY=$secretKey" + s"MINIO_ROOT_USER=$accessKey", + s"MINIO_ROOT_PASSWORD=$secretKey" ) .cmd("server", "/data") .build() diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Vertica.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Vertica.scala similarity index 98% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Vertica.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Vertica.scala index d3c651c..3c65610 100644 --- a/stream-loader-tests/src/it/scala/com/adform/streamloader/fixtures/Vertica.scala +++ b/stream-loader-tests/src/test/scala/com/adform/streamloader/fixtures/Vertica.scala @@ -23,7 +23,7 @@ case class VerticaConfig( dbName: String = "", user: String = "dbadmin", password: String = "", - image: String = "vertica/vertica-ce:12.0.3-0" + image: String = "vertica/vertica-ce:24.1.0-0" ) trait VerticaTestFixture extends Vertica with BeforeAndAfterAll { this: Suite with DockerTestFixture => diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/storage/ClickHouseStorageBackend.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/ClickHouseStorageBackend.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/storage/ClickHouseStorageBackend.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/storage/ClickHouseStorageBackend.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/storage/HdfsStorageBackend.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/HdfsStorageBackend.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/storage/HdfsStorageBackend.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/storage/HdfsStorageBackend.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/storage/JdbcStorageBackend.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/storage/S3StorageBackend.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/S3StorageBackend.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/storage/S3StorageBackend.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/storage/S3StorageBackend.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/storage/StorageBackend.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/StorageBackend.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/storage/StorageBackend.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/storage/StorageBackend.scala diff --git a/stream-loader-tests/src/it/scala/com/adform/streamloader/storage/VerticaStorageBackend.scala b/stream-loader-tests/src/test/scala/com/adform/streamloader/storage/VerticaStorageBackend.scala similarity index 100% rename from stream-loader-tests/src/it/scala/com/adform/streamloader/storage/VerticaStorageBackend.scala rename to stream-loader-tests/src/test/scala/com/adform/streamloader/storage/VerticaStorageBackend.scala