Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Replace spotify docker-client with forked library #44

Merged
merged 1 commit into from
Apr 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
30 changes: 16 additions & 14 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -49,13 +49,13 @@ lazy val `stream-loader-core` = project
"org.scala-lang" % "scala-reflect" % scalaVersion.value,
"org.apache.kafka" % "kafka-clients" % "3.7.0",
"org.log4s" %% "log4s" % "1.10.0",
"org.apache.commons" % "commons-compress" % "1.26.0",
"org.apache.commons" % "commons-compress" % "1.26.1",
"org.xerial.snappy" % "snappy-java" % "1.1.10.5",
"org.lz4" % "lz4-java" % "1.8.0",
"com.github.luben" % "zstd-jni" % "1.5.5-6",
"com.univocity" % "univocity-parsers" % "2.9.1",
"org.json4s" %% "json4s-native" % "4.0.7",
"io.micrometer" % "micrometer-core" % "1.12.3",
"io.micrometer" % "micrometer-core" % "1.12.4",
"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
"org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test",
"org.scalacheck" %% "scalacheck" % scalaCheckVersion % "test",
Expand Down Expand Up @@ -100,10 +100,10 @@ lazy val `stream-loader-s3` = project
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"software.amazon.awssdk" % "s3" % "2.25.1",
"software.amazon.awssdk" % "s3" % "2.25.22",
"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
"com.amazonaws" % "aws-java-sdk-s3" % "1.12.671" % "test",
"org.gaul" % "s3proxy" % "2.1.0" % "test"
"com.amazonaws" % "aws-java-sdk-s3" % "1.12.692" % "test",
"org.gaul" % "s3proxy" % "2.2.0" % "test"
)
)

Expand Down Expand Up @@ -141,15 +141,17 @@ lazy val `stream-loader-tests` = project
.settings(commonSettings)
.settings(
libraryDependencies ++= Seq(
"com.typesafe" % "config" % "1.4.3",
"ch.qos.logback" % "logback-classic" % "1.5.3",
"com.zaxxer" % "HikariCP" % "5.1.0",
"com.vertica.jdbc" % "vertica-jdbc" % verticaVersion,
"org.scalacheck" %% "scalacheck" % scalaCheckVersion,
"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
"org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test",
("com.spotify" % "docker-client" % "8.16.0" classifier "shaded") % "test",
"org.slf4j" % "log4j-over-slf4j" % "2.0.12" % "test"
"com.typesafe" % "config" % "1.4.3",
"ch.qos.logback" % "logback-classic" % "1.5.3",
"com.zaxxer" % "HikariCP" % "5.1.0",
"com.vertica.jdbc" % "vertica-jdbc" % verticaVersion,
"org.scalacheck" %% "scalacheck" % scalaCheckVersion,
"org.scalatest" %% "scalatest" % scalaTestVersion % "test",
"org.scalatestplus" %% "scalacheck-1-17" % scalaCheckTestVersion % "test",
"org.mandas" % "docker-client" % "7.0.8" % "test",
"org.jboss.resteasy" % "resteasy-client" % "6.2.8.Final" % "test",
"com.fasterxml.jackson.jakarta.rs" % "jackson-jakarta-rs-json-provider" % "2.17.0" % "test",
"org.slf4j" % "log4j-over-slf4j" % "2.0.12" % "test"
),
inConfig(IntegrationTest)(Defaults.testTasks),
publish := {},
Expand Down
2 changes: 1 addition & 1 deletion project/plugins.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ addSbtPlugin("com.github.sbt" % "sbt-git" % "2.0.1")

addSbtPlugin("org.xerial.sbt" % "sbt-pack" % "0.19")

addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.11.0")
addSbtPlugin("com.eed3si9n" % "sbt-buildinfo" % "0.12.0")

addSbtPlugin("org.scalameta" % "sbt-scalafmt" % "2.5.2")

Expand Down
2 changes: 1 addition & 1 deletion stream-loader-tests/src/test/resources/logback-test.xml
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
</appender>

<logger name="org.apache" level="ERROR" />
<logger name="com.spotify.docker" level="ERROR" />
<logger name="org.mandas.docker" level="ERROR" />
<logger name="com.zaxxer.hikari" level="ERROR" />

<logger name="com.adform.streamloader.clickhouse" level="WARN" />
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ package com.adform.streamloader.fixtures

import java.time.Duration

import com.spotify.docker.client.messages.ContainerConfig.Healthcheck
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.messages.ContainerConfig.Healthcheck
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import org.log4s.getLogger
import org.scalatest.{BeforeAndAfterAll, Suite}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,9 +11,10 @@ package com.adform.streamloader.fixtures
import java.time.Duration
import java.util.UUID

import com.spotify.docker.client.DefaultDockerClient
import com.spotify.docker.client.DockerClient.RemoveContainerParam
import com.spotify.docker.client.messages.{ContainerConfig, NetworkConfig, PortBinding}
import org.mandas.docker.client.DefaultDockerClient
import org.mandas.docker.client.DockerClient.RemoveContainerParam
import org.mandas.docker.client.builder.resteasy.ResteasyDockerClientBuilder
import org.mandas.docker.client.messages.{ContainerConfig, NetworkConfig, PortBinding}
import org.log4s.getLogger
import org.scalatest.{BeforeAndAfterAll, Suite}

Expand Down Expand Up @@ -52,7 +53,7 @@ trait Docker {

private var network: DockerNetwork = _

val docker: DefaultDockerClient = DefaultDockerClient.fromEnv().build()
val docker: DefaultDockerClient = new ResteasyDockerClientBuilder().fromEnv().build()
val dockerSandboxId: String = UUID.randomUUID().toString

val healthCheckTimeout: Duration = Duration.ofSeconds(60)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ package com.adform.streamloader.fixtures

import java.net.URI

import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import org.apache.hadoop.conf.Configuration
import org.apache.hadoop.fs.FileSystem
import org.log4s._
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import java.util.concurrent.atomic.AtomicBoolean
import java.util.concurrent.{TimeUnit, TimeoutException}
import java.util.{Properties, UUID}

import com.spotify.docker.client.messages.ContainerConfig.Healthcheck
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.messages.ContainerConfig.Healthcheck
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import org.apache.kafka.clients.admin.{AdminClient, AdminClientConfig, NewTopic}
import org.apache.kafka.clients.consumer.{ConsumerConfig, ConsumerRecord, KafkaConsumer}
import org.apache.kafka.clients.producer.{KafkaProducer, ProducerConfig}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -10,8 +10,8 @@ package com.adform.streamloader.fixtures

import java.net.URI

import com.spotify.docker.client.messages.ContainerConfig.Healthcheck
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.messages.ContainerConfig.Healthcheck
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import org.log4s.getLogger
import org.scalatest.{BeforeAndAfterAll, Suite}
import software.amazon.awssdk.auth.credentials.AwsBasicCredentials
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ package com.adform.streamloader.fixtures
import java.sql.DriverManager
import java.time.Duration

import com.spotify.docker.client.messages.ContainerConfig.Healthcheck
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.messages.ContainerConfig.Healthcheck
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import org.log4s.getLogger
import org.scalatest.{BeforeAndAfterAll, Suite}

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import com.adform.streamloader.source.KafkaContext
import com.adform.streamloader.util.Retry
import com.adform.streamloader.{BuildInfo, Loader}
import com.clickhouse.jdbc.ClickHouseArray
import com.spotify.docker.client.DockerClient
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.DockerClient
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import com.zaxxer.hikari.HikariConfig
import org.apache.kafka.common.TopicPartition
import org.scalacheck.Arbitrary
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,8 +15,8 @@ import com.adform.streamloader.model.{ExampleMessage, StreamPosition}
import com.adform.streamloader.sink.file.{FilePathFormatter, TimePartitioningFilePathFormatter}
import com.adform.streamloader.{BuildInfo, Loader}
import com.sksamuel.avro4s.{RecordFormat, ScalePrecision}
import com.spotify.docker.client.DockerClient
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.DockerClient
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import org.apache.avro.generic.{GenericData, GenericRecord}
import org.apache.hadoop.fs.{FileSystem, LocatedFileStatus, Path, RemoteIterator}
import org.apache.kafka.common.TopicPartition
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,8 +13,8 @@ import com.adform.streamloader.model.{StreamPosition, StringMessage, Timestamp}
import com.adform.streamloader.s3.S3FileStorage
import com.adform.streamloader.sink.file.{FilePathFormatter, TimePartitioningFilePathFormatter}
import com.adform.streamloader.{BuildInfo, Loader}
import com.spotify.docker.client.DockerClient
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.DockerClient
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import org.apache.kafka.common.TopicPartition
import org.scalacheck.Arbitrary
import software.amazon.awssdk.core.ResponseInputStream
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -12,7 +12,7 @@ import java.util.Properties
import com.adform.streamloader.fixtures.{Container, ContainerWithEndpoint, DockerNetwork}
import com.adform.streamloader.model.{StorageMessage, StreamPosition}
import com.adform.streamloader.source.{KafkaContext, LockingKafkaContext}
import com.spotify.docker.client.DockerClient
import org.mandas.docker.client.DockerClient
import org.apache.kafka.clients.consumer.{ConsumerConfig, KafkaConsumer, OffsetAndMetadata, OffsetAndTimestamp}
import org.apache.kafka.common.TopicPartition
import org.scalacheck.rng.Seed
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,8 +17,8 @@ import com.adform.streamloader.sink.batch.storage.RecordBatchStorage
import com.adform.streamloader.util.Retry
import com.adform.streamloader.vertica.{ExternalOffsetVerticaFileStorage, InRowOffsetVerticaFileStorage}
import com.adform.streamloader.{BuildInfo, Loader}
import com.spotify.docker.client.DockerClient
import com.spotify.docker.client.messages.{ContainerConfig, HostConfig}
import org.mandas.docker.client.DockerClient
import org.mandas.docker.client.messages.{ContainerConfig, HostConfig}
import com.zaxxer.hikari.HikariConfig

import javax.sql.DataSource
Expand Down