Skip to content

Commit

Permalink
Creation of sbt module spark-kinesis-dynamodb
Browse files Browse the repository at this point in the history
  • Loading branch information
julienrf committed Apr 16, 2024
1 parent bf7162c commit b9a15b3
Showing 1 changed file with 7 additions and 3 deletions.
10 changes: 7 additions & 3 deletions build.sbt
Original file line number Diff line number Diff line change
Expand Up @@ -11,6 +11,11 @@ inThisBuild(
)
)

// Adaptation of spark-streaming-kinesis-asl to work with DynamoDB Streams
lazy val `spark-kinesis-dynamodb` = project.in(file("spark-kinesis-dynamodb")).settings(
libraryDependencies += "org.apache.spark" %% "spark-streaming-kinesis-asl" % sparkVersion,
)

lazy val migrator = (project in file("migrator")).settings(
name := "scylla-migrator",
version := "0.0.1",
Expand All @@ -28,7 +33,6 @@ lazy val migrator = (project in file("migrator")).settings(
libraryDependencies ++= Seq(
"org.apache.spark" %% "spark-streaming" % sparkVersion % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
"org.apache.spark" %% "spark-sql" % sparkVersion % "provided",
"com.amazonaws" % "aws-java-sdk-sts" % awsSdkVersion,
"com.amazonaws" % "aws-java-sdk-dynamodb" % awsSdkVersion,
("com.amazonaws" % "dynamodb-streams-kinesis-adapter" % "1.5.2")
Expand Down Expand Up @@ -70,7 +74,7 @@ lazy val migrator = (project in file("migrator")).settings(
else
Some("releases" at nexus + "service/local/staging/deploy/maven2")
}
)
).dependsOn(`spark-kinesis-dynamodb`)

lazy val tests = project.in(file("tests")).settings(
libraryDependencies ++= Seq(
Expand All @@ -85,4 +89,4 @@ lazy val tests = project.in(file("tests")).settings(
)

lazy val root = project.in(file("."))
.aggregate(migrator, tests)
.aggregate(migrator, `spark-kinesis-dynamodb`, tests)

0 comments on commit b9a15b3

Please sign in to comment.