diff --git a/.github/workflows/cdm-integrationtest.yml b/.github/workflows/cdm-integrationtest.yml
index a7015962..a3cac927 100644
--- a/.github/workflows/cdm-integrationtest.yml
+++ b/.github/workflows/cdm-integrationtest.yml
@@ -14,12 +14,12 @@ jobs:
CDM-Integration-Test:
strategy:
matrix:
- jdk: [ 8 ]
+ jdk: [ 11, 17, 21 ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.jdk }}
diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml
index dbaa2df9..9c3da083 100644
--- a/.github/workflows/maven-publish.yml
+++ b/.github/workflows/maven-publish.yml
@@ -18,10 +18,10 @@ jobs:
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK
uses: actions/setup-java@v3
with:
- java-version: '8'
+ java-version: '11'
distribution: 'temurin'
cache: maven
server-id: github # Value of the distributionManagement/repository/id field of the pom.xml
diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml
index b1793e8e..fb6209ad 100644
--- a/.github/workflows/maven.yml
+++ b/.github/workflows/maven.yml
@@ -23,14 +23,14 @@ jobs:
build:
strategy:
matrix:
- jdk: [ 8 ]
+ jdk: [ 11, 17, 21 ]
os: [ ubuntu-latest ]
runs-on: ${{ matrix.os }}
steps:
- uses: actions/checkout@v3
- - name: Set up JDK 8
+ - name: Set up JDK
uses: actions/setup-java@v3
with:
java-version: ${{ matrix.jdk }}
diff --git a/Dockerfile b/Dockerfile
index b2a33144..c70dd81b 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -1,4 +1,4 @@
-FROM eclipse-temurin:8-jammy
+FROM eclipse-temurin:11-jammy
# Download all migration dependencies
RUN mkdir -p /assets/ && cd /assets && \
@@ -9,9 +9,9 @@ RUN mkdir -p /assets/ && cd /assets && \
curl -OL https://downloads.datastax.com/enterprise/cqlsh-astra.tar.gz && \
tar -xzf ./cqlsh-astra.tar.gz && \
rm ./cqlsh-astra.tar.gz && \
- curl -OL https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz && \
- tar -xzf ./spark-3.4.1-bin-hadoop3-scala2.13.tgz && \
- rm ./spark-3.4.1-bin-hadoop3-scala2.13.tgz
+ curl -OL https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz && \
+ tar -xzf ./spark-3.4.2-bin-hadoop3-scala2.13.tgz && \
+ rm ./spark-3.4.2-bin-hadoop3-scala2.13.tgz
RUN apt-get update && apt-get install -y openssh-server vim python3 --no-install-recommends && \
rm -rf /var/lib/apt/lists/* && \
@@ -46,7 +46,7 @@ RUN chmod +x ./get-latest-maven-version.sh && \
rm -rf "$USER_HOME_DIR/.m2"
# Add all migration tools to path
-ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.4.1-bin-hadoop3-scala2.13/bin/"
+ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.4.2-bin-hadoop3-scala2.13/bin/"
EXPOSE 22
diff --git a/README.md b/README.md
index 324c6a8b..b997ef81 100644
--- a/README.md
+++ b/README.md
@@ -7,7 +7,7 @@
Migrate and Validate Tables between Origin and Target Cassandra Clusters.
-> :warning: Please note this job has been tested with spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/)
+> :warning: Please note this job has been tested with spark version [3.4.2](https://archive.apache.org/dist/spark/spark-3.4.2/)
## Install as a Container
- Get the latest image that includes all dependencies from [DockerHub](https://hub.docker.com/r/datastax/cassandra-data-migrator)
@@ -17,11 +17,11 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters.
- Download the latest jar file from the GitHub [packages area here](https://github.com/datastax/cassandra-data-migrator/packages/1832128)
### Prerequisite
-- Install Java8 as spark binaries are compiled with it.
-- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
+- Install **Java11** (minimum) as Spark binaries are compiled with it.
+- Install Spark version [`3.4.2`](https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: -
```
-wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz
-tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz
+wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz
+tar -xvzf spark-3.4.2-bin-hadoop3-scala2.13.tgz
```
> :warning: If the above Spark and Scala version is not properly installed, you'll then see a similar exception like below when running the CDM jobs,
diff --git a/RELEASE.md b/RELEASE.md
index f792ea4f..e8965c57 100644
--- a/RELEASE.md
+++ b/RELEASE.md
@@ -1,4 +1,8 @@
# Release Notes
+## [4.1.12] - 2024-01-22
+- Upgraded to use Spark `3.4.2`.
+- Added Java `11` as the minimally required pre-requisite to run CDM jobs.
+
## [4.1.9 to 4.1.11] - 2023-12-11
- Code test & coverage changes
diff --git a/SIT/environment.sh b/SIT/environment.sh
index 21c31517..4d55898f 100755
--- a/SIT/environment.sh
+++ b/SIT/environment.sh
@@ -69,7 +69,8 @@ fi
# These variables are hard-coded for now
SUBNET=$(echo ${CIDR} | cut -d. -f1-3)
CASS_VERSION=4
-CDM_VERSION=latest
+#CDM_VERSION=latest
+CDM_VERSION=ms-jdk17
#==============================================================================================================================
# Helper Functions
#==============================================================================================================================
diff --git a/pom.xml b/pom.xml
index df86280d..336cd333 100644
--- a/pom.xml
+++ b/pom.xml
@@ -17,6 +17,9 @@
4.11.0
4.17.0
2.21.0
+ 11
+ 11
+ 11
@@ -182,7 +185,7 @@
scm:git:git@github.com:datastax/cassandra-data-migrator.git
scm:git:git@github.com:datastax/cassandra-data-migrator.git
https://github.com/datastax/cassandra-data-migrator
- 4.1.8
+ 4.1.11
@@ -246,10 +249,11 @@
org.apache.maven.plugins
maven-compiler-plugin
- 3.10.1
+ 3.12.1
- 1.8
- 1.8
+ ${maven.compiler.source}
+ ${maven.compiler.target}
+ ${maven.compiler.release}
diff --git a/src/resources/migrate_data.sh b/src/resources/migrate_data.sh
index 75aa7ba6..3147cba6 100644
--- a/src/resources/migrate_data.sh
+++ b/src/resources/migrate_data.sh
@@ -35,7 +35,7 @@
###########################################################################################################################
# Path to spark-submit
-SPARK_SUBMIT=/home/ubuntu/spark-3.4.1-bin-hadoop3-scala2.13/bin/spark-submit
+SPARK_SUBMIT=/home/ubuntu/spark-3.4.2-bin-hadoop3-scala2.13/bin/spark-submit
# Path to spark configuration for the table
PROPS_FILE=/home/ubuntu/sparkConf.properties