diff --git a/.github/workflows/cdm-integrationtest.yml b/.github/workflows/cdm-integrationtest.yml index a7015962..a3cac927 100644 --- a/.github/workflows/cdm-integrationtest.yml +++ b/.github/workflows/cdm-integrationtest.yml @@ -14,12 +14,12 @@ jobs: CDM-Integration-Test: strategy: matrix: - jdk: [ 8 ] + jdk: [ 11, 17, 21 ] os: [ ubuntu-latest ] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} diff --git a/.github/workflows/maven-publish.yml b/.github/workflows/maven-publish.yml index dbaa2df9..9c3da083 100644 --- a/.github/workflows/maven-publish.yml +++ b/.github/workflows/maven-publish.yml @@ -18,10 +18,10 @@ jobs: steps: - uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK uses: actions/setup-java@v3 with: - java-version: '8' + java-version: '11' distribution: 'temurin' cache: maven server-id: github # Value of the distributionManagement/repository/id field of the pom.xml diff --git a/.github/workflows/maven.yml b/.github/workflows/maven.yml index b1793e8e..fb6209ad 100644 --- a/.github/workflows/maven.yml +++ b/.github/workflows/maven.yml @@ -23,14 +23,14 @@ jobs: build: strategy: matrix: - jdk: [ 8 ] + jdk: [ 11, 17, 21 ] os: [ ubuntu-latest ] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 - - name: Set up JDK 8 + - name: Set up JDK uses: actions/setup-java@v3 with: java-version: ${{ matrix.jdk }} diff --git a/Dockerfile b/Dockerfile index b2a33144..c70dd81b 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM eclipse-temurin:8-jammy +FROM eclipse-temurin:11-jammy # Download all migration dependencies RUN mkdir -p /assets/ && cd /assets && \ @@ -9,9 +9,9 @@ RUN mkdir -p /assets/ && cd /assets && \ curl -OL https://downloads.datastax.com/enterprise/cqlsh-astra.tar.gz && \ tar -xzf ./cqlsh-astra.tar.gz && \ rm ./cqlsh-astra.tar.gz && \ - curl -OL https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz && \ - tar -xzf ./spark-3.4.1-bin-hadoop3-scala2.13.tgz && \ - rm ./spark-3.4.1-bin-hadoop3-scala2.13.tgz + curl -OL https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz && \ + tar -xzf ./spark-3.4.2-bin-hadoop3-scala2.13.tgz && \ + rm ./spark-3.4.2-bin-hadoop3-scala2.13.tgz RUN apt-get update && apt-get install -y openssh-server vim python3 --no-install-recommends && \ rm -rf /var/lib/apt/lists/* && \ @@ -46,7 +46,7 @@ RUN chmod +x ./get-latest-maven-version.sh && \ rm -rf "$USER_HOME_DIR/.m2" # Add all migration tools to path -ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.4.1-bin-hadoop3-scala2.13/bin/" +ENV PATH="${PATH}:/assets/dsbulk/bin/:/assets/cqlsh-astra/bin/:/assets/spark-3.4.2-bin-hadoop3-scala2.13/bin/" EXPOSE 22 diff --git a/README.md b/README.md index 324c6a8b..b997ef81 100644 --- a/README.md +++ b/README.md @@ -7,7 +7,7 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters. -> :warning: Please note this job has been tested with spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/) +> :warning: Please note this job has been tested with spark version [3.4.2](https://archive.apache.org/dist/spark/spark-3.4.2/) ## Install as a Container - Get the latest image that includes all dependencies from [DockerHub](https://hub.docker.com/r/datastax/cassandra-data-migrator) @@ -17,11 +17,11 @@ Migrate and Validate Tables between Origin and Target Cassandra Clusters. - Download the latest jar file from the GitHub [packages area here](https://github.com/datastax/cassandra-data-migrator/packages/1832128) ### Prerequisite -- Install Java8 as spark binaries are compiled with it. -- Install Spark version [3.4.1](https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: - +- Install **Java11** (minimum) as Spark binaries are compiled with it. +- Install Spark version [`3.4.2`](https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz) on a single VM (no cluster necessary) where you want to run this job. Spark can be installed by running the following: - ``` -wget https://archive.apache.org/dist/spark/spark-3.4.1/spark-3.4.1-bin-hadoop3-scala2.13.tgz -tar -xvzf spark-3.4.1-bin-hadoop3-scala2.13.tgz +wget https://archive.apache.org/dist/spark/spark-3.4.2/spark-3.4.2-bin-hadoop3-scala2.13.tgz +tar -xvzf spark-3.4.2-bin-hadoop3-scala2.13.tgz ``` > :warning: If the above Spark and Scala version is not properly installed, you'll then see a similar exception like below when running the CDM jobs, diff --git a/RELEASE.md b/RELEASE.md index f792ea4f..e8965c57 100644 --- a/RELEASE.md +++ b/RELEASE.md @@ -1,4 +1,8 @@ # Release Notes +## [4.1.12] - 2024-01-22 +- Upgraded to use Spark `3.4.2`. +- Added Java `11` as the minimally required pre-requisite to run CDM jobs. + ## [4.1.9 to 4.1.11] - 2023-12-11 - Code test & coverage changes diff --git a/SIT/environment.sh b/SIT/environment.sh index 21c31517..4d55898f 100755 --- a/SIT/environment.sh +++ b/SIT/environment.sh @@ -69,7 +69,8 @@ fi # These variables are hard-coded for now SUBNET=$(echo ${CIDR} | cut -d. -f1-3) CASS_VERSION=4 -CDM_VERSION=latest +#CDM_VERSION=latest +CDM_VERSION=ms-jdk17 #============================================================================================================================== # Helper Functions #============================================================================================================================== diff --git a/pom.xml b/pom.xml index df86280d..336cd333 100644 --- a/pom.xml +++ b/pom.xml @@ -17,6 +17,9 @@ 4.11.0 4.17.0 2.21.0 + 11 + 11 + 11 @@ -182,7 +185,7 @@ scm:git:git@github.com:datastax/cassandra-data-migrator.git scm:git:git@github.com:datastax/cassandra-data-migrator.git https://github.com/datastax/cassandra-data-migrator - 4.1.8 + 4.1.11 @@ -246,10 +249,11 @@ org.apache.maven.plugins maven-compiler-plugin - 3.10.1 + 3.12.1 - 1.8 - 1.8 + ${maven.compiler.source} + ${maven.compiler.target} + ${maven.compiler.release} diff --git a/src/resources/migrate_data.sh b/src/resources/migrate_data.sh index 75aa7ba6..3147cba6 100644 --- a/src/resources/migrate_data.sh +++ b/src/resources/migrate_data.sh @@ -35,7 +35,7 @@ ########################################################################################################################### # Path to spark-submit -SPARK_SUBMIT=/home/ubuntu/spark-3.4.1-bin-hadoop3-scala2.13/bin/spark-submit +SPARK_SUBMIT=/home/ubuntu/spark-3.4.2-bin-hadoop3-scala2.13/bin/spark-submit # Path to spark configuration for the table PROPS_FILE=/home/ubuntu/sparkConf.properties