Merge branch 'master' into locale-fix

.github/workflows/ci.yml

@@ -58,6 +58,37 @@ jobs:
           env-vars-for-codebuild: JAVA_ENV_VERSION
         env:
           JAVA_ENV_VERSION: ${{ matrix.platform.distribution }}${{ matrix.version }}
+  generateTestVectors:
+    name: Generate Vectors
+    runs-on: ubuntu-latest
+    strategy:
+      max-parallel: 1
+      fail-fast: true
+      matrix:
+        platform:
+          - distribution: openjdk
+            image: "aws/codebuild/standard:3.0"
+          - distribution: corretto
+            image: "aws/codebuild/amazonlinux2-x86_64-standard:3.0" # Corretto only runs on AL2
+        version: [ 8, 11 ]
+    steps:
+      - name: Configure AWS Credentials
+        uses: aws-actions/configure-aws-credentials@v2
+        with:
+          role-to-assume: ${{ secrets.CI_AWS_ROLE_ARN }}
+          aws-region: us-west-2
+          role-duration-seconds: 3600
+      - name: Generate Test Vectors
+        uses: aws-actions/aws-codebuild-run-build@v1
+        timeout-minutes: 60
+        with:
+          project-name: AWS-ESDK-Java-CI
+          buildspec-override: codebuild/ci/vectors-generator.yml
+          compute-type-override: BUILD_GENERAL1_LARGE
+          image-override: ${{ matrix.platform.image }}
+          env-vars-for-codebuild: JAVA_ENV_VERSION
+        env:
+          JAVA_ENV_VERSION: ${{ matrix.platform.distribution }}${{ matrix.version }}
   releaseCI:
     name: Release CI
     runs-on: ubuntu-latest

.gitmodules

@@ -4,3 +4,6 @@
 [submodule "aws-encryption-sdk-specification"]
 	path = aws-encryption-sdk-specification
 	url = https://github.com/awslabs/aws-encryption-sdk-specification.git
+[submodule "submodules/MaterialProviders"]
+	path = submodules/MaterialProviders
+	url = https://github.com/aws/aws-cryptographic-material-providers-library-java.git

CHANGELOG.md

@@ -1,5 +1,28 @@
 # Changelog
 
+## [3.0.0](https://github.com/aws/aws-encryption-sdk-java/compare/v2.4.1...v3.0.0) (2023-12-06)
+
+
+### ⚠ BREAKING CHANGES
+
+* This feature update includes a breaking change that requires AWS SDK v2 Java as a hard dependency.
+
+### Features
+
+* Integrate ESDK-Java with AWS Cryptographic Material Providers Library (MPL) for Keyring and CMM Support. ([#1864](https://github.com/aws/aws-encryption-sdk-java/issues/1864)) ([9297e1b](https://github.com/aws/aws-encryption-sdk-java/commit/9297e1bad3860301bcf52b73466ec4b762e4f817))
+
+
+### Maintenance
+
+* **CFN:** remove unused policy ([#1862](https://github.com/aws/aws-encryption-sdk-java/issues/1862)) ([43e078a](https://github.com/aws/aws-encryption-sdk-java/commit/43e078ae1a99c751dc59a354fbc2c7b70b4afd82))
+* **CFN:** update CFN stack to add managed policies to ci and release role ([#1856](https://github.com/aws/aws-encryption-sdk-java/issues/1856)) ([64c970f](https://github.com/aws/aws-encryption-sdk-java/commit/64c970f11ac7e14998c8939f937feeb48eee6c66))
+* **deps:** bump org.apache.commons:commons-lang3 from 3.12.0 to 3.13.0 ([#1717](https://github.com/aws/aws-encryption-sdk-java/issues/1717)) ([ec41ae2](https://github.com/aws/aws-encryption-sdk-java/commit/ec41ae2d492699bfbbc7d9ac477105cd2dccbabd))
+* fix release script ([#1796](https://github.com/aws/aws-encryption-sdk-java/issues/1796)) ([3617210](https://github.com/aws/aws-encryption-sdk-java/commit/3617210c1e60949aacd75f3c436b3b3010d0db71))
+* fix release-ci ([#1883](https://github.com/aws/aws-encryption-sdk-java/issues/1883)) ([92f29d0](https://github.com/aws/aws-encryption-sdk-java/commit/92f29d0c47e09807c46201bb2c100f2172c51294))
+* **tests:** update ESDK tests to replace sun.security.* API ([#1852](https://github.com/aws/aws-encryption-sdk-java/issues/1852)) ([ca4c763](https://github.com/aws/aws-encryption-sdk-java/commit/ca4c763fc8b6bf11df2914d37188aa5db5248042))
+* update release process ([#1888](https://github.com/aws/aws-encryption-sdk-java/issues/1888)) ([f222462](https://github.com/aws/aws-encryption-sdk-java/commit/f2224626ed5bb6cdc90ee6d8f8eb61f07d3422ee))
+* update the javadoc release script ([#1857](https://github.com/aws/aws-encryption-sdk-java/issues/1857)) ([1870a08](https://github.com/aws/aws-encryption-sdk-java/commit/1870a082358d59e32c60d74116d6f43c0efa466b))
+
 ## [2.4.1](https://github.com/aws/aws-encryption-sdk-java/compare/v2.4.0...v2.4.1) (2023-08-09)
 
 

README.md

@@ -58,7 +58,7 @@ You can get the latest release from Maven:
 <dependency>
   <groupId>com.amazonaws</groupId>
   <artifactId>aws-encryption-sdk-java</artifactId>
-  <version>2.4.1</version>
+  <version>3.0.0</version>
 </dependency>
 ```
 

cfn/ci_cd.yml

@@ -91,7 +91,7 @@ Resources:
         ## If this value is 0, greater than 25, or not provided then the full history is downloaded with each build project.
         GitCloneDepth: 0
         GitSubmodulesConfig:
-          FetchSubmodules: false
+          FetchSubmodules: true
         InsecureSsl: false
         ReportBuildStatus: false
         Type: GITHUB

codebuild/ci/release-ci.yml

@@ -17,13 +17,34 @@ phases:
   install:
     runtime-versions:
       java: openjdk11
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
   pre_build:
     commands:
       - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml
       - export CODEARTIFACT_TOKEN=$(aws codeartifact get-authorization-token --domain $DOMAIN --domain-owner $ACCOUNT --query authorizationToken --output text --region ${REGION})
       - export CODEARTIFACT_REPO_URL=https://${DOMAIN}-${ACCOUNT}.d.codeartifact.${REGION}.amazonaws.com/maven/${REPOSITORY}
       - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-CI --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz
       - tar -xvf ~/mvn_gpg.tgz -C ~
+
+      # Build and deploy to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
+
   build:
     commands:
       - VERSION_HASH="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)-$CODEBUILD_RESOLVED_SOURCE_VERSION-$GITHUB_EVENT_NAME"
@@ -37,15 +58,21 @@ phases:
           --package $PACKAGE \
           --versions $VERSION_HASH \
           --region $REGION;
-
+      # Assume Role to access non-prod resources
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
       # See https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
       - echo "Setting version in POM to $VERSION_HASH"
       - mvn versions:set -DnewVersion="$VERSION_HASH" --no-transfer-progress
       - echo "Version is now $(grep version pom.xml | head -n 1 | sed -n 's/[ \t]*<version>\(.*\)<\/version>/\1/p')"
       - |
         mvn deploy \
           -PpublishingCodeArtifact \
-          -Dmaven.test.skip=true \
+          -Pfast-tests-only \
           -DperformRelease \
           -Dgpg.homedir="$HOME/mvn_gpg" \
           -DautoReleaseAfterClose=true \

codebuild/ci/vectors-ci.yml

@@ -4,6 +4,35 @@ phases:
   install:
     runtime-versions:
       java: $JAVA_ENV_VERSION
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
+  pre_build:
+    commands:
+      # Assume Role to access non-prod resources
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+
+      # Build and deploy TestVectors to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
   build:
     commands:
       - mvn install -T 8 -Dgpg.skip=true -ntp "-DtestVectorZip=file://$CODEBUILD_SRC_DIR/src/test/resources/aws-encryption-sdk-test-vectors/vectors/awses-decrypt/python-2.3.0.zip"

codebuild/ci/vectors-generator.yml

@@ -0,0 +1,53 @@
+version: 0.2
+
+phases:
+  install:
+    runtime-versions:
+      java: $JAVA_ENV_VERSION
+    commands:
+      - n 16
+      # Install the Javascript ESDK run test vectors
+      - npm install -g @aws-crypto/integration-node
+
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
+  pre_build:
+    commands:
+      # Assume Role to access non-prod resources
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+
+      # Build and deploy to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
+  build:
+    commands:
+      - export VECTORS_ZIP="$CODEBUILD_SRC_DIR/generated_vectors.zip"
+      # Generate test vectors by encrypting with Keyrings
+      # Ignore Testing coverage requirement by skipping jacoco
+      - mvn -B -ntp install -Dgpg.skip=true -Djacoco.skip=true "-Dtest=TestVectorGenerator" "-DzipFilePath=$VECTORS_ZIP" "-DkeysManifest=$CODEBUILD_SRC_DIR/src/test/resources/keys.json"
+      # Decrypt generated vectors with Javascript ESDK
+      - integration-node decrypt -v $VECTORS_ZIP
+
+      - rm $VECTORS_ZIP
+      # Generate test vectors by encrypting with MasterKeys
+      - mvn -B -ntp install -Dgpg.skip=true -Djacoco.skip=true -Dmasterkey=true "-Dtest=TestVectorGenerator" "-DzipFilePath=$VECTORS_ZIP" "-DkeysManifest=$CODEBUILD_SRC_DIR/src/test/resources/keys.json"
+      # Decrypt generated vectors with Javascript ESDK
+      - integration-node decrypt -v $VECTORS_ZIP

codebuild/release/release-prod.yml

@@ -16,14 +16,40 @@ phases:
   install:
     runtime-versions:
       java: corretto11
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
   pre_build:
     commands:
       - git checkout $BRANCH
       - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml
       - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz
       - tar -xvf ~/mvn_gpg.tgz -C ~
+      # Build and deploy TestVectorsAwsCryptographicMaterialProviders to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
   build:
     commands:
+      # Assume Role to access non-prod resources
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
       - |
         mvn deploy \
           -Ppublishing \

codebuild/release/release-staging.yml

@@ -18,15 +18,42 @@ phases:
   install:
     runtime-versions:
       java: corretto11
+    commands:
+      - git submodule update --init submodules/MaterialProviders
+      # Get Dafny
+      - curl https://github.com/dafny-lang/dafny/releases/download/v4.2.0/dafny-4.2.0-x64-ubuntu-20.04.zip  -L -o dafny.zip
+      - unzip -qq dafny.zip && rm dafny.zip
+      - export PATH="$PWD/dafny:$PATH"
+      # Get Gradle 7.6
+      - curl https://services.gradle.org/distributions/gradle-7.6-all.zip -L -o gradle.zip
+      - unzip -qq gradle.zip && rm gradle.zip
+      - export PATH="$PWD/gradle-7.6/bin:$PATH"
   pre_build:
     commands:
       - export SETTINGS_FILE=$(pwd)/codebuild/release/settings.xml
       - export CODEARTIFACT_TOKEN=$(aws codeartifact get-authorization-token --domain $DOMAIN --domain-owner $ACCOUNT --query authorizationToken --output text --region ${REGION})
       - export CODEARTIFACT_REPO_URL=https://${DOMAIN}-${ACCOUNT}.d.codeartifact.${REGION}.amazonaws.com/maven/${REPOSITORY}
       - aws secretsmanager get-secret-value --region us-west-2 --secret-id Maven-GPG-Keys-Release --query SecretBinary --output text | base64 -d > ~/mvn_gpg.tgz
       - tar -xvf ~/mvn_gpg.tgz -C ~
+      # Build and deploy TestVectorsAwsCryptographicMaterialProviders to maven local
+      - cd submodules/MaterialProviders
+      - git checkout $BRANCH
+      - cd TestVectorsAwsCryptographicMaterialProviders/
+      # This works because `node` is installed by default on GHA runners
+      - CORES=$(node -e 'console.log(os.cpus().length)')
+      - make build_java CORES=$CORES
+      - ./runtimes/java/gradlew -p runtimes/java publishMavenLocalPublicationToMavenLocal
+      - cd $CODEBUILD_SRC_DIR
   build:
     commands:
+      # Assume Role to access non-prod resources
+      - TMP_ROLE=$(aws sts assume-role --role-arn "arn:aws:iam::370957321024:role/GitHub-CI-Public-ESDK-Java-Role-us-west-2" --role-session-name "CB-TestVectorResources")
+      - export TMP_ROLE
+      - export AWS_ACCESS_KEY_ID=$(echo "${TMP_ROLE}" | jq -r '.Credentials.AccessKeyId')
+      - export AWS_SECRET_ACCESS_KEY=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SecretAccessKey')
+      - export AWS_SESSION_TOKEN=$(echo "${TMP_ROLE}" | jq -r '.Credentials.SessionToken')
+      - aws sts get-caller-identity
+
       - VERSION_HASH="$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout)-$CODEBUILD_RESOLVED_SOURCE_VERSION"
 #      See https://docs.aws.amazon.com/codebuild/latest/userguide/build-env-ref-env-vars.html
       - echo "Setting version in POM to $VERSION_HASH"