From b40b8802d51718a48e6c1ea9626c195032cc2c78 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Wed, 28 Aug 2024 12:35:48 -0400 Subject: [PATCH 01/15] chore: Update to Protobuf-Java 4.27.4 --- .kokoro/build.sh | 75 ++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 75 insertions(+) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 351d0d40c..37848fab9 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -27,6 +27,81 @@ source ${scriptDir}/common.sh mvn -version echo ${JOB_TYPE} +# Store the current Java version since the version may change when installing sdk-platform-java +current_java_home=$JAVA_HOME + +# testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java +# needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. +if [ ! -z "${JAVA11_HOME}" ]; then + export JAVA_HOME="${JAVA11_HOME}" + export PATH=${JAVA_HOME}/bin:$PATH +fi + +# Get the current proto runtime version used in this repo +CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | +sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { + //{ + s/\(.*\)<\/version>/\1/p + q + } +}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') +echo "The current proto version is: ${CURRENT_PROTO_VERSION}" + +# Find the latest proto runtime version available +LATEST_PROTO_VERSION="4.27.4" +echo "The latest proto version is: ${LATEST_PROTO_VERSION}" + +# Only reinstall shared-deps again to test for a newer proto version +if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then + pushd /tmp + git clone https://github.com/googleapis/sdk-platform-java.git + pushd sdk-platform-java + pushd gapic-generator-java-pom-parent + sed -i "/.*<\/protobuf.version>/s/\(.*\).*\(<\/protobuf.version>\)/\1${LATEST_PROTO_VERSION}\2/" pom.xml + # sdk-platform-java + popd + + pushd sdk-platform-java-config + # Get current Shared-Deps version in sdk-platform-java + SHARED_DEPS_VERSION=$(mvn -ntp help:effective-pom | + sed -n "/sdk-platform-java-config<\/artifactId>/,/<\/dependency>/ { + //{ + s/\(.*\)<\/version>/\1/p + q + } + }" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + echo "Shared-Deps Version: ${SHARED_DEPS_VERSION}" + # sdk-platform-java + popd + + mvn clean install -q -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -T 1C + # /tmp + popd + + # Back to the original directory of the repo + popd + # Find all the poms with a reference to shared-deps and update to the new local version + poms=($(find . -name pom.xml)) + for pom in "${poms[@]}"; do + if grep -q "sdk-platform-java-config" "${pom}"; then + echo "Updating the pom: ${pom} to use shared-deps version: ${SHARED_DEPS_VERSION}" + sed -i "/sdk-platform-java-config<\/artifactId>/,/<\/parent>/ s/.*<\/version>/$SHARED_DEPS_VERSION<\/version>/" "${pom}" +# xmlstarlet ed --inplace -N x="http://maven.apache.org/POM/4.0.0" \ +# -u "//x:project/x:parent[x:artifactId='sdk-platform-java-config']/x:version" \ +# -v "${SHARED_DEPS_VERSION}" \ +# "${pom}" + fi + done +fi + +# Reset back to the original Java version if changed +export JAVA_HOME="${current_java_home}" +export PATH=${JAVA_HOME}/bin:$PATH + # attempt to install 3 times with exponential backoff (starting with 10 seconds) retry_with_backoff 3 10 \ mvn install -B -V -ntp \ From e9f9be7a1306d8dea416b4de09809d96179ad593 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 9 Sep 2024 15:07:26 -0400 Subject: [PATCH 02/15] chore: Test script --- .kokoro/build.sh | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 37848fab9..562832a9d 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -30,13 +30,6 @@ echo ${JOB_TYPE} # Store the current Java version since the version may change when installing sdk-platform-java current_java_home=$JAVA_HOME -# testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java -# needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. -if [ ! -z "${JAVA11_HOME}" ]; then - export JAVA_HOME="${JAVA11_HOME}" - export PATH=${JAVA_HOME}/bin:$PATH -fi - # Get the current proto runtime version used in this repo CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { @@ -53,6 +46,13 @@ echo "The latest proto version is: ${LATEST_PROTO_VERSION}" # Only reinstall shared-deps again to test for a newer proto version if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then + # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java + # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. + if [ ! -z "${JAVA11_HOME}" ]; then + export JAVA_HOME="${JAVA11_HOME}" + export PATH=${JAVA_HOME}/bin:$PATH + fi + pushd /tmp git clone https://github.com/googleapis/sdk-platform-java.git pushd sdk-platform-java @@ -96,11 +96,11 @@ if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then # "${pom}" fi done -fi -# Reset back to the original Java version if changed -export JAVA_HOME="${current_java_home}" -export PATH=${JAVA_HOME}/bin:$PATH + # Reset back to the original Java version if changed + export JAVA_HOME="${current_java_home}" + export PATH=${JAVA_HOME}/bin:$PATH +fi # attempt to install 3 times with exponential backoff (starting with 10 seconds) retry_with_backoff 3 10 \ From f74fb1c47371ddf8ad600a839527cf3217e64ef4 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 9 Sep 2024 15:14:04 -0400 Subject: [PATCH 03/15] chore: Test script --- .kokoro/build.sh | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 562832a9d..478a034b8 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -48,10 +48,10 @@ echo "The latest proto version is: ${LATEST_PROTO_VERSION}" if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. - if [ ! -z "${JAVA11_HOME}" ]; then - export JAVA_HOME="${JAVA11_HOME}" - export PATH=${JAVA_HOME}/bin:$PATH - fi +# if [ ! -z "${JAVA11_HOME}" ]; then +# export JAVA_HOME="${JAVA11_HOME}" +# export PATH=${JAVA_HOME}/bin:$PATH +# fi pushd /tmp git clone https://github.com/googleapis/sdk-platform-java.git @@ -97,9 +97,9 @@ if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then fi done - # Reset back to the original Java version if changed - export JAVA_HOME="${current_java_home}" - export PATH=${JAVA_HOME}/bin:$PATH +# # Reset back to the original Java version if changed +# export JAVA_HOME="${current_java_home}" +# export PATH=${JAVA_HOME}/bin:$PATH fi # attempt to install 3 times with exponential backoff (starting with 10 seconds) From 4d0814f44ad8527b13017a0336527d7a1755907b Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 9 Sep 2024 15:16:39 -0400 Subject: [PATCH 04/15] chore: Test script --- .kokoro/build.sh | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 478a034b8..6dfd72a69 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -30,6 +30,8 @@ echo ${JOB_TYPE} # Store the current Java version since the version may change when installing sdk-platform-java current_java_home=$JAVA_HOME +java -version + # Get the current proto runtime version used in this repo CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { From 0ff46beac2ed17b88fcec404c4f30ef0ee767d6f Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 9 Sep 2024 15:22:33 -0400 Subject: [PATCH 05/15] chore: Test script --- .kokoro/build.sh | 191 ++++++++++++++++++++++++++--------------------- 1 file changed, 106 insertions(+), 85 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 6dfd72a69..eb93ec67c 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -15,104 +15,116 @@ set -eo pipefail -## Get the directory of the build script -scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) -## cd to the parent directory, i.e. the root of the git repo -cd ${scriptDir}/.. - -# include common functions -source ${scriptDir}/common.sh - -# Print out Maven & Java version -mvn -version -echo ${JOB_TYPE} +function install_new_shared_deps() { + # Store the current Java version since the version may change when installing sdk-platform-java + current_java_home=$JAVA_HOME -# Store the current Java version since the version may change when installing sdk-platform-java -current_java_home=$JAVA_HOME - -java -version - -# Get the current proto runtime version used in this repo -CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | -sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { - //{ - s/\(.*\)<\/version>/\1/p - q - } -}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') -echo "The current proto version is: ${CURRENT_PROTO_VERSION}" - -# Find the latest proto runtime version available -LATEST_PROTO_VERSION="4.27.4" -echo "The latest proto version is: ${LATEST_PROTO_VERSION}" - -# Only reinstall shared-deps again to test for a newer proto version -if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then - # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java - # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. -# if [ ! -z "${JAVA11_HOME}" ]; then -# export JAVA_HOME="${JAVA11_HOME}" -# export PATH=${JAVA_HOME}/bin:$PATH -# fi - - pushd /tmp - git clone https://github.com/googleapis/sdk-platform-java.git - pushd sdk-platform-java - pushd gapic-generator-java-pom-parent - sed -i "/.*<\/protobuf.version>/s/\(.*\).*\(<\/protobuf.version>\)/\1${LATEST_PROTO_VERSION}\2/" pom.xml - # sdk-platform-java - popd - - pushd sdk-platform-java-config - # Get current Shared-Deps version in sdk-platform-java - SHARED_DEPS_VERSION=$(mvn -ntp help:effective-pom | - sed -n "/sdk-platform-java-config<\/artifactId>/,/<\/dependency>/ { + # Get the current proto runtime version used in this repo + CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | + sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { //{ s/\(.*\)<\/version>/\1/p q } }" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - echo "Shared-Deps Version: ${SHARED_DEPS_VERSION}" - # sdk-platform-java - popd + echo "The current proto version is: ${CURRENT_PROTO_VERSION}" + + # Find the latest proto runtime version available + LATEST_PROTO_VERSION="4.27.4" + echo "The latest proto version is: ${LATEST_PROTO_VERSION}" + + # Only reinstall shared-deps again to test for a newer proto version + if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then + # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java + # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. + if [ ! -z "${JAVA11_HOME}" ]; then + export JAVA_HOME="${JAVA11_HOME}" + export PATH=${JAVA_HOME}/bin:$PATH + fi + + pushd /tmp + git clone https://github.com/googleapis/sdk-platform-java.git + pushd sdk-platform-java + pushd gapic-generator-java-pom-parent + sed -i "/.*<\/protobuf.version>/s/\(.*\).*\(<\/protobuf.version>\)/\1${LATEST_PROTO_VERSION}\2/" pom.xml + # sdk-platform-java + popd + + pushd sdk-platform-java-config + # Get current Shared-Deps version in sdk-platform-java + SHARED_DEPS_VERSION=$(mvn -ntp help:effective-pom | + sed -n "/sdk-platform-java-config<\/artifactId>/,/<\/dependency>/ { + //{ + s/\(.*\)<\/version>/\1/p + q + } + }" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') + echo "Shared-Deps Version: ${SHARED_DEPS_VERSION}" + # sdk-platform-java + popd + + mvn clean install -q -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -T 1C + # /tmp + popd + + # Back to the original directory of the repo + popd + # Find all the poms with a reference to shared-deps and update to the new local version + poms=($(find . -name pom.xml)) + for pom in "${poms[@]}"; do + if grep -q "sdk-platform-java-config" "${pom}"; then + echo "Updating the pom: ${pom} to use shared-deps version: ${SHARED_DEPS_VERSION}" + sed -i "/sdk-platform-java-config<\/artifactId>/,/<\/parent>/ s/.*<\/version>/$SHARED_DEPS_VERSION<\/version>/" "${pom}" + # xmlstarlet ed --inplace -N x="http://maven.apache.org/POM/4.0.0" \ + # -u "//x:project/x:parent[x:artifactId='sdk-platform-java-config']/x:version" \ + # -v "${SHARED_DEPS_VERSION}" \ + # "${pom}" + fi + done - mvn clean install -q -ntp \ + # Reset back to the original Java version if changed + export JAVA_HOME="${current_java_home}" + export PATH=${JAVA_HOME}/bin:$PATH + fi + + # attempt to install 3 times with exponential backoff (starting with 10 seconds) + retry_with_backoff 3 10 \ + mvn install -B -V -ntp \ -DskipTests=true \ -Dclirr.skip=true \ -Denforcer.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dgcloud.download.skip=true \ -T 1C - # /tmp - popd - - # Back to the original directory of the repo - popd - # Find all the poms with a reference to shared-deps and update to the new local version - poms=($(find . -name pom.xml)) - for pom in "${poms[@]}"; do - if grep -q "sdk-platform-java-config" "${pom}"; then - echo "Updating the pom: ${pom} to use shared-deps version: ${SHARED_DEPS_VERSION}" - sed -i "/sdk-platform-java-config<\/artifactId>/,/<\/parent>/ s/.*<\/version>/$SHARED_DEPS_VERSION<\/version>/" "${pom}" -# xmlstarlet ed --inplace -N x="http://maven.apache.org/POM/4.0.0" \ -# -u "//x:project/x:parent[x:artifactId='sdk-platform-java-config']/x:version" \ -# -v "${SHARED_DEPS_VERSION}" \ -# "${pom}" - fi - done +} -# # Reset back to the original Java version if changed -# export JAVA_HOME="${current_java_home}" -# export PATH=${JAVA_HOME}/bin:$PATH -fi +function install_shared_deps() { + # attempt to install 3 times with exponential backoff (starting with 10 seconds) + retry_with_backoff 3 10 \ + mvn install -B -V -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dgcloud.download.skip=true \ + -T 1C +} + +## Get the directory of the build script +scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +## cd to the parent directory, i.e. the root of the git repo +cd ${scriptDir}/.. -# attempt to install 3 times with exponential backoff (starting with 10 seconds) -retry_with_backoff 3 10 \ - mvn install -B -V -ntp \ - -DskipTests=true \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dmaven.javadoc.skip=true \ - -Dgcloud.download.skip=true \ - -T 1C +# include common functions +source ${scriptDir}/common.sh + +# Print out Maven & Java version +mvn -version +echo ${JOB_TYPE} # if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then @@ -124,19 +136,23 @@ set +e case ${JOB_TYPE} in test) + install_new_shared_deps echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}" mvn test -B -ntp -Dclirr.skip=true -Denforcer.skip=true ${SUREFIRE_JVM_OPT} RETURN_CODE=$? ;; lint) + install_shared_deps mvn com.coveo:fmt-maven-plugin:check RETURN_CODE=$? ;; javadoc) + install_shared_deps mvn javadoc:javadoc javadoc:test-javadoc RETURN_CODE=$? ;; integration) + install_new_shared_deps mvn -B ${INTEGRATION_TEST_ARGS} \ -ntp \ -Penable-integration-tests \ @@ -148,16 +164,19 @@ integration) RETURN_CODE=$? ;; graalvm) + install_new_shared_deps # Run Unit and Integration Tests with Native Image mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test RETURN_CODE=$? ;; graalvm17) + install_new_shared_deps # Run Unit and Integration Tests with Native Image mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test RETURN_CODE=$? ;; samples) + install_new_shared_deps SAMPLES_DIR=samples # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. if [[ ! -z ${KOKORO_GITHUB_PULL_REQUEST_NUMBER} ]] @@ -188,6 +207,7 @@ samples) fi ;; presubmit-against-pubsublite-samples) + install_new_shared_deps ## cd to the directory one level above the root of the repo cd ${scriptDir}/../.. git clone https://github.com/googleapis/java-pubsublite.git @@ -223,6 +243,7 @@ presubmit-against-pubsublite-samples) fi ;; clirr) + install_shared_deps mvn -B -Denforcer.skip=true clirr:check RETURN_CODE=$? ;; From c9bc1cf9ca4f075fb90d2fad710a5131cefffa8d Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Mon, 9 Sep 2024 15:26:40 -0400 Subject: [PATCH 06/15] chore: Test script --- .kokoro/build.sh | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index eb93ec67c..2a051a994 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -142,12 +142,12 @@ test) RETURN_CODE=$? ;; lint) - install_shared_deps + install_new_shared_deps mvn com.coveo:fmt-maven-plugin:check RETURN_CODE=$? ;; javadoc) - install_shared_deps + install_new_shared_deps mvn javadoc:javadoc javadoc:test-javadoc RETURN_CODE=$? ;; @@ -243,7 +243,7 @@ presubmit-against-pubsublite-samples) fi ;; clirr) - install_shared_deps + install_new_shared_deps mvn -B -Denforcer.skip=true clirr:check RETURN_CODE=$? ;; From 720f3b113c2feeb4353a0963687f61a372a2bb00 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 15:38:29 -0400 Subject: [PATCH 07/15] chore: Add Clirr exemptions for Protobuf 4.27.4 runtime --- .../clirr-ignored-differences.xml | 66 +++++++++++++++++++ 1 file changed, 66 insertions(+) diff --git a/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml b/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml index 223719761..909185fc4 100644 --- a/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml +++ b/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml @@ -16,4 +16,70 @@ com/google/pubsub/v1/*OrBuilder boolean has*(*) + + 7006 + com/google/pubsub/v1/** + * getDefaultInstanceForType() + ** + + + 7006 + com/google/pubsub/v1/** + * addRepeatedField(*) + ** + + + 7006 + com/google/pubsub/v1/** + * clear() + ** + + + 7006 + com/google/pubsub/v1/** + * clear() + ** + + + 7006 + com/google/pubsub/v1/** + * clearField(*) + ** + + + 7006 + com/google/pubsub/v1/** + * clearOneof(*) + ** + + + 7006 + com/google/pubsub/v1/** + * clone() + ** + + + 7006 + com/google/pubsub/v1/** + * mergeUnknownFields(*) + ** + + + 7006 + com/google/pubsub/v1/** + * setField(*) + ** + + + 7006 + com/google/pubsub/v1/** + * setRepeatedField(*) + ** + + + 7006 + com/google/pubsub/v1/** + * setUnknownField(*) + ** + From 5f98d00f297726fd0835b4f777aad9681d0d569a Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 15:45:00 -0400 Subject: [PATCH 08/15] chore: Add Clirr exemptions for Protobuf 4.27.4 runtime --- google-cloud-pubsub/pom.xml | 1 + grpc-google-cloud-pubsub-v1/pom.xml | 1 + proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml | 2 +- proto-google-cloud-pubsub-v1/pom.xml | 1 + 4 files changed, 4 insertions(+), 1 deletion(-) diff --git a/google-cloud-pubsub/pom.xml b/google-cloud-pubsub/pom.xml index af2d034e5..77fb437c7 100644 --- a/google-cloud-pubsub/pom.xml +++ b/google-cloud-pubsub/pom.xml @@ -37,6 +37,7 @@ com.google.protobuf protobuf-java + com.google.api.grpc diff --git a/grpc-google-cloud-pubsub-v1/pom.xml b/grpc-google-cloud-pubsub-v1/pom.xml index 9b5e7af10..e24059c9b 100644 --- a/grpc-google-cloud-pubsub-v1/pom.xml +++ b/grpc-google-cloud-pubsub-v1/pom.xml @@ -28,6 +28,7 @@ com.google.protobuf protobuf-java + com.google.api.grpc diff --git a/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml b/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml index 909185fc4..bfec627ae 100644 --- a/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml +++ b/proto-google-cloud-pubsub-v1/clirr-ignored-differences.xml @@ -79,7 +79,7 @@ 7006 com/google/pubsub/v1/** - * setUnknownField(*) + * setUnknownFields(*) ** diff --git a/proto-google-cloud-pubsub-v1/pom.xml b/proto-google-cloud-pubsub-v1/pom.xml index 4d6605394..6ab83912b 100644 --- a/proto-google-cloud-pubsub-v1/pom.xml +++ b/proto-google-cloud-pubsub-v1/pom.xml @@ -16,6 +16,7 @@ com.google.protobuf protobuf-java + com.google.api.grpc From 35a57bc1281322f4902ccd05f411230b3a5a12a2 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 16:10:02 -0400 Subject: [PATCH 09/15] chore: Test script --- .kokoro/build.sh | 38 +++++++++++++++++++------------------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 2a051a994..df4260341 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -35,12 +35,12 @@ function install_new_shared_deps() { # Only reinstall shared-deps again to test for a newer proto version if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then - # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java - # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. - if [ ! -z "${JAVA11_HOME}" ]; then - export JAVA_HOME="${JAVA11_HOME}" - export PATH=${JAVA_HOME}/bin:$PATH - fi +# # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java +# # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. +# if [ ! -z "${JAVA11_HOME}" ]; then +# export JAVA_HOME="${JAVA11_HOME}" +# export PATH=${JAVA_HOME}/bin:$PATH +# fi pushd /tmp git clone https://github.com/googleapis/sdk-platform-java.git @@ -86,20 +86,20 @@ function install_new_shared_deps() { fi done - # Reset back to the original Java version if changed - export JAVA_HOME="${current_java_home}" - export PATH=${JAVA_HOME}/bin:$PATH - fi +# # Reset back to the original Java version if changed +# export JAVA_HOME="${current_java_home}" +# export PATH=${JAVA_HOME}/bin:$PATH - # attempt to install 3 times with exponential backoff (starting with 10 seconds) - retry_with_backoff 3 10 \ - mvn install -B -V -ntp \ - -DskipTests=true \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dmaven.javadoc.skip=true \ - -Dgcloud.download.skip=true \ - -T 1C + # attempt to install 3 times with exponential backoff (starting with 10 seconds) + retry_with_backoff 3 10 \ + mvn install -B -V -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dgcloud.download.skip=true \ + -T 1C + fi } function install_shared_deps() { From 8c65c40029193b4adec2fea404393c54ef9720fa Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 16:24:03 -0400 Subject: [PATCH 10/15] chore: Test script --- .kokoro/build.sh | 48 ++++++++++++++----------------------- .kokoro/presubmit/clirr.cfg | 2 +- 2 files changed, 19 insertions(+), 31 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index df4260341..86b1cdf42 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -35,12 +35,12 @@ function install_new_shared_deps() { # Only reinstall shared-deps again to test for a newer proto version if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then -# # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java -# # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. -# if [ ! -z "${JAVA11_HOME}" ]; then -# export JAVA_HOME="${JAVA11_HOME}" -# export PATH=${JAVA_HOME}/bin:$PATH -# fi + # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java + # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. + if [ ! -z "${JAVA11_HOME}" ]; then + export JAVA_HOME="${JAVA11_HOME}" + export PATH=${JAVA_HOME}/bin:$PATH + fi pushd /tmp git clone https://github.com/googleapis/sdk-platform-java.git @@ -86,33 +86,21 @@ function install_new_shared_deps() { fi done -# # Reset back to the original Java version if changed -# export JAVA_HOME="${current_java_home}" -# export PATH=${JAVA_HOME}/bin:$PATH - - # attempt to install 3 times with exponential backoff (starting with 10 seconds) - retry_with_backoff 3 10 \ - mvn install -B -V -ntp \ - -DskipTests=true \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dmaven.javadoc.skip=true \ - -Dgcloud.download.skip=true \ - -T 1C + # Reset back to the original Java version if changed + export JAVA_HOME="${current_java_home}" + export PATH=${JAVA_HOME}/bin:$PATH fi } -function install_shared_deps() { - # attempt to install 3 times with exponential backoff (starting with 10 seconds) - retry_with_backoff 3 10 \ - mvn install -B -V -ntp \ - -DskipTests=true \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -Dmaven.javadoc.skip=true \ - -Dgcloud.download.skip=true \ - -T 1C -} +# attempt to install 3 times with exponential backoff (starting with 10 seconds) +retry_with_backoff 3 10 \ + mvn install -B -V -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -Dmaven.javadoc.skip=true \ + -Dgcloud.download.skip=true \ + -T 1C ## Get the directory of the build script scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) diff --git a/.kokoro/presubmit/clirr.cfg b/.kokoro/presubmit/clirr.cfg index ec572442e..9f3ab2405 100644 --- a/.kokoro/presubmit/clirr.cfg +++ b/.kokoro/presubmit/clirr.cfg @@ -4,7 +4,7 @@ env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java8" + value: "gcr.io/cloud-devrel-kokoro-resources/java17" } env_vars: { From cc3c43b5a77e2d1f722c06bee5c5e7685ef64484 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 16:24:46 -0400 Subject: [PATCH 11/15] chore: Test script --- .github/workflows/ci.yaml | 2 +- .kokoro/presubmit/clirr.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index b91fa381f..a542929b6 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -116,7 +116,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 8 + java-version: 17 - run: java -version - run: .kokoro/build.sh env: diff --git a/.kokoro/presubmit/clirr.cfg b/.kokoro/presubmit/clirr.cfg index 9f3ab2405..ec572442e 100644 --- a/.kokoro/presubmit/clirr.cfg +++ b/.kokoro/presubmit/clirr.cfg @@ -4,7 +4,7 @@ env_vars: { key: "TRAMPOLINE_IMAGE" - value: "gcr.io/cloud-devrel-kokoro-resources/java17" + value: "gcr.io/cloud-devrel-kokoro-resources/java8" } env_vars: { From 3dccf7e62481ee018e1f0f1954223d98e0232689 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 16:33:02 -0400 Subject: [PATCH 12/15] chore: Test script --- .kokoro/build.sh | 141 ++++++++++++++++++++++------------------------- 1 file changed, 65 insertions(+), 76 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 86b1cdf42..ddcdc121b 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -15,82 +15,80 @@ set -eo pipefail -function install_new_shared_deps() { - # Store the current Java version since the version may change when installing sdk-platform-java - current_java_home=$JAVA_HOME +# Store the current Java version since the version may change when installing sdk-platform-java +current_java_home=$JAVA_HOME + +# Get the current proto runtime version used in this repo +CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | +sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { + //{ + s/\(.*\)<\/version>/\1/p + q + } +}" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') +echo "The current proto version is: ${CURRENT_PROTO_VERSION}" + +# Find the latest proto runtime version available +LATEST_PROTO_VERSION="4.27.4" +echo "The latest proto version is: ${LATEST_PROTO_VERSION}" + +# Only reinstall shared-deps again to test for a newer proto version +if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then + # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java + # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. + if [ ! -z "${JAVA11_HOME}" ]; then + export JAVA_HOME="${JAVA11_HOME}" + export PATH=${JAVA_HOME}/bin:$PATH + fi - # Get the current proto runtime version used in this repo - CURRENT_PROTO_VERSION=$(mvn -ntp help:effective-pom | - sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { + pushd /tmp + git clone https://github.com/googleapis/sdk-platform-java.git + pushd sdk-platform-java + pushd gapic-generator-java-pom-parent + sed -i "/.*<\/protobuf.version>/s/\(.*\).*\(<\/protobuf.version>\)/\1${LATEST_PROTO_VERSION}\2/" pom.xml + # sdk-platform-java + popd + + pushd sdk-platform-java-config + # Get current Shared-Deps version in sdk-platform-java + SHARED_DEPS_VERSION=$(mvn -ntp help:effective-pom | + sed -n "/sdk-platform-java-config<\/artifactId>/,/<\/dependency>/ { //{ s/\(.*\)<\/version>/\1/p q } }" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - echo "The current proto version is: ${CURRENT_PROTO_VERSION}" - - # Find the latest proto runtime version available - LATEST_PROTO_VERSION="4.27.4" - echo "The latest proto version is: ${LATEST_PROTO_VERSION}" + echo "Shared-Deps Version: ${SHARED_DEPS_VERSION}" + # sdk-platform-java + popd - # Only reinstall shared-deps again to test for a newer proto version - if [[ "${CURRENT_PROTO_VERSION}" != "${LATEST_PROTO_VERSION}" ]]; then - # testing-infra-docker has Java 11 installed in java8 docker container. Use this as sdk-platform-java - # needs Java 11+ to run with GraalVM. For GH actions, JAVA11_HOME does not exist and would skip this. - if [ ! -z "${JAVA11_HOME}" ]; then - export JAVA_HOME="${JAVA11_HOME}" - export PATH=${JAVA_HOME}/bin:$PATH + mvn clean install -q -ntp \ + -DskipTests=true \ + -Dclirr.skip=true \ + -Denforcer.skip=true \ + -T 1C + # /tmp + popd + + # Back to the original directory of the repo + popd + # Find all the poms with a reference to shared-deps and update to the new local version + poms=($(find . -name pom.xml)) + for pom in "${poms[@]}"; do + if grep -q "sdk-platform-java-config" "${pom}"; then + echo "Updating the pom: ${pom} to use shared-deps version: ${SHARED_DEPS_VERSION}" + sed -i "/sdk-platform-java-config<\/artifactId>/,/<\/parent>/ s/.*<\/version>/$SHARED_DEPS_VERSION<\/version>/" "${pom}" +# xmlstarlet ed --inplace -N x="http://maven.apache.org/POM/4.0.0" \ +# -u "//x:project/x:parent[x:artifactId='sdk-platform-java-config']/x:version" \ +# -v "${SHARED_DEPS_VERSION}" \ +# "${pom}" fi + done - pushd /tmp - git clone https://github.com/googleapis/sdk-platform-java.git - pushd sdk-platform-java - pushd gapic-generator-java-pom-parent - sed -i "/.*<\/protobuf.version>/s/\(.*\).*\(<\/protobuf.version>\)/\1${LATEST_PROTO_VERSION}\2/" pom.xml - # sdk-platform-java - popd - - pushd sdk-platform-java-config - # Get current Shared-Deps version in sdk-platform-java - SHARED_DEPS_VERSION=$(mvn -ntp help:effective-pom | - sed -n "/sdk-platform-java-config<\/artifactId>/,/<\/dependency>/ { - //{ - s/\(.*\)<\/version>/\1/p - q - } - }" | sed -e 's/^[[:space:]]*//' -e 's/[[:space:]]*$//') - echo "Shared-Deps Version: ${SHARED_DEPS_VERSION}" - # sdk-platform-java - popd - - mvn clean install -q -ntp \ - -DskipTests=true \ - -Dclirr.skip=true \ - -Denforcer.skip=true \ - -T 1C - # /tmp - popd - - # Back to the original directory of the repo - popd - # Find all the poms with a reference to shared-deps and update to the new local version - poms=($(find . -name pom.xml)) - for pom in "${poms[@]}"; do - if grep -q "sdk-platform-java-config" "${pom}"; then - echo "Updating the pom: ${pom} to use shared-deps version: ${SHARED_DEPS_VERSION}" - sed -i "/sdk-platform-java-config<\/artifactId>/,/<\/parent>/ s/.*<\/version>/$SHARED_DEPS_VERSION<\/version>/" "${pom}" - # xmlstarlet ed --inplace -N x="http://maven.apache.org/POM/4.0.0" \ - # -u "//x:project/x:parent[x:artifactId='sdk-platform-java-config']/x:version" \ - # -v "${SHARED_DEPS_VERSION}" \ - # "${pom}" - fi - done - - # Reset back to the original Java version if changed - export JAVA_HOME="${current_java_home}" - export PATH=${JAVA_HOME}/bin:$PATH - fi -} + # Reset back to the original Java version if changed + export JAVA_HOME="${current_java_home}" + export PATH=${JAVA_HOME}/bin:$PATH +fi # attempt to install 3 times with exponential backoff (starting with 10 seconds) retry_with_backoff 3 10 \ @@ -124,23 +122,19 @@ set +e case ${JOB_TYPE} in test) - install_new_shared_deps echo "SUREFIRE_JVM_OPT: ${SUREFIRE_JVM_OPT}" mvn test -B -ntp -Dclirr.skip=true -Denforcer.skip=true ${SUREFIRE_JVM_OPT} RETURN_CODE=$? ;; lint) - install_new_shared_deps mvn com.coveo:fmt-maven-plugin:check RETURN_CODE=$? ;; javadoc) - install_new_shared_deps mvn javadoc:javadoc javadoc:test-javadoc RETURN_CODE=$? ;; integration) - install_new_shared_deps mvn -B ${INTEGRATION_TEST_ARGS} \ -ntp \ -Penable-integration-tests \ @@ -152,19 +146,16 @@ integration) RETURN_CODE=$? ;; graalvm) - install_new_shared_deps # Run Unit and Integration Tests with Native Image mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test RETURN_CODE=$? ;; graalvm17) - install_new_shared_deps # Run Unit and Integration Tests with Native Image mvn -B ${INTEGRATION_TEST_ARGS} -ntp -Pnative -Penable-integration-tests test RETURN_CODE=$? ;; samples) - install_new_shared_deps SAMPLES_DIR=samples # only run ITs in snapshot/ on presubmit PRs. run ITs in all 3 samples/ subdirectories otherwise. if [[ ! -z ${KOKORO_GITHUB_PULL_REQUEST_NUMBER} ]] @@ -195,7 +186,6 @@ samples) fi ;; presubmit-against-pubsublite-samples) - install_new_shared_deps ## cd to the directory one level above the root of the repo cd ${scriptDir}/../.. git clone https://github.com/googleapis/java-pubsublite.git @@ -231,7 +221,6 @@ presubmit-against-pubsublite-samples) fi ;; clirr) - install_new_shared_deps mvn -B -Denforcer.skip=true clirr:check RETURN_CODE=$? ;; From 955efcd5d72c2f615bccd7fd07c3a837eb074266 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 16:35:39 -0400 Subject: [PATCH 13/15] chore: Test script --- .kokoro/build.sh | 24 ++++++++++++------------ 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index ddcdc121b..562832a9d 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -15,6 +15,18 @@ set -eo pipefail +## Get the directory of the build script +scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) +## cd to the parent directory, i.e. the root of the git repo +cd ${scriptDir}/.. + +# include common functions +source ${scriptDir}/common.sh + +# Print out Maven & Java version +mvn -version +echo ${JOB_TYPE} + # Store the current Java version since the version may change when installing sdk-platform-java current_java_home=$JAVA_HOME @@ -100,18 +112,6 @@ retry_with_backoff 3 10 \ -Dgcloud.download.skip=true \ -T 1C -## Get the directory of the build script -scriptDir=$(realpath $(dirname "${BASH_SOURCE[0]}")) -## cd to the parent directory, i.e. the root of the git repo -cd ${scriptDir}/.. - -# include common functions -source ${scriptDir}/common.sh - -# Print out Maven & Java version -mvn -version -echo ${JOB_TYPE} - # if GOOGLE_APPLICATION_CREDENTIALS is specified as a relative path, prepend Kokoro root directory onto it if [[ ! -z "${GOOGLE_APPLICATION_CREDENTIALS}" && "${GOOGLE_APPLICATION_CREDENTIALS}" != /* ]]; then export GOOGLE_APPLICATION_CREDENTIALS=$(realpath ${KOKORO_GFILE_DIR}/${GOOGLE_APPLICATION_CREDENTIALS}) From 25f08fa012150c2256e7460cdaf527f13db255f6 Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Tue, 10 Sep 2024 16:40:25 -0400 Subject: [PATCH 14/15] chore: Test script --- .github/workflows/ci.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index a542929b6..8aa5b7b9e 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -67,7 +67,7 @@ jobs: - uses: actions/setup-java@v4 with: distribution: temurin - java-version: 8 + java-version: 17 - run: java -version - run: .kokoro/build.bat env: From ebc88cd20a4f6ceb783168e3a077d69ccdef817d Mon Sep 17 00:00:00 2001 From: Lawrence Qiu Date: Thu, 19 Sep 2024 11:24:24 -0400 Subject: [PATCH 15/15] chore: Update to Protobuf 4.28.2 --- .kokoro/build.sh | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.kokoro/build.sh b/.kokoro/build.sh index 562832a9d..e66646e8b 100755 --- a/.kokoro/build.sh +++ b/.kokoro/build.sh @@ -41,7 +41,7 @@ sed -n "/protobuf-java<\/artifactId>/,/<\/dependency>/ { echo "The current proto version is: ${CURRENT_PROTO_VERSION}" # Find the latest proto runtime version available -LATEST_PROTO_VERSION="4.27.4" +LATEST_PROTO_VERSION="4.28.2" echo "The latest proto version is: ${LATEST_PROTO_VERSION}" # Only reinstall shared-deps again to test for a newer proto version