diff --git a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml index 8b1efeb01e..ed15f58201 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kms.yaml @@ -18,6 +18,7 @@ substitutions: _JAVA_SHARED_CONFIG_VERSION: '1.11.3' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ diff --git a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml index 9aab5964fd..a2455f3544 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-a-downstream-kmsinventory.yaml @@ -18,6 +18,7 @@ substitutions: _JAVA_SHARED_CONFIG_VERSION: '1.11.3' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ diff --git a/.cloudbuild/graalvm/cloudbuild-test-a.yaml b/.cloudbuild/graalvm/cloudbuild-test-a.yaml index 8ce2ac9a59..89df4c6f53 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-a.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-a.yaml @@ -14,10 +14,11 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.39.0' # {x-version-update:google-cloud-shared-dependencies:current} + _SHARED_DEPENDENCIES_VERSION: '3.39.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} _JAVA_SHARED_CONFIG_VERSION: '1.11.3' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ @@ -41,4 +42,5 @@ steps: entrypoint: bash args: [ './.kokoro/presubmit/showcase-native.sh' ] waitFor: [ "graalvm-a-build" ] - id: native-showcase \ No newline at end of file + id: native-showcase + diff --git a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml index 7d5bdc407f..e0ab82af8d 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kms.yaml @@ -18,6 +18,7 @@ substitutions: _JAVA_SHARED_CONFIG_VERSION: '1.11.3' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ diff --git a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml index 4f25eeef12..a0d0fecdc6 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-b-downstream-kmsinventory.yaml @@ -18,6 +18,7 @@ substitutions: _JAVA_SHARED_CONFIG_VERSION: '1.11.3' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ diff --git a/.cloudbuild/graalvm/cloudbuild-test-b.yaml b/.cloudbuild/graalvm/cloudbuild-test-b.yaml index 36fada606d..211b91a9df 100644 --- a/.cloudbuild/graalvm/cloudbuild-test-b.yaml +++ b/.cloudbuild/graalvm/cloudbuild-test-b.yaml @@ -14,10 +14,11 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.39.0' # {x-version-update:google-cloud-shared-dependencies:current} + _SHARED_DEPENDENCIES_VERSION: '3.39.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} _JAVA_SHARED_CONFIG_VERSION: '1.11.3' options: machineType: 'E2_HIGHCPU_8' + logging: CLOUD_LOGGING_ONLY steps: - name: gcr.io/cloud-builders/docker args: [ diff --git a/.cloudbuild/graalvm/cloudbuild.yaml b/.cloudbuild/graalvm/cloudbuild.yaml index 34572e8c0f..e9c573815b 100644 --- a/.cloudbuild/graalvm/cloudbuild.yaml +++ b/.cloudbuild/graalvm/cloudbuild.yaml @@ -14,7 +14,7 @@ timeout: 7200s # 2 hours substitutions: - _SHARED_DEPENDENCIES_VERSION: '3.39.0' # {x-version-update:google-cloud-shared-dependencies:current} + _SHARED_DEPENDENCIES_VERSION: '3.39.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current} _JAVA_SHARED_CONFIG_VERSION: '1.11.3' steps: # GraalVM A build @@ -45,6 +45,8 @@ steps: id: graalvm-b-build waitFor: [ "-" ] +options: + logging: CLOUD_LOGGING_ONLY images: - gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION} diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml index 81691fa572..1499c9053b 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml @@ -15,7 +15,7 @@ timeout: 7200s # 2 hours substitutions: _IMAGE_NAME: "us-docker.pkg.dev/java-hermetic-build-prod/private-resources/java-library-generation" - _GAPIC_GENERATOR_JAVA_VERSION: '2.46.2-SNAPSHOT' # {x-version-update:gapic-generator-java:current} + _GAPIC_GENERATOR_JAVA_VERSION: '2.49.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current} _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}" _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest" _VERSIONED_IMAGE_ID: "${_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}" @@ -30,6 +30,8 @@ steps: "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."] id: library-generation-build waitFor: ["-"] + env: + - 'DOCKER_BUILDKIT=1' options: logging: CLOUD_LOGGING_ONLY diff --git a/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml b/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml index a45b02b9bd..359c021f69 100644 --- a/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml +++ b/.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml @@ -15,7 +15,7 @@ timeout: 7200s # 2 hours substitutions: _IMAGE_NAME: "gcr.io/cloud-devrel-public-resources/java-library-generation" - _GAPIC_GENERATOR_JAVA_VERSION: '2.49.0' # {x-version-update:gapic-generator-java:current} + _GAPIC_GENERATOR_JAVA_VERSION: '2.49.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current} _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}" _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest" _VERSIONED_IMAGE_ID: "${_IMAGE_NAME}:${_GAPIC_GENERATOR_JAVA_VERSION}" @@ -30,6 +30,8 @@ steps: "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."] id: library-generation-build waitFor: ["-"] + env: + - 'DOCKER_BUILDKIT=1' options: logging: CLOUD_LOGGING_ONLY diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index 13fd53dd18..0c8442cb93 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -14,35 +14,72 @@ # install gapic-generator-java in a separate layer so we don't overload the image # with the transferred source code and jars -FROM gcr.io/cloud-devrel-public-resources/java21@sha256:2ceff5eeea72260258df56d42e44ed413e52ee421c1b77393c5f2c9c4d7c41da AS ggj-build + +# 3.9.9-eclipse-temurin-11-alpine +FROM docker.io/library/maven@sha256:006d25558f9d5244ed55b5d2bd8eaf34d883e447d0c4b940e67b9f44d21167bf AS ggj-build WORKDIR /sdk-platform-java COPY . . # {x-version-update-start:gapic-generator-java:current} -ENV DOCKER_GAPIC_GENERATOR_VERSION="2.49.0" +ENV DOCKER_GAPIC_GENERATOR_VERSION="2.49.1-SNAPSHOT" # {x-version-update-end} RUN mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip RUN cp "/root/.m2/repository/com/google/api/gapic-generator-java/${DOCKER_GAPIC_GENERATOR_VERSION}/gapic-generator-java-${DOCKER_GAPIC_GENERATOR_VERSION}.jar" \ "./gapic-generator-java.jar" -# build from the root of this repo: -FROM gcr.io/cloud-devrel-public-resources/python@sha256:9c5ea427632f195ad164054831968389d86fdde4a15abca651f3fcb2a71268cb +# alpine:3.20.3 +FROM docker.io/library/alpine@sha256:beefdbd8a1da6d2915566fde36db9db0b524eb737fc57cd1367effd16dc0d06d as glibc-compat + +RUN apk add git sudo +# This SHA is the latest known-to-work version of this binary compatibility tool +ARG GLIB_MUS_SHA=7717dd4dc26377dd9cedcc92b72ebf35f9e68a2d +WORKDIR /home + +# Install compatibility layer to run glibc-based programs (such as the +# grpc plugin). +# Alpine, by default, only supports musl-based binaries, and there is no public +# downloadable distribution of the grpc plugin that is Alpine (musl) compatible. +# This is one of the recommended approaches to ensure glibc-compatibility +# as per https://wiki.alpinelinux.org/wiki/Running_glibc_programs +RUN git clone https://gitlab.com/manoel-linux1/GlibMus-HQ.git +WORKDIR /home/GlibMus-HQ +# We lock the tool to the latest known-to-work version +RUN git checkout "${GLIB_MUS_SHA}" +RUN chmod a+x compile-x86_64-alpine-linux.sh +RUN sh compile-x86_64-alpine-linux.sh + +# python:3.12.7-alpine3.20 +FROM docker.io/library/python@sha256:38e179a0f0436c97ecc76bcd378d7293ab3ee79e4b8c440fdc7113670cb6e204 as final -SHELL [ "/bin/bash", "-c" ] ARG OWLBOT_CLI_COMMITTISH=38fe6f89a2339ee75c77739b31b371f601b01bb3 ARG PROTOC_VERSION=25.5 -ARG GRPC_VERSION=1.67.1 +ARG GRPC_VERSION=1.68.1 ARG JAVA_FORMAT_VERSION=1.7 ENV HOME=/home ENV OS_ARCHITECTURE="linux-x86_64" # install OS tools -RUN apt-get update && apt-get install -y \ - unzip openjdk-17-jdk rsync maven jq \ - && apt-get clean +RUN apk update && apk add unzip curl rsync openjdk11 jq bash nodejs npm git + +SHELL [ "/bin/bash", "-c" ] + +# Copy glibc shared objects to enable execution of the grpc plugin. +# This list was obtained via `libtree -pvvv /grpc/*` in the final container as +# well as inspecting the modifications done by compile-x86_64-alpine-linux.sh +# in the glibc-compat stage using the `dive` command. +COPY --from=glibc-compat /etc/libgcc* /etc/ +COPY --from=glibc-compat /lib64/ld-linux-x86-64.so.2 /lib64/ +COPY --from=glibc-compat /lib/GLIBCFAKE.so.0 /lib/ +COPY --from=glibc-compat /lib/ld-linux-x86-64.so.2 /lib/ +COPY --from=glibc-compat /lib/libpthread* /lib/ +COPY --from=glibc-compat /lib/libucontext* /lib/ +COPY --from=glibc-compat /lib/libc.* /lib/ +COPY --from=glibc-compat /usr/lib/libgcc* /usr/lib/ +COPY --from=glibc-compat /usr/lib/libstdc* /usr/lib/ +COPY --from=glibc-compat /usr/lib/libobstack* /usr/lib/ # copy source code COPY hermetic_build/common /src/common @@ -72,10 +109,6 @@ ENV DOCKER_GRPC_VERSION="${GRPC_VERSION}" COPY --from=ggj-build "/sdk-platform-java/gapic-generator-java.jar" "${HOME}/.library_generation/gapic-generator-java.jar" RUN chmod 755 "${HOME}/.library_generation/gapic-generator-java.jar" -# use python 3.12 (the base image has several python versions; here we define the default one) -RUN rm $(which python3) -RUN ln -s $(which python3.12) /usr/local/bin/python -RUN ln -s $(which python3.12) /usr/local/bin/python3 RUN python -m pip install --upgrade pip # install main scripts as a python package @@ -85,16 +118,6 @@ RUN python -m pip install src/common RUN python -m pip install --require-hashes -r src/library_generation/requirements.txt RUN python -m pip install src/library_generation -# Install nvm with node and npm -ENV NODE_VERSION 20.12.0 -WORKDIR /home -RUN curl -o- https://raw.githubusercontent.com/nvm-sh/nvm/v0.35.3/install.sh | bash -RUN chmod o+rx /home/.nvm -ENV NODE_PATH=/home/.nvm/versions/node/v${NODE_VERSION}/bin -ENV PATH=${PATH}:${NODE_PATH} -RUN node --version -RUN npm --version - # install the owl-bot CLI WORKDIR /tools RUN git clone https://github.com/googleapis/repo-automation-bots @@ -102,8 +125,7 @@ WORKDIR /tools/repo-automation-bots/packages/owl-bot RUN git checkout "${OWLBOT_CLI_COMMITTISH}" RUN npm i && npm run compile && npm link RUN owl-bot copy-code --version -RUN chmod -R o+rx ${NODE_PATH} -RUN ln -sf ${NODE_PATH}/* /usr/local/bin +RUN chmod o+rx $(which owl-bot) # download the Java formatter ADD https://maven-central.storage-download.googleapis.com/maven2/com/google/googlejavaformat/google-java-format/${JAVA_FORMAT_VERSION}/google-java-format-${JAVA_FORMAT_VERSION}-all-deps.jar \ @@ -120,7 +142,6 @@ RUN git config --system user.name "Cloud Java Bot" # allow read-write for /home and execution for binaries in /home/.nvm RUN chmod -R a+rw /home -RUN chmod -R a+rx /home/.nvm WORKDIR /workspace ENTRYPOINT [ "python", "/src/library_generation/cli/entry_point.py", "generate" ] diff --git a/.github/scripts/action.yaml b/.github/scripts/action.yaml index 7c4ae22f4e..5e8b55660f 100644 --- a/.github/scripts/action.yaml +++ b/.github/scripts/action.yaml @@ -60,8 +60,6 @@ runs: cd "${GITHUB_WORKSPACE}" pip install --require-hashes -r hermetic_build/common/requirements.txt pip install hermetic_build/common - pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - pip install hermetic_build/library_generation pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt pip install hermetic_build/release_note_generation - name: Generate changed libraries diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 59fc82fc12..3c2a7d0e47 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -81,9 +81,6 @@ git checkout "${current_branch}" # copy generation configuration from target branch to current branch. git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" -# get .m2 folder so it's mapped into the docker container -m2_folder=$(dirname "$(mvn help:evaluate -Dexpression=settings.localRepository -q -DforceStdout)") - # download api definitions from googleapis repository googleapis_commitish=$(grep googleapis_commitish "${generation_config}" | cut -d ":" -f 2 | xargs) api_def_dir=$(mktemp -d) @@ -92,18 +89,23 @@ pushd "${api_def_dir}" git checkout "${googleapis_commitish}" popd +# get changed library list. +changed_libraries=$(python hermetic_build/common/cli/get_changed_libraries.py create \ + --baseline-generation-config-path="${baseline_generation_config}" \ + --current-generation-config-path="${generation_config}") +echo "Changed libraries are: ${changed_libraries:-"No changed library"}." + # run hermetic code generation docker image. docker run \ --rm \ --quiet \ -u "$(id -u):$(id -g)" \ -v "$(pwd):${workspace_name}" \ - -v "${m2_folder}":/home/.m2 \ -v "${api_def_dir}:${workspace_name}/googleapis" \ -e GENERATOR_VERSION="${image_tag}" \ gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ - --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ - --current-generation-config-path="${workspace_name}/${generation_config}" \ + --generation-config-path="${workspace_name}/${generation_config}" \ + --library-names="${changed_libraries}" \ --api-definitions-path="${workspace_name}/googleapis" python hermetic_build/release_note_generation/cli/generate_release_note.py generate \ diff --git a/.gitignore b/.gitignore index 6768e8cd69..b1b8440011 100644 --- a/.gitignore +++ b/.gitignore @@ -25,4 +25,4 @@ target/ **/build/ **/dist/ library_generation/**/*.jar - +**/google-java-format.jar diff --git a/WORKSPACE b/WORKSPACE index 93d210c127..acc53e9842 100644 --- a/WORKSPACE +++ b/WORKSPACE @@ -58,7 +58,7 @@ load("@rules_jvm_external//:defs.bzl", "maven_install") load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_ARTIFACTS") load("@io_grpc_grpc_java//:repositories.bzl", "IO_GRPC_GRPC_JAVA_OVERRIDE_TARGETS") -_gapic_generator_java_version = "2.49.0" # {x-version-update:gapic-generator-java:current} +_gapic_generator_java_version = "2.49.1-SNAPSHOT" # {x-version-update:gapic-generator-java:current} maven_install( artifacts = [ diff --git a/api-common-java/pom.xml b/api-common-java/pom.xml index a7bf98f6c7..763bcb238d 100644 --- a/api-common-java/pom.xml +++ b/api-common-java/pom.xml @@ -5,14 +5,14 @@ com.google.api api-common jar - 2.40.0 + 2.40.1-SNAPSHOT API Common Common utilities for Google APIs in Java com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent diff --git a/coverage-report/pom.xml b/coverage-report/pom.xml index 65e30ed999..e66f4d066d 100644 --- a/coverage-report/pom.xml +++ b/coverage-report/pom.xml @@ -31,22 +31,22 @@ com.google.api gax - 2.57.0 + 2.57.1-SNAPSHOT com.google.api gax-grpc - 2.57.0 + 2.57.1-SNAPSHOT com.google.api gax-httpjson - 2.57.0 + 2.57.1-SNAPSHOT com.google.api api-common - 2.40.0 + 2.40.1-SNAPSHOT diff --git a/gapic-generator-java-bom/pom.xml b/gapic-generator-java-bom/pom.xml index 5899626b81..2c179d7f76 100644 --- a/gapic-generator-java-bom/pom.xml +++ b/gapic-generator-java-bom/pom.xml @@ -4,7 +4,7 @@ com.google.api gapic-generator-java-bom pom - 2.49.0 + 2.49.1-SNAPSHOT GAPIC Generator Java BOM BOM for the libraries in gapic-generator-java repository. Users should not @@ -15,7 +15,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -75,61 +75,61 @@ com.google.api api-common - 2.40.0 + 2.40.1-SNAPSHOT com.google.api gax-bom - 2.57.0 + 2.57.1-SNAPSHOT pom import com.google.api gapic-generator-java - 2.49.0 + 2.49.1-SNAPSHOT com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT com.google.api.grpc proto-google-iam-v1 - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc proto-google-iam-v2 - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc proto-google-iam-v2beta - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v1 - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2 - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2beta - 1.43.0 + 1.43.1-SNAPSHOT diff --git a/gapic-generator-java-pom-parent/pom.xml b/gapic-generator-java-pom-parent/pom.xml index d0c1ddc413..cbf713fe68 100644 --- a/gapic-generator-java-pom-parent/pom.xml +++ b/gapic-generator-java-pom-parent/pom.xml @@ -5,7 +5,7 @@ 4.0.0 com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT pom GAPIC Generator Java POM Parent https://github.com/googleapis/sdk-platform-java @@ -26,8 +26,8 @@ 1.3.2 - 1.67.1 - 1.29.0 + 1.68.1 + 1.30.0 1.45.0 2.11.0 33.3.1-jre diff --git a/gapic-generator-java/pom.xml b/gapic-generator-java/pom.xml index 9c163e79d4..37fa43c5b2 100644 --- a/gapic-generator-java/pom.xml +++ b/gapic-generator-java/pom.xml @@ -4,7 +4,7 @@ 4.0.0 com.google.api gapic-generator-java - 2.49.0 + 2.49.1-SNAPSHOT GAPIC Generator Java GAPIC generator Java @@ -22,7 +22,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -31,7 +31,7 @@ com.google.api gapic-generator-java-bom - 2.49.0 + 2.49.1-SNAPSHOT pom import diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java index ae23a0aedd..500d330d30 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/AbstractServiceStubSettingsClassComposer.java @@ -104,6 +104,7 @@ import com.google.longrunning.Operation; import com.google.protobuf.Empty; import java.io.IOException; +import java.time.Duration; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; @@ -120,7 +121,6 @@ import java.util.stream.Collectors; import javax.annotation.Generated; import javax.annotation.Nullable; -import org.threeten.bp.Duration; public abstract class AbstractServiceStubSettingsClassComposer implements ClassComposer { private static final Statement EMPTY_LINE_STATEMENT = EmptyLineStatement.create(); diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java index e5e4ad4195..6690c57674 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposer.java @@ -390,7 +390,7 @@ public static Expr createBatchingBuilderSettingsExpr( batchingSettingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(batchingSettingsBuilderExpr) - .setMethodName("setDelayThreshold") + .setMethodName("setDelayThresholdDuration") .setArguments( createDurationOfMillisExpr(toValExpr(batchingSettings.delayThresholdMillis()))) .build(); @@ -511,7 +511,7 @@ private static List createRetrySettingsExprs( settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) - .setMethodName("setInitialRetryDelay") + .setMethodName("setInitialRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(retryPolicy.getInitialBackoff()))) .build(); @@ -528,7 +528,7 @@ private static List createRetrySettingsExprs( settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) - .setMethodName("setMaxRetryDelay") + .setMethodName("setMaxRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(retryPolicy.getMaxBackoff()))) .build(); } @@ -537,7 +537,7 @@ private static List createRetrySettingsExprs( settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) - .setMethodName("setInitialRpcTimeout") + .setMethodName("setInitialRpcTimeoutDuration") .setArguments(createDurationOfMillisExpr(toValExpr(settings.timeout()))) .build(); } @@ -553,7 +553,8 @@ private static List createRetrySettingsExprs( .build(); if (!settings.kind().equals(GapicRetrySettings.Kind.NONE)) { - for (String setterMethodName : Arrays.asList("setMaxRpcTimeout", "setTotalTimeout")) { + for (String setterMethodName : + Arrays.asList("setMaxRpcTimeoutDuration", "setTotalTimeoutDuration")) { settingsBuilderExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingsBuilderExpr) @@ -614,7 +615,7 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setInitialRetryDelay") + .setMethodName("setInitialRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(initialPollDelayMillis))) .build(); @@ -628,7 +629,7 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setMaxRetryDelay") + .setMethodName("setMaxRetryDelayDuration") .setArguments(createDurationOfMillisExpr(toValExpr(maxPollDelayMillis))) .build(); @@ -638,7 +639,7 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setInitialRpcTimeout") + .setMethodName("setInitialRpcTimeoutDuration") .setArguments(zeroDurationExpr) .build(); @@ -654,14 +655,14 @@ private static Expr createLroRetrySettingsExpr( lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setMaxRpcTimeout") + .setMethodName("setMaxRpcTimeoutDuration") .setArguments(zeroDurationExpr) .build(); lroRetrySettingsExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(lroRetrySettingsExpr) - .setMethodName("setTotalTimeout") + .setMethodName("setTotalTimeoutDuration") .setArguments(createDurationOfMillisExpr(toValExpr(totalPollTimeoutMillis))) .build(); @@ -714,7 +715,7 @@ private static TypeStore createStaticTypes() { List> concreteClazzes = Arrays.asList( BatchingSettings.class, - org.threeten.bp.Duration.class, + java.time.Duration.class, FlowControlSettings.class, FlowController.LimitExceededBehavior.class, ImmutableMap.class, diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java index 460c8a443a..57d396847e 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposer.java @@ -70,15 +70,15 @@ public static Optional composeSettingsSample( .build(); // Builder with set value method - // e.g foobarSettingBuilder.fooSetting().setRetrySettings( - // echoSettingsBuilder.echoSettings().getRetrySettings().toBuilder().setTotalTimeout(Duration.ofSeconds(30)).build()); + // e.g. foobarSettingBuilder.fooSetting().setRetrySettings( + // echoSettingsBuilder.echoSettings().getRetrySettings().toBuilder() + // .setTotalTimeoutDuration(Duration.ofSeconds(30)).build()); MethodInvocationExpr settingBuilderMethodInvocationExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(localSettingsVarExpr) .setMethodName( JavaStyle.toLowerCamelCase(String.format("%sSettings", methodNameOpt.get()))) .build(); - String disambiguation = "Settings"; MethodInvocationExpr retrySettingsArgExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(settingBuilderMethodInvocationExpr) @@ -364,7 +364,7 @@ public static Optional composeLroSettingsSample( retrySettingsArgExpr = MethodInvocationExpr.builder() .setExprReferenceExpr(retrySettingsArgExpr) - .setMethodName("setMaxRetryDelay") + .setMethodName("setMaxRetryDelayDuration") .setArguments(ofFiveThousandMillisMethodInvocationExpr) .build(); retrySettingsArgExpr = diff --git a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java index 9a2da74747..2e17b9026b 100644 --- a/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java +++ b/gapic-generator-java/src/main/java/com/google/api/generator/gapic/protoparser/Parser.java @@ -14,6 +14,7 @@ package com.google.api.generator.gapic.protoparser; +import com.google.api.ClientLibrarySettings; import com.google.api.ClientProto; import com.google.api.DocumentationRule; import com.google.api.FieldBehavior; @@ -84,6 +85,7 @@ import java.util.Optional; import java.util.Set; import java.util.function.Function; +import java.util.logging.Level; import java.util.logging.Logger; import java.util.stream.Collectors; import java.util.stream.IntStream; @@ -160,11 +162,11 @@ public static GapicContext parse(CodeGeneratorRequest request) { messages = updateResourceNamesInMessages(messages, resourceNames.values()); // Contains only resource names that are actually used. Usage refers to the presence of a - // request message's field in an RPC's method_signature annotation. That is, resource name - // definitions - // or references that are simply defined, but not used in such a manner, will not have - // corresponding Java helper - // classes generated. + // request message's field in an RPC's method_signature annotation. That is, resource name + // definitions or references that are simply defined, but not used in such a manner, + // will not have corresponding Java helper classes generated. + // If selective api generation is configured via service yaml, Java helper classes are only + // generated if resource names are actually used by methods selected to generate. Set outputArgResourceNames = new HashSet<>(); List mixinServices = new ArrayList<>(); Transport transport = Transport.parse(transportOpt.orElse(Transport.GRPC.toString())); @@ -425,6 +427,71 @@ public static List parseService( Transport.GRPC); } + static boolean shouldIncludeMethodInGeneration( + MethodDescriptor method, + Optional serviceYamlProtoOpt, + String protoPackage) { + // default to include all when no service yaml or no library setting section. + if (!serviceYamlProtoOpt.isPresent() + || serviceYamlProtoOpt.get().getPublishing().getLibrarySettingsCount() == 0) { + return true; + } + List librarySettingsList = + serviceYamlProtoOpt.get().getPublishing().getLibrarySettingsList(); + // Validate for logging purpose, this should be validated upstream. + // If library_settings.version does not match with proto package name + // Give warnings and disregard this config. default to include all. + if (!librarySettingsList.get(0).getVersion().isEmpty() + && !protoPackage.equals(librarySettingsList.get(0).getVersion())) { + if (LOGGER.isLoggable(Level.WARNING)) { + LOGGER.warning( + String.format( + "Service yaml config is misconfigured. Version in " + + "publishing.library_settings (%s) does not match proto package (%s)." + + "Disregarding selective generation settings.", + librarySettingsList.get(0).getVersion(), protoPackage)); + } + return true; + } + // librarySettingsList is technically a list, but is processed upstream and + // only leave with 1 element. Otherwise, it is a misconfiguration and + // should be caught upstream. + List includeMethodsList = + librarySettingsList + .get(0) + .getJavaSettings() + .getCommon() + .getSelectiveGapicGeneration() + .getMethodsList(); + // default to include all when nothing specified, this could be no java section + // specified in library setting, or the method list is empty + if (includeMethodsList.isEmpty()) { + return true; + } + + return includeMethodsList.contains(method.getFullName()); + } + + private static boolean isEmptyService( + ServiceDescriptor serviceDescriptor, + Optional serviceYamlProtoOpt, + String protoPackage) { + List methodsList = serviceDescriptor.getMethods(); + List methodListSelected = + methodsList.stream() + .filter( + method -> + shouldIncludeMethodInGeneration(method, serviceYamlProtoOpt, protoPackage)) + .collect(Collectors.toList()); + if (methodListSelected.isEmpty()) { + LOGGER.log( + Level.WARNING, + "Service {0} has no RPC methods and will not be generated", + serviceDescriptor.getName()); + } + return methodListSelected.isEmpty(); + } + public static List parseService( FileDescriptor fileDescriptor, Map messageTypes, @@ -433,19 +500,11 @@ public static List parseService( Optional serviceConfigOpt, Set outputArgResourceNames, Transport transport) { - + String protoPackage = fileDescriptor.getPackage(); return fileDescriptor.getServices().stream() .filter( - serviceDescriptor -> { - List methodsList = serviceDescriptor.getMethods(); - if (methodsList.isEmpty()) { - LOGGER.warning( - String.format( - "Service %s has no RPC methods and will not be generated", - serviceDescriptor.getName())); - } - return !methodsList.isEmpty(); - }) + serviceDescriptor -> + !isEmptyService(serviceDescriptor, serviceYamlProtoOpt, protoPackage)) .map( s -> { // Workaround for a missing default_host and oauth_scopes annotation from a service @@ -498,6 +557,8 @@ public static List parseService( String pakkage = TypeParser.getPackage(fileDescriptor); String originalJavaPackage = pakkage; // Override Java package with that specified in gapic.yaml. + // this override is deprecated and legacy support only + // see go/client-user-guide#configure-long-running-operation-polling-timeouts-optional if (serviceConfigOpt.isPresent() && serviceConfigOpt.get().getLanguageSettingsOpt().isPresent()) { GapicLanguageSettings languageSettings = @@ -518,6 +579,7 @@ public static List parseService( .setMethods( parseMethods( s, + protoPackage, pakkage, messageTypes, resourceNames, @@ -709,6 +771,7 @@ public static Map parseResourceNames( @VisibleForTesting static List parseMethods( ServiceDescriptor serviceDescriptor, + String protoPackage, String servicePackage, Map messageTypes, Map resourceNames, @@ -721,8 +784,10 @@ static List parseMethods( // Parse the serviceYaml for autopopulated methods and fields once and put into a map Map> autoPopulatedMethodsWithFields = parseAutoPopulatedMethodsAndFields(serviceYamlProtoOpt); - for (MethodDescriptor protoMethod : serviceDescriptor.getMethods()) { + if (!shouldIncludeMethodInGeneration(protoMethod, serviceYamlProtoOpt, protoPackage)) { + continue; + } // Parse the method. TypeNode inputType = TypeParser.parseType(protoMethod.getInputType()); Method.Builder methodBuilder = Method.builder(); diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java index 717191842a..775a0b1d09 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/common/RetrySettingsComposerTest.java @@ -119,20 +119,20 @@ void paramDefinitionsBlock_basic() { "ImmutableMap.Builder definitions = ImmutableMap.builder();\n", "RetrySettings settings = null;\n", "settings =" - + " RetrySettings.newBuilder().setInitialRetryDelay(" + + " RetrySettings.newBuilder().setInitialRetryDelayDuration(" + "Duration.ofMillis(100L)).setRetryDelayMultiplier(2.0)" - + ".setMaxRetryDelay(Duration.ofMillis(3000L))" - + ".setInitialRpcTimeout(Duration.ofMillis(10000L))" + + ".setMaxRetryDelayDuration(Duration.ofMillis(3000L))" + + ".setInitialRpcTimeoutDuration(Duration.ofMillis(10000L))" + ".setRpcTimeoutMultiplier(1.0)" - + ".setMaxRpcTimeout(Duration.ofMillis(10000L))" - + ".setTotalTimeout(Duration.ofMillis(10000L)).build();\n", + + ".setMaxRpcTimeoutDuration(Duration.ofMillis(10000L))" + + ".setTotalTimeoutDuration(Duration.ofMillis(10000L)).build();\n", "definitions.put(\"retry_policy_1_params\", settings);\n", "settings =" + " RetrySettings.newBuilder()" - + ".setInitialRpcTimeout(Duration.ofMillis(5000L))" + + ".setInitialRpcTimeoutDuration(Duration.ofMillis(5000L))" + ".setRpcTimeoutMultiplier(1.0)" - + ".setMaxRpcTimeout(Duration.ofMillis(5000L))" - + ".setTotalTimeout(Duration.ofMillis(5000L)).build();\n", + + ".setMaxRpcTimeoutDuration(Duration.ofMillis(5000L))" + + ".setTotalTimeoutDuration(Duration.ofMillis(5000L)).build();\n", "definitions.put(\"no_retry_0_params\", settings);\n", "RETRY_PARAM_DEFINITIONS = definitions.build();\n", "}\n"); @@ -341,10 +341,10 @@ void lroBuilderExpr() { + "WaitResponse.class))" + ".setMetadataTransformer(ProtoOperationTransformers.MetadataTransformer.create(" + "WaitMetadata.class)).setPollingAlgorithm(OperationTimedPollAlgorithm.create(" - + "RetrySettings.newBuilder().setInitialRetryDelay(Duration.ofMillis(5000L))" - + ".setRetryDelayMultiplier(1.5).setMaxRetryDelay(Duration.ofMillis(45000L))" - + ".setInitialRpcTimeout(Duration.ZERO).setRpcTimeoutMultiplier(1.0)" - + ".setMaxRpcTimeout(Duration.ZERO).setTotalTimeout(Duration.ofMillis(300000L))" + + "RetrySettings.newBuilder().setInitialRetryDelayDuration(Duration.ofMillis(5000L))" + + ".setRetryDelayMultiplier(1.5).setMaxRetryDelayDuration(Duration.ofMillis(45000L))" + + ".setInitialRpcTimeoutDuration(Duration.ZERO).setRpcTimeoutMultiplier(1.0)" + + ".setMaxRpcTimeoutDuration(Duration.ZERO).setTotalTimeoutDuration(Duration.ofMillis(300000L))" + ".build()))"); assertEquals(expected, writerVisitor.write()); } @@ -394,7 +394,7 @@ void batchingSettings_minimalFlowControlSettings() { + "BatchingSettings.newBuilder()" + ".setElementCountThreshold(100L)" + ".setRequestByteThreshold(1048576L)" - + ".setDelayThreshold(Duration.ofMillis(10L))" + + ".setDelayThresholdDuration(Duration.ofMillis(10L))" + ".setFlowControlSettings(" + "FlowControlSettings.newBuilder()" + ".setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore)" @@ -451,7 +451,7 @@ void batchingSettings_fullFlowControlSettings() { + "BatchingSettings.newBuilder()" + ".setElementCountThreshold(1000L)" + ".setRequestByteThreshold(1048576L)" - + ".setDelayThreshold(Duration.ofMillis(50L))" + + ".setDelayThresholdDuration(Duration.ofMillis(50L))" + ".setFlowControlSettings(" + "FlowControlSettings.newBuilder()" + ".setMaxOutstandingElementCount(100000L)" diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden index 329be3111f..6d81887aed 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/DeprecatedServiceStubSettings.golden @@ -22,9 +22,9 @@ import com.google.common.collect.Lists; import com.google.protobuf.Empty; import com.google.testdata.v1.FibonacciRequest; import java.io.IOException; +import java.time.Duration; import java.util.List; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -206,10 +206,10 @@ public class DeprecatedServiceStubSettings extends StubSettings { RetrySettings settings = null; settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(2.0) - .setMaxRetryDelay(Duration.ofMillis(3000L)) - .setInitialRpcTimeout(Duration.ofMillis(10000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(3000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(10000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(10000L)) - .setTotalTimeout(Duration.ofMillis(10000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(10000L)) + .setTotalTimeoutDuration(Duration.ofMillis(10000L)) .build(); definitions.put("retry_policy_1_params", settings); settings = RetrySettings.newBuilder() - .setInitialRpcTimeout(Duration.ofMillis(5000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(5000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(5000L)) - .setTotalTimeout(Duration.ofMillis(5000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(5000L)) + .setTotalTimeoutDuration(Duration.ofMillis(5000L)) .build(); definitions.put("no_retry_0_params", settings); RETRY_PARAM_DEFINITIONS = definitions.build(); @@ -575,13 +575,13 @@ public class EchoStubSettings extends StubSettings { .setPollingAlgorithm( OperationTimedPollAlgorithm.create( RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(5000L)) + .setInitialRetryDelayDuration(Duration.ofMillis(5000L)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(45000L)) - .setInitialRpcTimeout(Duration.ZERO) + .setMaxRetryDelayDuration(Duration.ofMillis(45000L)) + .setInitialRpcTimeoutDuration(Duration.ZERO) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ZERO) - .setTotalTimeout(Duration.ofMillis(300000L)) + .setMaxRpcTimeoutDuration(Duration.ZERO) + .setTotalTimeoutDuration(Duration.ofMillis(300000L)) .build())); return builder; diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden index 3b10c57af5..c199db3fee 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/LoggingServiceV2StubSettings.golden @@ -54,10 +54,10 @@ import com.google.logging.v2.WriteLogEntriesRequest; import com.google.logging.v2.WriteLogEntriesResponse; import com.google.protobuf.Empty; import java.io.IOException; +import java.time.Duration; import java.util.Collection; import java.util.List; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -557,24 +557,24 @@ public class LoggingServiceV2StubSettings extends StubSettings { RetrySettings settings = null; settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(1.3) - .setMaxRetryDelay(Duration.ofMillis(60000L)) - .setInitialRpcTimeout(Duration.ofMillis(60000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(60000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(60000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(60000L)) - .setTotalTimeout(Duration.ofMillis(60000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(60000L)) + .setTotalTimeoutDuration(Duration.ofMillis(60000L)) .build(); definitions.put("retry_policy_0_params", settings); settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(1.3) - .setMaxRetryDelay(Duration.ofMillis(60000L)) - .setInitialRpcTimeout(Duration.ofMillis(60000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(60000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(60000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(60000L)) - .setTotalTimeout(Duration.ofMillis(60000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(60000L)) + .setTotalTimeoutDuration(Duration.ofMillis(60000L)) .build(); definitions.put("retry_policy_1_params", settings); settings = RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(100L)) + .setInitialRetryDelayDuration(Duration.ofMillis(100L)) .setRetryDelayMultiplier(1.3) - .setMaxRetryDelay(Duration.ofMillis(60000L)) - .setInitialRpcTimeout(Duration.ofMillis(60000L)) + .setMaxRetryDelayDuration(Duration.ofMillis(60000L)) + .setInitialRpcTimeoutDuration(Duration.ofMillis(60000L)) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ofMillis(60000L)) - .setTotalTimeout(Duration.ofMillis(60000L)) + .setMaxRpcTimeoutDuration(Duration.ofMillis(60000L)) + .setTotalTimeoutDuration(Duration.ofMillis(60000L)) .build(); definitions.put("retry_policy_2_params", settings); RETRY_PARAM_DEFINITIONS = definitions.build(); @@ -697,7 +697,7 @@ public class PublisherStubSettings extends StubSettings { BatchingSettings.newBuilder() .setElementCountThreshold(100L) .setRequestByteThreshold(1048576L) - .setDelayThreshold(Duration.ofMillis(10L)) + .setDelayThresholdDuration(Duration.ofMillis(10L)) .setFlowControlSettings( FlowControlSettings.newBuilder() .setLimitExceededBehavior(FlowController.LimitExceededBehavior.Ignore) diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden index 71cf9529ac..cd31f6581a 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/SyncWait.golden @@ -41,7 +41,7 @@ public class SyncWait { RetrySettings.newBuilder() .setInitialRetryDelayDuration(Duration.ofMillis(500)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(5000)) + .setMaxRetryDelayDuration(Duration.ofMillis(5000)) .setTotalTimeoutDuration(Duration.ofHours(24)) .build()); echoSettingsBuilder diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden index ee1c010647..bd2264892e 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpc/goldens/samples/servicesettings/stub/SyncWait.golden @@ -41,7 +41,7 @@ public class SyncWait { RetrySettings.newBuilder() .setInitialRetryDelayDuration(Duration.ofMillis(500)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(5000)) + .setMaxRetryDelayDuration(Duration.ofMillis(5000)) .setTotalTimeoutDuration(Duration.ofHours(24)) .build()); echoSettingsBuilder diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden index c94624b679..8ef290e405 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoSettings.golden @@ -90,7 +90,7 @@ import javax.annotation.Generated; * RetrySettings.newBuilder() * .setInitialRetryDelayDuration(Duration.ofMillis(500)) * .setRetryDelayMultiplier(1.5) - * .setMaxRetryDelay(Duration.ofMillis(5000)) + * .setMaxRetryDelayDuration(Duration.ofMillis(5000)) * .setTotalTimeoutDuration(Duration.ofHours(24)) * .build()); * echoSettingsBuilder diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden index 41cfa15e00..4f2603041f 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/grpcrest/goldens/EchoStubSettings.golden @@ -54,9 +54,9 @@ import com.google.showcase.grpcrest.v1beta1.WaitMetadata; import com.google.showcase.grpcrest.v1beta1.WaitRequest; import com.google.showcase.grpcrest.v1beta1.WaitResponse; import java.io.IOException; +import java.time.Duration; import java.util.List; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -123,7 +123,7 @@ import org.threeten.bp.Duration; * RetrySettings.newBuilder() * .setInitialRetryDelayDuration(Duration.ofMillis(500)) * .setRetryDelayMultiplier(1.5) - * .setMaxRetryDelay(Duration.ofMillis(5000)) + * .setMaxRetryDelayDuration(Duration.ofMillis(5000)) * .setTotalTimeoutDuration(Duration.ofHours(24)) * .build()); * echoSettingsBuilder @@ -630,13 +630,13 @@ public class EchoStubSettings extends StubSettings { .setPollingAlgorithm( OperationTimedPollAlgorithm.create( RetrySettings.newBuilder() - .setInitialRetryDelay(Duration.ofMillis(5000L)) + .setInitialRetryDelayDuration(Duration.ofMillis(5000L)) .setRetryDelayMultiplier(1.5) - .setMaxRetryDelay(Duration.ofMillis(45000L)) - .setInitialRpcTimeout(Duration.ZERO) + .setMaxRetryDelayDuration(Duration.ofMillis(45000L)) + .setInitialRpcTimeoutDuration(Duration.ZERO) .setRpcTimeoutMultiplier(1.0) - .setMaxRpcTimeout(Duration.ZERO) - .setTotalTimeout(Duration.ofMillis(300000L)) + .setMaxRpcTimeoutDuration(Duration.ZERO) + .setTotalTimeoutDuration(Duration.ofMillis(300000L)) .build())); return builder; diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java index 7745f3aff8..eb81a73a8e 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/composer/samplecode/SettingsSampleComposerTest.java @@ -129,7 +129,7 @@ void composeSettingsSample_serviceSettingsClass_LroMethod() { " RetrySettings.newBuilder()\n", " .setInitialRetryDelayDuration(Duration.ofMillis(500))\n", " .setRetryDelayMultiplier(1.5)\n", - " .setMaxRetryDelay(Duration.ofMillis(5000))\n", + " .setMaxRetryDelayDuration(Duration.ofMillis(5000))\n", " .setTotalTimeoutDuration(Duration.ofHours(24))\n", " .build());\n", "waitSettingsBuilder\n", @@ -159,7 +159,7 @@ void composeSettingsSample_serviceStubClass_LroMethod() { " RetrySettings.newBuilder()\n", " .setInitialRetryDelayDuration(Duration.ofMillis(500))\n", " .setRetryDelayMultiplier(1.5)\n", - " .setMaxRetryDelay(Duration.ofMillis(5000))\n", + " .setMaxRetryDelayDuration(Duration.ofMillis(5000))\n", " .setTotalTimeoutDuration(Duration.ofHours(24))\n", " .build());\n", "waitSettingsBuilder\n", diff --git a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java index 2776fec687..6e8ffa7232 100644 --- a/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java +++ b/gapic-generator-java/src/test/java/com/google/api/generator/gapic/protoparser/ParserTest.java @@ -21,9 +21,11 @@ import static org.junit.jupiter.api.Assertions.assertThrows; import static org.junit.jupiter.api.Assertions.assertTrue; +import com.google.api.ClientLibrarySettings; import com.google.api.FieldInfo.Format; import com.google.api.MethodSettings; import com.google.api.Publishing; +import com.google.api.PythonSettings; import com.google.api.Service; import com.google.api.generator.engine.ast.ConcreteReference; import com.google.api.generator.engine.ast.Reference; @@ -46,6 +48,7 @@ import com.google.protobuf.Descriptors.MethodDescriptor; import com.google.protobuf.Descriptors.ServiceDescriptor; import com.google.protobuf.compiler.PluginProtos.CodeGeneratorRequest; +import com.google.selective.generate.v1beta1.SelectiveApiGenerationOuterClass; import com.google.showcase.v1beta1.EchoOuterClass; import com.google.showcase.v1beta1.TestingOuterClass; import com.google.testgapic.v1beta1.LockerProto; @@ -58,6 +61,8 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.stream.Collectors; +import org.junit.Assert; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; @@ -137,6 +142,7 @@ void parseMethods_basic() { Parser.parseMethods( echoService, ECHO_PACKAGE, + ECHO_PACKAGE, messageTypes, resourceNames, Optional.empty(), @@ -200,6 +206,7 @@ void parseMethods_basicLro() { Parser.parseMethods( echoService, ECHO_PACKAGE, + ECHO_PACKAGE, messageTypes, resourceNames, Optional.empty(), @@ -705,6 +712,128 @@ void parseServiceWithNoMethodsTest() { assertEquals("EchoWithMethods", services.get(0).overriddenName()); } + @Test + void selectiveGenerationTest_shouldExcludeUnusedResourceNames() { + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + Map messageTypes = Parser.parseMessages(fileDescriptor); + Map resourceNames = Parser.parseResourceNames(fileDescriptor); + + String serviceYamlFilename = "selective_api_generation_v1beta1.yaml"; + String testFilesDirectory = "src/test/resources/"; + Path serviceYamlPath = Paths.get(testFilesDirectory, serviceYamlFilename); + Optional serviceYamlOpt = + ServiceYamlParser.parse(serviceYamlPath.toString()); + Assert.assertTrue(serviceYamlOpt.isPresent()); + + Set helperResourceNames = new HashSet<>(); + Parser.parseService( + fileDescriptor, messageTypes, resourceNames, serviceYamlOpt, helperResourceNames); + // resource Name Foobarbaz is not present + assertEquals(2, helperResourceNames.size()); + assertTrue( + helperResourceNames.stream() + .map(ResourceName::variableName) + .collect(Collectors.toSet()) + .containsAll(ImmutableList.of("foobar", "anythingGoes"))); + } + + @Test + void selectiveGenerationTest_shouldGenerateOnlySelectiveMethods() { + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + Map messageTypes = Parser.parseMessages(fileDescriptor); + Map resourceNames = Parser.parseResourceNames(fileDescriptor); + + // test with service yaml file to show usage of this feature, test itself + // can be done without this file and build a Service object from code. + String serviceYamlFilename = "selective_api_generation_v1beta1.yaml"; + String testFilesDirectory = "src/test/resources/"; + Path serviceYamlPath = Paths.get(testFilesDirectory, serviceYamlFilename); + Optional serviceYamlOpt = + ServiceYamlParser.parse(serviceYamlPath.toString()); + Assert.assertTrue(serviceYamlOpt.isPresent()); + + List services = + Parser.parseService( + fileDescriptor, messageTypes, resourceNames, serviceYamlOpt, new HashSet<>()); + assertEquals(1, services.size()); + assertEquals("EchoServiceShouldGeneratePartial", services.get(0).overriddenName()); + assertEquals(3, services.get(0).methods().size()); + for (Method method : services.get(0).methods()) { + assertTrue(method.name().contains("ShouldInclude")); + } + } + + @Test + void selectiveGenerationTest_shouldGenerateAllIfNoPublishingSectionInServiceYaml() { + Service service = + Service.newBuilder() + .setTitle("Selective generation testing with no publishing section") + .build(); + Publishing publishing = service.getPublishing(); + Assert.assertEquals(0, publishing.getLibrarySettingsCount()); + + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + List methods = fileDescriptor.getServices().get(0).getMethods(); + String protoPackage = "google.selective.generate.v1beta1"; + + assertTrue( + Parser.shouldIncludeMethodInGeneration(methods.get(0), Optional.of(service), protoPackage)); + } + + @Test + void selectiveGenerationTest_shouldIncludeMethodInGenerationWhenProtoPackageMismatch() { + String protoPackage = "google.selective.generate.v1beta1"; + + // situation where service yaml has different version stated + ClientLibrarySettings clientLibrarySettings = + ClientLibrarySettings.newBuilder().setVersion("google.selective.generate.v1").build(); + Publishing publishing = + Publishing.newBuilder().addLibrarySettings(clientLibrarySettings).build(); + Service service = + Service.newBuilder() + .setTitle( + "Selective generation test when proto package " + + "does not match library_settings version from service yaml") + .setPublishing(publishing) + .build(); + + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + List methods = fileDescriptor.getServices().get(0).getMethods(); + + assertTrue( + Parser.shouldIncludeMethodInGeneration(methods.get(0), Optional.of(service), protoPackage)); + } + + @Test + void selectiveGenerationTest_shouldGenerateAllIfNoJavaSectionInServiceYaml() { + String protoPackage = "google.selective.generate.v1beta1"; + + // situation where service yaml has other language settings but no + // java settings in library_settings. + ClientLibrarySettings clientLibrarySettings = + ClientLibrarySettings.newBuilder() + .setVersion(protoPackage) + .setPythonSettings(PythonSettings.newBuilder().build()) + .build(); + Publishing publishing = + Publishing.newBuilder().addLibrarySettings(clientLibrarySettings).build(); + Service service = + Service.newBuilder() + .setTitle( + "Selective generation test when no java section in " + + "library_settings from service yaml") + .setPublishing(publishing) + .build(); + + Assert.assertEquals(1, publishing.getLibrarySettingsCount()); + + FileDescriptor fileDescriptor = SelectiveApiGenerationOuterClass.getDescriptor(); + List methods = fileDescriptor.getServices().get(0).getMethods(); + + assertTrue( + Parser.shouldIncludeMethodInGeneration(methods.get(0), Optional.of(service), protoPackage)); + } + private void assertMethodArgumentEquals( String name, TypeNode type, List nestedFields, MethodArgument argument) { assertEquals(name, argument.name()); diff --git a/gapic-generator-java/src/test/proto/selective_api_generation.proto b/gapic-generator-java/src/test/proto/selective_api_generation.proto new file mode 100644 index 0000000000..06da2c2e41 --- /dev/null +++ b/gapic-generator-java/src/test/proto/selective_api_generation.proto @@ -0,0 +1,163 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +import "google/api/annotations.proto"; +import "google/api/client.proto"; +import "google/api/field_behavior.proto"; +import "google/api/field_info.proto"; +import "google/api/resource.proto"; +import "google/longrunning/operations.proto"; +import "google/protobuf/duration.proto"; +import "google/protobuf/timestamp.proto"; +import "google/rpc/status.proto"; + +package google.selective.generate.v1beta1; + +option java_package = "com.google.selective.generate.v1beta1"; +option java_multiple_files = true; +option java_outer_classname = "SelectiveApiGenerationOuterClass"; + +// resource not tied to message +option (google.api.resource_definition) = { + type: "showcase.googleapis.com/AnythingGoes" + pattern: "*" +}; + +// This proto is used to test selective api generation +// covered scenarios: +// - A service with several rpcs, part of them should be generated +// - A service with several rpcs, none of them should be generated +// This proto should be tested side-by-side with yaml file: +// - selective_api_generation_v1beta1.yaml + +service EchoServiceShouldGeneratePartial { + option (google.api.default_host) = "localhost:7469"; + + rpc EchoShouldInclude(EchoRequest) returns (EchoResponse) { + option (google.api.http) = { + post: "/v1beta1/echo:echo" + body: "*" + }; + option (google.api.method_signature) = "name"; + option (google.api.method_signature) = ""; + } + + rpc ChatShouldInclude(stream EchoRequest) returns (stream EchoResponse); + + rpc ChatAgainShouldInclude(stream EchoRequest) returns (stream EchoResponse) { + option (google.api.method_signature) = "content"; + } + + rpc AnExcludedMethod(stream EchoRequestWithFoobarbaz) returns (stream EchoResponse); + + rpc AnotherExcludedMethod(stream EchoRequest) returns (stream EchoResponse); + +} + +service EchoServiceShouldGenerateNone { + option (google.api.default_host) = "localhost:7469"; + option (google.api.oauth_scopes) = + "https://www.googleapis.com/auth/cloud-platform"; + + rpc Echo(EchoRequest) returns (EchoResponse) { + option (google.api.method_signature) = "content"; + } + + rpc ChatAgain(stream EchoRequest) returns (stream EchoResponse) { + option (google.api.method_signature) = "content"; + } +} + +// resource name used for message EchoRequest. +message Foobar { + option (google.api.resource) = { + type: "showcase.googleapis.com/Foobar" + pattern: "projects/{project}/foobars/{foobar}" + }; + + string name = 1; + string info = 2; +} + +// resource name used only for message EchoRequestWithFoobarbaz. +// should not be generated with selective generation when +// AnExcludedMethod is not config as included. +message Foobarbaz { + option (google.api.resource) = { + type: "showcase.googleapis.com/Foobarbaz" + pattern: "projects/{project}/foobarsbaz/{foobarbaz}" + pattern: "projects/{project}/chocolate/variants/{variant}/foobars/{foobar}" + }; + + string name = 1; + string info = 2; +} + +// RPCs in inclusion list and not in the list both relies on this request message. +message EchoRequest { + string name = 5 [ + (google.api.resource_reference).type = "showcase.googleapis.com/Foobar", + (google.api.field_behavior) = REQUIRED + ]; + + string parent = 6 [ + (google.api.resource_reference).child_type = + "showcase.googleapis.com/AnythingGoes", + (google.api.field_behavior) = REQUIRED + ]; + + oneof response { + // The content to be echoed by the server. + string content = 1; + + // The error to be thrown by the server. + google.rpc.Status error = 2; + } + + Foobar foobar = 4; +} + +// This request message is used by AnExcludedMethod rpc. +// To demonstrate that if AnExcludedMethod is not included in generation, +// then the resource name Foobarbaz, which is only used by this method, +// should not be generated. +message EchoRequestWithFoobarbaz { + string name = 5 [ + (google.api.resource_reference).type = "showcase.googleapis.com/Foobarbaz", + (google.api.field_behavior) = REQUIRED + ]; + + string parent = 6 [ + (google.api.resource_reference).child_type = + "showcase.googleapis.com/AnythingGoes", + (google.api.field_behavior) = REQUIRED + ]; + + oneof response { + // The content to be echoed by the server. + string content = 1; + + // The error to be thrown by the server. + google.rpc.Status error = 2; + } + + Foobarbaz foobar = 4; +} + +message EchoResponse { + // The content specified in the request. + string content = 1; +} diff --git a/gapic-generator-java/src/test/resources/selective_api_generation_v1beta1.yaml b/gapic-generator-java/src/test/resources/selective_api_generation_v1beta1.yaml new file mode 100644 index 0000000000..021e257c50 --- /dev/null +++ b/gapic-generator-java/src/test/resources/selective_api_generation_v1beta1.yaml @@ -0,0 +1,23 @@ +type: google.api.Service +config_version: 3 +name: selective_api_generation_testing.googleapis.com +title: Selective Generation Testing API + +publishing: + # ... + library_settings: + - version: google.selective.generate.v1beta1 + java_settings: + common: + selective_gapic_generation: + methods: + - google.selective.generate.v1beta1.EchoServiceShouldGeneratePartial.EchoShouldInclude + - google.selective.generate.v1beta1.EchoServiceShouldGeneratePartial.ChatShouldInclude + - google.selective.generate.v1beta1.EchoServiceShouldGeneratePartial.ChatAgainShouldInclude + reference_docs_uri: www.abc.net + destinations: + - PACKAGE_MANAGER + python_settings: + common: + destinations: + - PACKAGE_MANAGER diff --git a/gax-java/dependencies.properties b/gax-java/dependencies.properties index 144965463b..27509afd5c 100644 --- a/gax-java/dependencies.properties +++ b/gax-java/dependencies.properties @@ -8,16 +8,16 @@ # Versions of oneself # {x-version-update-start:gax:current} -version.gax=2.57.0 +version.gax=2.57.1-SNAPSHOT # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_grpc=2.57.0 +version.gax_grpc=2.57.1-SNAPSHOT # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_bom=2.57.0 +version.gax_bom=2.57.1-SNAPSHOT # {x-version-update-end} # {x-version-update-start:gax:current} -version.gax_httpjson=2.57.0 +version.gax_httpjson=2.57.1-SNAPSHOT # {x-version-update-end} # Versions for dependencies which actual artifacts differ between Bazel and Gradle. @@ -28,7 +28,7 @@ version.gax_httpjson=2.57.0 version.com_google_protobuf=3.25.5 version.google_java_format=1.15.0 -version.io_grpc=1.67.1 +version.io_grpc=1.68.1 # Maven artifacts. # Note, the actual name of each property matters (bazel build scripts depend on it). @@ -38,7 +38,7 @@ version.io_grpc=1.67.1 maven.com_google_api_grpc_proto_google_common_protos=com.google.api.grpc:proto-google-common-protos:2.46.0 maven.com_google_api_grpc_grpc_google_common_protos=com.google.api.grpc:grpc-google-common-protos:2.46.0 maven.com_google_auth_google_auth_library_oauth2_http=com.google.auth:google-auth-library-oauth2-http:1.29.0 -maven.com_google_auth_google_auth_library_credentials=com.google.auth:google-auth-library-credentials:1.29.0 +maven.com_google_auth_google_auth_library_credentials=com.google.auth:google-auth-library-credentials:1.30.0 maven.io_opentelemetry_opentelemetry_api=io.opentelemetry:opentelemetry-api:1.42.1 maven.io_opencensus_opencensus_api=io.opencensus:opencensus-api:0.31.1 maven.io_opencensus_opencensus_contrib_grpc_metrics=io.opencensus:opencensus-contrib-grpc-metrics:0.31.1 diff --git a/gax-java/gax-bom/pom.xml b/gax-java/gax-bom/pom.xml index 08c2149ac8..671ef04c3a 100644 --- a/gax-java/gax-bom/pom.xml +++ b/gax-java/gax-bom/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.api gax-bom - 2.57.0 + 2.57.1-SNAPSHOT pom GAX (Google Api eXtensions) for Java (BOM) Google Api eXtensions for Java (BOM) @@ -43,55 +43,55 @@ com.google.api gax - 2.57.0 + 2.57.1-SNAPSHOT com.google.api gax - 2.57.0 + 2.57.1-SNAPSHOT test-jar testlib com.google.api gax - 2.57.0 + 2.57.1-SNAPSHOT testlib com.google.api gax-grpc - 2.57.0 + 2.57.1-SNAPSHOT com.google.api gax-grpc - 2.57.0 + 2.57.1-SNAPSHOT test-jar testlib com.google.api gax-grpc - 2.57.0 + 2.57.1-SNAPSHOT testlib com.google.api gax-httpjson - 2.57.0 + 2.57.1-SNAPSHOT com.google.api gax-httpjson - 2.57.0 + 2.57.1-SNAPSHOT test-jar testlib com.google.api gax-httpjson - 2.57.0 + 2.57.1-SNAPSHOT testlib diff --git a/gax-java/gax-grpc/pom.xml b/gax-java/gax-grpc/pom.xml index 6992eea261..cde6985523 100644 --- a/gax-java/gax-grpc/pom.xml +++ b/gax-java/gax-grpc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax-grpc - 2.57.0 + 2.57.1-SNAPSHOT jar GAX (Google Api eXtensions) for Java (gRPC) Google Api eXtensions for Java (gRPC) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.57.0 + 2.57.1-SNAPSHOT diff --git a/gax-java/gax-httpjson/pom.xml b/gax-java/gax-httpjson/pom.xml index 6ab196acce..8799283814 100644 --- a/gax-java/gax-httpjson/pom.xml +++ b/gax-java/gax-httpjson/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax-httpjson - 2.57.0 + 2.57.1-SNAPSHOT jar GAX (Google Api eXtensions) for Java (HTTP JSON) Google Api eXtensions for Java (HTTP JSON) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.57.0 + 2.57.1-SNAPSHOT diff --git a/gax-java/gax/pom.xml b/gax-java/gax/pom.xml index 7aa543eaf0..d2e40b6a48 100644 --- a/gax-java/gax/pom.xml +++ b/gax-java/gax/pom.xml @@ -3,7 +3,7 @@ 4.0.0 gax - 2.57.0 + 2.57.1-SNAPSHOT jar GAX (Google Api eXtensions) for Java (Core) Google Api eXtensions for Java (Core) @@ -11,7 +11,7 @@ com.google.api gax-parent - 2.57.0 + 2.57.1-SNAPSHOT diff --git a/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java b/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java index f15046afcb..994ba2eb82 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/core/GaxProperties.java @@ -49,7 +49,7 @@ public class GaxProperties { private static final String GAX_VERSION = getLibraryVersion(GaxProperties.class, "version.gax"); private static final String JAVA_VERSION = getRuntimeVersion(); private static final String PROTOBUF_VERSION = - getBundleVersion(Any.class).orElse(DEFAULT_VERSION); + getProtobufVersion(Any.class, "com.google.protobuf.RuntimeVersion");; private GaxProperties() {} @@ -148,4 +148,29 @@ static Optional getBundleVersion(Class clazz) { return Optional.empty(); } } + + /** + * Returns the Protobuf runtime version as reported by com.google.protobuf.RuntimeVersion, if + * class is available, otherwise by reading from MANIFEST file. If niether option is available + * defaults to protobuf version 3 as RuntimeVersion class is available in protobuf version 4+ + */ + @VisibleForTesting + static String getProtobufVersion(Class clazz, String protobufRuntimeVersionClassName) { + try { + Class protobufRuntimeVersionClass = Class.forName(protobufRuntimeVersionClassName); + return protobufRuntimeVersionClass.getField("MAJOR").get(null) + + "." + + protobufRuntimeVersionClass.getField("MINOR").get(null) + + "." + + protobufRuntimeVersionClass.getField("PATCH").get(null); + } catch (ClassNotFoundException + | NoSuchFieldException + | IllegalAccessException + | SecurityException + | NullPointerException e) { + // If manifest file is not available default to protobuf generic version 3 as we know + // RuntimeVersion class is available in protobuf jar 4+. + return getBundleVersion(clazz).orElse("3"); + } + } } diff --git a/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java b/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java index 35307764d2..52f923e111 100644 --- a/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java +++ b/gax-java/gax/src/main/java/com/google/api/gax/rpc/ApiClientHeaderProvider.java @@ -88,7 +88,7 @@ private static String checkAndAppendProtobufVersionIfNecessary( // TODO(b/366417603): appending protobuf version to existing client library token until resolved Pattern pattern = Pattern.compile("(gccl|gapic)\\S*"); Matcher matcher = pattern.matcher(apiClientHeaderValue); - if (matcher.find()) { + if (matcher.find() && GaxProperties.getProtobufVersion() != null) { return apiClientHeaderValue.substring(0, matcher.end()) + "--" + PROTOBUF_HEADER_VERSION_KEY diff --git a/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/reflect-config.json b/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/reflect-config.json new file mode 100644 index 0000000000..3a38d53361 --- /dev/null +++ b/gax-java/gax/src/main/resources/META-INF/native-image/com.google.api/gax/reflect-config.json @@ -0,0 +1,10 @@ +[ + { + "name": "com.google.protobuf.RuntimeVersion", + "fields" : [ + { "name" : "MAJOR" }, + { "name" : "MINOR" }, + { "name" : "PATCH" } + ] + } +] \ No newline at end of file diff --git a/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java b/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java index 1369ec35ae..6560df4bc1 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/core/GaxPropertiesTest.java @@ -35,6 +35,7 @@ import static org.junit.jupiter.api.Assertions.assertTrue; import com.google.common.base.Strings; +import com.google.protobuf.Any; import java.io.IOException; import java.util.Optional; import java.util.regex.Pattern; @@ -160,12 +161,8 @@ void testGetJavaRuntimeInfo_nullJavaVersion() { @Test public void testGetProtobufVersion() throws IOException { - Version version = readVersion(GaxProperties.getProtobufVersion()); - - assertTrue(version.major >= 3); - if (version.major == 3) { - assertTrue(version.minor >= 25); - } + assertTrue( + Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(GaxProperties.getProtobufVersion()).find()); } @Test @@ -175,6 +172,36 @@ public void testGetBundleVersion_noManifestFile() throws IOException { assertFalse(version.isPresent()); } + @Test + void testGetProtobufVersion_success() { + String version = + GaxProperties.getProtobufVersion( + Any.class, "com.google.api.gax.core.GaxPropertiesTest$RuntimeVersion"); + + assertEquals("3.13.6", version); + } + + @Test + void testGetProtobufVersion_classNotFoundException() throws Exception { + String version = GaxProperties.getProtobufVersion(Any.class, "foo.NonExistantClass"); + + assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find()); + } + + @Test + void testgetProtobufVersion_noSuchFieldException() throws Exception { + String version = GaxProperties.getProtobufVersion(Any.class, "java.lang.Class"); + + assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find()); + } + + @Test + void testGetProtobufVersion_noManifest() throws Exception { + String version = GaxProperties.getProtobufVersion(GaxProperties.class, "foo.NonExistantClass"); + + assertEquals("3", version); + } + private Version readVersion(String version) { assertTrue(Pattern.compile("^\\d+\\.\\d+\\.\\d+").matcher(version).find()); String[] versionComponents = version.split("\\."); @@ -194,4 +221,11 @@ public Version(int major, int minor) { this.minor = minor; } } + + // Test class that emulates com.google.protobuf.RuntimeVersion for reflection lookup of fields + class RuntimeVersion { + public static final int MAJOR = 3; + public static final int MINOR = 13; + public static final int PATCH = 6; + } } diff --git a/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java b/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java index 53c1707290..5a65895ef0 100644 --- a/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java +++ b/gax-java/gax/src/test/java/com/google/api/gax/retrying/ScheduledRetryingExecutorTest.java @@ -195,13 +195,13 @@ void testCancelGetAttempt(boolean withCustomRetrySettings) throws Exception { setUp(withCustomRetrySettings); for (int executionsCount = 0; executionsCount < EXECUTIONS_COUNT; executionsCount++) { ScheduledExecutorService localExecutor = Executors.newSingleThreadScheduledExecutor(); - final int maxRetries = 100; + final int maxRetries = 20; FailingCallable callable = new FailingCallable(maxRetries - 1, "request", "SUCCESS", tracer); RetrySettings retrySettings = FAST_RETRY_SETTINGS .toBuilder() - .setTotalTimeoutDuration(java.time.Duration.ofMillis(1000L)) + .setTotalTimeoutDuration(java.time.Duration.ofMillis(5000L)) .setMaxAttempts(maxRetries) .build(); @@ -259,10 +259,11 @@ void testCancelOuterFutureAfterStart() throws Exception { .toBuilder() // These params were selected to ensure that future tries to run and fail (at least // once) but does not complete before it is cancelled. Assuming no computation time, - // it would take 25 + 100 + 400 + 1000 = 1525ms for the future to complete, which should + // it would take 2500 + 10000 + 10000 + 10000 = 32500ms for the future to complete, + // which should // be more than enough time to cancel the future. - .setInitialRetryDelayDuration(java.time.Duration.ofMillis(25L)) - .setMaxRetryDelayDuration(java.time.Duration.ofMillis(1000L)) + .setInitialRetryDelayDuration(java.time.Duration.ofMillis(2500L)) + .setMaxRetryDelayDuration(java.time.Duration.ofMillis(10000L)) .setRetryDelayMultiplier(4.0) .setTotalTimeoutDuration(java.time.Duration.ofMillis(60000L)) // Set this test to not use jitter as the randomized retry delay (RRD) may introduce diff --git a/gax-java/pom.xml b/gax-java/pom.xml index a5d9a10384..1963fd0b3f 100644 --- a/gax-java/pom.xml +++ b/gax-java/pom.xml @@ -4,14 +4,14 @@ com.google.api gax-parent pom - 2.57.0 + 2.57.1-SNAPSHOT GAX (Google Api eXtensions) for Java (Parent) Google Api eXtensions for Java (Parent) com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -50,7 +50,7 @@ com.google.api api-common - 2.40.0 + 2.40.1-SNAPSHOT com.google.auth @@ -98,24 +98,24 @@ com.google.api gax - 2.57.0 + 2.57.1-SNAPSHOT com.google.api gax - 2.57.0 + 2.57.1-SNAPSHOT test-jar testlib com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT io.grpc diff --git a/hermetic_build/DEVELOPMENT.md b/hermetic_build/DEVELOPMENT.md new file mode 100644 index 0000000000..1969bbcdfb --- /dev/null +++ b/hermetic_build/DEVELOPMENT.md @@ -0,0 +1,230 @@ +> [!IMPORTANT] +> All examples assume you are inside the repository root folder. + + +# Linting + +When contributing, ensure your changes to python code have a valid format. + +``` +python -m pip install black +black {source_file_or_directory} +``` + +# Install package dependencies + +```shell +python -m pip install --require-hashes -r hermetic_build/common/requirements.txt +python -m pip install hermetic_build/common +python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt +python -m pip install hermetic_build/library_generation +python -m pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt +python -m pip install hermetic_build/release_note_generation +``` + +# Run the integration tests + +The integration tests build the docker image declared in +`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC +repositories, generate the libraries and compare the results with the source +code declared in a "golden branch" of the repo. + +It requires docker and python (>= 3.12.0) to be installed. + +```shell +python -m unittest hermetic_build/library_generation/tests/integration_tests.py +``` + +# Run the unit tests + +There is one unit test file per component. +Every unit test script ends with `unit_tests.py`. +To avoid specifying them individually, we can use the following command: + +```shell +python -m unittest discover -s hermetic_build -p "*unit_tests.py" +``` + +> [!NOTE] +> The output of this command may look erratic during the first 30 seconds. +> This is normal. After the tests are done, an "OK" message should be shown. + +# Run the library generation scripts in your local environment + +Although the scripts are designed to run in a Docker container, you can also +run them directly. +This section explains how to run the entrypoint script +(`hermetic_build/library_generation/cli/entry_point.py`). + +## Assumptions made by the scripts + +### The Hermetic Build's well-known folder +Located in `${HOME}/.library_generation`, this folder is assumed by the scripts +to contain certain tools. + +Developers must make sure this folder is properly configured before running the +scripts locally. +Note that this relies on the `HOME` environment variable which is always defined +as per [POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). + +#### Put the gapic-generator-java jar in its well-known location + +1. Run the following command to install gapic-generator-java. + + ```shell + mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip + ``` + This will generate a jar located in `~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` + +2. Move the jar into its well-known location. + + ```shell + mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" + ``` + +#### Put the java formatter jar in its well-known location + +1. Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format) +or [GitHub releases](https://github.com/google/google-java-format/releases). +2. Move the jar into its well-known location. + + ```shell + mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar" + ``` + +## Installing prerequisites + +In order to run the generation scripts directly, there are a few tools we +need to install beforehand. + +### Install the owl-bot CLI + +This requires node.js to be installed. +Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) +for NVM, Node.js's version manager. + +After you install it, you can install the owl-bot CLI with the following +commands: + +```shell +git clone https://github.com/googleapis/repo-automation-bots +cd repo-automation-bots/packages/owl-bot +npm i && npm run compile && npm link +owl-bot copy-code --version +``` + +The key step is `npm link`, which will make the command available in you current +shell session. + +## Run the script +The entrypoint script (`hermetic_build/library_generation/cli/entry_point.py`) +allows you to generate a GAPIC repository with a given api definition (proto, +service yaml). + +### Download the api definition +For example, from googleapis + +```shell +git clone https://github.com/googleapis/googleapis +export api_definitions_path="$(pwd)/googleapis" +``` + +### Download the repo +For example, google-cloud-java +```shell +git clone https://github.com/googleapis/google-cloud-java +export path_to_repo="$(pwd)/google-cloud-java" +``` + +### Install the scripts + +You can skip this step if you've installed the packages in [Install package dependencies](#install-package-dependencies). + +```shell +python -m pip install --require-hashes -r hermetic_build/common/requirements.txt +python -m pip install hermetic_build/common +python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt +python -m pip install hermetic_build/library_generation +``` + +### Run the script + +```shell +python hermetic_build/library_generation/cli/entry_point.py generate \ + --repository-path="${path_to_repo}" \ + --api-definitions-path="${api_definitions_path}" +``` + +# Build the image from source + +1. Run the following command to build the image from source + + ```shell + DOCKER_BUILDKIT=1 docker build \ + -f .cloudbuild/library_generation/library_generation.Dockerfile \ + -t local:image-tag \ + . + ``` + Please note that the build only works when using the new [Docker BuildKit](https://docs.docker.com/build/buildkit/) + (enabled through the `DOCKER_BUILDKIT` variable). + +2. Set the version of gapic-generator-java + + ```shell + LOCAL_GENERATOR_VERSION=$(mvn \ + org.apache.maven.plugins:maven-help-plugin:evaluate \ + -Dexpression=project.version \ + -pl gapic-generator-java \ + -DforceStdout \ + -q) + ``` + +3. Run the image + + ```shell + # Assume you want to generate the library in the current working directory + # and the generation configuration is in the same directory. + docker run \ + --rm \ + --quiet \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + -v /path/to/api-definitions:/workspace/apis \ + -e GENERATOR_VERSION="${LOCAL_GENERATOR_VERSION}" \ + local:image-tag \ + --generation-config-path=/workspace/generation_config_file \ + --library-names=apigee-connect,asset \ + --repository-path=/workspace \ + --api-definitions-path=/workspace/apis + ``` + Note that if you specify the generator version using environment variable, + `-e GENERATOR_VERSION="${LOCAL_GENERATOR_VERSION}"` in the above example, + you should not set `gapic_generator_version` and `protoc_version` in the + generation configuration because values in the generation configuration will + take precedence. + +# Debug the library generation container +If you are working on changing the way the containers are created, you may want +to inspect the containers to check the setup. +It would be convenient in such case to have a text editor/viewer available. +You can achieve this by modifying the Dockerfile as follows: + +```dockerfile +# install OS tools +RUN apk update && apk add \ + unzip curl rsync openjdk11 jq bash nodejs npm git less vim +``` + +We add `less` and `vim` as text tools for further inspection. + +You can also run a shell in a new container by running: + +```shell +docker run \ + --rm \ + -it \ + -u $(id -u):$(id -g) \ + -v /path/to/google-cloud-java:/workspace \ + --entrypoint="bash" \ + $(cat image-id) +``` diff --git a/hermetic_build/library_generation/README.md b/hermetic_build/README.md similarity index 67% rename from hermetic_build/library_generation/README.md rename to hermetic_build/README.md index 0b4208ac3e..583e766bdf 100644 --- a/hermetic_build/library_generation/README.md +++ b/hermetic_build/README.md @@ -1,44 +1,43 @@ +> [!IMPORTANT] +> All scripts/examples assume you are inside the repository root folder. + # Generate a repository containing GAPIC Client Libraries -The script, `entry_point.py`, allows you to generate a repository containing -GAPIC client libraries (a monorepo, for example, google-cloud-java) from a -configuration file. +Running the docker image built from `hermetic_build/library_generation` +directory, you can generate a repository containing GAPIC client libraries (a +monorepo, for example, google-cloud-java) from a configuration file. + +Instead of running the docker image, if you prefer running the underlying python +scripts directly, please refer to the [development guide](DEVELOPMENT.md#run-the-script) +for additional instructions. ## Environment - OS: Linux -- Java runtime environment (8 or above) -- Python (3.12 or above) -- Docker -- Git +- Docker -## Prerequisite +## Prerequisites In order to generate a version for each library, a versions.txt has to exist -in `repository_path`. -Please refer to [Repository path](#repository-path--repositorypath---optional) for more information. - -## Parameters to generate a repository using `entry_point.py` - -### Baseline generation configuration yaml (`baseline_generation_config`) +in `repository-path`. +Please refer to [Repository path](#repository-path--repositorypath---optional) +for more information. -An absolute or relative path to a generation_config.yaml. -This config file is used for computing changed libraries, not library -generation. +## Parameters to generate a repository using the docker image -### Current generation configuration yaml (`current_generation_config`) +### Generation configuration yaml (`generation-config-path`) An absolute or relative path to a configuration file containing parameters to generate the repository. -Please refer [Configuration to generate a repository](#configuration-to-generate-a-repository) +Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) for more information. -### Repository path (`repository_path`), optional +### Repository path (`repository-path`), optional The path to where the generated repository goes. The default value is the current working directory when running the script. -For example, `cd google-cloud-java && python entry_point.py ...` without +For example, `cd google-cloud-java && python /path/to/entry_point.py ...` without specifying the `--repository_path` option will modify the `google-cloud-java` repository the user `cd`'d into. @@ -47,28 +46,37 @@ right version for each library. Please refer [here](go/java-client-releasing#versionstxt-manifest) for more info of versions.txt. -### Api definitions path (`api_definitions_path`), optional +### A list of library names (`library-names`), optional + +A list of library names that will be generated, separated by comma. +The library name of a library is the value of `library_name` or `api_shortname`, +if `library_name` is not specified, in the generation configuration. + +If not specified, all libraries in the generation +configuration will be generated. + +### Api definitions path (`api-definitions-path`), optional The path to where the api definition (proto, service yaml) resides. The default value is the current working directory when running the script. -Note that you need not only the protos defined the service, but also the transitive -dependencies of those protos. +**Note that you need not only the protos defined the service, but also the +transitive dependencies of those protos.** Any missing dependencies will cause `File not found` error. -For example, if your service is defined in `example_service.proto` and it imports -`google/api/annotations.proto`, you need the `annotations.proto` resides in a -folder that has the exact structure of the import statement (`google/api` in this -case), and set `api_definitions_path` to the path contains the root folder (`google` -in this case). +For example, if your service is defined in `example_service.proto` and it +imports `google/api/annotations.proto`, you need the `annotations.proto` resides +in a folder that has the exact structure of the import statement (`google/api` +in this case), and set `api_definitions_path` to the path contains the root +folder (`google` in this case). -## Output of `entry_point.py` +## Output ### GAPIC libraries -For each module (e.g. `google-cloud-java/java-asset`), the following files/folders -will be created/modified: +For each module (e.g. `google-cloud-java/java-asset`), the following +files/folders will be created/modified: | Name | Notes | |:------------------------------------|:-------------------------------------------------------------------------| @@ -185,32 +193,58 @@ libraries: - proto_path: google/cloud/asset/v1p7beta1 ``` -# Local Environment Setup before running `entry_point.py` - -1. Assuming Python 3 is installed, follow official guide from [Python.org](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments) to create a virtual environment. -The virtual environment can be installed to any folder, usually it is recommended to be installed under the root folder of the project(`sdk-platform-java` in this case). -2. Assuming the virtual environment is installed under `sdk-platform-java`. -Run the following command under the root folder of `sdk-platform-java` to install `library_generation` and its dependencies. +# Run the library generation image + +1. Download the API definitions to a local directory, e.g., from [googleapis](https://github.com/googleapis/googleapis). + +2. Run the docker image. + ```shell + # Assume you want to generate the library in the current working directory + # and the generation configuration is in the same directory. + docker run \ + --rm \ + --quiet \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + -v /path/to/api_definition:/workspace \ + gcr.io/cloud-devrel-public-resources/java-library-generation:image-tag + ``` - ```bash - python -m pip install --require-hashes -r hermetic_build/common/requirements.txt - python -m pip install hermetic_build/common - python -m pip install --require-hashes -r hermetic_build/library_generation/requirements.txt - python -m pip install hermetic_build/library_generation + * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating + yourself. + This avoids folder ownership changes since it runs as root by default. + * `-v "$(pwd):/workspace"` maps the host machine's current working directory + to the /workspace folder. + The image is configured to perform changes in this directory. + * `-v /path/to/api_definition:/workspace` maps the host machine's API + definitions folder to `/workspace/apis` folder. + +3. An advanced example: + ```shell + docker run \ + --rm \ + --quiet \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):/workspace" \ + -v /path/to/api_definition:/workspace/apis \ + gcr.io/cloud-devrel-public-resources/java-library-generation:image-tag \ + --generation-config-path=/workspace/generation_config_file \ + --library-names=apigee-connect,asset \ + --repository-path=/workspace \ + --api-definitions-path=/workspace/apis ``` + + * `--generation-config-path=/workspace/generation_config_file` set the + generation configuration to `/workspace/generation_config_file`. + * `--api-definitions-path=/workspace/apis` set the API definition path to + `/workspace/apis`. -3. Download api definition to a local directory +To debug the image, please refer to [development guide](DEVELOPMENT.md#debug-the-library-generation-container) +for more info. -## An example to generate a repository using `entry_point.py` +## An example to generate a repository using the docker image -```bash -python hermetic_build/library_generation/cli/entry_point.py generate \ - --baseline-generation-config-path=/path/to/baseline_config_file \ - --current-generation-config-path=/path/to/current_config_file \ - --repository-path=path/to/repository \ - --api-definitions-path=path/to/api_definition -``` -If you run `entry_point.py` with the example [configuration](#an-example-of-generation-configuration) +If you run the docker image with the example [configuration](#an-example-of-generation-configuration) shown above, the repository structure is: ``` $repository_path @@ -284,7 +318,70 @@ $repository_path |_versions.txt ``` -# Owlbot Java Postprocessor +# Generate release notes from API definition changes + +The script, `hermetic_build/release_note_generation/cli/generate_release_note.py` +allows you to generate a file containing release notes from API definition +changes in [googleapis](https://github.com/googleapis/googleapis) GitHub +repository. + +## Environment + +- OS: Linux +- Python (3.12.0 or above) + +## Parameters to generate a release note + +### Baseline generation configuration path (`baseline-generation-config-path`) + +Absolute or relative path to a generation configuration. +Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. + +Note that the `googleapis_commitish` in this configuration is used to retrieve +the first commit, exclusively, to generate the release notes. + +### Current generation configuration path (`current-generation-config-path`) + +Absolute or relative path to a generation configuration. +The release notes will be generated from commits that are related to the +libraries specified in this configuration. +Please refer to [Configuration to generate a repository](#configuration-to-generate-a-repository) +for more information. + +Note that the `googleapis_commitish` entry in this configuration is used to +retrieve the last commit, inclusively, to generate the release notes. + +### Repository path (`repository-path`), optional + +The path to which the file, `pr_description.txt` containing the release notes +will be sent. +If not specified, the file will be generated to the current working directory. + +## Generate a release notes file in a local environment + +1. Install python (>= 3.12.0). +It is recommended to create a python virtual environment through the +[official guide](https://packaging.python.org/en/latest/guides/installing-using-pip-and-virtual-environments/#create-and-use-virtual-environments). + +2. Run the following commands to install python packages + ```shell + cd /path/to/sdk-platform-java + pip install --require-hashes -r hermetic_build/common/requirements.txt + pip install hermetic_build/common + pip install --require-hashes -r hermetic_build/release_note_generation/requirements.txt + pip install hermetic_build/release_note_generation + ``` +3. Run the following commands to generate a release note + ```shell + cd /path/to/sdk-platform-java + python hermetic_build/release_note_generation/cli/generate_release_note.py generate \ + --baseline-generation-config-path=/path/to/baseline_generation_config \ + --current-generation-config-path=/path/to/current_generation_config \ + --repository-path=/path/to/send/release_note + ``` + +# OwlBot Java Postprocessor We have transferred the [implementation](https://github.com/googleapis/synthtool/tree/59fe44fde9866a26e7ee4e4450fd79f67f8cf599/docker/owlbot/java) diff --git a/hermetic_build/common/cli/get_changed_libraries.py b/hermetic_build/common/cli/get_changed_libraries.py new file mode 100644 index 0000000000..cf92cf0853 --- /dev/null +++ b/hermetic_build/common/cli/get_changed_libraries.py @@ -0,0 +1,83 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os + +import click as click + +from common.model.generation_config import from_yaml +from common.utils.generation_config_comparator import compare_config + + +@click.group(invoke_without_command=False) +@click.pass_context +@click.version_option(message="%(version)s") +def main(ctx): + pass + + +@main.command() +@click.option( + "--baseline-generation-config-path", + required=True, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml. + This config file is used for computing changed library list. + """, +) +@click.option( + "--current-generation-config-path", + required=True, + type=str, + help=""" + Absolute or relative path to a generation_config.yaml that contains the + metadata about library generation. + """, +) +def create( + baseline_generation_config_path: str, + current_generation_config_path: str, +) -> None: + """ + Compares baseline generation config with current generation config and + generates changed library names (a comma separated string) based on current + generation config. + """ + baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) + if not os.path.isfile(baseline_generation_config_path): + raise FileNotFoundError( + f"{baseline_generation_config_path} does not exist. " + "A valid generation config has to be passed in as " + "baseline-generation-config-path." + ) + current_generation_config_path = os.path.abspath(current_generation_config_path) + if not os.path.isfile(current_generation_config_path): + raise FileNotFoundError( + f"{current_generation_config_path} does not exist. " + "A valid generation config has to be passed in as " + "current-generation-config-path." + ) + config_change = compare_config( + baseline_config=from_yaml(baseline_generation_config_path), + current_config=from_yaml(current_generation_config_path), + ) + changed_libraries = config_change.get_changed_libraries() + if changed_libraries is None: + print("No changed library.") + return + click.echo(",".join(config_change.get_changed_libraries())) + + +if __name__ == "__main__": + main() diff --git a/hermetic_build/library_generation/model/config_change.py b/hermetic_build/common/model/config_change.py similarity index 86% rename from hermetic_build/library_generation/model/config_change.py rename to hermetic_build/common/model/config_change.py index 018aee8ccd..7ddc338448 100644 --- a/hermetic_build/library_generation/model/config_change.py +++ b/hermetic_build/common/model/config_change.py @@ -11,17 +11,14 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import os -import shutil +import tempfile from enum import Enum from typing import Optional from git import Commit, Repo - from common.model.gapic_inputs import parse_build_str from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig -from library_generation.utils.utilities import sh_util -from library_generation.utils.proto_path_utils import find_versioned_proto_path +from common.utils.proto_path_utils import find_versioned_proto_path INSERTIONS = "insertions" LINES = "lines" @@ -109,25 +106,22 @@ def get_qualified_commits( :param repo_url: the repository contains the commit history. :return: QualifiedCommit objects. """ - tmp_dir = sh_util("get_output_folder") - shutil.rmtree(tmp_dir, ignore_errors=True) - os.mkdir(tmp_dir) - # we only need commit history, thus shadow clone is enough. - repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) - commit = repo.commit(self.current_config.googleapis_commitish) - proto_paths = self.current_config.get_proto_path_to_library_name() - qualified_commits = [] - while str(commit.hexsha) != self.baseline_config.googleapis_commitish: - qualified_commit = ConfigChange.__create_qualified_commit( - proto_paths=proto_paths, commit=commit - ) - if qualified_commit is not None: - qualified_commits.append(qualified_commit) - commit_parents = commit.parents - if len(commit_parents) == 0: - break - commit = commit_parents[0] - shutil.rmtree(tmp_dir, ignore_errors=True) + with tempfile.TemporaryDirectory() as tmp_dir: + # we only need commit history, thus a shadow clone is enough. + repo = Repo.clone_from(url=repo_url, to_path=tmp_dir, filter=["blob:none"]) + commit = repo.commit(self.current_config.googleapis_commitish) + proto_paths = self.current_config.get_proto_path_to_library_name() + qualified_commits = [] + while str(commit.hexsha) != self.baseline_config.googleapis_commitish: + qualified_commit = ConfigChange.__create_qualified_commit( + proto_paths=proto_paths, commit=commit + ) + if qualified_commit is not None: + qualified_commits.append(qualified_commit) + commit_parents = commit.parents + if len(commit_parents) == 0: + break + commit = commit_parents[0] return qualified_commits def __get_library_names_from_qualified_commits(self) -> list[str]: diff --git a/hermetic_build/common/requirements.in b/hermetic_build/common/requirements.in index a34205e5fa..21607220f6 100644 --- a/hermetic_build/common/requirements.in +++ b/hermetic_build/common/requirements.in @@ -1,3 +1,4 @@ black==24.8.0 +GitPython==3.1.43 parameterized==0.9.0 PyYAML==6.0.2 \ No newline at end of file diff --git a/hermetic_build/common/requirements.txt b/hermetic_build/common/requirements.txt index d952506bb5..9b79817c1b 100644 --- a/hermetic_build/common/requirements.txt +++ b/hermetic_build/common/requirements.txt @@ -32,6 +32,14 @@ click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via black +gitdb==4.0.11 \ + --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ + --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b + # via gitpython +gitpython==3.1.43 \ + --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ + --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff + # via -r hermetic_build/common/requirements.in mypy-extensions==1.0.0 \ --hash=sha256:4392f6c0eb8a5668a69e23d168ffa70f0be9ccfd32b5cc2d26a34ae5b844552d \ --hash=sha256:75dbf8955dc00442a438fc4d0666508a9a97b6bd41aa2f0ffe9d2f2725af0782 @@ -107,3 +115,7 @@ pyyaml==6.0.2 \ --hash=sha256:f753120cb8181e736c57ef7636e83f31b9c0d1722c516f7e86cf15b7aa57ff12 \ --hash=sha256:ff3824dc5261f50c9b0dfb3be22b4567a6f938ccce4587b38952d85fd9e9afe4 # via -r hermetic_build/common/requirements.in +smmap==5.0.1 \ + --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ + --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da + # via gitdb diff --git a/hermetic_build/common/tests/cli/__init__.py b/hermetic_build/common/tests/cli/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/common/tests/cli/config_change_unit_tests.py b/hermetic_build/common/tests/cli/config_change_unit_tests.py new file mode 100644 index 0000000000..e3bdd753de --- /dev/null +++ b/hermetic_build/common/tests/cli/config_change_unit_tests.py @@ -0,0 +1,71 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +from click.testing import CliRunner +import unittest + +from common.cli.get_changed_libraries import create + +script_dir = os.path.dirname(os.path.realpath(__file__)) +test_resource_dir = os.path.join(script_dir, "..", "resources", "cli") + + +class GetChangedLibrariesTest(unittest.TestCase): + def test_entry_point_without_baseline_config_raise_system_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke(create) + self.assertEqual(2, result.exit_code) + self.assertEqual(SystemExit, result.exc_info[0]) + + def test_entry_point_without_current_config_raise_system_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + create, ["--baseline-generation-config-path=/invalid/path/file"] + ) + self.assertEqual(2, result.exit_code) + self.assertEqual(SystemExit, result.exc_info[0]) + + def test_entry_point_with_invalid_baseline_config_raise_file_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + create, + [ + "--baseline-generation-config-path=/invalid/path/file", + "--current-generation-config-path=/invalid/path/file", + ], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex(result.exception.args[0], "baseline-generation-config-path") + + def test_entry_point_with_invalid_current_config_raise_file_exception(self): + os.chdir(script_dir) + runner = CliRunner() + # noinspection PyTypeChecker + result = runner.invoke( + create, + [ + f"--baseline-generation-config-path={test_resource_dir}/empty_config.yaml", + "--current-generation-config-path=/invalid/path/file", + ], + ) + self.assertEqual(1, result.exit_code) + self.assertEqual(FileNotFoundError, result.exc_info[0]) + self.assertRegex(result.exception.args[0], "current-generation-config-path") diff --git a/hermetic_build/library_generation/tests/model/config_change_unit_tests.py b/hermetic_build/common/tests/model/config_change_unit_tests.py similarity index 98% rename from hermetic_build/library_generation/tests/model/config_change_unit_tests.py rename to hermetic_build/common/tests/model/config_change_unit_tests.py index 6e0a088e75..3abc603141 100644 --- a/hermetic_build/library_generation/tests/model/config_change_unit_tests.py +++ b/hermetic_build/common/tests/model/config_change_unit_tests.py @@ -13,9 +13,9 @@ # limitations under the License. import unittest -from library_generation.model.config_change import ChangeType -from library_generation.model.config_change import ConfigChange -from library_generation.model.config_change import LibraryChange +from common.model.config_change import ChangeType +from common.model.config_change import ConfigChange +from common.model.config_change import LibraryChange from common.model.gapic_config import GapicConfig from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig diff --git a/hermetic_build/common/tests/resources/cli/empty_config.yaml b/hermetic_build/common/tests/resources/cli/empty_config.yaml new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/common/tests/utils/__init__.py b/hermetic_build/common/tests/utils/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py b/hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py similarity index 99% rename from hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py rename to hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py index f88f71d40e..00edb511eb 100644 --- a/hermetic_build/library_generation/tests/utils/generation_config_comparator_unit_tests.py +++ b/hermetic_build/common/tests/utils/generation_config_comparator_unit_tests.py @@ -16,8 +16,8 @@ from common.model.gapic_config import GapicConfig from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig -from library_generation.utils.generation_config_comparator import ChangeType -from library_generation.utils.generation_config_comparator import compare_config +from common.utils.generation_config_comparator import ChangeType +from common.utils.generation_config_comparator import compare_config class GenerationConfigComparatorTest(unittest.TestCase): diff --git a/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py b/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py new file mode 100644 index 0000000000..90b3dd3f55 --- /dev/null +++ b/hermetic_build/common/tests/utils/proto_path_utils_unit_tests.py @@ -0,0 +1,39 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import os +import unittest +from pathlib import Path +from common.utils.proto_path_utils import find_versioned_proto_path + +script_dir = os.path.dirname(os.path.realpath(__file__)) +resources_dir = os.path.join(script_dir, "..", "resources") +test_config_dir = Path(os.path.join(resources_dir, "test-config")).resolve() + + +class ProtoPathsUtilsTest(unittest.TestCase): + def test_find_versioned_proto_path_nested_version_success(self): + proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" + expected = "google/cloud/aiplatform/v1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_path_success(self): + proto_path = "google/cloud/asset/v1p2beta1/assets.proto" + expected = "google/cloud/asset/v1p2beta1" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) + + def test_find_versioned_proto_without_version_return_itself(self): + proto_path = "google/type/color.proto" + expected = "google/type/color.proto" + self.assertEqual(expected, find_versioned_proto_path(proto_path)) diff --git a/hermetic_build/library_generation/utils/generation_config_comparator.py b/hermetic_build/common/utils/generation_config_comparator.py similarity index 97% rename from hermetic_build/library_generation/utils/generation_config_comparator.py rename to hermetic_build/common/utils/generation_config_comparator.py index d0851c7f31..f41299ddc2 100644 --- a/hermetic_build/library_generation/utils/generation_config_comparator.py +++ b/hermetic_build/common/utils/generation_config_comparator.py @@ -15,10 +15,10 @@ from typing import Any from typing import Dict from typing import List -from library_generation.model.config_change import ChangeType -from library_generation.model.config_change import ConfigChange -from library_generation.model.config_change import LibraryChange -from library_generation.model.config_change import HashLibrary +from common.model.config_change import ChangeType +from common.model.config_change import ConfigChange +from common.model.config_change import LibraryChange +from common.model.config_change import HashLibrary from common.model.gapic_config import GapicConfig from common.model.generation_config import GenerationConfig from common.model.library_config import LibraryConfig diff --git a/hermetic_build/common/utils/proto_path_utils.py b/hermetic_build/common/utils/proto_path_utils.py new file mode 100644 index 0000000000..49a86dcbd2 --- /dev/null +++ b/hermetic_build/common/utils/proto_path_utils.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +import re + + +def find_versioned_proto_path(proto_path: str) -> str: + """ + Returns a versioned proto_path from a given proto_path; or proto_path itself + if it doesn't contain a versioned proto_path. + :param proto_path: a proto file path + :return: the versioned proto_path + """ + version_regex = re.compile(r"^v[1-9].*") + directories = proto_path.split("/") + for directory in directories: + result = version_regex.search(directory) + if result: + version = result[0] + idx = proto_path.find(version) + return proto_path[:idx] + version + return proto_path diff --git a/hermetic_build/library_generation/DEVELOPMENT.md b/hermetic_build/library_generation/DEVELOPMENT.md deleted file mode 100644 index df7843aeaa..0000000000 --- a/hermetic_build/library_generation/DEVELOPMENT.md +++ /dev/null @@ -1,207 +0,0 @@ -> [!IMPORTANT] -> All examples assume you are inside the `hermetic_build` folder. - - -# Linting - -When contributing, ensure your changes to python code have a valid format. - -``` -python -m pip install black -black . -``` - -# Running the integration tests - -The integration tests build the docker image declared in -`.cloudbuild/library_generation/library_generation.Dockerfile`, pull GAPIC -repositories, generate the libraries and compares the results with the source -code declared in a "golden branch" of the repo. - -It requires docker and python 3.x to be installed. - -``` -python -m pip install --require-hashes -r library_generation/requirements.txt -python -m pip install library_generation -python -m unittest library_generation/tests/integration_tests.py -``` - -# Running the unit tests - -The unit tests of the hermetic build scripts are contained in several scripts, -corresponding to a specific component. -Every unit test script ends with `unit_tests.py`. -To avoid them specifying them individually, we can use the following command: - -```bash -python -m unittest discover -s library_generation/tests/ -p "*unit_tests.py" -``` - -> [!NOTE] -> The output of this command may look erratic during the first 30 seconds. -> This is normal. After the tests are done, an "OK" message should be shown. - -# Running the scripts in your local environment - -Although the scripts are designed to be run in a Docker container, you can also -run them directly. -This section explains how to run the entrypoint script -(`library_generation/cli/entry_point.py`). - -## Assumptions made by the scripts -### The Hermetic Build's well-known folder -Located in `${HOME}/.library_generation`, this folder is assumed by the scripts -to contain certain tools. - -Developers must make sure this folder is properly configured before running the -scripts locally. -Note that this relies on the `HOME` en var which is always defined as per -[POSIX env var definition](https://pubs.opengroup.org/onlinepubs/9699919799/basedefs/V1_chap08.html). - -#### Put the gapic-generator-java jar in its well-known location - -Run `cd sdk-platform-java && mvn install -DskipTests -Dclirr.skip --Dcheckstyle.skip`. -This will generate a jar located in -`~/.m2/repository/com/google/api/gapic-generator-java/{version}/gapic-generator-java-{version}.jar` - -Then `mv` the jar into the well-known location of the jar. -The generation scripts will assume the jar is there. - -```shell -mv /path/to/jar "${HOME}/.library_generation/gapic-generator-java.jar" -``` - -#### Put the java formatter jar in its well-known location - -Download google-java-format-{version}-all-deps.jar from [Maven Central](https://central.sonatype.com/artifact/com.google.googlejavaformat/google-java-format) -or [GitHub releases](https://github.com/google/google-java-format/releases). -Then `mv` the jar into the well-known location of the jar. -The generation scripts will assume the jar is there. - -```shell -mv /path/to/jar "${HOME}/.library_generation/google-java-format.jar" -``` - -## Installing prerequisites - -In order to run the generation scripts directly, there are a few tools we -need to install beforehand. - -### Install the owl-bot CLI - -Requires node.js to be installed. -Check this [installation guide](https://github.com/nvm-sh/nvm?tab=readme-ov-file#install--update-script) -for NVM, Node.js's version manager. - -After you install it, you can install the owl-bot CLI with the following -commands: -```bash -git clone https://github.com/googleapis/repo-automation-bots -cd repo-automation-bots/packages/owl-bot -npm i && npm run compile && npm link -owl-bot copy-code --version -``` - -The key step is `npm link`, which will make the command available in you current -shell session. - - -## Running the script -The entrypoint script (`library_generation/cli/entry_point.py`) allows you to -generate a GAPIC repository with a given api definition (proto, service yaml). - -### Download the api definition -For example, googleapis -``` -git clone https://github.com/googleapis/googleapis -export api_definitions_path="$(pwd)/googleapis" -``` - -### Download the repo -For example, google-cloud-java -``` -git clone https://github.com/googleapis/google-cloud-java -export path_to_repo="$(pwd)/google-cloud-java" -``` - -### Install the scripts -``` -python -m pip install . -``` - -### Run the script -``` -python library_generation/cli/entry_point.py generate \ - --repository-path="${path_to_repo}" \ - --api-definitions-path="${api_definitions_path}" -``` - - -# Running the scripts using the docker container image -This is convenient in order to avoid installing the dependencies manually. - -> [!IMPORTANT] -> From now, the examples assume you are in the root of your sdk-platform-java -> folder. - -## Build the docker image -```bash -docker build --file .cloudbuild/library_generation/library_generation.Dockerfile --iidfile image-id . -``` - -This will create an `image-id` file at the root of the repo with the hash ID of -the image. - -## Run the docker image -The docker image will perform changes on its internal `/workspace` folder, -to which you need to map a folder on your host machine (i.e. map your downloaded -repo to this folder). - -To run the docker container on the google-cloud-java repo, you must run: -```bash -docker run \ - -u "$(id -u)":"$(id -g)" \ - -v /path/to/google-cloud-java:/workspace \ - -v /path/to/api-definition:/workspace/apis \ - $(cat image-id) \ - --api-definitions-path=/workspace/apis -``` - - * `-u "$(id -u)":"$(id -g)"` makes docker run the container impersonating - yourself. This avoids folder ownership changes since it runs as root by - default. - * `-v /path/to/google-cloud-java:/workspace` maps the host machine's - google-cloud-java folder to the /workspace folder. - The image is configured to perform changes in this directory. - * `-v /path/to/api-definition:/workspace/apis` maps the host machine's - api-definition folder to /workspace/apis folder. - * `$(cat image-id)` obtains the image ID created in the build step. - * `--api-definitions-path=/workspace/apis` set the API definition path to - `/workspace/apis`. - -## Debug the created containers -If you are working on changing the way the containers are created, you may want -to inspect the containers to check the setup. -It would be convenient in such case to have a text editor/viewer available. -You can achieve this by modifying the Dockerfile as follows: - -```docker -# install OS tools -RUN apt-get update && apt-get install -y \ - unzip openjdk-17-jdk rsync maven jq less vim \ - && apt-get clean -``` - -We add `less` and `vim` as text tools for further inspection. - -You can also run a shell in a new container by running: - -```bash -docker run \ - --rm -it \ - -u $(id -u):$(id -g) \ - -v /path/to/google-cloud-java:/workspace \ - --entrypoint="bash" \ - $(cat image-id) -``` diff --git a/hermetic_build/library_generation/cli/entry_point.py b/hermetic_build/library_generation/cli/entry_point.py index b15880f06d..e568f831ab 100644 --- a/hermetic_build/library_generation/cli/entry_point.py +++ b/hermetic_build/library_generation/cli/entry_point.py @@ -16,9 +16,7 @@ from typing import Optional import click as click from library_generation.generate_repo import generate_from_yaml -from library_generation.model.config_change import ConfigChange -from common.model.generation_config import from_yaml -from library_generation.utils.generation_config_comparator import compare_config +from common.model.generation_config import from_yaml, GenerationConfig @click.group(invoke_without_command=False) @@ -30,18 +28,7 @@ def main(ctx): @main.command() @click.option( - "--baseline-generation-config-path", - required=False, - default=None, - type=str, - help=""" - Absolute or relative path to a generation_config.yaml. - This config file is used for commit history generation, not library - generation. - """, -) -@click.option( - "--current-generation-config-path", + "--generation-config-path", required=False, default=None, type=str, @@ -85,8 +72,7 @@ def main(ctx): """, ) def generate( - baseline_generation_config_path: str, - current_generation_config_path: str, + generation_config_path: Optional[str], library_names: Optional[str], repository_path: str, api_definitions_path: str, @@ -115,103 +101,63 @@ def generate( Raise FileNotFoundError if the default config does not exist. """ - __generate_repo_and_pr_description_impl( - baseline_generation_config_path=baseline_generation_config_path, - current_generation_config_path=current_generation_config_path, + __generate_repo_impl( + generation_config_path=generation_config_path, library_names=library_names, repository_path=repository_path, api_definitions_path=api_definitions_path, ) -def __generate_repo_and_pr_description_impl( - baseline_generation_config_path: str, - current_generation_config_path: str, +def __generate_repo_impl( + generation_config_path: Optional[str], library_names: Optional[str], repository_path: str, api_definitions_path: str, ): """ Implementation method for generate(). - The decoupling of generate and __generate_repo_and_pr_description_impl is + The decoupling of generate and __generate_repo_impl is meant to allow testing of this implementation function. """ default_generation_config_path = f"{os.getcwd()}/generation_config.yaml" - - if ( - baseline_generation_config_path is None - and current_generation_config_path is None - ): - if not os.path.isfile(default_generation_config_path): - raise FileNotFoundError( - f"{default_generation_config_path} does not exist. " - "A valid generation config has to be passed in as " - "current_generation_config or exist in the current working " - "directory." - ) - current_generation_config_path = default_generation_config_path - elif current_generation_config_path is None: + if generation_config_path is None: + generation_config_path = default_generation_config_path + generation_config_path = os.path.abspath(generation_config_path) + if not os.path.isfile(generation_config_path): raise FileNotFoundError( - "current_generation_config is not specified when " - "baseline_generation_config is specified. " - "current_generation_config should be the source of truth of " - "library generation." + f"Generation config {generation_config_path} does not exist." ) - - current_generation_config_path = os.path.abspath(current_generation_config_path) repository_path = os.path.abspath(repository_path) api_definitions_path = os.path.abspath(api_definitions_path) - include_library_names = _parse_library_name_from(library_names) - - if not baseline_generation_config_path: - # Execute selective generation based on current_generation_config if - # baseline_generation_config is not specified. - generate_from_yaml( - config=from_yaml(current_generation_config_path), - repository_path=repository_path, - api_definitions_path=api_definitions_path, - target_library_names=include_library_names, - ) - return - - # Compare two generation configs to get changed libraries. - baseline_generation_config_path = os.path.abspath(baseline_generation_config_path) - config_change = compare_config( - baseline_config=from_yaml(baseline_generation_config_path), - current_config=from_yaml(current_generation_config_path), - ) - # Pass None if we want to fully generate the repository. - changed_library_names = ( - config_change.get_changed_libraries() - if not _needs_full_repo_generation(config_change=config_change) - else None - ) - # Include library names takes preference if specified. - target_library_names = ( - include_library_names - if include_library_names is not None - else changed_library_names + generation_config = from_yaml(generation_config_path) + include_library_names = _parse_library_name_from( + includes=library_names, generation_config=generation_config ) generate_from_yaml( - config=config_change.current_config, + config=generation_config, repository_path=repository_path, api_definitions_path=api_definitions_path, - target_library_names=target_library_names, + target_library_names=include_library_names, ) -def _needs_full_repo_generation(config_change: ConfigChange) -> bool: +def _needs_full_repo_generation(generation_config: GenerationConfig) -> bool: """ Whether you should need a full repo generation, i.e., generate all libraries in the generation configuration. """ - current_config = config_change.current_config - return not current_config.is_monorepo() or current_config.contains_common_protos() + return ( + not generation_config.is_monorepo() + or generation_config.contains_common_protos() + ) -def _parse_library_name_from(includes: str) -> Optional[list[str]]: - if includes is None: +def _parse_library_name_from( + includes: Optional[str], generation_config: GenerationConfig +) -> Optional[list[str]]: + if includes is None or _needs_full_repo_generation(generation_config): return None return [library_name.strip() for library_name in includes.split(",")] diff --git a/hermetic_build/library_generation/generate_library.sh b/hermetic_build/library_generation/generate_library.sh index b8d22aca54..f5cae1ba5a 100755 --- a/hermetic_build/library_generation/generate_library.sh +++ b/hermetic_build/library_generation/generate_library.sh @@ -115,11 +115,11 @@ if [ -z "${os_architecture}" ]; then os_architecture=$(detect_os_architecture) fi -temp_destination_path="${output_folder}/temp_preprocessed" +temp_destination_path="${output_folder}/temp_preprocessed-$RANDOM" mkdir -p "${output_folder}/${destination_path}" if [ -d "${temp_destination_path}" ]; then # we don't want the preprocessed sources of a previous run - rm -rd "${temp_destination_path}" + rm -r "${temp_destination_path}" fi mkdir -p "${temp_destination_path}" ##################### Section 0 ##################### @@ -274,5 +274,5 @@ rm -rf java_gapic_srcjar java_gapic_srcjar_raw.srcjar.zip java_grpc.jar java_pro popd # destination path cp -r ${temp_destination_path}/* "${output_folder}/${destination_path}" -rm -rdf "${temp_destination_path}" +rm -rf "${temp_destination_path}" exit 0 diff --git a/hermetic_build/library_generation/owlbot/bin/entrypoint.sh b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh index 3d38677678..3118f32a2b 100755 --- a/hermetic_build/library_generation/owlbot/bin/entrypoint.sh +++ b/hermetic_build/library_generation/owlbot/bin/entrypoint.sh @@ -36,7 +36,7 @@ library_version=$5 if [[ "${is_monorepo}" == "true" ]]; then mv owl-bot-staging/* temp - rm -rd owl-bot-staging/ + rm -rf owl-bot-staging/ mv temp owl-bot-staging fi diff --git a/hermetic_build/library_generation/owlbot/bin/format_source.sh b/hermetic_build/library_generation/owlbot/bin/format_source.sh index 9402be778b..efc51b8940 100755 --- a/hermetic_build/library_generation/owlbot/bin/format_source.sh +++ b/hermetic_build/library_generation/owlbot/bin/format_source.sh @@ -36,6 +36,9 @@ do elif [[ $file =~ .*/samples/snippets/src/.*/java/com/example/spanner/.*.java ]]; then echo "File skipped formatting: $file" + elif [[ $file =~ .*/test-jdk17/java/com/google/cloud/firestore/.*java ]]; + then + echo "File skipped formatting: $file" else echo $file >> $tmp_file fi diff --git a/hermetic_build/library_generation/requirements.in b/hermetic_build/library_generation/requirements.in index 4f8ad9709b..7ab992ea62 100644 --- a/hermetic_build/library_generation/requirements.in +++ b/hermetic_build/library_generation/requirements.in @@ -1,6 +1,5 @@ attrs==24.2.0 click==8.1.7 -GitPython==3.1.43 jinja2==3.1.4 lxml==5.3.0 PyYAML==6.0.2 diff --git a/hermetic_build/library_generation/requirements.txt b/hermetic_build/library_generation/requirements.txt index 87ac0ba921..ef3d97bedc 100644 --- a/hermetic_build/library_generation/requirements.txt +++ b/hermetic_build/library_generation/requirements.txt @@ -123,14 +123,6 @@ click==8.1.7 \ --hash=sha256:ae74fb96c20a0277a1d615f1e4d73c8414f5a98db8b799a7931d1582f3390c28 \ --hash=sha256:ca9853ad459e787e2192211578cc907e7594e294c7ccc834310722b41b9ca6de # via -r hermetic_build/library_generation/requirements.in -gitdb==4.0.11 \ - --hash=sha256:81a3407ddd2ee8df444cbacea00e2d038e40150acfa3001696fe0dcf1d3adfa4 \ - --hash=sha256:bf5421126136d6d0af55bc1e7c1af1c397a34f5b7bd79e776cd3e89785c2b04b - # via gitpython -gitpython==3.1.43 \ - --hash=sha256:35f314a9f878467f5453cc1fee295c3e18e52f1b99f10f6cf5b1682e968a9e7c \ - --hash=sha256:eec7ec56b92aad751f9912a73404bc02ba212a23adb2c7098ee668417051a1ff - # via -r hermetic_build/library_generation/requirements.in idna==3.10 \ --hash=sha256:12f65c9b470abda6dc35cf8e63cc574b1c52b11df2c86030af0ac09b01b13ea9 \ --hash=sha256:946d195a0d259cbba61165e88e65941f16e9b36ea6ddb97f00452bae8b1287d3 @@ -407,10 +399,6 @@ requests-mock==1.12.1 \ --hash=sha256:b1e37054004cdd5e56c84454cc7df12b25f90f382159087f4b6915aaeef39563 \ --hash=sha256:e9e12e333b525156e82a3c852f22016b9158220d2f47454de9cae8a77d371401 # via -r hermetic_build/library_generation/requirements.in -smmap==5.0.1 \ - --hash=sha256:dceeb6c0028fdb6734471eb07c0cd2aae706ccaecab45965ee83f11c8d3b1f62 \ - --hash=sha256:e6d8668fa5f93e706934a62d7b4db19c8d9eb8cf2adbb75ef1b675aa332b69da - # via gitdb urllib3==2.2.3 \ --hash=sha256:ca899ca043dcb1bafa3e262d73aa25c465bfb49e0bd9dd5d59f1d0acba2f8fac \ --hash=sha256:e7d814a81dad81e6caf2ec9fdedb284ecc9c73076b62654547cc64ccdcae26e9 diff --git a/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py index b7e7088cf9..82f6ec1c13 100644 --- a/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py +++ b/hermetic_build/library_generation/tests/cli/entry_point_unit_tests.py @@ -18,7 +18,7 @@ from library_generation.cli.entry_point import ( generate, validate_generation_config, - __generate_repo_and_pr_description_impl as generate_impl, + __generate_repo_impl as generate_impl, ) from common.model.generation_config import from_yaml @@ -27,34 +27,27 @@ class EntryPointTest(unittest.TestCase): - def test_entry_point_without_config_raise_file_exception(self): + def test_entry_point_without_default_config_raise_file_exception(self): os.chdir(script_dir) runner = CliRunner() # noinspection PyTypeChecker - result = runner.invoke(generate, ["--repository-path=."]) + result = runner.invoke(generate) self.assertEqual(1, result.exit_code) self.assertEqual(FileNotFoundError, result.exc_info[0]) self.assertRegex( result.exception.args[0], "generation_config.yaml does not exist." ) - def test_entry_point_with_baseline_without_current_raise_file_exception(self): + def test_entry_point_with_invalid_config_raise_file_exception(self): + os.chdir(script_dir) runner = CliRunner() # noinspection PyTypeChecker result = runner.invoke( - generate, - [ - "--baseline-generation-config-path=path/to/config.yaml", - "--repository-path=.", - ], + generate, ["--generation-config-path=/non-existent/file"] ) self.assertEqual(1, result.exit_code) self.assertEqual(FileNotFoundError, result.exc_info[0]) - self.assertRegex( - result.exception.args[0], - "current_generation_config is not specified when " - "baseline_generation_config is specified.", - ) + self.assertRegex(result.exception.args[0], "/non-existent/file does not exist.") def test_validate_generation_config_succeeds( self, @@ -86,7 +79,7 @@ def test_validate_generation_config_with_duplicate_library_name_raise_file_excep ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_without_changes_triggers_full_generation( + def test_generate_non_monorepo_without_library_names_full_generation( self, generate_from_yaml, ): @@ -100,8 +93,7 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names=None, repository_path=".", api_definitions_path=".", @@ -114,7 +106,7 @@ def test_generate_non_monorepo_without_changes_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_generation( + def test_generate_non_monorepo_with_library_names_full_generation( self, generate_from_yaml, ): @@ -129,41 +121,8 @@ def test_generate_non_monorepo_without_changes_with_includes_triggers_selective_ # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names="cloudasset,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudasset", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_with_changes_triggers_full_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). generate() should call generate_from_yaml() - with target_library_names=None in order to trigger the full generation - """ - baseline_config_path = f"{test_resource_dir}/generation_config.yaml" - current_config_path = ( - f"{test_resource_dir}/generation_config_library_modified.yaml" - ) - self.assertFalse(from_yaml(current_config_path).is_monorepo()) - self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names=None, + generation_config_path=config_path, + library_names="non-existent-library", repository_path=".", api_definitions_path=".", ) @@ -175,40 +134,7 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_non_monorepo_with_changes_with_includes_triggers_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of non monorepos - (HW libraries). - generate() should call generate_from_yaml() with - target_library_names equals includes - """ - baseline_config_path = f"{test_resource_dir}/generation_config.yaml" - current_config_path = ( - f"{test_resource_dir}/generation_config_library_modified.yaml" - ) - self.assertFalse(from_yaml(current_config_path).is_monorepo()) - self.assertFalse(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names="cloudasset,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudasset", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_common_protos_triggers_full_generation( + def test_generate_monorepo_with_common_protos_without_library_names_triggers_full_generation( self, generate_from_yaml, ): @@ -223,8 +149,7 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names=None, repository_path=".", api_definitions_path=".", @@ -237,7 +162,7 @@ def test_generate_monorepo_with_common_protos_triggers_full_generation( ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_generation( + def test_generate_monorepo_with_common_protos_with_library_names_triggers_full_generation( self, generate_from_yaml, ): @@ -251,8 +176,7 @@ def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_g # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names="iam,non-existent-library", repository_path=".", api_definitions_path=".", @@ -261,11 +185,11 @@ def test_generate_monorepo_with_common_protos_with_includes_triggers_selective_g config=ANY, repository_path=ANY, api_definitions_path=ANY, - target_library_names=["iam", "non-existent-library"], + target_library_names=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_change_does_not_trigger_generation( + def test_generate_monorepo_without_library_names_trigger_full_generation( self, generate_from_yaml, ): @@ -281,8 +205,7 @@ def test_generate_monorepo_without_change_does_not_trigger_generation( # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names=None, repository_path=".", api_definitions_path=".", @@ -291,11 +214,11 @@ def test_generate_monorepo_without_change_does_not_trigger_generation( config=ANY, repository_path=ANY, api_definitions_path=ANY, - target_library_names=[], + target_library_names=None, ) @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_change_with_includes_trigger_selective_generation( + def test_generate_monorepo_with_library_names_trigger_selective_generation( self, generate_from_yaml, ): @@ -311,8 +234,7 @@ def test_generate_monorepo_without_change_with_includes_trigger_selective_genera # we call the implementation method directly since click # does special handling when a method is annotated with @main.command() generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, + generation_config_path=config_path, library_names="asset", repository_path=".", api_definitions_path=".", @@ -323,94 +245,3 @@ def test_generate_monorepo_without_change_with_includes_trigger_selective_genera api_definitions_path=ANY, target_library_names=["asset"], ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_changed_config_without_includes_trigger_changed_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the changed libraries if includes - is not specified. - """ - current_config_path = f"{test_resource_dir}/monorepo_current.yaml" - baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" - self.assertTrue(from_yaml(current_config_path).is_monorepo()) - self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["asset"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_with_changed_config_and_includes_trigger_selective_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the same as include libraries, regardless - the library exists or not. - """ - current_config_path = f"{test_resource_dir}/monorepo_current.yaml" - baseline_config_path = f"{test_resource_dir}/monorepo_baseline.yaml" - self.assertTrue(from_yaml(current_config_path).is_monorepo()) - self.assertTrue(from_yaml(baseline_config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=baseline_config_path, - current_generation_config_path=current_config_path, - library_names="cloudbuild,non-existent-library", - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=["cloudbuild", "non-existent-library"], - ) - - @patch("library_generation.cli.entry_point.generate_from_yaml") - def test_generate_monorepo_without_changed_config_without_includes_does_not_trigger_generation( - self, - generate_from_yaml, - ): - """ - this tests confirms the behavior of generation of a monorepo without - common protos. - target_library_names should be the changed libraries if includes - is not specified. - """ - config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" - self.assertTrue(from_yaml(config_path).is_monorepo()) - # we call the implementation method directly since click - # does special handling when a method is annotated with @main.command() - generate_impl( - baseline_generation_config_path=config_path, - current_generation_config_path=config_path, - library_names=None, - repository_path=".", - api_definitions_path=".", - ) - generate_from_yaml.assert_called_with( - config=ANY, - repository_path=ANY, - api_definitions_path=ANY, - target_library_names=[], - ) diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.py b/hermetic_build/library_generation/tests/generate_library_unit_tests.py index 7bc14e3e20..eb2249908c 100644 --- a/hermetic_build/library_generation/tests/generate_library_unit_tests.py +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.py @@ -53,7 +53,7 @@ def setUp(self): bash_call(f"mkdir {self.output_folder}") def tearDown(self): - bash_call(f"rm -rdf {self.simulated_home}") + bash_call(f"rm -rf {self.simulated_home}") def _run_command(self, command, **kwargs): env = os.environ.copy() diff --git a/hermetic_build/library_generation/tests/generate_library_unit_tests.sh b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh index 639abd8677..a3d95c1be7 100755 --- a/hermetic_build/library_generation/tests/generate_library_unit_tests.sh +++ b/hermetic_build/library_generation/tests/generate_library_unit_tests.sh @@ -138,7 +138,7 @@ download_tools_succeed_with_baked_protoc() { download_tools "99.99" "${test_grpc_version}" "linux-x86_64" assertEquals "${protoc_bin_folder}" "${protoc_path}" - rm -rdf "${output_folder}" + rm -rf "${output_folder}" unset DOCKER_PROTOC_LOCATION unset DOCKER_PROTOC_VERSION unset output_folder @@ -159,7 +159,7 @@ download_tools_succeed_with_baked_grpc() { download_tools "${test_protoc_version}" "99.99" "linux-x86_64" assertEquals "${DOCKER_GRPC_LOCATION}" "${grpc_path}" - rm -rdf "${output_folder}" + rm -rf "${output_folder}" unset DOCKER_GRPC_LOCATION unset DOCKER_GRPC_VERSION unset output_folder @@ -243,7 +243,7 @@ copy_directory_if_exists_valid_folder_succeeds() { mkdir -p "${destination}" copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) - rm -rdf "${destination}" + rm -rf "${destination}" assertEquals 1 ${n_matching_folders} } @@ -253,7 +253,7 @@ copy_directory_if_exists_invalid_folder_does_not_copy() { mkdir -p "${destination}" copy_directory_if_exists "${source_folder}" "gapic" "${destination}/copied-folder" n_matching_folders=$(ls "${destination}" | grep -e 'copied-folder' | wc -l) || res=$? - rm -rdf "${destination}" + rm -rf "${destination}" assertEquals 0 ${n_matching_folders} } diff --git a/hermetic_build/library_generation/tests/integration_tests.py b/hermetic_build/library_generation/tests/integration_tests.py index fd534ff207..a940069baa 100644 --- a/hermetic_build/library_generation/tests/integration_tests.py +++ b/hermetic_build/library_generation/tests/integration_tests.py @@ -43,7 +43,6 @@ "google-cloud-java": "chore/test-hermetic-build", "java-bigtable": "chore/test-hermetic-build", } -baseline_config_name = "baseline_generation_config.yaml" current_config_name = "current_generation_config.yaml" googleapis_commitish = "113a378d5aad5018876ec0a8cbfd4d6a4f746809" # This variable is used to override the jar created by building the image @@ -88,8 +87,7 @@ def test_entry_point_running_in_container(self): self.__run_entry_point_in_docker_container( repo_location=repo_location, config_location=config_location, - baseline_config=baseline_config_name, - current_config=current_config_name, + generation_config=current_config_name, api_definition=api_definitions_path, ) # 4. compare generation result with golden files @@ -204,6 +202,7 @@ def __build_image(cls, docker_file: str, cwd: str): subprocess.check_call( ["docker", "build", "-f", docker_file, "-t", image_tag, "."], cwd=cwd, + env=dict(os.environ, DOCKER_BUILDKIT="1"), ) @classmethod @@ -222,6 +221,8 @@ def __download_generator_jar(cls, coordinates_file: str) -> None: [ "mvn", "dependency:copy", + "-B", + "-ntp", f"-Dartifact={coordinates}", f"-DoutputDirectory={config_dir}", ] @@ -287,8 +288,7 @@ def __run_entry_point_in_docker_container( cls, repo_location: str, config_location: str, - baseline_config: str, - current_config: str, + generation_config: str, api_definition: str, ): # we use the calling user to prevent the mapped volumes from changing @@ -314,8 +314,7 @@ def __run_entry_point_in_docker_container( "-w", "/workspace/repo", image_tag, - f"--baseline-generation-config-path=/workspace/config/{baseline_config}", - f"--current-generation-config-path=/workspace/config/{current_config}", + f"--generation-config-path=/workspace/config/{generation_config}", f"--api-definitions-path=/workspace/api", ], ) diff --git a/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py b/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py index 2e23c2b403..76b663c6cf 100644 --- a/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py +++ b/hermetic_build/library_generation/tests/utils/proto_path_utils_unit_tests.py @@ -15,10 +15,7 @@ import os import unittest from pathlib import Path -from library_generation.utils.proto_path_utils import ( - find_versioned_proto_path, - remove_version_from, -) +from library_generation.utils.proto_path_utils import remove_version_from script_dir = os.path.dirname(os.path.realpath(__file__)) resources_dir = os.path.join(script_dir, "..", "resources") @@ -26,21 +23,6 @@ class ProtoPathsUtilsTest(unittest.TestCase): - def test_find_versioned_proto_path_nested_version_success(self): - proto_path = "google/cloud/aiplatform/v1/schema/predict/params/image_classification.proto" - expected = "google/cloud/aiplatform/v1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_path_success(self): - proto_path = "google/cloud/asset/v1p2beta1/assets.proto" - expected = "google/cloud/asset/v1p2beta1" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - - def test_find_versioned_proto_without_version_return_itself(self): - proto_path = "google/type/color.proto" - expected = "google/type/color.proto" - self.assertEqual(expected, find_versioned_proto_path(proto_path)) - def test_remove_version_from_returns_non_versioned_path(self): proto_path = "google/cloud/aiplatform/v1" self.assertEqual("google/cloud/aiplatform", remove_version_from(proto_path)) diff --git a/hermetic_build/library_generation/utils/proto_path_utils.py b/hermetic_build/library_generation/utils/proto_path_utils.py index d2ae25f602..27e3f8aa38 100644 --- a/hermetic_build/library_generation/utils/proto_path_utils.py +++ b/hermetic_build/library_generation/utils/proto_path_utils.py @@ -27,21 +27,3 @@ def remove_version_from(proto_path: str) -> str: if re.match(version_pattern, version): return proto_path[:index] return proto_path - - -def find_versioned_proto_path(proto_path: str) -> str: - """ - Returns a versioned proto_path from a given proto_path; or proto_path itself - if it doesn't contain a versioned proto_path. - :param proto_path: a proto file path - :return: the versioned proto_path - """ - version_regex = re.compile(r"^v[1-9].*") - directories = proto_path.split("/") - for directory in directories: - result = version_regex.search(directory) - if result: - version = result[0] - idx = proto_path.find(version) - return proto_path[:idx] + version - return proto_path diff --git a/hermetic_build/release_note_generation/cli/generate_release_note.py b/hermetic_build/release_note_generation/cli/generate_release_note.py index adc93d9ea7..a54e228f62 100644 --- a/hermetic_build/release_note_generation/cli/generate_release_note.py +++ b/hermetic_build/release_note_generation/cli/generate_release_note.py @@ -16,7 +16,7 @@ import click as click from release_note_generation.generate_pr_description import generate_pr_descriptions from common.model.generation_config import from_yaml -from library_generation.utils.generation_config_comparator import compare_config +from common.utils.generation_config_comparator import compare_config @click.group(invoke_without_command=False) @@ -54,8 +54,9 @@ def main(ctx): default=".", show_default=True, help=""" - The repository path to which the generated files will be sent. - If not specified, the repository will be generated to the current working + The path where the file `pr_description.txt`, which contains the release + notes, will be sent. + If not specified, the file will be generated to the current working directory. """, ) diff --git a/hermetic_build/release_note_generation/commit_message_formatter.py b/hermetic_build/release_note_generation/commit_message_formatter.py index 48c060c571..c86c1df192 100644 --- a/hermetic_build/release_note_generation/commit_message_formatter.py +++ b/hermetic_build/release_note_generation/commit_message_formatter.py @@ -14,7 +14,7 @@ import re from git import Commit -from library_generation.model.config_change import ConfigChange, ChangeType +from common.model.config_change import ConfigChange, ChangeType from common.model.generation_config import ( GAPIC_GENERATOR_VERSION, LIBRARIES_BOM_VERSION, diff --git a/hermetic_build/release_note_generation/generate_pr_description.py b/hermetic_build/release_note_generation/generate_pr_description.py index b71facd7a4..d544268092 100755 --- a/hermetic_build/release_note_generation/generate_pr_description.py +++ b/hermetic_build/release_note_generation/generate_pr_description.py @@ -18,8 +18,8 @@ from typing import Dict from git import Commit, Repo -from library_generation.model.config_change import ConfigChange -from library_generation.utils.proto_path_utils import find_versioned_proto_path +from common.model.config_change import ConfigChange +from common.utils.proto_path_utils import find_versioned_proto_path from release_note_generation.commit_message_formatter import ( format_commit_message, format_repo_level_change, diff --git a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py index ac888cea7b..b7891495ee 100644 --- a/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/commit_message_formatter_unit_tests.py @@ -13,7 +13,7 @@ # limitations under the License. import unittest from unittest.mock import patch -from library_generation.model.config_change import ( +from common.model.config_change import ( ConfigChange, ChangeType, LibraryChange, diff --git a/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py index 7c6f0db9ac..9659af5b2d 100644 --- a/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py +++ b/hermetic_build/release_note_generation/tests/generate_pr_description_unit_tests.py @@ -18,7 +18,7 @@ get_repo_level_commit_messages, generate_pr_descriptions, ) -from library_generation.model.config_change import ( +from common.model.config_change import ( ConfigChange, ChangeType, LibraryChange, diff --git a/java-common-protos/grpc-google-common-protos/pom.xml b/java-common-protos/grpc-google-common-protos/pom.xml index 7abb58d191..88dc6f3061 100644 --- a/java-common-protos/grpc-google-common-protos/pom.xml +++ b/java-common-protos/grpc-google-common-protos/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT grpc-google-common-protos GRPC library for grpc-google-common-protos com.google.api.grpc google-common-protos-parent - 2.48.0 + 2.48.1-SNAPSHOT diff --git a/java-common-protos/pom.xml b/java-common-protos/pom.xml index b498a25e9f..87415aa2fb 100644 --- a/java-common-protos/pom.xml +++ b/java-common-protos/pom.xml @@ -4,7 +4,7 @@ com.google.api.grpc google-common-protos-parent pom - 2.48.0 + 2.48.1-SNAPSHOT Google Common Protos Parent Java idiomatic client for Google Cloud Platform services. @@ -13,7 +13,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -61,7 +61,7 @@ com.google.cloud third-party-dependencies - 3.39.0 + 3.39.1-SNAPSHOT pom import @@ -75,7 +75,7 @@ com.google.api.grpc grpc-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT io.grpc @@ -87,7 +87,7 @@ com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT com.google.guava diff --git a/java-common-protos/proto-google-common-protos/pom.xml b/java-common-protos/proto-google-common-protos/pom.xml index 3ce2807f44..79cdcc52c4 100644 --- a/java-common-protos/proto-google-common-protos/pom.xml +++ b/java-common-protos/proto-google-common-protos/pom.xml @@ -3,13 +3,13 @@ 4.0.0 com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT proto-google-common-protos PROTO library for proto-google-common-protos com.google.api.grpc google-common-protos-parent - 2.48.0 + 2.48.1-SNAPSHOT diff --git a/java-core/google-cloud-core-bom/pom.xml b/java-core/google-cloud-core-bom/pom.xml index a29076962e..8d2c1d39d5 100644 --- a/java-core/google-cloud-core-bom/pom.xml +++ b/java-core/google-cloud-core-bom/pom.xml @@ -3,13 +3,13 @@ 4.0.0 com.google.cloud google-cloud-core-bom - 2.47.0 + 2.47.1-SNAPSHOT pom com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../../gapic-generator-java-pom-parent @@ -23,17 +23,17 @@ com.google.cloud google-cloud-core - 2.47.0 + 2.47.1-SNAPSHOT com.google.cloud google-cloud-core-grpc - 2.47.0 + 2.47.1-SNAPSHOT com.google.cloud google-cloud-core-http - 2.47.0 + 2.47.1-SNAPSHOT diff --git a/java-core/google-cloud-core-grpc/pom.xml b/java-core/google-cloud-core-grpc/pom.xml index 784f52e334..e4660dd8cb 100644 --- a/java-core/google-cloud-core-grpc/pom.xml +++ b/java-core/google-cloud-core-grpc/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-core-grpc - 2.47.0 + 2.47.1-SNAPSHOT jar Google Cloud Core gRPC @@ -12,7 +12,7 @@ com.google.cloud google-cloud-core-parent - 2.47.0 + 2.47.1-SNAPSHOT google-cloud-core-grpc diff --git a/java-core/google-cloud-core-http/pom.xml b/java-core/google-cloud-core-http/pom.xml index 753801f14a..728a988ad3 100644 --- a/java-core/google-cloud-core-http/pom.xml +++ b/java-core/google-cloud-core-http/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-core-http - 2.47.0 + 2.47.1-SNAPSHOT jar Google Cloud Core HTTP @@ -12,7 +12,7 @@ com.google.cloud google-cloud-core-parent - 2.47.0 + 2.47.1-SNAPSHOT google-cloud-core-http diff --git a/java-core/google-cloud-core/pom.xml b/java-core/google-cloud-core/pom.xml index fadc83f63c..3db0823119 100644 --- a/java-core/google-cloud-core/pom.xml +++ b/java-core/google-cloud-core/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud google-cloud-core - 2.47.0 + 2.47.1-SNAPSHOT jar Google Cloud Core @@ -12,7 +12,7 @@ com.google.cloud google-cloud-core-parent - 2.47.0 + 2.47.1-SNAPSHOT google-cloud-core diff --git a/java-core/pom.xml b/java-core/pom.xml index d1b558fe97..e1bf7614c8 100644 --- a/java-core/pom.xml +++ b/java-core/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-core-parent pom - 2.47.0 + 2.47.1-SNAPSHOT Google Cloud Core Parent Java idiomatic client for Google Cloud Platform services. @@ -13,7 +13,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -33,7 +33,7 @@ com.google.cloud google-cloud-shared-dependencies - 3.39.0 + 3.39.1-SNAPSHOT pom import diff --git a/java-iam/grpc-google-iam-v1/pom.xml b/java-iam/grpc-google-iam-v1/pom.xml index 6600488873..1da2c10e5c 100644 --- a/java-iam/grpc-google-iam-v1/pom.xml +++ b/java-iam/grpc-google-iam-v1/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-iam-v1 - 1.43.0 + 1.43.1-SNAPSHOT grpc-google-iam-v1 GRPC library for grpc-google-iam-v1 com.google.cloud google-iam-parent - 1.43.0 + 1.43.1-SNAPSHOT diff --git a/java-iam/grpc-google-iam-v2/pom.xml b/java-iam/grpc-google-iam-v2/pom.xml index 397daefb9f..5485e68593 100644 --- a/java-iam/grpc-google-iam-v2/pom.xml +++ b/java-iam/grpc-google-iam-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-iam-v2 - 1.43.0 + 1.43.1-SNAPSHOT grpc-google-iam-v2 GRPC library for proto-google-iam-v2 com.google.cloud google-iam-parent - 1.43.0 + 1.43.1-SNAPSHOT diff --git a/java-iam/grpc-google-iam-v2beta/pom.xml b/java-iam/grpc-google-iam-v2beta/pom.xml index 5a3a881579..9260769009 100644 --- a/java-iam/grpc-google-iam-v2beta/pom.xml +++ b/java-iam/grpc-google-iam-v2beta/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc grpc-google-iam-v2beta - 1.43.0 + 1.43.1-SNAPSHOT grpc-google-iam-v2beta GRPC library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.43.0 + 1.43.1-SNAPSHOT diff --git a/java-iam/pom.xml b/java-iam/pom.xml index d64821dc60..0ab1f965fd 100644 --- a/java-iam/pom.xml +++ b/java-iam/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-iam-parent pom - 1.43.0 + 1.43.1-SNAPSHOT Google IAM Parent Java idiomatic client for Google Cloud Platform services. @@ -13,7 +13,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -60,7 +60,7 @@ com.google.cloud third-party-dependencies - 3.39.0 + 3.39.1-SNAPSHOT pom import @@ -88,44 +88,44 @@ com.google.api gax-bom - 2.57.0 + 2.57.1-SNAPSHOT pom import com.google.api.grpc proto-google-iam-v2 - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2 - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc proto-google-common-protos - 2.48.0 + 2.48.1-SNAPSHOT com.google.api.grpc proto-google-iam-v2beta - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v1 - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc grpc-google-iam-v2beta - 1.43.0 + 1.43.1-SNAPSHOT com.google.api.grpc proto-google-iam-v1 - 1.43.0 + 1.43.1-SNAPSHOT javax.annotation diff --git a/java-iam/proto-google-iam-v1/pom.xml b/java-iam/proto-google-iam-v1/pom.xml index 56e0b12453..ce7c3e5c5b 100644 --- a/java-iam/proto-google-iam-v1/pom.xml +++ b/java-iam/proto-google-iam-v1/pom.xml @@ -3,13 +3,13 @@ 4.0.0 com.google.api.grpc proto-google-iam-v1 - 1.43.0 + 1.43.1-SNAPSHOT proto-google-iam-v1 PROTO library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.43.0 + 1.43.1-SNAPSHOT diff --git a/java-iam/proto-google-iam-v2/pom.xml b/java-iam/proto-google-iam-v2/pom.xml index caadea7aa1..ea6a02a26c 100644 --- a/java-iam/proto-google-iam-v2/pom.xml +++ b/java-iam/proto-google-iam-v2/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-iam-v2 - 1.43.0 + 1.43.1-SNAPSHOT proto-google-iam-v2 Proto library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.43.0 + 1.43.1-SNAPSHOT diff --git a/java-iam/proto-google-iam-v2beta/pom.xml b/java-iam/proto-google-iam-v2beta/pom.xml index fc12546e97..d03e162bce 100644 --- a/java-iam/proto-google-iam-v2beta/pom.xml +++ b/java-iam/proto-google-iam-v2beta/pom.xml @@ -4,13 +4,13 @@ 4.0.0 com.google.api.grpc proto-google-iam-v2beta - 1.43.0 + 1.43.1-SNAPSHOT proto-google-iam-v2beta Proto library for proto-google-iam-v1 com.google.cloud google-iam-parent - 1.43.0 + 1.43.1-SNAPSHOT diff --git a/java-shared-dependencies/dependency-convergence-check/pom.xml b/java-shared-dependencies/dependency-convergence-check/pom.xml index 44260c53ed..fe190abc7a 100644 --- a/java-shared-dependencies/dependency-convergence-check/pom.xml +++ b/java-shared-dependencies/dependency-convergence-check/pom.xml @@ -3,7 +3,7 @@ 4.0.0 com.google.cloud shared-dependencies-dependency-convergence-test - 3.39.0 + 3.39.1-SNAPSHOT Dependency convergence test for certain artifacts in Google Cloud Shared Dependencies An dependency convergence test case for the shared dependencies BOM. A failure of this test case means diff --git a/java-shared-dependencies/first-party-dependencies/pom.xml b/java-shared-dependencies/first-party-dependencies/pom.xml index 5bfcad13cd..166decace2 100644 --- a/java-shared-dependencies/first-party-dependencies/pom.xml +++ b/java-shared-dependencies/first-party-dependencies/pom.xml @@ -6,7 +6,7 @@ com.google.cloud first-party-dependencies pom - 3.39.0 + 3.39.1-SNAPSHOT Google Cloud First-party Shared Dependencies Shared first-party dependencies for Google Cloud Java libraries. @@ -33,7 +33,7 @@ com.google.api gapic-generator-java-bom - 2.49.0 + 2.49.1-SNAPSHOT pom import @@ -45,7 +45,7 @@ com.google.cloud google-cloud-core-bom - 2.47.0 + 2.47.1-SNAPSHOT pom import @@ -69,13 +69,13 @@ com.google.cloud google-cloud-core - 2.47.0 + 2.47.1-SNAPSHOT test-jar com.google.cloud google-cloud-core - 2.47.0 + 2.47.1-SNAPSHOT tests diff --git a/java-shared-dependencies/pom.xml b/java-shared-dependencies/pom.xml index e1d8b0ac8d..c684e8cdb6 100644 --- a/java-shared-dependencies/pom.xml +++ b/java-shared-dependencies/pom.xml @@ -4,7 +4,7 @@ com.google.cloud google-cloud-shared-dependencies pom - 3.39.0 + 3.39.1-SNAPSHOT first-party-dependencies third-party-dependencies @@ -17,7 +17,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../gapic-generator-java-pom-parent @@ -31,14 +31,14 @@ com.google.cloud first-party-dependencies - 3.39.0 + 3.39.1-SNAPSHOT pom import com.google.cloud third-party-dependencies - 3.39.0 + 3.39.1-SNAPSHOT pom import diff --git a/java-shared-dependencies/third-party-dependencies/pom.xml b/java-shared-dependencies/third-party-dependencies/pom.xml index f2b2489c2c..6b7314303a 100644 --- a/java-shared-dependencies/third-party-dependencies/pom.xml +++ b/java-shared-dependencies/third-party-dependencies/pom.xml @@ -6,7 +6,7 @@ com.google.cloud third-party-dependencies pom - 3.39.0 + 3.39.1-SNAPSHOT Google Cloud Third-party Shared Dependencies Shared third-party dependencies for Google Cloud Java libraries. @@ -15,7 +15,7 @@ com.google.api gapic-generator-java-pom-parent - 2.49.0 + 2.49.1-SNAPSHOT ../../gapic-generator-java-pom-parent diff --git a/java-shared-dependencies/upper-bound-check/pom.xml b/java-shared-dependencies/upper-bound-check/pom.xml index 60ffedc992..801d22eb12 100644 --- a/java-shared-dependencies/upper-bound-check/pom.xml +++ b/java-shared-dependencies/upper-bound-check/pom.xml @@ -4,7 +4,7 @@ com.google.cloud shared-dependencies-upper-bound-test pom - 3.39.0 + 3.39.1-SNAPSHOT Upper bound test for Google Cloud Shared Dependencies An upper bound test case for the shared dependencies BOM. A failure of this test case means @@ -30,7 +30,7 @@ com.google.cloud google-cloud-shared-dependencies - 3.39.0 + 3.39.1-SNAPSHOT pom import diff --git a/release-please-config.json b/release-please-config.json index df5cc02468..c264303ad1 100644 --- a/release-please-config.json +++ b/release-please-config.json @@ -11,6 +11,7 @@ ".cloudbuild/graalvm/cloudbuild-test-a.yaml", ".cloudbuild/graalvm/cloudbuild-test-b.yaml", ".cloudbuild/library_generation/cloudbuild-library-generation-push.yaml", + ".cloudbuild/library_generation/cloudbuild-library-generation-push-prod.yaml", ".cloudbuild/library_generation/library_generation.Dockerfile" ] } diff --git a/sdk-platform-java-config/pom.xml b/sdk-platform-java-config/pom.xml index 2f46519bce..1bd948a324 100644 --- a/sdk-platform-java-config/pom.xml +++ b/sdk-platform-java-config/pom.xml @@ -4,7 +4,7 @@ com.google.cloud sdk-platform-java-config pom - 3.39.0 + 3.39.1-SNAPSHOT SDK Platform For Java Configurations Shared build configuration for Google Cloud Java libraries. @@ -17,6 +17,6 @@ - 3.39.0 + 3.39.1-SNAPSHOT \ No newline at end of file diff --git a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java index a04432db8b..7a04ee146d 100644 --- a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java +++ b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/EchoSettings.java @@ -116,7 +116,7 @@ * RetrySettings.newBuilder() * .setInitialRetryDelayDuration(Duration.ofMillis(500)) * .setRetryDelayMultiplier(1.5) - * .setMaxRetryDelay(Duration.ofMillis(5000)) + * .setMaxRetryDelayDuration(Duration.ofMillis(5000)) * .setTotalTimeoutDuration(Duration.ofHours(24)) * .build()); * echoSettingsBuilder diff --git a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java index e253bad663..cfa46cdc9c 100644 --- a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java +++ b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/MessagingSettings.java @@ -117,7 +117,7 @@ * RetrySettings.newBuilder() * .setInitialRetryDelayDuration(Duration.ofMillis(500)) * .setRetryDelayMultiplier(1.5) - * .setMaxRetryDelay(Duration.ofMillis(5000)) + * .setMaxRetryDelayDuration(Duration.ofMillis(5000)) * .setTotalTimeoutDuration(Duration.ofHours(24)) * .build()); * messagingSettingsBuilder diff --git a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java index c339fa1e7e..93765029b6 100644 --- a/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java +++ b/showcase/gapic-showcase/src/main/java/com/google/showcase/v1beta1/stub/EchoStubSettings.java @@ -82,10 +82,10 @@ import com.google.showcase.v1beta1.WaitRequest; import com.google.showcase.v1beta1.WaitResponse; import java.io.IOException; +import java.time.Duration; import java.util.List; import java.util.Map; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -152,7 +152,7 @@ * RetrySettings.newBuilder() * .setInitialRetryDelayDuration(Duration.ofMillis(500)) * .setRetryDelayMultiplier(1.5) - * .setMaxRetryDelay(Duration.ofMillis(5000)) + * .setMaxRetryDelayDuration(Duration.ofMillis(5000)) * .setTotalTimeoutDuration(Duration.ofHours(24)) * .build()); * echoSettingsBuilder @@ -624,21 +624,21 @@ public static class Builder extends StubSettings.Builder headerValues = (ArrayList) diff --git a/showcase/pom.xml b/showcase/pom.xml index 85af0cd696..6cbcdf92ab 100644 --- a/showcase/pom.xml +++ b/showcase/pom.xml @@ -34,7 +34,7 @@ com.google.cloud google-cloud-shared-dependencies - 3.39.0 + 3.39.1-SNAPSHOT pom import diff --git a/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java b/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java index ee81471625..7d7f145b1a 100644 --- a/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java +++ b/test/integration/goldens/apigeeconnect/src/com/google/cloud/apigeeconnect/v1/stub/ConnectionServiceStubSettings.java @@ -52,9 +52,9 @@ import com.google.common.collect.ImmutableSet; import com.google.common.collect.Lists; import java.io.IOException; +import java.time.Duration; import java.util.List; import javax.annotation.Generated; -import org.threeten.bp.Duration; // AUTO-GENERATED DOCUMENTATION AND CLASS. /** @@ -319,13 +319,13 @@ public static class Builder extends StubSettings.Builder