Merge remote-tracking branch 'origin/main' into renovate/protobuf-dep…

…endencies

.cloudbuild/cloudbuild-test-a.yaml → .cloudbuild/graalvm/cloudbuild-test-a.yaml

@@ -14,14 +14,14 @@
 
 timeout: 7200s # 2 hours
 substitutions:
-  _SHARED_DEPENDENCIES_VERSION: '3.25.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.7.1'
+  _SHARED_DEPENDENCIES_VERSION: '3.27.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
+  _JAVA_SHARED_CONFIG_VERSION: '1.7.4'
 
 steps:
   # GraalVM A build
   - name: gcr.io/cloud-builders/docker
     args: ["build", "-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION}", "--file", "graalvm_a.Dockerfile", "--build-arg", "JAVA_SHARED_CONFIG_VERSION=$_JAVA_SHARED_CONFIG_VERSION", "."]
-    dir: .cloudbuild
+    dir: .cloudbuild/graalvm
     id: graalvm-a-build
     waitFor: ["-"]
   - name: gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION}
@@ -30,4 +30,4 @@ steps:
     waitFor: [ "graalvm-a-build" ]
     env:
       - 'MODULES_UNDER_TEST=java-kms'
-      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'
+      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'

.cloudbuild/cloudbuild-test-b.yaml → .cloudbuild/graalvm/cloudbuild-test-b.yaml

@@ -14,14 +14,14 @@
 
 timeout: 7200s # 2 hours
 substitutions:
-  _SHARED_DEPENDENCIES_VERSION: '3.25.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.7.1'
+  _SHARED_DEPENDENCIES_VERSION: '3.27.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
+  _JAVA_SHARED_CONFIG_VERSION: '1.7.4'
 
 steps:
   # GraalVM A build
   - name: gcr.io/cloud-builders/docker
     args: ["build", "-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:${_SHARED_DEPENDENCIES_VERSION}", "--file", "graalvm_b.Dockerfile", "--build-arg", "JAVA_SHARED_CONFIG_VERSION=$_JAVA_SHARED_CONFIG_VERSION", "."]
-    dir: .cloudbuild
+    dir: .cloudbuild/graalvm
     id: graalvm-b-build
     waitFor: ["-"]
   - name: gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:${_SHARED_DEPENDENCIES_VERSION}
@@ -30,4 +30,4 @@ steps:
     waitFor: [ "graalvm-b-build" ]
     env:
       - 'MODULES_UNDER_TEST=java-kms'
-      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'
+      - 'GOOGLE_CLOUD_PROJECT=cloud-java-ci-test'

.cloudbuild/cloudbuild.yaml → .cloudbuild/graalvm/cloudbuild.yaml

@@ -14,20 +14,20 @@
 
 timeout: 7200s # 2 hours
 substitutions:
-  _SHARED_DEPENDENCIES_VERSION: '3.25.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
-  _JAVA_SHARED_CONFIG_VERSION: '1.7.1'
+  _SHARED_DEPENDENCIES_VERSION: '3.27.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
+  _JAVA_SHARED_CONFIG_VERSION: '1.7.4'
 steps:
   # GraalVM A build
   - name: gcr.io/cloud-builders/docker
     args: ["build", "-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION}", "--file", "graalvm_a.Dockerfile", "--build-arg", "JAVA_SHARED_CONFIG_VERSION=$_JAVA_SHARED_CONFIG_VERSION", "."]
-    dir: .cloudbuild
+    dir: .cloudbuild/graalvm
     id: graalvm-a-build
     waitFor: ["-"]
 
   # GraalVM B build
   - name: gcr.io/cloud-builders/docker
     args: [ "build", "-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:${_SHARED_DEPENDENCIES_VERSION}", "--file", "graalvm_b.Dockerfile",  "--build-arg", "JAVA_SHARED_CONFIG_VERSION=$_JAVA_SHARED_CONFIG_VERSION", "." ]
-    dir: .cloudbuild
+    dir: .cloudbuild/graalvm
     id: graalvm-b-build
     waitFor: [ "-" ]
 

.cloudbuild/library_generation/cloudbuild-library-generation-push.yaml

@@ -0,0 +1,33 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 7200s # 2 hours
+substitutions:
+  _IMAGE_NAME: "gcr.io/cloud-devrel-public-resources/java-library-generation"
+  _SHA_IMAGE_ID: "${_IMAGE_NAME}:${COMMIT_SHA}"
+  _LATEST_IMAGE_ID: "${_IMAGE_NAME}:latest"
+steps:
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: [
+      "build",
+      "-t", "${_SHA_IMAGE_ID}",
+      "-t", "${_LATEST_IMAGE_ID}",
+      "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
+    id: library-generation-build
+    waitFor: ["-"]
+
+images:
+  - ${_SHA_IMAGE_ID}
+  - ${_LATEST_IMAGE_ID}

.cloudbuild/library_generation/cloudbuild-library-generation-release.yaml

@@ -0,0 +1,31 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 7200s # 2 hours
+substitutions:
+  _GAPIC_GENERATOR_JAVA_VERSION: '2.37.1-SNAPSHOT' # {x-version-update:gapic-generator-java:current}
+  _IMAGE_ID: "gcr.io/cloud-devrel-public-resources/java-library-generation:${_GAPIC_GENERATOR_JAVA_VERSION}"
+steps:
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: [
+      "build",
+      "-t", "${_IMAGE_ID}",
+      "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
+    id: library-generation-build
+    waitFor: ["-"]
+
+images:
+  - ${_IMAGE_ID}
+

.cloudbuild/library_generation/cloudbuild-test-library-generation.yaml

@@ -0,0 +1,31 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 7200s # 2 hours
+substitutions:
+  _TEST_IMAGE_ID: 'gcr.io/cloud-devrel-public-resources/java-library-generation:${COMMIT_SHA}'
+
+steps:
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: ["build", "-t", "${_TEST_IMAGE_ID}", "--file", ".cloudbuild/library_generation/library_generation.Dockerfile", "."]
+    id: library-generation-build
+    waitFor: ["-"]
+  - name: ${_TEST_IMAGE_ID}
+    entrypoint: bash
+    args: [ './library_generation/test/container_integration_tests.sh' ]
+    waitFor: [ "library-generation-build" ]
+    env:
+      - 'TEST_IMAGE_ID=${_TEST_IMAGE_ID}'
+

.cloudbuild/library_generation/library_generation.Dockerfile

@@ -0,0 +1,40 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# build from the root of this repo:
+FROM gcr.io/cloud-devrel-public-resources/python
+
+# install tools
+RUN apt-get update && apt-get install -y \
+	unzip openjdk-17-jdk rsync maven jq \
+	&& apt-get clean
+
+COPY library_generation /src
+
+RUN rm $(which python3)
+RUN ln -s $(which python3.11) /usr/local/bin/python
+RUN ln -s $(which python3.11) /usr/local/bin/python3
+RUN python -m pip install --upgrade pip
+RUN cd /src && python -m pip install -r requirements.in
+RUN cd /src && python -m pip install .
+
+# set dummy git credentials for empty commit used in postprocessing
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "Cloud Java Bot"
+
+WORKDIR /workspace
+RUN chmod 750 /workspace
+RUN chmod 750 /src/generate_repo.py
+
+CMD [ "/src/generate_repo.py" ]

.github/release-please.yml

@@ -3,4 +3,4 @@ bumpMinorPreMajor: true
 handleGHRelease: true
 primaryBranch: main
 manifest: true
-extraFiles: ["WORKSPACE", ".cloudbuild/cloudbuild.yaml", ".cloudbuild/cloudbuild-test-a.yaml", ".cloudbuild/cloudbuild-test-b.yaml" ]
+extraFiles: ["WORKSPACE", ".cloudbuild/graalvm/cloudbuild.yaml", ".cloudbuild/graalvm/cloudbuild-test-a.yaml", ".cloudbuild/graalvm/cloudbuild-test-b.yaml", ".cloudbuild/library_generation/cloudbuild-library-generation-release.yaml" ]

.github/workflows/ci.yaml

@@ -22,6 +22,14 @@ jobs:
         run: |
           mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
               -Dfmt.skip -DenableTestCoverage
+      # The `envVarTest` profile runs tests that require an environment variable
+      - name: Env Var Tests
+        run: |
+          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
+              -Dfmt.skip -DenableTestCoverage -PenvVarTest
+        # Set the Env Var for this step only
+        env:
+          GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com
       - run: bazelisk version
       - name: Install Maven modules
         run: |
@@ -64,6 +72,14 @@ jobs:
         run: |
           mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
               -Dfmt.skip -DenableTestCoverage
+      # The `envVarTest` profile runs tests that require an environment variable
+      - name: Env Var Tests
+        run: |
+          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
+              -Dfmt.skip -DenableTestCoverage -PenvVarTest
+        # Set the Env Var for this step only
+        env:
+          GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com
       - run: bazelisk version
       - name: Install Maven modules
         run: |
@@ -98,6 +114,22 @@ jobs:
           mvn verify --batch-mode --no-transfer-progress -Dcheckstyle.skip \
               -Dfmt.skip \
               -Djvm="${JAVA8_HOME}/bin/java"
+      # The `envVarTest` profile runs tests that require an environment variable
+      - name: Compile with Java 17 and run tests with Java 8 (Env Var Tests)
+        shell: bash
+        run: |
+          set -x
+          export JAVA_HOME=$JAVA_HOME
+          export PATH=${JAVA_HOME}/bin:$PATH
+          # Maven surefire plugin lets us to specify the JVM when running tests via
+          # the "jvm" system property.
+          export GOOGLE_CLOUD_UNIVERSE_DOMAIN=random.com
+          mvn test --batch-mode --no-transfer-progress -Dcheckstyle.skip \
+              -Dfmt.skip -DenableTestCoverage -Dsurefire.failIfNoSpecifiedTests=false \
+              -PenvVarTest
+        # Set the Env Var for this step only
+        env:
+          GOOGLE_CLOUD_UNIVERSE_DOMAIN: random.com
 
   build-java8-gapic-generator-java:
     name: "build(8) for gapic-generator-java"
@@ -295,7 +327,7 @@ jobs:
       run: |
         mvn install -B -ntp -DskipTests -Dclirr.skip -Dcheckstyle.skip
     - name: Validate gapic-generator-java-bom
-      uses: googleapis/java-cloud-bom/tests/validate-bom@ac9893c4ba759cda192b11c2e8bac0d5a6bd60a5
+      uses: googleapis/java-cloud-bom/tests/validate-bom@3f93d58a049cf6089a930c55cc23ff0c0869a12f
       with:
         bom-path: gapic-generator-java-bom/pom.xml
 

.github/workflows/sonar.yaml

@@ -47,6 +47,10 @@ jobs:
             tar -xf showcase-*
             ./gapic-showcase run &
             cd -
+      # Intentionally do not run the Env Var Tests (no -PenvVarTests) as setting the Env Var
+      # may alter the results for other tests that use Env Var in the logic. Adding a Sonar
+      # step for a few tests (env var tests) may be overkill and should be better covered
+      # when we can upgrade to JUnit 5 (https://github.com/googleapis/sdk-platform-java/issues/1611#issuecomment-1970079325)
       - name: Build and analyze for full test coverage
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any
@@ -59,7 +63,6 @@ jobs:
               -Dsonar.projectKey=googleapis_gapic-generator-java \
               -Dsonar.organization=googleapis \
               -Dsonar.host.url=https://sonarcloud.io
-
       - name: Build and analyze Showcase Integration Tests Coverage
         env:
           GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}  # Needed to get PR information, if any

.github/workflows/verify_library_generation.yaml

@@ -15,13 +15,13 @@ jobs:
         java: [ 11 ]
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
-    - uses: actions/setup-java@v3
+    - uses: actions/checkout@v4
+    - uses: actions/setup-java@v4
       with:
         java-version: ${{ matrix.java }}
         distribution: temurin
         cache: maven
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.11
     - name: install pyenv
@@ -58,14 +58,14 @@ jobs:
         os: [ ubuntu-22.04, macos-12 ]
     runs-on: ${{ matrix.os }}
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: install utils (macos)
       if: matrix.os == 'macos-12'
       shell: bash
       run: |
         brew update --preinstall
         brew install coreutils
-    - uses: actions/setup-python@v4
+    - uses: actions/setup-python@v5
       with:
         python-version: 3.11
     - name: install python dependencies
@@ -97,11 +97,11 @@ jobs:
     - name: Run python unit tests
       run: |
         set -x
-        python -m unittest library_generation/test/unit_tests.py
+        python -m unittest discover -s library_generation/test/ -p "*unit_tests.py"
   lint-shell:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Run ShellCheck
       uses: ludeeus/[email protected]
       with:
@@ -111,7 +111,7 @@ jobs:
   lint-python:
     runs-on: ubuntu-22.04
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: install python dependencies
       shell: bash
       run: |

.gitignore

@@ -25,6 +25,7 @@ target/
 **/output/
 **/googleapis
 library_generation/test/**/golden*/
-library_generation/test/resources/test_repo_level_postprocess/
+library_generation/test/resources/test_monorepo_postprocessing/
 **/*egg-info/
 **/build/
+**/dist/

.release-please-manifest.json

@@ -1,3 +1,3 @@
 {
-  ".": "2.35.0"
+  ".": "2.37.0"
 }