diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh index 1448ab7917..861f28d9bd 100755 --- a/.github/scripts/hermetic_library_generation.sh +++ b/.github/scripts/hermetic_library_generation.sh @@ -64,17 +64,14 @@ fi image_tag=local workspace_name="/workspace" -docker_file="library_generation.Dockerfile" +baseline_generation_config="baseline_generation_config.yaml" message="chore: generate libraries at $(date)" +git checkout "${target_branch}" git checkout "${current_branch}" -# if the last commit doesn't contain changes to generation configuration -# or Dockerfile, do not generate again as the result will be the same. -change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)" -if [[ ! ("${change_of_last_commit}" == *"${generation_config}"* || "${change_of_last_commit}" == *"${docker_file}"*) ]]; then - echo "The last commit doesn't contain any changes to the generation_config.yaml or Dockerfile, skipping the whole generation process." || true - exit 0 -fi + +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" generator_version=$(mvn help:evaluate -Dexpression=project.version -q -DforceStdout -pl gapic-generator-java) echo "Local generator version: ${generator_version}" @@ -94,10 +91,12 @@ docker run \ -v "$(pwd):${workspace_name}" \ -v "$HOME"/.m2:/home/.m2 \ -e GENERATOR_VERSION="${generator_version}" \ - gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" # commit the change to the pull request. -rm -rdf output googleapis +rm -rdf output googleapis "${baseline_generation_config}" git add --all -- ':!pr_description.txt' changed_files=$(git diff --cached --name-only) if [[ "${changed_files}" == "" ]]; then diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml index ebc9ab770c..9a2cf40d03 100644 --- a/.github/workflows/hermetic_library_generation.yaml +++ b/.github/workflows/hermetic_library_generation.yaml @@ -17,10 +17,11 @@ name: Hermetic library generation upon generation config change through pull req on: pull_request: +env: + REPO_FULL_NAME: ${{ github.event.pull_request.head.repo.full_name }} + GITHUB_REPOSITORY: ${{ github.repository }} jobs: library_generation: - # skip pull requests come from a forked repository - if: github.event.pull_request.head.repo.full_name == github.repository runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 @@ -31,6 +32,10 @@ jobs: shell: bash run: | set -x + if [[ "${GITHUB_REPOSITORY}" != "${REPO_FULL_NAME}" ]]; then + echo "This PR comes from a fork. Skip library generation." + exit 0 + fi [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" bash .github/scripts/hermetic_library_generation.sh \ diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 7eefeeca05..d69dde1b77 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -17,6 +17,7 @@ import click as click from library_generation.generate_pr_description import generate_pr_descriptions from library_generation.generate_repo import generate_from_yaml +from library_generation.model.config_change import ConfigChange from library_generation.model.generation_config import from_yaml from library_generation.utils.generation_config_comparator import compare_config @@ -145,11 +146,10 @@ def __generate_repo_and_pr_description_impl( baseline_config=from_yaml(baseline_generation_config_path), current_config=from_yaml(current_generation_config_path), ) - # pass None if this is not a monorepo in order to trigger the full - # generation + # pass None if we want to fully generate the repository. target_library_names = ( config_change.get_changed_libraries() - if config_change.current_config.is_monorepo() + if not _needs_full_repo_generation(config_change=config_change) else None ) generate_from_yaml( @@ -163,6 +163,15 @@ def __generate_repo_and_pr_description_impl( ) +def _needs_full_repo_generation(config_change: ConfigChange) -> bool: + """ + Whether you should need a full repo generation, i.e., generate all + libraries in the generation configuration. + """ + current_config = config_change.current_config + return not current_config.is_monorepo() or current_config.contains_common_protos() + + @main.command() @click.option( "--generation-config-path", diff --git a/library_generation/test/cli/entry_point_unit_tests.py b/library_generation/test/cli/entry_point_unit_tests.py index 66756be500..e05c64de55 100644 --- a/library_generation/test/cli/entry_point_unit_tests.py +++ b/library_generation/test/cli/entry_point_unit_tests.py @@ -138,3 +138,56 @@ def test_generate_non_monorepo_with_changes_triggers_full_generation( generate_from_yaml.assert_called_with( config=ANY, repository_path=ANY, target_library_names=None ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_monorepo_with_common_protos_triggers_full_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo with + common protos. + generate() should call generate_from_yaml() with + target_library_names=None in order to trigger the full generation + """ + config_path = f"{test_resource_dir}/monorepo_with_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + repository_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, repository_path=ANY, target_library_names=None + ) + + @patch("library_generation.cli.entry_point.generate_from_yaml") + @patch("library_generation.cli.entry_point.generate_pr_descriptions") + def test_generate_monorepo_without_common_protos_does_not_trigger_full_generation( + self, + generate_pr_descriptions, + generate_from_yaml, + ): + """ + this tests confirms the behavior of generation of a monorepo without + common protos. + generate() should call generate_from_yaml() with + target_library_names=changed libraries which does not trigger the full + generation. + """ + config_path = f"{test_resource_dir}/monorepo_without_common_protos.yaml" + self.assertTrue(from_yaml(config_path).is_monorepo()) + # we call the implementation method directly since click + # does special handling when a method is annotated with @main.command() + generate_impl( + baseline_generation_config_path=config_path, + current_generation_config_path=config_path, + repository_path=".", + ) + generate_from_yaml.assert_called_with( + config=ANY, repository_path=ANY, target_library_names=[] + ) diff --git a/library_generation/test/generate_library_unit_tests.py b/library_generation/test/generate_library_unit_tests.py index 86a6a5d637..7bc14e3e20 100644 --- a/library_generation/test/generate_library_unit_tests.py +++ b/library_generation/test/generate_library_unit_tests.py @@ -1,3 +1,16 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. import subprocess import unittest import os @@ -66,13 +79,13 @@ def _run_command_and_get_sdout(self, command, **kwargs): def test_get_grpc_version_with_no_env_var_fails(self): # the absence of DOCKER_GRPC_VERSION will make this function to fail result = self._run_command("get_grpc_version") - self.assertEquals(1, result.returncode) + self.assertEqual(1, result.returncode) self.assertRegex(result.stdout.decode(), "DOCKER_GRPC_VERSION is not set") def test_get_protoc_version_with_no_env_var_fails(self): # the absence of DOCKER_PROTOC_VERSION will make this function to fail result = self._run_command("get_protoc_version") - self.assertEquals(1, result.returncode) + self.assertEqual(1, result.returncode) self.assertRegex(result.stdout.decode(), "DOCKER_PROTOC_VERSION is not set") def test_download_tools_without_baked_generator_fails(self): @@ -92,5 +105,5 @@ def test_download_tools_without_baked_generator_fails(self): result = self._run_command( f"download_tools {test_protoc_version} {test_grpc_version} {self.TEST_ARCHITECTURE}" ) - self.assertEquals(1, result.returncode) + self.assertEqual(1, result.returncode) self.assertRegex(result.stdout.decode(), "Please configure your environment") diff --git a/library_generation/test/resources/test-config/monorepo_with_common_protos.yaml b/library_generation/test/resources/test-config/monorepo_with_common_protos.yaml new file mode 100644 index 0000000000..6d4c94444a --- /dev/null +++ b/library_generation/test/resources/test-config/monorepo_with_common_protos.yaml @@ -0,0 +1,43 @@ +googleapis_commitish: 6a474b31c53cc1797710206824a17b364a835d2d +gapic_generator_version: 2.34.0 +# the libraries are ordered with respect to library name, which is +# java-{library.library_name} or java-{library.api-shortname} when +# library.library_name is not defined. +libraries: +- api_shortname: common-protos + name_pretty: Common Protos + product_documentation: https://github.com/googleapis/api-common-protos + api_description: Protobuf classes for Google's common protos. + release_level: stable + client_documentation: https://cloud.google.com/java/docs/reference/proto-google-common-protos/latest/history + distribution_name: com.google.api.grpc:proto-google-common-protos + excluded_dependencies: "proto-google-common-protos,grpc-google-common-protos,proto-google-common-protos-parent" + excluded_poms: "proto-google-common-protos-bom,proto-google-common-protos" + library_type: OTHER + GAPICs: + - proto_path: google/api + - proto_path: google/apps/card/v1 + - proto_path: google/cloud + - proto_path: google/cloud/audit + - proto_path: google/cloud/location + - proto_path: google/geo/type + - proto_path: google/logging/type + - proto_path: google/longrunning + - proto_path: google/rpc + - proto_path: google/rpc/context + - proto_path: google/shopping/type + - proto_path: google/type +- api_shortname: iam + name_pretty: IAM + product_documentation: https://cloud.google.com/iam + api_description: Manages access control for Google Cloud Platform resources + release_level: stable + client_documentation: https://cloud.google.com/java/docs/reference/proto-google-iam-v1/latest/overview + distribution_name: com.google.api.grpc:proto-google-iam-v1 + excluded_dependencies: "grpc-google-iam-v1" + excluded_poms: "proto-google-iam-v1-bom,google-iam-policy,proto-google-iam-v1" + library_type: OTHER + GAPICs: + - proto_path: google/iam/v1 + - proto_path: google/iam/v2 + - proto_path: google/iam/v2beta \ No newline at end of file diff --git a/library_generation/test/resources/test-config/monorepo_without_common_protos.yaml b/library_generation/test/resources/test-config/monorepo_without_common_protos.yaml new file mode 100644 index 0000000000..ca21ccfb01 --- /dev/null +++ b/library_generation/test/resources/test-config/monorepo_without_common_protos.yaml @@ -0,0 +1,33 @@ +gapic_generator_version: 2.34.0 +protoc_version: 25.2 +googleapis_commitish: 1a45bf7393b52407188c82e63101db7dc9c72026 +libraries_bom_version: 26.37.0 +libraries: + - api_shortname: cloudasset + name_pretty: Cloud Asset Inventory + product_documentation: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + api_description: "provides inventory services based on a time series database." + library_name: "asset" + client_documentation: "https://cloud.google.com/java/docs/reference/google-cloud-asset/latest/overview" + distribution_name: "com.google.cloud:google-cloud-asset" + release_level: "stable" + issue_tracker: "https://issuetracker.google.com/issues/new?component=187210&template=0" + api_reference: "https://cloud.google.com/resource-manager/docs/cloud-asset-inventory/overview" + GAPICs: + - proto_path: google/cloud/asset/v1 + - proto_path: google/cloud/asset/v1p1beta1 + - proto_path: google/cloud/asset/v1p2beta1 + - proto_path: google/cloud/asset/v1p5beta1 + - proto_path: google/cloud/asset/v1p7beta1 + - api_shortname: cloudbuild + name_pretty: Cloud Build + product_documentation: https://cloud.google.com/cloud-build/ + api_description: lets you build software quickly across all languages. Get complete + control over defining custom workflows for building, testing, and deploying across + multiple environments such as VMs, serverless, Kubernetes, or Firebase. + release_level: stable + distribution_name: com.google.cloud:google-cloud-build + issue_tracker: https://issuetracker.google.com/savedsearches/5226584 + GAPICs: + - proto_path: google/devtools/cloudbuild/v1 + - proto_path: google/devtools/cloudbuild/v2