From 283a5e89837071678f8dd94b8b8adfad91a6766c Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Tue, 2 Jul 2024 20:11:20 -0400 Subject: [PATCH] feat: enable hermetic library generation (#2048) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat: enable hermetic library generation * fix config yaml syntax * do not map runners home folder * try dummy proto_path * use copyright update comittish * correct proto_path * update protoc * preserve pr_description * update gapic_generator_version to 2.41.0 * infer image tag from config yaml * correct workflow name * update config scripts and yamls * remove old update_googleapis_committish workflow * sync config structure with that of google-cloud-java * remove quotes from config yamls * fix typo in update_generation_config.yaml * correct * quote codeowners_team in generation config * update generator version * fix library info * retrigger jobs * fix path to hermetic_library_generation * 🦉 Updates from OwlBot post-processor See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md * fixes to hermetic_library_generation --------- Co-authored-by: Owl Bot --- .../{.OwlBot.yaml => .OwlBot-hermetic.yaml} | 0 .../scripts/hermetic_library_generation.sh | 117 +++++++++++++++++ .github/scripts/update_generation_config.sh | 121 ++++++++++++++++++ .../hermetic_library_generation.yaml | 40 ++++++ .../workflows/update_generation_config.yaml | 42 ++++++ generation_config.yaml | 23 ++++ 6 files changed, 343 insertions(+) rename .github/{.OwlBot.yaml => .OwlBot-hermetic.yaml} (100%) create mode 100644 .github/scripts/hermetic_library_generation.sh create mode 100644 .github/scripts/update_generation_config.sh create mode 100644 .github/workflows/hermetic_library_generation.yaml create mode 100644 .github/workflows/update_generation_config.yaml create mode 100644 generation_config.yaml diff --git a/.github/.OwlBot.yaml b/.github/.OwlBot-hermetic.yaml similarity index 100% rename from .github/.OwlBot.yaml rename to .github/.OwlBot-hermetic.yaml diff --git a/.github/scripts/hermetic_library_generation.sh b/.github/scripts/hermetic_library_generation.sh new file mode 100644 index 000000000..6c3f22d8f --- /dev/null +++ b/.github/scripts/hermetic_library_generation.sh @@ -0,0 +1,117 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to, when a pull request changes the generation +# configuration (generation_config.yaml by default): +# 1. Find whether the last commit in this pull request contains changes to +# the generation configuration and exit early if it doesn't have such a change +# since the generation result would be the same. +# 2. Compare generation configurations in the current branch (with which the +# pull request associated) and target branch (into which the pull request is +# merged); +# 3. Generate changed libraries using library_generation image; +# 4. Commit the changes to the pull request, if any. +# 5. Edit the PR body with generated pull request description, if applicable. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. docker + +# The parameters of this script is: +# 1. target_branch, the branch into which the pull request is merged. +# 2. current_branch, the branch with which the pull request is associated. +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --target_branch) + target_branch="$2" + shift + ;; + --current_branch) + current_branch="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${target_branch}" ]; then + echo "missing required argument --target_branch" + exit 1 +fi + +if [ -z "${current_branch}" ]; then + echo "missing required argument --current_branch" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config=generation_config.yaml + echo "Using default generation config: ${generation_config}" +fi + +workspace_name="/workspace" +baseline_generation_config="baseline_generation_config.yaml" +message="chore: generate libraries at $(date)" + +git checkout "${target_branch}" +git checkout "${current_branch}" +# if the last commit doesn't contain changes to generation configuration, +# do not generate again as the result will be the same. +change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)" +if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then + echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true + exit 0 +fi +# copy generation configuration from target branch to current branch. +git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}" +config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true) + +# parse image tag from the generation configuration. +image_tag=$(grep "gapic_generator_version" "${generation_config}" | cut -d ':' -f 2 | xargs) + +# run hermetic code generation docker image. +docker run \ + --rm \ + -u "$(id -u):$(id -g)" \ + -v "$(pwd):${workspace_name}" \ + gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \ + --baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \ + --current-generation-config-path="${workspace_name}/${generation_config}" + + +# commit the change to the pull request. +if [[ $(basename $(pwd)) == "google-cloud-java" ]]; then + git add java-* pom.xml gapic-libraries-bom/pom.xml versions.txt +else + # The image leaves intermediate folders and files it works with. Here we remove them + rm -rdf output googleapis "${baseline_generation_config}" + git add --all -- ':!pr_description.txt' +fi +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "There is no generated code change with the generation config change ${config_diff}." + echo "Skip committing to the pull request." + exit 0 +fi + +echo "Configuration diff:" +echo "${config_diff}" +git commit -m "${message}" +git push +# set pr body if pr_description.txt is generated. +if [[ -f "pr_description.txt" ]]; then + pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") + gh pr edit "${pr_num}" --body "$(cat pr_description.txt)" +fi diff --git a/.github/scripts/update_generation_config.sh b/.github/scripts/update_generation_config.sh new file mode 100644 index 000000000..561a31304 --- /dev/null +++ b/.github/scripts/update_generation_config.sh @@ -0,0 +1,121 @@ +#!/bin/bash +set -e +# This script should be run at the root of the repository. +# This script is used to update googleapis_commitish, gapic_generator_version, +# and libraries_bom_version in generation configuration at the time of running +# and create a pull request. + +# The following commands need to be installed before running the script: +# 1. git +# 2. gh +# 3. jq + +# Utility functions +# Get the latest released version of a Maven artifact. +function get_latest_released_version() { + local group_id=$1 + local artifact_id=$2 + latest=$(curl -s "https://search.maven.org/solrsearch/select?q=g:${group_id}+AND+a:${artifact_id}&core=gav&rows=500&wt=json" | jq -r '.response.docs[] | select(.v | test("^[0-9]+(\\.[0-9]+)*$")) | .v' | sort -V | tail -n 1) + echo "${latest}" +} + +# Update a key to a new value in the generation config. +function update_config() { + local key_word=$1 + local new_value=$2 + local file=$3 + echo "Update ${key_word} to ${new_value} in ${file}" + sed -i -e "s/^${key_word}.*$/${key_word}: ${new_value}/" "${file}" +} + +# The parameters of this script is: +# 1. base_branch, the base branch of the result pull request. +# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java +# 3. [optional] generation_config, the path to the generation configuration, +# the default value is generation_config.yaml in the repository root. +while [[ $# -gt 0 ]]; do +key="$1" +case "${key}" in + --base_branch) + base_branch="$2" + shift + ;; + --repo) + repo="$2" + shift + ;; + --generation_config) + generation_config="$2" + shift + ;; + *) + echo "Invalid option: [$1]" + exit 1 + ;; +esac +shift +done + +if [ -z "${base_branch}" ]; then + echo "missing required argument --base_branch" + exit 1 +fi + +if [ -z "${repo}" ]; then + echo "missing required argument --repo" + exit 1 +fi + +if [ -z "${generation_config}" ]; then + generation_config="generation_config.yaml" + echo "Use default generation config: ${generation_config}" +fi + +current_branch="generate-libraries-${base_branch}" +title="chore: Update generation configuration at $(date)" + +# try to find a open pull request associated with the branch +pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number") +# create a branch if there's no open pull request associated with the +# branch; otherwise checkout the pull request. +if [ -z "${pr_num}" ]; then + git checkout -b "${current_branch}" +else + gh pr checkout "${pr_num}" +fi + +mkdir tmp-googleapis +# use partial clone because only commit history is needed. +git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis +pushd tmp-googleapis +git pull +latest_commit=$(git rev-parse HEAD) +popd +rm -rf tmp-googleapis +update_config "googleapis_commitish" "${latest_commit}" "${generation_config}" + +# update gapic-generator-java version to the latest +latest_version=$(get_latest_released_version "com.google.api" "gapic-generator-java") +update_config "gapic_generator_version" "${latest_version}" "${generation_config}" + +# update libraries-bom version to the latest +latest_version=$(get_latest_released_version "com.google.cloud" "libraries-bom") +update_config "libraries_bom_version" "${latest_version}" "${generation_config}" + +git add "${generation_config}" +changed_files=$(git diff --cached --name-only) +if [[ "${changed_files}" == "" ]]; then + echo "The latest generation config is not changed." + echo "Skip committing to the pull request." + exit 0 +fi +git commit -m "${title}" +if [ -z "${pr_num}" ]; then + git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git" + git fetch -q --unshallow remote_repo + git push -f remote_repo "${current_branch}" + gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}" +else + git push + gh pr edit "${pr_num}" --title "${title}" --body "${title}" +fi diff --git a/.github/workflows/hermetic_library_generation.yaml b/.github/workflows/hermetic_library_generation.yaml new file mode 100644 index 000000000..7146cc3dc --- /dev/null +++ b/.github/workflows/hermetic_library_generation.yaml @@ -0,0 +1,40 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Hermetic library generation upon generation config change through pull requests +on: + pull_request: + +jobs: + library_generation: + # skip pull requests coming from a forked repository + if: github.event.pull_request.head.repo.full_name == github.repository + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + with: + fetch-depth: 0 + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Generate changed libraries + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash .github/scripts/hermetic_library_generation.sh \ + --target_branch ${{ github.base_ref }} \ + --current_branch ${{ github.head_ref }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/.github/workflows/update_generation_config.yaml b/.github/workflows/update_generation_config.yaml new file mode 100644 index 000000000..3cf773992 --- /dev/null +++ b/.github/workflows/update_generation_config.yaml @@ -0,0 +1,42 @@ +# Copyright 2024 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# GitHub action job to test core java library features on +# downstream client libraries before they are released. +name: Update generation configuration +on: + schedule: + - cron: '0 2 * * *' + workflow_dispatch: + +jobs: + update-generation-config: + runs-on: ubuntu-22.04 + env: + # the branch into which the pull request is merged + base_branch: main + steps: + - uses: actions/checkout@v4 + with: + token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} + - name: Update params in generation config to latest + shell: bash + run: | + set -x + [ -z "$(git config user.email)" ] && git config --global user.email "cloud-java-bot@google.com" + [ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot" + bash .github/scripts/update_generation_config.sh \ + --base_branch "${base_branch}"\ + --repo ${{ github.repository }} + env: + GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }} diff --git a/generation_config.yaml b/generation_config.yaml new file mode 100644 index 000000000..e7b53b6d9 --- /dev/null +++ b/generation_config.yaml @@ -0,0 +1,23 @@ +gapic_generator_version: 2.42.0 +googleapis_commitish: 6f289d775912966eb0cf04bda91e5e355c998d30 +libraries_bom_version: 26.38.0 +libraries: + - api_shortname: pubsub + name_pretty: Cloud Pub/Sub + api_reference: https://cloud.google.com/pubsub/ + product_documentation: https://cloud.google.com/pubsub/docs/ + client_documentation: https://cloud.google.com/java/docs/reference/google-cloud-pubsub/latest/history + api_description: is designed to provide reliable, many-to-many, asynchronous messaging between applications. Publisher applications can send messages to a topic and other applications can subscribe to that topic to receive the messages. By decoupling senders and receivers, Google Cloud Pub/Sub allows developers to communicate between independently written applications. + issue_tracker: https://issuetracker.google.com/savedsearches/559741 + release_level: stable + language: java + repo: googleapis/java-pubsub + repo_short: java-pubsub + distribution_name: com.google.cloud:google-cloud-pubsub + codeowner_team: '@googleapis/api-pubsub' + api_id: pubsub.googleapis.com + library_type: GAPIC_COMBO + requires_billing: true + recommended_package: com.google.cloud.pubsub.v1 + GAPICs: + - proto_path: google/pubsub/v1