Skip to content

Commit

Permalink
Merge branch 'main' into renovate/org.codehaus.mojo-exec-maven-plugin…
Browse files Browse the repository at this point in the history
…-3.x
  • Loading branch information
diegomarquezp authored May 30, 2024
2 parents d9aed8f + 1638f99 commit a1fe148
Show file tree
Hide file tree
Showing 474 changed files with 1,019 additions and 666 deletions.
4 changes: 4 additions & 0 deletions .cloudbuild/graalvm/cloudbuild.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ steps:
args: [
"build",
"-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION}",
"-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:infrastructure-public-image-$SHORT_SHA",
"--file", "graalvm_a.Dockerfile",
"--build-arg", "JAVA_SHARED_CONFIG_VERSION=$_JAVA_SHARED_CONFIG_VERSION",
"."
Expand All @@ -35,6 +36,7 @@ steps:
args: [
"build",
"-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:${_SHARED_DEPENDENCIES_VERSION}",
"-t", "gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:infrastructure-public-image-$SHORT_SHA",
"--file", "graalvm_b.Dockerfile",
"--build-arg", "JAVA_SHARED_CONFIG_VERSION=$_JAVA_SHARED_CONFIG_VERSION",
"."
Expand All @@ -47,3 +49,5 @@ steps:
images:
- gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION}
- gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:${_SHARED_DEPENDENCIES_VERSION}
- gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:infrastructure-public-image-$SHORT_SHA
- gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:infrastructure-public-image-$SHORT_SHA
19 changes: 15 additions & 4 deletions .cloudbuild/library_generation/library_generation.Dockerfile
Original file line number Diff line number Diff line change
Expand Up @@ -15,24 +15,35 @@
# build from the root of this repo:
FROM gcr.io/cloud-devrel-public-resources/python

ARG SYNTHTOOL_COMMITTISH=a2c9b4a5da2d7f583c8a1869fd2843c206145834
SHELL [ "/bin/bash", "-c" ]

ARG SYNTHTOOL_COMMITTISH=e36d2f164ca698f0264fb6f79ddc4b0fa024a940
ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550
ARG PROTOC_VERSION=25.3
ENV HOME=/home

# install OS tools
RUN apt-get update && apt-get install -y \
unzip openjdk-17-jdk rsync maven jq \
&& apt-get clean

# copy source code
COPY library_generation /src

# install protoc
WORKDIR /protoc
RUN source /src/utils/utilities.sh \
&& download_protoc "${PROTOC_VERSION}" "linux-x86_64"
# we indicate protoc is available in the container via env vars
ENV DOCKER_PROTOC_LOCATION=/protoc
ENV DOCKER_PROTOC_VERSION="${PROTOC_VERSION}"

# use python 3.11 (the base image has several python versions; here we define the default one)
RUN rm $(which python3)
RUN ln -s $(which python3.11) /usr/local/bin/python
RUN ln -s $(which python3.11) /usr/local/bin/python3
RUN python -m pip install --upgrade pip

# copy source code
COPY library_generation /src

# install scripts as a python package
WORKDIR /src
RUN python -m pip install -r requirements.txt
Expand Down
2 changes: 1 addition & 1 deletion .github/release-please.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,4 +3,4 @@ bumpMinorPreMajor: true
handleGHRelease: true
primaryBranch: main
manifest: true
extraFiles: ["WORKSPACE", ".cloudbuild/graalvm/cloudbuild.yaml", ".cloudbuild/graalvm/cloudbuild-test-a.yaml", ".cloudbuild/graalvm/cloudbuild-test-b.yaml", ".cloudbuild/library_generation/cloudbuild-library-generation-release.yaml" ]
extraFiles: ["WORKSPACE", ".cloudbuild/graalvm/cloudbuild.yaml", ".cloudbuild/graalvm/cloudbuild-test-a.yaml", ".cloudbuild/graalvm/cloudbuild-test-b.yaml", ".cloudbuild/library_generation/cloudbuild-library-generation-release.yaml", "generation_config.yaml"]
119 changes: 119 additions & 0 deletions .github/scripts/hermetic_library_generation.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,119 @@
#!/bin/bash
set -e
# This script should be run at the root of the repository.
# This script is used to, when a pull request changes the generation
# configuration (generation_config.yaml by default):
# 1. Find whether the last commit in this pull request contains changes to
# the generation configuration and exit early if it doesn't have such a change
# since the generation result would be the same.
# 2. Compare generation configurations in the current branch (with which the
# pull request associated) and target branch (into which the pull request is
# merged);
# 3. Generate changed libraries using library_generation image;
# 4. Commit the changes to the pull request, if any.
# 5. Edit the PR body with generated pull request description, if applicable.

# The following commands need to be installed before running the script:
# 1. git
# 2. gh
# 3. docker

# The parameters of this script is:
# 1. target_branch, the branch into which the pull request is merged.
# 2. current_branch, the branch with which the pull request is associated.
# 3. image_tag, the tag of gcr.io/cloud-devrel-public-resources/java-library-generation.
# 3. [optional] generation_config, the path to the generation configuration,
# the default value is generation_config.yaml in the repository root.
while [[ $# -gt 0 ]]; do
key="$1"
case "${key}" in
--target_branch)
target_branch="$2"
shift
;;
--current_branch)
current_branch="$2"
shift
;;
--image_tag)
image_tag="$2"
shift
;;
--generation_config)
generation_config="$2"
shift
;;
*)
echo "Invalid option: [$1]"
exit 1
;;
esac
shift
done

if [ -z "${target_branch}" ]; then
echo "missing required argument --target_branch"
exit 1
fi

if [ -z "${current_branch}" ]; then
echo "missing required argument --current_branch"
exit 1
fi

if [ -z "${image_tag}" ]; then
echo "missing required argument --image_tag"
exit 1
fi

if [ -z "${generation_config}" ]; then
generation_config=generation_config.yaml
echo "Use default generation config: ${generation_config}"
fi

workspace_name="/workspace"
baseline_generation_config="baseline_generation_config.yaml"
message="chore: generate libraries at $(date)"

git checkout "${target_branch}"
git checkout "${current_branch}"
# if the last commit doesn't contain changes to generation configuration,
# do not generate again as the result will be the same.
change_of_last_commit="$(git diff-tree --no-commit-id --name-only HEAD~1..HEAD -r)"
if [[ ! ("${change_of_last_commit}" == *"${generation_config}"*) ]]; then
echo "The last commit doesn't contain any changes to the generation_config.yaml, skipping the whole generation process." || true
exit 0
fi
# copy generation configuration from target branch to current branch.
git show "${target_branch}":"${generation_config}" > "${baseline_generation_config}"
config_diff=$(diff "${generation_config}" "${baseline_generation_config}" || true)

# run hermetic code generation docker image.
docker run \
--rm \
-u "$(id -u):$(id -g)" \
-v "$(pwd):${workspace_name}" \
-v "$HOME"/.m2:/home/.m2 \
gcr.io/cloud-devrel-public-resources/java-library-generation:"${image_tag}" \
--baseline-generation-config-path="${workspace_name}/${baseline_generation_config}" \
--current-generation-config-path="${workspace_name}/${generation_config}"

# commit the change to the pull request.
rm -rdf output googleapis "${baseline_generation_config}"
git add --all -- ':!pr_description.txt'
changed_files=$(git diff --cached --name-only)
if [[ "${changed_files}" == "" ]]; then
echo "There is no generated code change with the generation config change ${config_diff}."
echo "Skip committing to the pull request."
exit 0
fi

echo "Configuration diff:"
echo "${config_diff}"
git commit -m "${message}"
git push
# set pr body if pr_description.txt is generated.
if [[ -f "pr_description.txt" ]]; then
pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
gh pr edit "${pr_num}" --body "$(cat pr_description.txt)"
fi
92 changes: 92 additions & 0 deletions .github/scripts/update_googleapis_commit.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
#!/bin/bash
set -e
# This script should be run at the root of the repository.
# This script is used to update googleapis commit to latest in generation
# configuration at the time of running and create a pull request.

# The following commands need to be installed before running the script:
# 1. git
# 2. gh

# The parameters of this script is:
# 1. base_branch, the base branch of the result pull request.
# 2. repo, organization/repo-name, e.g., googleapis/google-cloud-java
# 3. [optional] generation_config, the path to the generation configuration,
# the default value is generation_config.yaml in the repository root.
while [[ $# -gt 0 ]]; do
key="$1"
case "${key}" in
--base_branch)
base_branch="$2"
shift
;;
--repo)
repo="$2"
shift
;;
--generation_config)
generation_config="$2"
shift
;;
*)
echo "Invalid option: [$1]"
exit 1
;;
esac
shift
done

if [ -z "${base_branch}" ]; then
echo "missing required argument --base_branch"
exit 1
fi

if [ -z "${repo}" ]; then
echo "missing required argument --repo"
exit 1
fi

if [ -z "${generation_config}" ]; then
generation_config="generation_config.yaml"
echo "Use default generation config: ${generation_config}"
fi

current_branch="generate-libraries-${base_branch}"
title="chore: update googleapis commit at $(date)"

# try to find a open pull request associated with the branch
pr_num=$(gh pr list -s open -H "${current_branch}" -q . --json number | jq ".[] | .number")
# create a branch if there's no open pull request associated with the
# branch; otherwise checkout the pull request.
if [ -z "${pr_num}" ]; then
git checkout -b "${current_branch}"
else
gh pr checkout "${pr_num}"
fi

mkdir tmp-googleapis
# use partial clone because only commit history is needed.
git clone --filter=blob:none https://github.com/googleapis/googleapis.git tmp-googleapis
pushd tmp-googleapis
git pull
latest_commit=$(git rev-parse HEAD)
popd
rm -rf tmp-googleapis
sed -i -e "s/^googleapis_commitish.*$/googleapis_commitish: ${latest_commit}/" "${generation_config}"

git add "${generation_config}"
changed_files=$(git diff --cached --name-only)
if [[ "${changed_files}" == "" ]]; then
echo "The latest googleapis commit is not changed."
echo "Skip committing to the pull request."
exit 0
fi
git commit -m "${title}"
if [ -z "${pr_num}" ]; then
git remote add remote_repo https://cloud-java-bot:"${GH_TOKEN}@github.com/${repo}.git"
git fetch -q --unshallow remote_repo
git push -f remote_repo "${current_branch}"
gh pr create --title "${title}" --head "${current_branch}" --body "${title}" --base "${base_branch}"
else
git push
fi
54 changes: 54 additions & 0 deletions .github/workflows/hermetic_library_generation.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,54 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# GitHub action job to test core java library features on
# downstream client libraries before they are released.
name: Hermetic library generation upon generation config change through pull requests
on:
pull_request:

jobs:
library_generation:
# skip pull requests come from a forked repository
if: github.event.pull_request.head.repo.full_name == github.repository
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
with:
fetch-depth: 0
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
- name: Build image
shell: bash
run: |
docker build \
-f .cloudbuild/library_generation/library_generation.Dockerfile \
-t gcr.io/cloud-devrel-public-resources/java-library-generation:latest \
.
- name: Install gapic-generator-java-pom-parent
shell: bash
run: |
mvn clean install -pl gapic-generator-java-pom-parent
- name: Generate changed libraries
shell: bash
run: |
set -x
[ -z "$(git config user.email)" ] && git config --global user.email "[email protected]"
[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
bash .github/scripts/hermetic_library_generation.sh \
--target_branch "${base_ref}" \
--current_branch "${head_ref}" \
--image_tag latest
env:
base_ref: ${{ github.base_ref }}
head_ref: ${{ github.head_ref }}
GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
42 changes: 42 additions & 0 deletions .github/workflows/update_googleapis_commit.yaml
Original file line number Diff line number Diff line change
@@ -0,0 +1,42 @@
# Copyright 2024 Google LLC
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# GitHub action job to test core java library features on
# downstream client libraries before they are released.
name: Update googleapis commit
on:
schedule:
- cron: '0 2 * * *'
workflow_dispatch:

jobs:
update-googleapis-commit:
runs-on: ubuntu-22.04
env:
# the branch into which the pull request is merged
base_branch: main
steps:
- uses: actions/checkout@v4
with:
token: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
- name: Update googleapis commit to latest
shell: bash
run: |
set -x
[ -z "$(git config user.email)" ] && git config --global user.email "[email protected]"
[ -z "$(git config user.name)" ] && git config --global user.name "cloud-java-bot"
bash .github/scripts/update_googleapis_commit.sh \
--base_branch "${base_branch}"\
--repo ${{ github.repository }}
env:
GH_TOKEN: ${{ secrets.CLOUD_JAVA_BOT_TOKEN }}
Loading

0 comments on commit a1fe148

Please sign in to comment.