chore: add docker image for hermetic build scripts

.cloudbuild/cloudbuild-test-library-generation.yaml

@@ -0,0 +1,31 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+timeout: 7200s # 2 hours
+substitutions:
+  _SHARED_DEPENDENCIES_VERSION: '3.25.1-SNAPSHOT' # {x-version-update:google-cloud-shared-dependencies:current}
+
+steps:
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: ["build", "-t", "gcr.io/cloud-devrel-public-resources/java-library-generation:${_SHARED_DEPENDENCIES_VERSION}", "--file", ".cloudbuild/library_generation.Dockerfile", "."]
+    id: library-generation-build
+    waitFor: ["-"]
+  - name: gcr.io/cloud-devrel-public-resources/java-library-generation:${_SHARED_DEPENDENCIES_VERSION}
+    entrypoint: bash
+    args: [ './library_generation/test/container_integration_tests.sh' ]
+    waitFor: [ "library-generation-build" ]
+    env:
+      - 'SHARED_DEPENDENCIES_VERSION=${_SHARED_DEPENDENCIES_VERSION}'
+

.cloudbuild/cloudbuild.yaml

@@ -31,7 +31,14 @@ steps:
     id: graalvm-b-build
     waitFor: [ "-" ]
 
+  # Library generation build
+  - name: gcr.io/cloud-builders/docker
+    args: ["build", "-t", "gcr.io/cloud-devrel-public-resources/java-library-generation:${_SHARED_DEPENDENCIES_VERSION}", "--file", ".cloudbuild/library_generation.Dockerfile", "."]
+    id: library-generation-build
+    waitFor: ["-"]
+
 
 images:
   - gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_a:${_SHARED_DEPENDENCIES_VERSION}
   - gcr.io/cloud-devrel-public-resources/graalvm_sdk_platform_b:${_SHARED_DEPENDENCIES_VERSION}
+  - gcr.io/cloud-devrel-public-resources/java-library-generation:${_SHARED_DEPENDENCIES_VERSION}

.cloudbuild/library_generation.Dockerfile

@@ -0,0 +1,40 @@
+# Copyright 2024 Google LLC
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     https://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# build from the root of this repo:
+FROM gcr.io/cloud-devrel-public-resources/python
+
+# install tools
+RUN apt-get update && apt-get install -y \
+	unzip openjdk-17-jdk rsync maven \
+	&& apt-get clean
+
+COPY library_generation /src
+
+RUN rm $(which python3)
+RUN ln -s $(which python3.11) /usr/local/bin/python
+RUN ln -s $(which python3.11) /usr/local/bin/python3
+RUN python -m pip install --upgrade pip
+RUN cd /src && python -m pip install -r requirements.in
+RUN cd /src && python -m pip install .
+
+# set dummy git credentials for empty commit used in postprocessing
+RUN git config --global user.email "[email protected]"
+RUN git config --global user.name "Cloud Java Bot"
+
+WORKDIR /workspace
+RUN chmod 750 /workspace
+RUN chmod 750 /src/generate_repo.py
+
+CMD [ "/src/generate_repo.py" ]

library_generation/dockerignore

@@ -0,0 +1,6 @@
+README.md
+**/__pycache__/
+**/*.egg-info/
+**/output/
+**/build/
+**/google-cloud-java/

library_generation/postprocess_library.sh

@@ -22,7 +22,7 @@
 # different logic
 # 8 - configuration_yaml_path: path to the configuration yaml containing library
 # generation information for this library
-set -eo pipefail
+set -eox pipefail
 scripts_root=$(dirname "$(readlink -f "$0")")
 
 postprocessing_target=$1
@@ -61,22 +61,47 @@ fi
 
 # we determine the location of the .OwlBot.yaml file by checking if the target
 # folder is a monorepo folder or not
-if [[ "${postprocessing_target}" == *google-cloud-java* ]]; then
+if [[ "${is_monorepo}" == "true" ]]; then
   owlbot_yaml_relative_path=".OwlBot.yaml"
 else
   owlbot_yaml_relative_path=".github/.OwlBot.yaml"
 fi
 
+# Default values for running copy-code directly from host
+repo_binding="${postprocessing_target}"
+repo_workspace="/repo"
+preprocessed_libraries_binding="${owlbot_cli_source_folder}"
+
+# When running docker inside docker, we run into the issue of volume bindings
+# being mapped from the host machine to the child container (instead of the
+# parent container to child container) because we bind the `docker.sock` socket
+# to the parent container (i.e. docker calls use the host's filesystem context)
+# We solve this by referencing environment variables that will be
+# set to produce the correct volume mapping.
+#
+# The workflow is: to check if we are in a docker container (via passed env var)
+# and use managed volumes (docker volume create) instead of bindings
+# (-v /path:/other-path). The volume names are also received as env vars.
+
+if [[ -n "${RUNNING_IN_DOCKER}" ]]; then
+  set -u # temporarily fail on unset variables
+  repo_binding="${REPO_BINDING_VOLUME}"
+  set +u
+  if [[ "${is_monorepo}" == "true" ]]; then
+    repo_workspace="/repo/$(echo "${postprocessing_target}" | rev | cut -d'/' -f1 | rev)"
+  fi
+fi
+
 docker run --rm \
   --user "$(id -u)":"$(id -g)" \
-  -v "${postprocessing_target}:/repo" \
-  -v "${owlbot_cli_source_folder}:/pre-processed-libraries" \
-  -w /repo \
+  -v "${repo_binding}:/repo" \
+  -v "/tmp:/tmp" \
+  -w "${repo_workspace}" \
   --env HOME=/tmp \
   gcr.io/cloud-devrel-public-resources/owlbot-cli@"${owlbot_cli_image_sha}" \
   copy-code \
   --source-repo-commit-hash=none \
-  --source-repo=/pre-processed-libraries \
+  --source-repo="${preprocessed_libraries_binding}" \
   --config-file="${owlbot_yaml_relative_path}"
 
 # we clone the synthtool library and manually build it
@@ -86,6 +111,7 @@ pushd /tmp/synthtool
 if [ ! -d "synthtool" ]; then
   git clone https://github.com/googleapis/synthtool.git
 fi
+git config --global --add safe.directory /tmp/synthtool/synthtool
 pushd "synthtool"
 
 git reset --hard "${synthtool_commitish}"

library_generation/test/container_integration_tests.sh

@@ -0,0 +1,31 @@
+#!/bin/bash
+# This is a wrapper of integration_tests.py that sets up the environment to run
+# the script in a docker container
+
+set -xe
+if [[ -z "${SHARED_DEPENDENCIES_VERSION}" ]]; then
+  echo "required environemnt variable SHARED_DEPENDENCIES_VERSION is not set"
+  exit 1
+fi
+
+if [[ ! -d google-cloud-java ]]; then
+  git clone https://github.com/googleapis/google-cloud-java
+fi
+pushd google-cloud-java
+git reset --hard main
+popd
+if [[ $(docker volume inspect repo) != '[]' ]]; then
+  docker volume rm repo
+fi
+docker volume create --name "repo" --opt "type=none" --opt "device=$(pwd)/google-cloud-java" --opt "o=bind"
+
+image_id="gcr.io/cloud-devrel-public-resources/java-library-generation:${SHARED_DEPENDENCIES_VERSION}"
+docker run --rm \
+  -v repo:/workspace \
+  -v /tmp:/tmp \
+  -v /var/run/docker.sock:/var/run/docker.sock \
+  -e "RUNNING_IN_DOCKER=true" \
+  -e "REPO_BINDING_VOLUME=repo" \
+  -w "/src" \
+  "${image_id}" \
+  python -m unittest /src/test/integration_tests.py

library_generation/test/integration_tests.py

@@ -26,7 +26,7 @@
 from library_generation.generate_repo import generate_from_yaml
 from library_generation.model.generation_config import from_yaml
 from library_generation.test.compare_poms import compare_xml
-from library_generation.utilities import get_library_name
+from library_generation.utilities import get_library_name, run_process_and_print_output
 
 config_name = "generation_config.yaml"
 script_dir = os.path.dirname(os.path.realpath(__file__))
@@ -46,8 +46,9 @@ def test_generate_repo(self):
         config_files = self.__get_config_files(config_dir)
         i = 0
         for repo, config_file in config_files.items():
-            repo_dest = f"{golden_dir}/{repo}"
-            self.__pull_repo_to(Path(repo_dest), repo, committish_list[i])
+            repo_dest = self.__pull_repo_to(
+                Path(f"{golden_dir}/{repo}"), repo, committish_list[i]
+            )
             library_names = self.__get_library_names_from_config(config_file)
             # prepare golden files
             for library_name in library_names:
@@ -106,11 +107,24 @@ def test_generate_repo(self):
             i += 1
 
     @classmethod
-    def __pull_repo_to(cls, dest: Path, repo: str, committish: str):
-        repo_url = f"{repo_prefix}/{repo}"
-        print(f"Cloning repository {repo_url}")
-        repo = Repo.clone_from(repo_url, dest)
+    def __pull_repo_to(cls, default_dest: Path, repo: str, committish: str) -> str:
+        if "RUNNING_IN_DOCKER" in os.environ:
+            # the docker image expects the repo to be in /workspace
+            dest_str = "/workspace"
+            run_process_and_print_output(
+                ["git", "config", "--global", "--add", "safe.directory", dest_str],
+                "Add /workspace to safe directories",
+            )
+            dest = Path(dest_str)
+            repo = Repo(dest)
+        else:
+            dest = default_dest
+            repo_dest = f"{golden_dir}/{repo}"
+            repo_url = f"{repo_prefix}/{repo}"
+            print(f"Cloning repository {repo_url}")
+            repo = Repo.clone_from(repo_url, dest)
         repo.git.checkout(committish)
+        return str(dest)
 
     @classmethod
     def __get_library_names_from_config(cls, config_path: str) -> List[str]: