diff --git a/.cloudbuild/library_generation/library_generation.Dockerfile b/.cloudbuild/library_generation/library_generation.Dockerfile index f6ef6f1609..e962dc5e51 100644 --- a/.cloudbuild/library_generation/library_generation.Dockerfile +++ b/.cloudbuild/library_generation/library_generation.Dockerfile @@ -15,7 +15,7 @@ # build from the root of this repo: FROM gcr.io/cloud-devrel-public-resources/python -ARG SYNTHTOOL_COMMITTISH=63cc541da2c45fcfca2136c43e638da1fbae174d +ARG SYNTHTOOL_COMMITTISH=a2c9b4a5da2d7f583c8a1869fd2843c206145834 ARG OWLBOT_CLI_COMMITTISH=ac84fa5c423a0069bbce3d2d869c9730c8fdf550 ENV HOME=/home diff --git a/library_generation/cli/entry_point.py b/library_generation/cli/entry_point.py index 960fae8693..0bb499e9f2 100644 --- a/library_generation/cli/entry_point.py +++ b/library_generation/cli/entry_point.py @@ -117,7 +117,6 @@ def generate( # baseline_generation_config is not specified. # Do not generate pull request description. generate_from_yaml( - config_path=current_generation_config_path, config=from_yaml(current_generation_config_path), repository_path=repository_path, ) @@ -131,7 +130,6 @@ def generate( current_config=from_yaml(current_generation_config_path), ) generate_from_yaml( - config_path=current_generation_config_path, config=config_change.current_config, repository_path=repository_path, target_library_names=config_change.get_changed_libraries(), diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index ae5b535e3e..11c98e2dd9 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -41,7 +41,6 @@ def generate_composed_library( - config_path: str, config: GenerationConfig, library_path: str, library: LibraryConfig, @@ -51,7 +50,6 @@ def generate_composed_library( """ Generate libraries composed of more than one service or service version - :param config_path: Path to generation configuration. :param config: a GenerationConfig object representing a parsed configuration yaml :param library_path: the path to which the generated file goes @@ -113,7 +111,7 @@ def generate_composed_library( versions_file, owlbot_cli_source_folder, str(config.is_monorepo()).lower(), - config_path, + config.libraries_bom_version, ], "Library postprocessing", ) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index 214cd78648..b5cf9229fb 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -20,7 +20,6 @@ def generate_from_yaml( - config_path: str, config: GenerationConfig, repository_path: str, target_library_names: list[str] = None, @@ -29,7 +28,6 @@ def generate_from_yaml( Based on the generation config, generates libraries via generate_composed_library.py - :param config_path: Path to generation configuration. :param config: a GenerationConfig object. :param repository_path: The repository path to which the generated files will be sent. @@ -49,7 +47,6 @@ def generate_from_yaml( for library_path, library in repo_config.libraries.items(): print(f"generating library {library.get_library_name()}") generate_composed_library( - config_path=config_path, config=config, library_path=library_path, library=library, diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 3557cf63d2..70e4adeec8 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -40,7 +40,7 @@ def __init__( ): self.gapic_generator_version = gapic_generator_version self.googleapis_commitish = googleapis_commitish - self.libraris_bom_version = libraries_bom_version + self.libraries_bom_version = libraries_bom_version self.template_excludes = template_excludes self.libraries = libraries self.grpc_version = grpc_version diff --git a/library_generation/owlbot/bin/entrypoint.sh b/library_generation/owlbot/bin/entrypoint.sh index 51c0b619d9..b5b84cf1ac 100755 --- a/library_generation/owlbot/bin/entrypoint.sh +++ b/library_generation/owlbot/bin/entrypoint.sh @@ -16,9 +16,12 @@ # This is the entrypoint script for java owlbot. This is not intended to be # called directly but rather be called from postproces_library.sh # For reference, the positional arguments are -# 1: scripts_root: location of postprocess_library.sh +# 1: scripts_root: location of the root of the library_generation scripts. When +# in a Docker container, this value should be /src/ # 2: versions_file: points to a versions.txt containing versions to be applied # both to README and pom.xml files +# 3: is_monorepo: whether we are postprocessing a monorepo +# 4: libraries_bom_version: used to render the readme # The scripts assumes the CWD is the folder where postprocessing is going to be # applied @@ -26,8 +29,8 @@ set -ex scripts_root=$1 versions_file=$2 -configuration_yaml=$3 -is_monorepo=$4 +is_monorepo=$3 +libraries_bom_version=$4 if [[ "${is_monorepo}" == "true" ]]; then @@ -36,22 +39,19 @@ if [[ "${is_monorepo}" == "true" ]]; then mv temp owl-bot-staging fi - -# Runs template and etc in current working directory - -# apply repo templates -echo "Rendering templates" -python3 "${scripts_root}/owlbot/src/apply_repo_templates.py" "${configuration_yaml}" "${is_monorepo}" - # templates as well as retrieving files from owl-bot-staging echo "Retrieving files from owl-bot-staging directory..." if [ -f "owlbot.py" ] then - # we use an empty synthtool folder to prevent cached templates from being used - export SYNTHTOOL_TEMPLATES=$(mktemp -d) + # we copy the templates to a temp folder because we need to do a special + # modification regarding libraries_bom_version that can't be handled by the + # synthtool library considering the way owlbot.py files are written + export SYNTHTOOL_TEMPLATES="${scripts_root}/owlbot/templates" + export SYNTHTOOL_LIBRARIES_BOM_VERSION="${libraries_bom_version}" # defaults to run owlbot.py python3 owlbot.py - export SYNTHTOOL_TEMPLATES="" + unset SYNTHTOOL_TEMPLATES + unset SYNTHTOOL_LIBRARIES_BOM_VERSION fi echo "...done" diff --git a/library_generation/owlbot/src/apply_repo_templates.py b/library_generation/owlbot/src/apply_repo_templates.py deleted file mode 100644 index 8860586640..0000000000 --- a/library_generation/owlbot/src/apply_repo_templates.py +++ /dev/null @@ -1,43 +0,0 @@ -""" -This script parses an owlbot.py file, specifically the call to `java.common_templates` in -order to extract the excluded files so it can be called with a custom template path -pointing to the templates hosted in `sdk-platform-java/library_generation/owlbot/templates`. -Briefly, this wraps the call to synthtool's common templates using a custom template folder. -""" - -import os -import sys -from collections.abc import Sequence -from synthtool.languages.java import common_templates -from pathlib import Path -from library_generation.model.generation_config import from_yaml - -script_dir = os.path.dirname(os.path.realpath(__file__)) -repo_templates_path = os.path.normpath(os.path.join(script_dir, "..", "templates")) - - -def apply_repo_templates(configuration_yaml_path: str, monorepo: bool) -> None: - config = from_yaml(configuration_yaml_path) - print(f"repo_templates_path: {repo_templates_path}") - print(f"excludes: {config.template_excludes}") - common_templates( - excludes=config.template_excludes, - template_path=Path(repo_templates_path), - monorepo=monorepo, - libraries_bom_version=config.libraris_bom_version, - ) - - -def main(argv: Sequence[str]) -> None: - if len(argv) != 3: - raise ValueError( - "Usage: python apply-repo-templates.py configuration_yaml_path monorepo" - ) - - configuration_yaml_path = argv[1] - monorepo = argv[2] - apply_repo_templates(configuration_yaml_path, monorepo.lower() == "true") - - -if __name__ == "__main__": - main(sys.argv) diff --git a/library_generation/postprocess_library.sh b/library_generation/postprocess_library.sh index 411c93fd08..392301814f 100755 --- a/library_generation/postprocess_library.sh +++ b/library_generation/postprocess_library.sh @@ -17,8 +17,8 @@ # googleapis-gen. It will be used instead of preprocessed_sources_path if # 5 - is_monorepo: whether this library is a monorepo, which implies slightly # different logic -# 6 - configuration_yaml_path: path to the configuration yaml containing library -# generation information for this library +# 6 - libraries_bom_version: used by our implementation of owlbot to render the +# readme set -exo pipefail scripts_root=$(dirname "$(readlink -f "$0")") @@ -27,7 +27,7 @@ preprocessed_sources_path=$2 versions_file=$3 owlbot_cli_source_folder=$4 is_monorepo=$5 -configuration_yaml_path=$6 +libraries_bom_version=$6 owlbot_yaml_file_name=".OwlBot-hermetic.yaml" source "${scripts_root}"/utils/utilities.sh @@ -98,5 +98,10 @@ fi # run the postprocessor echo 'running owl-bot post-processor' pushd "${postprocessing_target}" -bash "${scripts_root}/owlbot/bin/entrypoint.sh" "${scripts_root}" "${versions_file}" "${configuration_yaml_path}" "${is_monorepo}" +bash "${scripts_root}/owlbot/bin/entrypoint.sh" \ + "${scripts_root}" \ + "${versions_file}" \ + "${is_monorepo}" \ + "${libraries_bom_version}" + popd # postprocessing_target diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 47ad22b6dc..e4d298cd9c 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -183,8 +183,10 @@ def test_entry_point_running_in_container(self): @classmethod def __build_image(cls, docker_file: str, cwd: str): + # we build the docker image without removing intermediate containers so + # we can re-test more quickly subprocess.check_call( - ["docker", "build", "--rm", "-f", docker_file, "-t", image_tag, "."], + ["docker", "build", "-f", docker_file, "-t", image_tag, "."], cwd=cwd, )