From 9829721b5d98330f699b4be6b96da5f442965005 Mon Sep 17 00:00:00 2001 From: Diego Marquez Date: Tue, 18 Jun 2024 23:24:31 -0400 Subject: [PATCH] chore: adjust repo-metadata generation to fit existing files in HW repos (#2923) This PR introduces a couple of new keys in the repo-metadata.json [model](https://github.com/googleapis/sdk-platform-java/blob/main/library_generation/model/library_config.py). * `min_java_version`, as seen in [pubsublite](https://github.com/googleapis/java-pubsublite/blob/11c6540e2dec642492f5bd324ee810ca037d5a86/.repo-metadata.json#L11), [spanner](https://github.com/googleapis/java-spanner/blob/9deb34846b57e238642b6aa13f3107b2204cf969/.repo-metadata.json#L10) and others. * `recommended_package`, as seen in [spanner](https://github.com/googleapis/java-spanner/blob/9deb34846b57e238642b6aa13f3107b2204cf969/.repo-metadata.json#L20), [storage](https://github.com/googleapis/java-storage/blob/87bf7371b6c4300b0f306ca36d1918d52adf721b/.repo-metadata.json#L19) and others. This constitutes an effort to fit our library generation to the existing hand-written libraries. --- .../generate_composed_library.py | 4 +-- library_generation/model/generation_config.py | 2 ++ library_generation/model/library_config.py | 8 +++++ .../.repo-metadata-monorepo-golden.json | 4 ++- .../.repo-metadata-non-monorepo-golden.json | 4 ++- .../.repo-metadata-proto-only-golden.json | 4 ++- .../test/utilities_unit_tests.py | 30 +++++++++++-------- library_generation/utils/utilities.py | 13 ++++---- 8 files changed, 47 insertions(+), 22 deletions(-) diff --git a/library_generation/generate_composed_library.py b/library_generation/generate_composed_library.py index 11c98e2dd9..2056bd6611 100755 --- a/library_generation/generate_composed_library.py +++ b/library_generation/generate_composed_library.py @@ -70,10 +70,10 @@ def generate_composed_library( build_file_folder = Path(f"{output_folder}/{gapic.proto_path}").resolve() print(f"build_file_folder: {build_file_folder}") gapic_inputs = parse_build_file(build_file_folder, gapic.proto_path) - # generate prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml, + # generate postprocessing prerequisite files (.repo-metadata.json, .OwlBot-hermetic.yaml, # owlbot.py) here because transport is parsed from BUILD.bazel, # which lives in a versioned proto_path. - util.generate_prerequisite_files( + util.generate_postprocessing_prerequisite_files( config=config, library=library, proto_path=util.remove_version_from(gapic.proto_path), diff --git a/library_generation/model/generation_config.py b/library_generation/model/generation_config.py index 3a924600e5..8a2b0829c4 100644 --- a/library_generation/model/generation_config.py +++ b/library_generation/model/generation_config.py @@ -141,6 +141,8 @@ def from_yaml(path_to_yaml: str) -> GenerationConfig: extra_versioned_modules=__optional( library, "extra_versioned_modules", None ), + recommended_package=__optional(library, "recommended_package", None), + min_java_version=__optional(library, "min_java_version", None), ) parsed_libraries.append(new_library) diff --git a/library_generation/model/library_config.py b/library_generation/model/library_config.py index 30d923da81..f379ec9c50 100644 --- a/library_generation/model/library_config.py +++ b/library_generation/model/library_config.py @@ -48,6 +48,8 @@ def __init__( cloud_api: Optional[bool] = True, requires_billing: Optional[bool] = True, extra_versioned_modules: Optional[str] = None, + recommended_package: Optional[str] = None, + min_java_version: Optional[int] = None, ): self.api_shortname = api_shortname self.api_description = api_description @@ -72,6 +74,8 @@ def __init__( self.cloud_api = cloud_api self.requires_billing = requires_billing self.extra_versioned_modules = extra_versioned_modules + self.recommended_package = recommended_package + self.min_java_version = min_java_version def get_library_name(self) -> str: """ @@ -105,6 +109,8 @@ def __eq__(self, other): and self.cloud_api == other.cloud_api and self.requires_billing == other.requires_billing and self.extra_versioned_modules == other.extra_versioned_modules + and self.recommended_package == other.recommended_package + and self.min_java_version == other.min_java_version ) def __hash__(self): @@ -134,6 +140,8 @@ def __hash__(self): self.cloud_api, self.requires_billing, self.extra_versioned_modules, + self.recommended_package, + self.min_java_version, ] + [config.proto_path for config in self.gapic_configs] ).encode("utf-8") diff --git a/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json b/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json index 88ee68b2e1..0b3bbc2b55 100644 --- a/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json +++ b/library_generation/test/resources/goldens/.repo-metadata-monorepo-golden.json @@ -14,5 +14,7 @@ "library_type": "GAPIC_AUTO", "requires_billing": true, "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc" + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "recommended_package": "com.google.example", + "min_java_version": 8 } \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json b/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json index 9e4a878d67..b39c297a85 100644 --- a/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json +++ b/library_generation/test/resources/goldens/.repo-metadata-non-monorepo-golden.json @@ -15,5 +15,7 @@ "requires_billing": true, "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", - "extra_versioned_modules": "test-module" + "extra_versioned_modules": "test-module", + "recommended_package": "com.google.example", + "min_java_version": 8 } \ No newline at end of file diff --git a/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json b/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json index 995607bae4..7730cd6987 100644 --- a/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json +++ b/library_generation/test/resources/goldens/.repo-metadata-proto-only-golden.json @@ -13,5 +13,7 @@ "library_type": "OTHER", "requires_billing": true, "rest_documentation": "https://cloud.google.com/bare-metal/docs/reference/rest", - "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc" + "rpc_documentation": "https://cloud.google.com/bare-metal/docs/reference/rpc", + "recommended_package": "com.google.example", + "min_java_version": 8 } \ No newline at end of file diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index 1d522114cc..c38016fcba 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -40,6 +40,8 @@ library_name="bare-metal-solution", rest_documentation="https://cloud.google.com/bare-metal/docs/reference/rest", rpc_documentation="https://cloud.google.com/bare-metal/docs/reference/rpc", + recommended_package="com.google.example", + min_java_version=8, ) library_2 = LibraryConfig( api_shortname="secretmanager", @@ -167,8 +169,8 @@ def test_eprint_valid_input_succeeds(self): # print() appends a `\n` each time it's called self.assertEqual(test_input + "\n", result) - def test_generate_prerequisite_files_non_monorepo_success(self): - library_path = self.__setup_prerequisite_files( + def test_generate_postprocessing_prerequisite_files_non_monorepo_success(self): + library_path = self.__setup_postprocessing_prerequisite_files( combination=1, library_type="GAPIC_COMBO" ) @@ -185,10 +187,12 @@ def test_generate_prerequisite_files_non_monorepo_success(self): file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) - self.__remove_prerequisite_files(path=library_path, is_monorepo=False) + self.__remove_postprocessing_prerequisite_files( + path=library_path, is_monorepo=False + ) - def test_generate_prerequisite_files_monorepo_success(self): - library_path = self.__setup_prerequisite_files(combination=2) + def test_generate_postprocessing_prerequisite_files_monorepo_success(self): + library_path = self.__setup_postprocessing_prerequisite_files(combination=2) file_comparator.compare_files( f"{library_path}/.repo-metadata.json", @@ -201,10 +205,10 @@ def test_generate_prerequisite_files_monorepo_success(self): file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) - self.__remove_prerequisite_files(path=library_path) + self.__remove_postprocessing_prerequisite_files(path=library_path) - def test_generate_prerequisite_files_proto_only_repo_success(self): - library_path = self.__setup_prerequisite_files( + def test_generate_postprocessing_prerequisite_files_proto_only_repo_success(self): + library_path = self.__setup_postprocessing_prerequisite_files( combination=3, library_type="OTHER" ) @@ -219,7 +223,7 @@ def test_generate_prerequisite_files_proto_only_repo_success(self): file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) - self.__remove_prerequisite_files(path=library_path) + self.__remove_postprocessing_prerequisite_files(path=library_path) def test_prepare_repo_monorepo_success(self): gen_config = self.__get_a_gen_config(2) @@ -256,7 +260,7 @@ def test_prepare_repo_split_repo_success(self): self.assertEqual(["misc"], library_path) shutil.rmtree(repo_config.output_folder) - def __setup_prerequisite_files( + def __setup_postprocessing_prerequisite_files( self, combination: int, library_type: str = "GAPIC_AUTO", @@ -273,7 +277,7 @@ def __setup_prerequisite_files( config = self.__get_a_gen_config(combination, library_type=library_type) proto_path = "google/cloud/baremetalsolution/v2" transport = "grpc" - util.generate_prerequisite_files( + util.generate_postprocessing_prerequisite_files( config=config, library=library, proto_path=proto_path, @@ -319,7 +323,9 @@ def __get_a_gen_config( ) @staticmethod - def __remove_prerequisite_files(path: str, is_monorepo: bool = True) -> None: + def __remove_postprocessing_prerequisite_files( + path: str, is_monorepo: bool = True + ) -> None: os.remove(f"{path}/.repo-metadata.json") os.remove(f"{path}/owlbot.py") if is_monorepo: diff --git a/library_generation/utils/utilities.py b/library_generation/utils/utilities.py index f21c35bf0e..0490ad9e2a 100755 --- a/library_generation/utils/utilities.py +++ b/library_generation/utils/utilities.py @@ -183,7 +183,7 @@ def pull_api_definition( ) -def generate_prerequisite_files( +def generate_postprocessing_prerequisite_files( config: GenerationConfig, library: LibraryConfig, proto_path: str, @@ -192,14 +192,13 @@ def generate_prerequisite_files( language: str = "java", ) -> None: """ - Generate prerequisite files for a library. - - Note that the version, if any, in the proto_path will be removed. + Generates the postprocessing prerequisite files for a library. :param config: a GenerationConfig object representing a parsed configuration yaml :param library: the library configuration - :param proto_path: the proto path + :param proto_path: the path from the root of googleapis to the location of the service + protos. If the path contains a version, it will be removed :param transport: transport supported by the library :param library_path: the path to which the generated file goes :param language: programming language of the library @@ -275,6 +274,10 @@ def generate_prerequisite_files( repo_metadata["rpc_documentation"] = library.rpc_documentation if library.extra_versioned_modules: repo_metadata["extra_versioned_modules"] = library.extra_versioned_modules + if library.recommended_package: + repo_metadata["recommended_package"] = library.recommended_package + if library.min_java_version: + repo_metadata["min_java_version"] = library.min_java_version # generate .repo-meta.json json_file = ".repo-metadata.json"