From 6b4c48248301d2fe1f70067fa74e9a31b3e3122a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Tue, 16 Apr 2024 14:21:09 -0400 Subject: [PATCH 01/13] chore: remove generated files in IT --- library_generation/test/integration_tests.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 08aef860f0..f240a421bb 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -51,9 +51,16 @@ class IntegrationTest(unittest.TestCase): def setUpClass(cls) -> None: IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) - def test_entry_point_running_in_container(self): + @classmethod + def setUp(cls) -> None: shutil.rmtree(f"{golden_dir}", ignore_errors=True) os.makedirs(f"{golden_dir}", exist_ok=True) + + @classmethod + def tearDown(cls) -> None: + shutil.rmtree(f"{golden_dir}", ignore_errors=True) + + def test_entry_point_running_in_container(self): config_files = self.__get_config_files(config_dir) for repo, config_file in config_files: config = from_yaml(config_file) From 2236ec6b6009d9b2cc9628021c9c840bc279f8c9 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Tue, 16 Apr 2024 20:15:01 +0000 Subject: [PATCH 02/13] remove output folder --- library_generation/test/integration_tests.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index f240a421bb..ea01379a02 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -53,12 +53,12 @@ def setUpClass(cls) -> None: @classmethod def setUp(cls) -> None: - shutil.rmtree(f"{golden_dir}", ignore_errors=True) + cls.cleanUp() os.makedirs(f"{golden_dir}", exist_ok=True) @classmethod def tearDown(cls) -> None: - shutil.rmtree(f"{golden_dir}", ignore_errors=True) + cls.cleanUp() def test_entry_point_running_in_container(self): config_files = self.__get_config_files(config_dir) @@ -183,6 +183,11 @@ def __build_image(cls, docker_file: str, cwd: str): cwd=cwd, ) + @classmethod + def cleanUp(cls): + shutil.rmtree(f"{output_dir}", ignore_errors=True) + shutil.rmtree(f"{golden_dir}", ignore_errors=True) + @classmethod def __pull_repo_to(cls, dest: Path, repo: str, committish: str) -> str: shutil.rmtree(dest, ignore_errors=True) From b001648fa65714fb1606ddfd2ebb01fac7549e3f Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Tue, 16 Apr 2024 20:19:20 +0000 Subject: [PATCH 03/13] remove image --- library_generation/test/integration_tests.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index ea01379a02..3e0e98c2d9 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -51,6 +51,10 @@ class IntegrationTest(unittest.TestCase): def setUpClass(cls) -> None: IntegrationTest.__build_image(docker_file=build_file, cwd=repo_root_dir) + @classmethod + def tearDownClass(cls) -> None: + cls.__remove_docker_image() + @classmethod def setUp(cls) -> None: cls.cleanUp() @@ -335,3 +339,7 @@ def __recursive_diff_files( cls.__recursive_diff_files( sub_dcmp, diff_files, left_only, right_only, dirname + sub_dirname + "/" ) + + @classmethod + def __remove_docker_image(cls): + subprocess.check_call(["docker", "image", "rmi", image_tag]) From 35c143e573ed48c743ae69e18a7e1c82202a7d68 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Tue, 16 Apr 2024 21:15:25 +0000 Subject: [PATCH 04/13] change ignore --- .gitignore | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.gitignore b/.gitignore index 05901afab0..bcbee64aad 100644 --- a/.gitignore +++ b/.gitignore @@ -15,14 +15,12 @@ target/ # Vscode Settings .vscode/settings.json -*.iml - # Python **/__pycache__/ .venv # library generation -**/output/ +output/ **/googleapis library_generation/test/resources/test_monorepo_postprocessing/ **/*egg-info/ From e446c9bd9970d808c3e950df38e031c76b9248da Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Tue, 16 Apr 2024 21:48:53 +0000 Subject: [PATCH 05/13] remove generated files in unit tests --- library_generation/test/integration_tests.py | 6 +++--- library_generation/test/utilities_unit_tests.py | 13 ++++++++++++- 2 files changed, 15 insertions(+), 4 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 3e0e98c2d9..267a5d84f2 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -57,12 +57,12 @@ def tearDownClass(cls) -> None: @classmethod def setUp(cls) -> None: - cls.cleanUp() + cls.__remove_generated_files() os.makedirs(f"{golden_dir}", exist_ok=True) @classmethod def tearDown(cls) -> None: - cls.cleanUp() + cls.__remove_generated_files() def test_entry_point_running_in_container(self): config_files = self.__get_config_files(config_dir) @@ -188,7 +188,7 @@ def __build_image(cls, docker_file: str, cwd: str): ) @classmethod - def cleanUp(cls): + def __remove_generated_files(cls): shutil.rmtree(f"{output_dir}", ignore_errors=True) shutil.rmtree(f"{golden_dir}", ignore_errors=True) diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index bc8b965c2c..4d84b844e3 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -15,7 +15,7 @@ """ Unit tests for python scripts """ - +import shutil import unittest import os import io @@ -313,6 +313,7 @@ def test_generate_prerequisite_files_non_monorepo_success(self): file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) + self.__remove_generated_files(path=library_path, is_monorepo=False) def test_generate_prerequisite_files_monorepo_success(self): library_path = self.__setup_prerequisite_files(num_libraries=2) @@ -328,6 +329,7 @@ def test_generate_prerequisite_files_monorepo_success(self): file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) + self.__remove_generated_files(path=library_path) def test_prepare_repo_monorepo_success(self): gen_config = self.__get_a_gen_config(2) @@ -448,6 +450,15 @@ def __get_a_gen_config( libraries=libraries, ) + @staticmethod + def __remove_generated_files(path: str, is_monorepo: bool = True) -> None: + os.remove(f"{path}/.repo-metadata.json") + os.remove(f"{path}/owlbot.py") + if is_monorepo: + os.remove(f"{path}/.OwlBot-hermetic.yaml") + return + shutil.rmtree(f"{path}/.github", ignore_errors=True) + if __name__ == "__main__": unittest.main() From afec3256240e146206d176878c6e84f6d43c8038 Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Tue, 16 Apr 2024 22:25:51 +0000 Subject: [PATCH 06/13] ignore output --- .gitignore | 1 - library_generation/test/utilities_unit_tests.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index bcbee64aad..3cdba0d1cd 100644 --- a/.gitignore +++ b/.gitignore @@ -20,7 +20,6 @@ target/ .venv # library generation -output/ **/googleapis library_generation/test/resources/test_monorepo_postprocessing/ **/*egg-info/ diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index 4d84b844e3..4178e3d1f1 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -375,6 +375,7 @@ def test_prepare_repo_split_repo_success(self): self.assertEqual("output", Path(repo_config.output_folder).name) library_path = sorted([Path(key).name for key in repo_config.libraries]) self.assertEqual(["misc"], library_path) + shutil.rmtree(repo_config.output_folder, ignore_errors=True) def __setup_prerequisite_files( self, num_libraries: int, library_type: str = "GAPIC_AUTO" From df6e53fd1c895160fde28472af19e63b1d23a9e3 Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 18 Apr 2024 14:27:04 -0400 Subject: [PATCH 07/13] comment out remove --- library_generation/test/integration_tests.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 267a5d84f2..ea4319d4f5 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -189,8 +189,9 @@ def __build_image(cls, docker_file: str, cwd: str): @classmethod def __remove_generated_files(cls): - shutil.rmtree(f"{output_dir}", ignore_errors=True) - shutil.rmtree(f"{golden_dir}", ignore_errors=True) + # uncomment this line when the generated files don't owned by root. + # shutil.rmtree(f"{output_dir}", ignore_errors=True) + shutil.rmtree(f"{golden_dir}") @classmethod def __pull_repo_to(cls, dest: Path, repo: str, committish: str) -> str: From fa588e4faab55a751e51e02e8cbd646bef3c562a Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 18 Apr 2024 14:33:25 -0400 Subject: [PATCH 08/13] remove generated pom --- .gitignore | 1 - .../test/utils/monorepo_postprocessor_unit_tests.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/.gitignore b/.gitignore index 3cdba0d1cd..98c7b77b4a 100644 --- a/.gitignore +++ b/.gitignore @@ -21,7 +21,6 @@ target/ # library generation **/googleapis -library_generation/test/resources/test_monorepo_postprocessing/ **/*egg-info/ **/build/ **/dist/ diff --git a/library_generation/test/utils/monorepo_postprocessor_unit_tests.py b/library_generation/test/utils/monorepo_postprocessor_unit_tests.py index c7df83acaf..5dfafe2086 100644 --- a/library_generation/test/utils/monorepo_postprocessor_unit_tests.py +++ b/library_generation/test/utils/monorepo_postprocessor_unit_tests.py @@ -43,3 +43,4 @@ def test_monorepo_postprocessing_valid_repository_success(self): expect=f"{repository_path}/gapic-libraries-bom/pom-golden.xml", actual=f"{repository_path}/gapic-libraries-bom/pom.xml", ) + cleanup(files) From 9dab038a9639dd79bfbc850c90f732efa0234d7f Mon Sep 17 00:00:00 2001 From: Joe Wang Date: Thu, 18 Apr 2024 14:36:19 -0400 Subject: [PATCH 09/13] lint python --- library_generation/generate_repo.py | 8 +++++--- library_generation/test/compare_poms.py | 1 + 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/library_generation/generate_repo.py b/library_generation/generate_repo.py index c133c620dd..9a97e3555b 100755 --- a/library_generation/generate_repo.py +++ b/library_generation/generate_repo.py @@ -77,9 +77,11 @@ def generate( config_path=generation_config_yaml, config=config, repository_path=repository_path, - target_library_names=target_library_names.split(",") - if target_library_names is not None - else target_library_names, + target_library_names=( + target_library_names.split(",") + if target_library_names is not None + else target_library_names + ), ) diff --git a/library_generation/test/compare_poms.py b/library_generation/test/compare_poms.py index a1ef443799..11cc77c8b4 100644 --- a/library_generation/test/compare_poms.py +++ b/library_generation/test/compare_poms.py @@ -4,6 +4,7 @@ The only comparison points are: element path (e.g. project/dependencies) and element text There is a special case for `dependency`, where the maven coordinates are prepared as well """ + from library_generation.utils.utilities import eprint import xml.etree.ElementTree as et from collections import Counter From 2cd421ebfa9f29a4583ae8c9766e5111f31cb5bb Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 18 Apr 2024 19:04:19 +0000 Subject: [PATCH 10/13] remove dir if exists --- library_generation/test/integration_tests.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index ea4319d4f5..52c7b93bab 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -191,7 +191,8 @@ def __build_image(cls, docker_file: str, cwd: str): def __remove_generated_files(cls): # uncomment this line when the generated files don't owned by root. # shutil.rmtree(f"{output_dir}", ignore_errors=True) - shutil.rmtree(f"{golden_dir}") + if os.path.isdir(f"{golden_dir}"): + shutil.rmtree(f"{golden_dir}") @classmethod def __pull_repo_to(cls, dest: Path, repo: str, committish: str) -> str: From d4afa4d17d2116d83d954def5f3d79a1f190f54f Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 18 Apr 2024 19:41:37 +0000 Subject: [PATCH 11/13] change method name --- library_generation/test/utilities_unit_tests.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/library_generation/test/utilities_unit_tests.py b/library_generation/test/utilities_unit_tests.py index 4178e3d1f1..5e5869d0c1 100644 --- a/library_generation/test/utilities_unit_tests.py +++ b/library_generation/test/utilities_unit_tests.py @@ -313,7 +313,7 @@ def test_generate_prerequisite_files_non_monorepo_success(self): file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) - self.__remove_generated_files(path=library_path, is_monorepo=False) + self.__remove_prerequisite_files(path=library_path, is_monorepo=False) def test_generate_prerequisite_files_monorepo_success(self): library_path = self.__setup_prerequisite_files(num_libraries=2) @@ -329,7 +329,7 @@ def test_generate_prerequisite_files_monorepo_success(self): file_comparator.compare_files( f"{library_path}/owlbot.py", f"{library_path}/owlbot-golden.py" ) - self.__remove_generated_files(path=library_path) + self.__remove_prerequisite_files(path=library_path) def test_prepare_repo_monorepo_success(self): gen_config = self.__get_a_gen_config(2) @@ -375,7 +375,7 @@ def test_prepare_repo_split_repo_success(self): self.assertEqual("output", Path(repo_config.output_folder).name) library_path = sorted([Path(key).name for key in repo_config.libraries]) self.assertEqual(["misc"], library_path) - shutil.rmtree(repo_config.output_folder, ignore_errors=True) + shutil.rmtree(repo_config.output_folder) def __setup_prerequisite_files( self, num_libraries: int, library_type: str = "GAPIC_AUTO" @@ -452,13 +452,14 @@ def __get_a_gen_config( ) @staticmethod - def __remove_generated_files(path: str, is_monorepo: bool = True) -> None: + def __remove_prerequisite_files(path: str, is_monorepo: bool = True) -> None: os.remove(f"{path}/.repo-metadata.json") os.remove(f"{path}/owlbot.py") if is_monorepo: os.remove(f"{path}/.OwlBot-hermetic.yaml") return - shutil.rmtree(f"{path}/.github", ignore_errors=True) + if os.path.isdir(f"{path}/.github"): + shutil.rmtree(f"{path}/.github", ignore_errors=True) if __name__ == "__main__": From d4079e233e0069d57bd339e566eebf33717dc5cc Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 18 Apr 2024 22:56:10 +0000 Subject: [PATCH 12/13] remove files after succeeds --- library_generation/test/integration_tests.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index 52c7b93bab..df9a692fad 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -60,10 +60,6 @@ def setUp(cls) -> None: cls.__remove_generated_files() os.makedirs(f"{golden_dir}", exist_ok=True) - @classmethod - def tearDown(cls) -> None: - cls.__remove_generated_files() - def test_entry_point_running_in_container(self): config_files = self.__get_config_files(config_dir) for repo, config_file in config_files: @@ -179,6 +175,7 @@ def test_entry_point_running_in_container(self): "The generated PR description does not match the expected golden file", ) print(" PR description comparison succeed.") + self.__remove_generated_files() @classmethod def __build_image(cls, docker_file: str, cwd: str): From 819e75647c2e7f0992606b9661a14182644604dd Mon Sep 17 00:00:00 2001 From: JoeWang1127 Date: Thu, 18 Apr 2024 22:59:07 +0000 Subject: [PATCH 13/13] change intention --- library_generation/test/integration_tests.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/library_generation/test/integration_tests.py b/library_generation/test/integration_tests.py index df9a692fad..9c4663e373 100644 --- a/library_generation/test/integration_tests.py +++ b/library_generation/test/integration_tests.py @@ -175,7 +175,7 @@ def test_entry_point_running_in_container(self): "The generated PR description does not match the expected golden file", ) print(" PR description comparison succeed.") - self.__remove_generated_files() + self.__remove_generated_files() @classmethod def __build_image(cls, docker_file: str, cwd: str):