From 73e95bd5f2dfd7688824ca7540de64788bda9c82 Mon Sep 17 00:00:00 2001 From: Dominik Wernberger Date: Sun, 7 Jul 2024 20:28:34 +0200 Subject: [PATCH 1/4] Build on Windows, Linux and Mac Run tests on Linux and Mac --- .github/workflows/main.yml | 31 +++++++++++++++++++++++++++++-- run_tests.py | 16 ++++++++++++---- test/py/requirements.txt | 5 +++-- test/py/testing.py | 9 ++++++--- test/src/Helper.hpp | 2 +- 5 files changed, 51 insertions(+), 12 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d50bd061..f817d3ad 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -38,6 +38,7 @@ jobs: run: | python3 run_tests.py + build-macos: runs-on: macos-14 steps: @@ -48,13 +49,27 @@ jobs: git clone https://github.com/microsoft/vcpkg.git ./vcpkg/bootstrap-vcpkg.sh + - name: Download Unit Test Designs + run: | + python -m pip install -r test/py/requirements.txt + python test/py/testing.py --download_repos true --database_file_path repos.yaml --third_party_path test/designs + + - name: Create Unit Tests + run: | + python test/py/testing.py --generate_unit_tests true --database_file_path repos.yaml --unit_test_path test/src/generated_tests --third_party_path test/designs + + - name: Build Library and CLI Application run: | export VCPKG_ROOT=$(realpath ./vcpkg) export CMAKE_BUILD_PARALLEL_LEVEL=2 - cmake --preset release -DENABLE_UNIT_TESTING=OFF + cmake --preset release -DENABLE_UNIT_TESTING=ON cmake --build --preset release + - name: Run Tests + run: | + python3 run_tests.py + build-windows: runs-on: windows-2022 @@ -65,10 +80,22 @@ jobs: git clone https://github.com/microsoft/vcpkg.git .\vcpkg\bootstrap-vcpkg.bat + - name: Download Unit Test Designs + run: | + ${ENV:PYTHONUTF8}=1 + git config --system core.longpaths true + python -m pip install -r test/py/requirements.txt + python test/py/testing.py --download_repos true --database_file_path repos.yaml --third_party_path test\designs + + - name: Create Unit Tests + run: | + ${ENV:PYTHONUTF8}=1 + git config --system core.longpaths true + python test/py/testing.py --generate_unit_tests true --database_file_path repos.yaml --unit_test_path test\src\generated_tests --third_party_path test\designs - name: Build Library and CLI Application run: | ${ENV:VCPKG_ROOT}=$(Resolve-Path ./vcpkg) ${ENV:CMAKE_BUILD_PARALLEL_LEVEL}=2 - cmake --preset release -DENABLE_UNIT_TESTING=OFF + cmake --preset release -DENABLE_UNIT_TESTING=ON cmake --build --preset release diff --git a/run_tests.py b/run_tests.py index c86b1c44..c21458cc 100644 --- a/run_tests.py +++ b/run_tests.py @@ -35,10 +35,11 @@ class TestThread(threading.Thread): - def __init__(self, shard_count: int, shard_index: int): + def __init__(self, test_file: Path, shard_count: int, shard_index: int): threading.Thread.__init__(self) self.shard_count = shard_count self.shard_index = shard_index + self.test_file = test_file def run(self): global tests_ran_successfully @@ -48,9 +49,16 @@ def run(self): print(f"Starting shard {self.shard_index + 1} / {self.shard_count}") lock.release() + cmd = [ + str(self.test_file), + "--shard-count", + str(self.shard_count), + "--shard-index", + str(self.shard_index), + ] + result = subprocess.run( - f"{test_file} --shard-count {self.shard_count} --shard-index {self.shard_index}", - shell=True, + cmd, capture_output=True, text=True, ) @@ -79,7 +87,7 @@ def run(self): TEST_SHARDS = 8 for idx in range(TEST_SHARDS): - threads += [TestThread(TEST_SHARDS, idx)] + threads += [TestThread(test_file, TEST_SHARDS, idx)] start_time = datetime.now() diff --git a/test/py/requirements.txt b/test/py/requirements.txt index daf6ab24..71537aa7 100644 --- a/test/py/requirements.txt +++ b/test/py/requirements.txt @@ -1,2 +1,3 @@ -GitPython==3.1.26 -pydantic==1.9.0 \ No newline at end of file +GitPython==3.1.43 +pydantic==1.9.0 +pyyaml==6.0.1 \ No newline at end of file diff --git a/test/py/testing.py b/test/py/testing.py index ea46ac45..505af78d 100644 --- a/test/py/testing.py +++ b/test/py/testing.py @@ -10,6 +10,9 @@ from FileErrorDatabase import RepoFile, Repository, FileErrorDatabase +import logging + +logging.basicConfig(level=logging.DEBUG) ExtensionsOfInterest = ['*.OBK', '*.OLB', '*.DSN', '*.DBK'] @@ -113,15 +116,15 @@ def create_repo_cpp_unit_test(repo: Repository, path_repo: str, path_unit_test: for file in repo.files: file_name = os.path.basename(file.path) - full_path = os.path.normpath(os.path.join(path_repo, file.path)) + full_path = Path(os.path.normpath(path_repo / file.path)) unit_test = f""" -TEST_CASE("{repo.author} - {repo.project} - {file_name} : Check File {full_path}", "[Misc]") +TEST_CASE("{repo.author} - {repo.project} - {file_name} : Check File {full_path.as_posix()}", "[Misc]") {{ configure_spdlog(); - const fs::path inputFile{{"{full_path}"}}; + const fs::path inputFile{{"{full_path.as_posix()}"}}; OOCP::ParserConfig cfg = get_parser_config(); diff --git a/test/src/Helper.hpp b/test/src/Helper.hpp index 9b3d7581..92f22f49 100644 --- a/test/src/Helper.hpp +++ b/test/src/Helper.hpp @@ -76,7 +76,7 @@ inline void check_error_count(const fs::path& aFilePath, size_t aActualErrCnt, s // that want to write to this file. while(true) { - std::FILE* lockFile = std::fopen(lockFilePath.c_str(), "ax"); + std::FILE* lockFile = std::fopen(lockFilePath.string().c_str(), "ax"); if(nullptr != lockFile) { From d925c4c613fbf150069fe90c601f89a237bdedaf Mon Sep 17 00:00:00 2001 From: Dominik Wernberger Date: Sun, 7 Jul 2024 20:40:38 +0200 Subject: [PATCH 2/4] Avoid bare except --- test/py/FileErrorDatabase.py | 4 ++-- test/py/testing.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/test/py/FileErrorDatabase.py b/test/py/FileErrorDatabase.py index 22a3bb0b..57f8330d 100644 --- a/test/py/FileErrorDatabase.py +++ b/test/py/FileErrorDatabase.py @@ -114,8 +114,8 @@ def cloneRepoTo(self, repo: Repository, base_path: str) -> None: try: fs_repo = Repo.clone_from(repo.url, export_dir) - except: - raise Exception(f'Failed cloning repository from `{repo.url}`!') + except Exception as ex: + raise Exception(f'Failed cloning repository from `{repo.url}`!\n{ex}') # Commit differs from currently downloaded master if str(fs_repo.head.commit) != repo.commit: diff --git a/test/py/testing.py b/test/py/testing.py index 505af78d..366ac787 100644 --- a/test/py/testing.py +++ b/test/py/testing.py @@ -50,8 +50,8 @@ def add_new_repo_to_database(db: FileErrorDatabase, url: str, base_path: str) -> else: try: fs_repo = Repo.clone_from(url, base_path) - except: - raise Exception(f'Failed cloning repository from `{url}`!') + except Exception as ex: + raise Exception(f'Failed cloning repository from `{url}`!\n{ex}') commit = fs_repo.head.commit From f5d3867351c93d204242a814f03c576a8057a511 Mon Sep 17 00:00:00 2001 From: Dominik Wernberger Date: Sat, 13 Jul 2024 10:29:46 +0200 Subject: [PATCH 3/4] Use Path instead of str --- test/py/testing.py | 42 ++++++++++++++++++++---------------------- 1 file changed, 20 insertions(+), 22 deletions(-) diff --git a/test/py/testing.py b/test/py/testing.py index 366ac787..c4c80171 100644 --- a/test/py/testing.py +++ b/test/py/testing.py @@ -16,13 +16,13 @@ ExtensionsOfInterest = ['*.OBK', '*.OLB', '*.DSN', '*.DBK'] -def get_files_of_interest(path: str, file_extensions: List[str]) -> List[str]: +def get_files_of_interest(path: Path, file_extensions: List[str]) -> List[Path]: """ Get a list of absolute paths to files with specified extension. This function searches recursively through the path. Usage e.g. - >>> files = get_files_of_interest('/home/some_dir', ['*.txt', '*.log']) + >>> files = get_files_of_interest(Path('/home/some_dir'), ['*.txt', '*.log']) >>> print(files) >>> ['/home/som_dir/sub_dir/foo.txt', '/home/some_dir/bar.log'] """ @@ -30,21 +30,19 @@ def get_files_of_interest(path: str, file_extensions: List[str]) -> List[str]: files = [] for extension in file_extensions: - for file in Path(path).rglob(extension): + for file in path.rglob(extension): files += [file] return files -def add_new_repo_to_database(db: FileErrorDatabase, url: str, base_path: str) -> None: +def add_new_repo_to_database(db: FileErrorDatabase, url: str, base_path: Path) -> None: author = url.split('.git')[0].split('/')[-2] project = url.split('.git')[0].split('/')[-1] - base_path = os.path.join(base_path, author, project) + base_path = base_path / author / project - exists_on_fs = os.path.exists(base_path) - - if exists_on_fs: + if base_path.exists(): print(f'Repository {base_path} does already exist, do not clone.') fs_repo = Repo(base_path) else: @@ -60,7 +58,7 @@ def add_new_repo_to_database(db: FileErrorDatabase, url: str, base_path: str) -> files = get_files_of_interest(base_path, ExtensionsOfInterest) for file in files: - file = os.path.join('.', os.path.relpath(file, base_path)) + file = Path(os.path.join('.', os.path.relpath(file, base_path))) repo_file = RepoFile(path=file, errors=99999) if not db.isFileInRepo(repo, repo_file): @@ -68,8 +66,8 @@ def add_new_repo_to_database(db: FileErrorDatabase, url: str, base_path: str) -> db.addRepo(repo) -def write_back_err_count(db: FileErrorDatabase, path_err_cnt_log: str, path_thirdparty_designs: str) -> None: - with open(path_err_cnt_log, 'r') as f: +def write_back_err_count(db: FileErrorDatabase, path_err_cnt_log: Path, path_thirdparty_designs: Path) -> None: + with path_err_cnt_log.open() as f: log = f.readlines() for line in log: @@ -81,14 +79,14 @@ def write_back_err_count(db: FileErrorDatabase, path_err_cnt_log: str, path_thir expect_err = int(matches[1]) file_path = str(matches[2]) - author = file_path[len(path_thirdparty_designs):].split(os.path.sep)[1] - project = file_path[len(path_thirdparty_designs):].split(os.path.sep)[2] + author = file_path[len(str(path_thirdparty_designs)):].split(os.path.sep)[1] + project = file_path[len(str(path_thirdparty_designs)):].split(os.path.sep)[2] - path_repo = os.path.join(path_thirdparty_designs, author, project) + path_repo = path_thirdparty_designs / author / project # Make path relative file_path = os.path.relpath(file_path, path_repo) - file_path = os.path.join('.', file_path) + file_path = Path(os.path.join('.', file_path)) repo = db.getRepoByNames(author, project) file = db.getFileByPath(repo, file_path) @@ -96,8 +94,8 @@ def write_back_err_count(db: FileErrorDatabase, path_err_cnt_log: str, path_thir db.updateErrorCounter(file, actual_err) -def create_repo_cpp_unit_test(repo: Repository, path_repo: str, path_unit_test: str) -> None: - with open(path_unit_test, 'w') as f: +def create_repo_cpp_unit_test(repo: Repository, path_repo: Path, path_unit_test: Path) -> None: + with path_unit_test.open('w', encoding='utf-8') as f: preamble = """// THIS FILE IS AUTOMATICALLY GENERATED! DO NOT MODIFY IT! @@ -198,8 +196,8 @@ def create_repo_cpp_unit_test(repo: Repository, path_repo: str, path_unit_test: do_write_back = args.write_back do_download = args.do_download - path_thirdparty_designs = args.path_thirdparty_designs - path_unit_tests = args.path_unit_tests + path_thirdparty_designs = Path(args.path_thirdparty_designs) + path_unit_tests = Path(args.path_unit_tests) path_err_cnt_log = args.path_err_cnt_log path_db = args.path_db @@ -207,10 +205,10 @@ def create_repo_cpp_unit_test(repo: Repository, path_repo: str, path_unit_test: if not do_generate_unit_tests and not do_write_back and not do_download: parser.print_help() - def generate_unit_tests(db: FileErrorDatabase, path_thirdparty_designs: str, path_unit_tests: str) -> None: + def generate_unit_tests(db: FileErrorDatabase, path_thirdparty_designs: Path, path_unit_tests: Path) -> None: for repo in db.data.repositories: - path_repo = os.path.join(path_thirdparty_designs, repo.author, repo.project) - path_unit_test = os.path.join(path_unit_tests, f'Test_{repo.author}_{repo.project}.cpp') + path_repo = path_thirdparty_designs / repo.author / repo.project + path_unit_test = path_unit_tests / f'Test_{repo.author}_{repo.project}.cpp' create_repo_cpp_unit_test(repo, path_repo, path_unit_test) From d024e794bde060c315428ac7570fd8354bf37ffa Mon Sep 17 00:00:00 2001 From: Dominik Wernberger Date: Sat, 13 Jul 2024 09:50:47 +0200 Subject: [PATCH 4/4] Deactivate some warning --- lib/CMakeLists.txt | 5 +++++ test/CMakeLists.txt | 5 +++++ 2 files changed, 10 insertions(+) diff --git a/lib/CMakeLists.txt b/lib/CMakeLists.txt index 84df2d03..fed49648 100644 --- a/lib/CMakeLists.txt +++ b/lib/CMakeLists.txt @@ -97,6 +97,11 @@ set(SOURCES # Create library file from sources add_library(${NAME_LIB} SHARED ${SOURCES}) +# Disable warnings +# warning C4996: 'x': This function or variable may be unsafe. Consider using x instead. +# To disable deprecation, use _CRT_SECURE_NO_WARNINGS. See online help for details. +target_compile_options(${NAME_LIB} PRIVATE $<$:/wd4996>) + target_include_directories(${NAME_LIB} PRIVATE ${COMPOUNDFILEREADER_INCLUDE_DIRS} ${LIB_INCLUDE_DIR} diff --git a/test/CMakeLists.txt b/test/CMakeLists.txt index 29712ffb..9abab7d3 100644 --- a/test/CMakeLists.txt +++ b/test/CMakeLists.txt @@ -15,6 +15,11 @@ set(HEADERS # Create executable file from sources add_executable(${NAME_TEST} ${SOURCES} ${HEADERS}) +# Disable warnings +# warning C4996: 'x': This function or variable may be unsafe. Consider using x instead. +# To disable deprecation, use _CRT_SECURE_NO_WARNINGS. See online help for details. +target_compile_options(${NAME_TEST} PRIVATE $<$:/wd4996>) + target_include_directories(${NAME_TEST} PRIVATE ${LIB_INCLUDE_DIR} ${TEST_INCLUDE_DIR}