From f49b5f3c69c9e55e0eff6f6f4dedaaab2539b146 Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Thu, 14 Nov 2024 16:07:37 +0200 Subject: [PATCH] Added test cases for utils and fixed bugs with parallelization --- .github/workflows/macos_tests.yml | 23 +++++ .github/workflows/ubuntu_tests.yml | 23 +++++ NiChart_DLMUSE/utils.py | 9 +- requirements.txt | 7 +- tests/test_utils.py | 147 +++++++++++++++++++++++++++++ 5 files changed, 207 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/macos_tests.yml create mode 100644 .github/workflows/ubuntu_tests.yml create mode 100644 tests/test_utils.py diff --git a/.github/workflows/macos_tests.yml b/.github/workflows/macos_tests.yml new file mode 100644 index 0000000..9375dc1 --- /dev/null +++ b/.github/workflows/macos_tests.yml @@ -0,0 +1,23 @@ +name: macos build + +# workflow dispatch has been added for testing purposes +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: ["macos-13"] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install spare scores + run: | + python -m pip cache purge + pip install -r requirements.txt + pip install setuptools twine wheel + python -m pip install . + - name: Run unit tests + run: | + cd tests/ && pytest --cov=../ diff --git a/.github/workflows/ubuntu_tests.yml b/.github/workflows/ubuntu_tests.yml new file mode 100644 index 0000000..3d66e55 --- /dev/null +++ b/.github/workflows/ubuntu_tests.yml @@ -0,0 +1,23 @@ +name: ubuntu tests + +# workflow dispatch has been added for testing purposes +on: [push, pull_request, workflow_dispatch] + +jobs: + build: + runs-on: ["ubuntu-latest"] + + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.12" + - name: Install spare scores + run: | + python -m pip cache purge + pip install -r requirements.txt + pip install setuptools twine wheel + python -m pip install . + - name: Run unit tests + run: | + cd tests && pytest --cov=../ diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index f62fa5a..edd2844 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -249,10 +249,17 @@ def split_data(in_dir: str, N: int) -> list: os.system(f"cp {file} {in_dir}/split_{current_folder}") current_file += 1 - if current_file < no_files_in_folders: + if current_file <= no_files_in_folders: # Don't forget the last split if it has less files than the maximum files in a subfolder subfolders.append(f"{in_dir}/split_{current_folder}") + for subfldr in os.listdir(in_dir): + joined_folder = os.path.join(in_dir, subfldr) + if os.path.isdir(joined_folder): + if len(os.listdir(joined_folder)) == 0: + os.system(f"rm -r {joined_folder}") + subfolders.remove(f"{joined_folder}") + return subfolders diff --git a/requirements.txt b/requirements.txt index 4cf1b7d..cdaaf67 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,13 @@ +# Main pipeline torch==2.2.1 DLICV DLMUSE nibabel>=5.2 +scipy + +# Developer tools +pytest +pytest-cov huggingface_hub argparse -scipy pathlib diff --git a/tests/test_utils.py b/tests/test_utils.py new file mode 100644 index 0000000..1d40357 --- /dev/null +++ b/tests/test_utils.py @@ -0,0 +1,147 @@ +from dateutil.relativedelta import WE +import pytest +import os + +from NiChart_DLMUSE.utils import * + + +def testing_get_basename(): + test_file: str = "test.nii.gz" + assert get_basename(test_file, "", [".nii", ".nii.gz"]) == "test" + + test_file: str = "test.nii" + assert get_basename(test_file, "", [".nii", ".nii.gz"]) == "test" + +def testing_remove_common_suffix(): + test_files: list = ["test1.nii.gz", "test2.nii.gz", "test3.nii.gz", "test4.nii.gz"] + correct_res: list = ["test1", "test2", "test3", "test4"] + assert remove_common_suffix(test_files) == correct_res + + test_files = ["test1.nii.gz"] # WARNING: Single case, review if needed + correct_res = ["test1.nii.gz"] + + assert remove_common_suffix(test_files) == correct_res + +def testing_make_img_list(): + os.system("mkdir test_dataset") + for i in range(3): + os.system(f"touch test_dataset/IXI10{i}-Guys-0000-T1.nii.gz") + + df_img: pd.DataFrame = make_img_list("test_dataset") + df_img = df_img.sort_values(by='MRID') + info = { + "MRID": [f"IXI10{i}" for i in range(3)], + "img_path": [os.path.abspath(f"test_dataset/IXI10{i}-Guys-0000-T1.nii.gz") for i in range(3)], + "img_base": [f"IXI10{i}-Guys-0000-T1.nii.gz" for i in range(3)], + "img_prefix": [f"IXI10{i}-Guys-0000-T1" for i in range(3)] + } + df_test:pd.DataFrame = pd.DataFrame( + info + ) + + assert list(df_img["MRID"]) == list(df_test["MRID"]) + assert list(df_img["img_path"]) == list(df_test["img_path"]) + assert list(df_img["img_base"] == list(df_test["img_base"])) + assert list(df_img["img_prefix"] == list(df_test["img_prefix"])) + + os.system("rm -r test_dataset") + + +def testing_get_bids_prefix(): + pass + +def testing_collect_T1(): + os.system("mkdir test_collect_T1") + + # Generate the BIDS input folder for testing + for i in range(9): + os.system(f"mkdir test_collect_T1/sub-0{i}") + os.system(f"mkdir test_collect_T1/sub-0{i}/anat") + os.system(f"touch test_collect_T1/sub-0{i}/anat/IXI-10{i}-Guys-0000-T1.nii.gz") + + for subfolder in os.listdir("test_collect_T1"): + assert get_bids_prefix(subfolder) == "sub" + + os.system("rm -r test_collect_T1") + +def testing_split_data(): + if os.path.exists("test_split_data"): + os.system("rm -r test_split_data") + + def generate_random_test_folders(no_files: int = 15): + os.system("mkdir test_split_data") + for i in range(no_files): + if i < 10: + os.system(f"touch test_split_data/IXI-10{i}-Guys-0000-T1.nii.gz") + else: + os.system(f"touch test_split_data/IXI-1{i}-Guys-0000-T1.nii.gz") + + def check_subfolder_count(no_folders: int, count_in: int, count_last: int): + subfldr_files = [] + for (idx, subfldr) in enumerate(os.listdir("test_split_data")): + joined = os.path.join("test_split_data", subfldr) + if idx < no_folders and os.path.isdir(joined): + subfldr_files.append(len(os.listdir(joined))) + elif idx >= no_folders and os.path.isdir(joined): + subfldr_files.append(len(os.listdir(joined))) + + in_count: int = 0 + last_count: int = 0 + + for files in subfldr_files: + if files == count_in: + in_count += 1 + else: + last_count += 1 + + assert (last_count == 1 if count_in != count_last else last_count == 0) and (in_count == no_folders - 1 if count_in != count_last else in_count == no_folders) + + generate_random_test_folders(12) + subfolders: list = split_data("test_split_data", 4) + + assert len(subfolders) == 4 + check_subfolder_count(4, 3, 3) + os.system("rm -r test_split_data") + + generate_random_test_folders(20) + subfolders = split_data("test_split_data", 4) + + assert len(subfolders) == 4 + check_subfolder_count(4, 5, 5) + os.system("rm -r test_split_data") + + generate_random_test_folders(35) + subfolders = split_data("test_split_data", 4) + + assert len(subfolders) == 4 + check_subfolder_count(4, 9, 8) + os.system("rm -r test_split_data") + + generate_random_test_folders(1) + subfolders = split_data("test_split_data", 4) + + assert len(subfolders) == 1 + print(len(subfolders)) + check_subfolder_count(1, 1, 1) + os.system("rm -r test_split_data") + +def testing_remove_subfolders(): + def generate_random_test_folders(no_files: int = 15): + os.system("mkdir test_split_data") + for i in range(no_files): + if i < 10: + os.system(f"touch test_split_data/IXI-10{i}-Guys-0000-T1.nii.gz") + else: + os.system(f"touch test_split_data/IXI-1{i}-Guys-0000-T1.nii.gz") + + generate_random_test_folders(10) + remove_subfolders("test_split_data") + + assert len(os.listdir("test_split_data")) == 10 + os.system("rm -r test_split_data") + + generate_random_test_folders(1) + remove_subfolders("test_split_data") + assert len(os.listdir("test_split_data")) == 1 + + os.system("rm -r test_split_data")