From aef47f704a7c85fffde1ac9a34d2623df16540e1 Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Mon, 28 Oct 2024 13:19:42 +0200 Subject: [PATCH 1/7] Minor fixes to parallelization --- .gitattributes | 35 ----------------------------------- NiChart_DLMUSE/__main__.py | 2 +- NiChart_DLMUSE/utils.py | 2 +- requirements.txt | 6 +++--- 4 files changed, 5 insertions(+), 40 deletions(-) delete mode 100644 .gitattributes diff --git a/.gitattributes b/.gitattributes deleted file mode 100644 index a6344aa..0000000 --- a/.gitattributes +++ /dev/null @@ -1,35 +0,0 @@ -*.7z filter=lfs diff=lfs merge=lfs -text -*.arrow filter=lfs diff=lfs merge=lfs -text -*.bin filter=lfs diff=lfs merge=lfs -text -*.bz2 filter=lfs diff=lfs merge=lfs -text -*.ckpt filter=lfs diff=lfs merge=lfs -text -*.ftz filter=lfs diff=lfs merge=lfs -text -*.gz filter=lfs diff=lfs merge=lfs -text -*.h5 filter=lfs diff=lfs merge=lfs -text -*.joblib filter=lfs diff=lfs merge=lfs -text -*.lfs.* filter=lfs diff=lfs merge=lfs -text -*.mlmodel filter=lfs diff=lfs merge=lfs -text -*.model filter=lfs diff=lfs merge=lfs -text -*.msgpack filter=lfs diff=lfs merge=lfs -text -*.npy filter=lfs diff=lfs merge=lfs -text -*.npz filter=lfs diff=lfs merge=lfs -text -*.onnx filter=lfs diff=lfs merge=lfs -text -*.ot filter=lfs diff=lfs merge=lfs -text -*.parquet filter=lfs diff=lfs merge=lfs -text -*.pb filter=lfs diff=lfs merge=lfs -text -*.pickle filter=lfs diff=lfs merge=lfs -text -*.pkl filter=lfs diff=lfs merge=lfs -text -*.pt filter=lfs diff=lfs merge=lfs -text -*.pth filter=lfs diff=lfs merge=lfs -text -*.rar filter=lfs diff=lfs merge=lfs -text -*.safetensors filter=lfs diff=lfs merge=lfs -text -saved_model/**/* filter=lfs diff=lfs merge=lfs -text -*.tar.* filter=lfs diff=lfs merge=lfs -text -*.tar filter=lfs diff=lfs merge=lfs -text -*.tflite filter=lfs diff=lfs merge=lfs -text -*.tgz filter=lfs diff=lfs merge=lfs -text -*.wasm filter=lfs diff=lfs merge=lfs -text -*.xz filter=lfs diff=lfs merge=lfs -text -*.zip filter=lfs diff=lfs merge=lfs -text -*.zst filter=lfs diff=lfs merge=lfs -text -*tfevents* filter=lfs diff=lfs merge=lfs -text diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index a2fae8e..5375377 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -138,7 +138,7 @@ def main() -> None: os.system("DLMUSE -i ./dummy -o ./dummy --clear_cache") # Run pipeline - no_threads = args.cores # for now + no_threads = int(args.cores) # for now subfolders = split_data(in_data, no_threads) threads = [] diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index 2862510..66e0f68 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -154,7 +154,7 @@ def split_data(in_dir: str, N: int) -> list: """ assert N > 0 data_size = dir_size(in_dir) - no_files_in_folders = data_size / N if (data_size % N == 0) else (data_size / N) + 1 + no_files_in_folders = data_size // N if (data_size % N == 0) else (data_size // N) + 1 assert no_files_in_folders > 0 subfolders = [] diff --git a/requirements.txt b/requirements.txt index 8fda497..9ce60f8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,8 @@ torch>=2.2.1 -DLICV>=1.0.1 -DLMUSE>=1.0.1 +DLICV +DLMUSE nibabel>=5.2 huggingface_hub argparse scipy -pathlib \ No newline at end of file +pathlib From 513be2f41ccf159be533fed6dd39aba0eb6724dd Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Mon, 28 Oct 2024 13:56:31 +0200 Subject: [PATCH 2/7] [Potential bug] Don't remove the subfolders twice --- NiChart_DLMUSE/utils.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index 66e0f68..0866167 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -150,7 +150,8 @@ def dir_size(in_dir: str) -> int: def split_data(in_dir: str, N: int) -> list: """ - Splits the input data directory into subfolders of size N + Splits the input data directory into subfolders of size. + N should be > 0 and the number of files in each subfolder should be > 0 as well. """ assert N > 0 data_size = dir_size(in_dir) @@ -177,10 +178,17 @@ def split_data(in_dir: str, N: int) -> list: def remove_subfolders(in_dir: str) -> None: + """ + Removes all the split_* subolders from the input folder + """ os.system(f"rm -r {in_dir}/split_*") def merge_output_data(in_dir: str) -> None: + """ + Takes all the results from the temp_working_fir and moves them into + the results/ subfolder in the output folder + """ os.system(f"mkdir {in_dir}/results") os.system(f"mkdir {in_dir}/results/s1_reorient_lps") os.system(f"mkdir {in_dir}/results/s2_dlicv") @@ -212,5 +220,3 @@ def merge_output_data(in_dir: str) -> None: f"mv {in_dir}/{dir}/temp_working_dir/s6_combined/* {in_dir}/results/s6_combined/" ) os.system(f"mv {in_dir}/{dir}/*.nii.gz {in_dir}/results/") - - os.system(f"rm -r {in_dir}/split_*") From a9aaa1ad5ed5f93289b962dfa5d6127453d3af7d Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Mon, 28 Oct 2024 14:03:21 +0200 Subject: [PATCH 3/7] Remove split_* subfolders from output/ as well --- NiChart_DLMUSE/utils.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index 0866167..819e0ab 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -220,3 +220,5 @@ def merge_output_data(in_dir: str) -> None: f"mv {in_dir}/{dir}/temp_working_dir/s6_combined/* {in_dir}/results/s6_combined/" ) os.system(f"mv {in_dir}/{dir}/*.nii.gz {in_dir}/results/") + + os.system(f"rm -r {in_dir}/split_*") From 65952cb251e29b58fdd1936ae2f7704604d2d390 Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Mon, 28 Oct 2024 14:12:20 +0200 Subject: [PATCH 4/7] Remove all files from the output/ folder if it's not empty --- NiChart_DLMUSE/utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index 819e0ab..a95ba3f 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -189,6 +189,10 @@ def merge_output_data(in_dir: str) -> None: Takes all the results from the temp_working_fir and moves them into the results/ subfolder in the output folder """ + + if len(os.listdir(in_dir)) != 0: + os.system(f"rm -r {in_dir}/*") + os.system(f"mkdir {in_dir}/results") os.system(f"mkdir {in_dir}/results/s1_reorient_lps") os.system(f"mkdir {in_dir}/results/s2_dlicv") From 316651a3e28ebe39a3f48eff672b08d599c7805a Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Mon, 28 Oct 2024 14:20:02 +0200 Subject: [PATCH 5/7] Move the subfolder removing operation to remove_subfolders function --- NiChart_DLMUSE/__main__.py | 1 + NiChart_DLMUSE/utils.py | 2 -- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index 5375377..a56d396 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -163,6 +163,7 @@ def main() -> None: merge_output_data(out_dir) remove_subfolders(in_data) + remove_subfolders(out_dir) if __name__ == "__main__": diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index a95ba3f..788afde 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -224,5 +224,3 @@ def merge_output_data(in_dir: str) -> None: f"mv {in_dir}/{dir}/temp_working_dir/s6_combined/* {in_dir}/results/s6_combined/" ) os.system(f"mv {in_dir}/{dir}/*.nii.gz {in_dir}/results/") - - os.system(f"rm -r {in_dir}/split_*") From 8f3f008d22c05937cf6fdd1c115a98bb2ee5c9f1 Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Mon, 28 Oct 2024 14:26:01 +0200 Subject: [PATCH 6/7] subfolder removing for output dir update --- NiChart_DLMUSE/__main__.py | 5 +++++ NiChart_DLMUSE/utils.py | 3 --- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index a56d396..8058f19 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -133,6 +133,11 @@ def main() -> None: print(args) print() + + if len(os.listdir(out_dir)) != 0: + print(f"Emptying output folder: {out_dir}...") + os.system(f"rm -r {out_dir}/*") + if args.clear_cache: os.system("DLICV -i ./dummy -o ./dummy --clear_cache") os.system("DLMUSE -i ./dummy -o ./dummy --clear_cache") diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index 788afde..04d7072 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -190,9 +190,6 @@ def merge_output_data(in_dir: str) -> None: the results/ subfolder in the output folder """ - if len(os.listdir(in_dir)) != 0: - os.system(f"rm -r {in_dir}/*") - os.system(f"mkdir {in_dir}/results") os.system(f"mkdir {in_dir}/results/s1_reorient_lps") os.system(f"mkdir {in_dir}/results/s2_dlicv") From a9b23e0611ae3f54c20014b5a030c4674732dd76 Mon Sep 17 00:00:00 2001 From: Spiros Maggioros Date: Mon, 28 Oct 2024 14:33:06 +0200 Subject: [PATCH 7/7] code fixing for pre-commit --- NiChart_DLMUSE/__main__.py | 1 - NiChart_DLMUSE/utils.py | 14 ++++++++------ setup.py | 5 ++--- 3 files changed, 10 insertions(+), 10 deletions(-) diff --git a/NiChart_DLMUSE/__main__.py b/NiChart_DLMUSE/__main__.py index 8058f19..b012cc0 100644 --- a/NiChart_DLMUSE/__main__.py +++ b/NiChart_DLMUSE/__main__.py @@ -133,7 +133,6 @@ def main() -> None: print(args) print() - if len(os.listdir(out_dir)) != 0: print(f"Emptying output folder: {out_dir}...") os.system(f"rm -r {out_dir}/*") diff --git a/NiChart_DLMUSE/utils.py b/NiChart_DLMUSE/utils.py index 04d7072..f23825c 100644 --- a/NiChart_DLMUSE/utils.py +++ b/NiChart_DLMUSE/utils.py @@ -150,12 +150,14 @@ def dir_size(in_dir: str) -> int: def split_data(in_dir: str, N: int) -> list: """ - Splits the input data directory into subfolders of size. - N should be > 0 and the number of files in each subfolder should be > 0 as well. + Splits the input data directory into subfolders of size. + N should be > 0 and the number of files in each subfolder should be > 0 as well. """ assert N > 0 data_size = dir_size(in_dir) - no_files_in_folders = data_size // N if (data_size % N == 0) else (data_size // N) + 1 + no_files_in_folders = ( + data_size // N if (data_size % N == 0) else (data_size // N) + 1 + ) assert no_files_in_folders > 0 subfolders = [] @@ -179,15 +181,15 @@ def split_data(in_dir: str, N: int) -> list: def remove_subfolders(in_dir: str) -> None: """ - Removes all the split_* subolders from the input folder + Removes all the split_* subolders from the input folder """ os.system(f"rm -r {in_dir}/split_*") def merge_output_data(in_dir: str) -> None: """ - Takes all the results from the temp_working_fir and moves them into - the results/ subfolder in the output folder + Takes all the results from the temp_working_fir and moves them into + the results/ subfolder in the output folder """ os.system(f"mkdir {in_dir}/results") diff --git a/setup.py b/setup.py index d020ad4..d8c56a1 100644 --- a/setup.py +++ b/setup.py @@ -1,12 +1,11 @@ from pathlib import Path -import setuptools from setuptools import find_packages, setup this_directory = Path(__file__).parent long_description = (this_directory / "README.md").read_text() -with open('requirements.txt') as f: +with open("requirements.txt") as f: required = f.read().splitlines() @@ -42,6 +41,6 @@ "nnunet", ], package_data={ - 'NiChart_DLMUSE': ['**/*.csv', '**/*.json'], + "NiChart_DLMUSE": ["**/*.csv", "**/*.json"], }, )