Merge pull request #44 from spirosmaggioros/logger

Output logger for parallelization

NiChart_DLMUSE/CalcROIVol.py

@@ -1,11 +1,15 @@
 import csv as csv
+import logging
 import os
 from typing import Any
 
 import nibabel as nib
 import numpy as np
 import pandas as pd
 
+logger = logging.getLogger(__name__)
+logging.basicConfig(filename="pipeline.log", encoding="utf-8", level=logging.DEBUG)
+
 
 def calc_roi_volumes(mrid: Any, in_img: Any, label_indices: Any) -> pd.DataFrame:
     """
@@ -150,7 +154,7 @@ def combine_roi_csv(
             df_tmp = pd.read_csv(in_csv)
             dfs.append(df_tmp)
         except:
-            print("Skip subject, out csv missing: " + in_csv)
+            logging.info("Skip subject, out csv missing: " + in_csv)
     if len(dfs) > 0:
         df_out = pd.concat(dfs)
     df_out.to_csv(out_csv, index=False)

NiChart_DLMUSE/RelabelROI.py

@@ -11,7 +11,9 @@ def relabel_rois(
 ) -> None:
     """
     Convert labels in input roi image to new labels based on the mapping
+    print()
     The mapping file should contain numeric indices for the mapping
+    irint()
     between the input roi image (from) and output roi image (to)
     """
 

NiChart_DLMUSE/ReorientImage.py

@@ -1,3 +1,4 @@
+import logging
 import os
 from typing import Any
 
@@ -7,13 +8,16 @@
 
 IMG_EXT = ".nii.gz"
 
+logger = logging.getLogger(__name__)
+logging.basicConfig(filename="pipeline.log", encoding="utf-8", level=logging.DEBUG)
+
 
 def reorient_img(in_img: Any, ref: Any, out_img: Any) -> None:
     """
     Reorient image
     """
     if os.path.exists(out_img):
-        print("Out file exists, skip reorientation ...")
+        logging.info("Out file exists, skip reorientation ...")
 
     else:
         # Read input img

NiChart_DLMUSE/SegmentImage.py

@@ -13,10 +13,8 @@ def run_dlicv(
     extra_args: str = "",
 ) -> None:
     # Call DLICV
-    print("Running DLICV")
     os.system(f"DLICV -i {in_dir} -o {out_dir} -device {device} " + extra_args)
 
-    print("Rename dlicv out file")
     for fname in glob.glob(os.path.join(out_dir, "label_*.nii.gz")):
         new_fname = fname.replace("label_", "", 1).replace(in_suff, out_suff)
         shutil.copyfile(fname, new_fname)
@@ -31,13 +29,9 @@ def run_dlmuse(
     extra_args: str = "",
 ) -> None:
     # Call DLMUSE
-    print("Running DLMUSE")
     os.system(f"DLMUSE -i {in_dir} -o {out_dir} -device {device} " + extra_args)
 
-    print("Rename dlmuse out file")
     for fname in glob.glob(os.path.join(out_dir, "DLMUSE_mask_*.nii.gz")):
         new_fname = fname.replace("DLMUSE_mask_", "", 1).replace(in_suff, out_suff)
         # os.rename(fname, new_fname)
         shutil.copyfile(fname, new_fname)
-        # print(fname)
-        # print(new_fname)

NiChart_DLMUSE/__main__.py

@@ -152,6 +152,7 @@ def main() -> None:
                 device,
                 dlmuse_extra_args,
                 dlicv_extra_args,
+                i,
             ),
         )
         curr_thread.start()

NiChart_DLMUSE/dlmuse_pipeline.py

@@ -1,3 +1,4 @@
+import logging
 import os
 
 import pkg_resources  # type: ignore
@@ -18,50 +19,38 @@
 
 REF_ORIENT = "LPS"
 
-# Dictionary for mapping consecutive dlmuse indices back to regular MUSE indices
-# DICT_MUSE_NNUNET_MAP = os.path.join(
-#     os.path.dirname(os.getcwd()),
-#     "shared",
-#     "dicts",
-#     "MUSE_mapping_consecutive_indices.csv",
-# )
-# DICT_MUSE_NNUNET_MAP = os.path.join(
-#    Path(__file__).parent.parent,
-#    "shared",
-#    "dicts",
-#   "MUSE_mapping_consecutive_indices.csv",
-# )
 DICT_MUSE_NNUNET_MAP = pkg_resources.resource_filename(
     "NiChart_DLMUSE", "shared/dicts/MUSE_mapping_consecutive_indices.csv"
 )
 LABEL_FROM = "IndexConsecutive"
 LABEL_TO = "IndexMUSE"
 
 DICT_MUSE_SINGLE = DICT_MUSE_NNUNET_MAP
-# DICT_MUSE_DERIVED = os.path.join(
-#     os.path.dirname(os.getcwd()), "shared", "dicts", "MUSE_mapping_derived_rois.csv"
-# )
-# DICT_MUSE_DERIVED = os.path.join(
-#    Path(__file__).parent.parent, "shared", "dicts", "MUSE_mapping_derived_rois.csv"
-# )
+
 DICT_MUSE_DERIVED = pkg_resources.resource_filename(
     "NiChart_DLMUSE", "shared/dicts/MUSE_mapping_derived_rois.csv"
 )
 
+logger = logging.getLogger(__name__)
+logging.basicConfig(filename="pipeline.log", encoding="utf-8", level=logging.DEBUG)
+
 
 def run_pipeline(
     in_data: str,
     out_dir: str,
     device: str,
     dlmuse_extra_args: str,
     dlicv_extra_args: str,
+    sub_fldr: int,
 ) -> None:
     """
     NiChart pipeline
     """
-
+    logging.info(f"Starting the pipeline on folder {sub_fldr}")
+    logging.info(f"Detecting input images for batch [{sub_fldr}]...")
     # Detect input images
     df_img = make_img_list(in_data)
+    logging.info(f"Detecting input images for batch [{sub_fldr}] done")
 
     # Set init paths and envs
     out_dir = os.path.abspath(out_dir)
@@ -72,33 +61,31 @@ def run_pipeline(
     # Create working dir (FIXME: created within the output dir for now)
     working_dir = os.path.join(out_dir_final, "temp_working_dir")
 
-    # FIXME remove tmp working dir
-    # if os.path.exists(working_dir):
-    # shutil.rmtree(working_dir)
-
     os.makedirs(working_dir, exist_ok=True)
 
+    logging.info(f"Reorient images to LPS for batch [{sub_fldr}]...")
     # Reorient image to LPS
-    print("------------------------\n   Reorient images")
     out_dir = os.path.join(working_dir, "s1_reorient_lps")
     ref = REF_ORIENT
     out_suff = SUFF_LPS
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
     apply_reorient_img(df_img, ref, out_dir, out_suff)
+    logging.info(f"Reorient images to LPS for batch [{sub_fldr}] done")
 
+    logging.info(f"Applying DLICV for batch [{sub_fldr}]...")
     # Apply DLICV
-    print("------------------------\n   Apply DLICV")
     in_dir = os.path.join(working_dir, "s1_reorient_lps")
     out_dir = os.path.join(working_dir, "s2_dlicv")
     in_suff = SUFF_LPS
     out_suff = SUFF_DLICV
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
     run_dlicv(in_dir, in_suff, out_dir, out_suff, device, dlicv_extra_args)
+    logging.info(f"Applying DLICV for batch [{sub_fldr}] done")
 
+    logging.info(f"Applying mask for batch [{sub_fldr}]...")
     # Mask image
-    print("------------------------\n   Apply DLICV mask")
     in_dir = os.path.join(working_dir, "s1_reorient_lps")
     mask_dir = os.path.join(working_dir, "s2_dlicv")
     out_dir = os.path.join(working_dir, "s3_masked")
@@ -108,19 +95,21 @@ def run_pipeline(
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
     apply_mask_img(df_img, in_dir, in_suff, mask_dir, mask_suff, out_dir, out_suff)
+    logging.info(f"Applying mask for batch [{sub_fldr}] done")
 
+    logging.info(f"Applying DLMUSE for batch [{sub_fldr}]...")
     # Apply DLMUSE
-    print("------------------------\n   Apply DLMUSE")
     in_dir = os.path.join(working_dir, "s3_masked")
     out_dir = os.path.join(working_dir, "s4_dlmuse")
     in_suff = SUFF_DLICV
     out_suff = SUFF_DLMUSE
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
     run_dlmuse(in_dir, in_suff, out_dir, out_suff, device, dlmuse_extra_args)
+    logging.info(f"Applying DLMUSE for batch [{sub_fldr}] done")
 
+    logging.info(f"Relabeling DLMUSE for batch [{sub_fldr}]...")
     # Relabel DLMUSE
-    print("------------------------\n   Relabel DLMUSE")
     in_dir = os.path.join(working_dir, "s4_dlmuse")
     out_dir = os.path.join(working_dir, "s5_relabeled")
     in_suff = SUFF_DLMUSE
@@ -137,9 +126,10 @@ def run_pipeline(
         LABEL_FROM,
         LABEL_TO,
     )
+    logging.info(f"Applying DLMUSE for batch [{sub_fldr}] done")
 
+    logging.info(f"Combining DLICV and MUSE masks for batch [{sub_fldr}]...")
     # Combine DLICV and MUSE masks
-    print("------------------------\n   Combine DLICV and DLMUSE masks")
     in_dir = os.path.join(working_dir, "s5_relabeled")
     mask_dir = os.path.join(working_dir, "s2_dlicv")
     out_dir = os.path.join(working_dir, "s6_combined")
@@ -149,32 +139,33 @@ def run_pipeline(
     if not os.path.exists(out_dir):
         os.makedirs(out_dir)
     apply_combine_masks(df_img, in_dir, in_suff, mask_dir, mask_suff, out_dir, out_suff)
+    logging.info(f"Combining DLICV and MUSE masks for batch [{sub_fldr}] done")
 
+    logging.info(f"Reorienting to initial orientation for batch [{sub_fldr}]...")
     # Reorient to initial orientation
-    print("------------------------\n   Reorient to initial")
     in_dir = os.path.join(working_dir, "s6_combined")
     out_dir = out_dir_final
     in_suff = SUFF_DLMUSE
     out_suff = SUFF_DLMUSE
     apply_reorient_to_init(df_img, in_dir, in_suff, out_dir, out_suff)
+    logging.info(f"Reorienting to initial orientation for batch [{sub_fldr}] done")
 
+    logging.info(f"Create ROI csv for batch [{sub_fldr}]...")
     # Create roi csv
-    print("------------------------\n   Create csv")
     in_dir = out_dir_final
     out_dir = out_dir_final
     in_suff = SUFF_DLMUSE
     out_suff = SUFF_ROI
     apply_create_roi_csv(
         df_img, in_dir, in_suff, DICT_MUSE_SINGLE, DICT_MUSE_DERIVED, out_dir, out_suff
     )
+    logging.info(f"Create ROI csv for batch [{sub_fldr}] done")
 
+    logging.info(f"Combine ROI csv for batch [{sub_fldr}]...")
     # Combine roi csv
-    print("------------------------\n   Combine csv")
     in_dir = out_dir_final
     out_dir = out_dir_final
     in_suff = SUFF_ROI
     out_name = OUT_CSV
     combine_roi_csv(df_img, in_dir, in_suff, out_dir, out_name)
-
-    # Remove working dir
-    # rm -rf working_dir
+    logging.info(f"Combine ROI csv for batch [{sub_fldr}] done")

NiChart_DLMUSE/utils.py

@@ -1,4 +1,5 @@
 import glob
+import logging
 import os
 import re
 from typing import Any
@@ -8,6 +9,9 @@
 
 LIST_IMG_EXT = [".nii", ".nii.gz"]
 
+logger = logging.getLogger(__name__)
+logging.basicConfig(filename="pipeline.log", encoding="utf-8", level=logging.DEBUG)
+
 
 def get_basename(
     in_file: Any, suffix_to_remove: Any, ext_to_remove: Any = LIST_IMG_EXT
@@ -102,7 +106,6 @@ def make_img_list(in_data: str) -> pd.DataFrame:
                     nii_files.append(os.path.abspath(line.strip()))
 
     nii_files = np.array(nii_files)
-    print(f"Detected {nii_files.shape[0]} images ...")  # type:ignore
 
     # Check if images exist
     if len(nii_files) > 0:
@@ -112,7 +115,9 @@ def make_img_list(in_data: str) -> pd.DataFrame:
                 flag[i] = 1
         nii_files = nii_files[flag == 1]
 
-    print(f"Number of valid images is {len(nii_files)} ...")
+    logging.info(
+        f"Detected {nii_files.shape[0]} images. Valid images are {len(nii_files)}..."  # type:ignore
+    )
     # Create a dataframe
     df_out = pd.DataFrame(data=nii_files, columns=["img_path"])