diff --git a/Readme.md b/Readme.md index 6f19e01..29e6df0 100644 --- a/Readme.md +++ b/Readme.md @@ -78,6 +78,8 @@ optional arguments: -f, --force force overwriting of existing files -b BATCH, --batch BATCH break results file into subsets of this size + -s SEED, --seed SEED, + set a seed used to produce a random number in all modules -n NPROCESSES, --nprocesses NPROCESSES number of processes to launch --symlink TARGET_DIR create symlink to outdir in TARGET_DIR diff --git a/histoqc/BaseImage.py b/histoqc/BaseImage.py index 947f7c7..16bd764 100644 --- a/histoqc/BaseImage.py +++ b/histoqc/BaseImage.py @@ -18,7 +18,7 @@ class BaseImage(dict): - def __init__(self, fname, fname_outdir, params): + def __init__(self, fname, fname_outdir, seed, params): dict.__init__(self) self.in_memory_compression = strtobool(params.get("in_memory_compression", "False")) @@ -31,6 +31,7 @@ def __init__(self, fname, fname_outdir, params): self.addToPrintList("comments", " ") self["outdir"] = fname_outdir + self["seed"] = seed self["dir"] = os.path.dirname(fname) self["os_handle"] = openslide.OpenSlide(fname) diff --git a/histoqc/ClassificationModule.py b/histoqc/ClassificationModule.py index 9567a49..660145d 100644 --- a/histoqc/ClassificationModule.py +++ b/histoqc/ClassificationModule.py @@ -194,6 +194,11 @@ def byExampleWithFeatures(s, params): if nsamples_per_example != -1: #sub sambling required nitems = nsamples_per_example if nsamples_per_example > 1 else int(mask.shape[0]*nsamples_per_example) + + # set seed to random function if seed is not None + if s["seed"] is not None: + np.random.seed(int(s["seed"])) + idxkeep = np.random.choice(mask.shape[0], size=int(nitems)) eximg = eximg[idxkeep, :] mask = mask[idxkeep] diff --git a/histoqc/LocalTextureEstimationModule.py b/histoqc/LocalTextureEstimationModule.py index 6d3733f..cbe7cb2 100644 --- a/histoqc/LocalTextureEstimationModule.py +++ b/histoqc/LocalTextureEstimationModule.py @@ -8,6 +8,7 @@ def estimateGreyComatrixFeatures(s, params): + prefix = params.get("prefix", None) prefix = prefix+"_" if prefix else "" @@ -19,6 +20,9 @@ def estimateGreyComatrixFeatures(s, params): invert = strtobool(params.get("invert", "False")) mask_name = params.get("mask_name","img_mask_use") + # set seed to random function if seed is not None + if s["seed"] is not None: + np.random.seed(int(s["seed"])) img = s.getImgThumb(s["image_work_size"]) img = color.rgb2gray(img) diff --git a/histoqc/__main__.py b/histoqc/__main__.py index c0bd0c0..d86ff7d 100644 --- a/histoqc/__main__.py +++ b/histoqc/__main__.py @@ -54,6 +54,10 @@ def main(argv=None): help="break results file into subsets of this size", type=int, default=None) + parser.add_argument('-s', '--seed', + help="set a seed used to produce a random number in all modules", + type=int, + default=None) parser.add_argument('-n', '--nprocesses', help="number of processes to launch", type=int, @@ -158,6 +162,7 @@ def main(argv=None): 'shared_dict': mpm.dict(), 'num_files': num_files, 'force': args.force, + 'seed': args.seed } failed = mpm.list() setup_plotting_backend(lm.logger) diff --git a/histoqc/_worker.py b/histoqc/_worker.py index 97e5ff6..e62c635 100644 --- a/histoqc/_worker.py +++ b/histoqc/_worker.py @@ -1,7 +1,7 @@ """histoqc worker functions""" import os import shutil - +import numpy as np from histoqc.BaseImage import BaseImage from histoqc._pipeline import load_pipeline from histoqc._pipeline import setup_plotting_backend @@ -16,9 +16,11 @@ def worker_setup(c): def worker(idx, file_name, *, - process_queue, config, outdir, log_manager, lock, shared_dict, num_files, force): + process_queue, config, outdir, log_manager, lock, shared_dict, num_files, force, seed): """pipeline worker function""" - + # set the seed + if seed is not None: + np.random.seed(seed) # --- output directory preparation -------------------------------- fname_outdir = os.path.join(outdir, os.path.basename(file_name)) if os.path.isdir(fname_outdir): # directory exists @@ -37,7 +39,7 @@ def worker(idx, file_name, *, log_manager.logger.info(f"-----Working on:\t{file_name}\t\t{idx+1} of {num_files}") try: - s = BaseImage(file_name, fname_outdir, dict(config.items("BaseImage.BaseImage"))) + s = BaseImage(file_name, fname_outdir, seed, dict(config.items("BaseImage.BaseImage"))) for process, process_params in process_queue: process_params["lock"] = lock diff --git a/histoqc/config/config_v2.1.ini b/histoqc/config/config_v2.1.ini index 0aad2dd..f0a74f4 100644 --- a/histoqc/config/config_v2.1.ini +++ b/histoqc/config/config_v2.1.ini @@ -247,6 +247,7 @@ npatches: 1000 feats: contrast:dissimilarity:homogeneity:ASM:energy:correlation invert: False mask_name: img_mask_use + [LightDarkModule.minimumPixelIntensityNeighborhoodFiltering] disk_size: 5 upper_threshold: 210 diff --git a/histoqc/tests/coverage.xml b/histoqc/tests/coverage.xml new file mode 100644 index 0000000..99bdf26 --- /dev/null +++ b/histoqc/tests/coverage.xml @@ -0,0 +1,2740 @@ + + + + + + /Users/nanli/Documents/github/HistoQC/histoqc + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/histoqc/tests/readme.md b/histoqc/tests/readme.md new file mode 100644 index 0000000..7fac7e2 --- /dev/null +++ b/histoqc/tests/readme.md @@ -0,0 +1,11 @@ +# HistoQC Unit Test +--- + +Tested with Python 3.7 and 3.8 + +Running the unit test with coverage report is now done via a python module: + +``` +python3 -m pytest --cov-report xml:./histoqc/tests/coverage.xml --cov=histoqc histoqc/tests/ +``` + diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_areathresh.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_areathresh.png new file mode 100644 index 0000000..975c39c Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_areathresh.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_blurry.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_blurry.png new file mode 100644 index 0000000..d8f07de Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_blurry.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_bright.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_bright.png new file mode 100644 index 0000000..766b5bb Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_bright.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_dark.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_dark.png new file mode 100644 index 0000000..9f95435 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_dark.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c0.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c0.png new file mode 100644 index 0000000..4590c45 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c0.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c1.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c1.png new file mode 100644 index 0000000..de54a5e Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c1.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c2.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c2.png new file mode 100644 index 0000000..3413e90 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_deconv_c2.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_equalized_thumb.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_equalized_thumb.png new file mode 100644 index 0000000..17ff36d Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_equalized_thumb.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_fatlike.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_fatlike.png new file mode 100644 index 0000000..975c39c Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_fatlike.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_flat.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_flat.png new file mode 100644 index 0000000..2e7fb89 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_flat.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_fuse.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_fuse.png new file mode 100644 index 0000000..158ec1e Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_fuse.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_hist.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_hist.png new file mode 100644 index 0000000..32ef99b Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_hist.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_mask_use.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_mask_use.png new file mode 100644 index 0000000..ee17006 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_mask_use.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_small_fill.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_small_fill.png new file mode 100644 index 0000000..9f95435 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_small_fill.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_small_remove.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_small_remove.png new file mode 100644 index 0000000..5c72958 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_small_remove.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_spur.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_spur.png new file mode 100644 index 0000000..975c39c Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_spur.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_thumb.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_thumb.png new file mode 100644 index 0000000..2b5efff Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_thumb.png differ diff --git a/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_thumb_small.png b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_thumb_small.png new file mode 100644 index 0000000..74e3f36 Binary files /dev/null and b/histoqc/tests/target/TCGA-EJ-5509-01A-01-BS1_ROI.svs/TCGA-EJ-5509-01A-01-BS1_ROI.svs_thumb_small.png differ diff --git a/histoqc/tests/target/error.log b/histoqc/tests/target/error.log new file mode 100644 index 0000000..e2facdf --- /dev/null +++ b/histoqc/tests/target/error.log @@ -0,0 +1 @@ +2023-12-06 11:00:45,553 - WARNING - TCGA-EJ-5509-01A-01-BS1_ROI.svs- saveMacro Can't Read 'macro' Image from Slide's Associated Images diff --git a/histoqc/tests/target/results.tsv b/histoqc/tests/target/results.tsv new file mode 100644 index 0000000..544d9fc --- /dev/null +++ b/histoqc/tests/target/results.tsv @@ -0,0 +1,7 @@ +#start_time: 2023-12-06 11:00:41.788659 +#pipeline: BasicModule.getBasicStats LightDarkModule.saveEqualisedImage LightDarkModule.minimumPixelIntensityNeighborhoodFiltering LightDarkModule.getIntensityThresholdPercent:darktissue BubbleRegionByRegion.detectSmoothness MorphologyModule.removeFatlikeTissue MorphologyModule.fillSmallHoles MorphologyModule.removeSmallObjects LocalTextureEstimationModule.estimateGreyComatrixFeatures:background BrightContrastModule.getContrast:background BrightContrastModule.getBrightnessGray:background BrightContrastModule.getBrightnessByChannelinColorSpace:RGB_background BlurDetectionModule.identifyBlurryRegions BasicModule.finalProcessingSpur BasicModule.finalProcessingArea HistogramModule.compareToTemplates HistogramModule.getHistogram LocalTextureEstimationModule.estimateGreyComatrixFeatures:final BrightContrastModule.getContrast BrightContrastModule.getBrightnessGray BrightContrastModule.getBrightnessByChannelinColorSpace:RGB BrightContrastModule.getBrightnessByChannelinColorSpace:YUV DeconvolutionModule.separateStains SaveModule.saveFinalMask SaveModule.saveMacro SaveModule.saveThumbnails BasicModule.finalComputations +#outdir: /Users/nanli/Documents/github/HistoQC/histoqc/tests/new +#config_file: /Users/nanli/Documents/github/HistoQC/histoqc/config/config_v2.1.ini +#command_line_args: -c ./histoqc/config/config_v2.1.ini -o ./histoqc/tests/new -s 123 ./histoqc/tests/data/TCGA-EJ-5509-01A-01-BS1_ROI.svs +#dataset:filename comments image_bounding_box base_mag type levels height width mpp_x mpp_y comment brightestPixels dark flat_areas fatlike_tissue_removed_num_regions fatlike_tissue_removed_mean_area fatlike_tissue_removed_max_area fatlike_tissue_removed_percent small_tissue_filled_num_regions small_tissue_filled_mean_area small_tissue_filled_max_area small_tissue_filled_percent small_tissue_removed_num_regions small_tissue_removed_mean_area small_tissue_removed_max_area small_tissue_removed_percent background_contrast background_contrast_std background_dissimilarity background_dissimilarity_std background_homogeneity background_homogeneity_std background_ASM background_ASM_std background_energy background_energy_std background_correlation background_correlation_std background_tenenGrad_contrast background_michelson_contrast background_rms_contrast background_grayscale_brightness background_grayscale_brightness_std background_chan1_brightness background_chan1_brightness_std background_chan2_brightness background_chan2_brightness_std background_chan3_brightness background_chan3_brightness_std blurry_removed_num_regions blurry_removed_mean_area blurry_removed_max_area blurry_removed_percent spur_pixels areaThresh template1_MSE_hist template2_MSE_hist template3_MSE_hist template4_MSE_hist final_contrast final_contrast_std final_dissimilarity final_dissimilarity_std final_homogeneity final_homogeneity_std final_ASM final_ASM_std final_energy final_energy_std final_correlation final_correlation_std tenenGrad_contrast michelson_contrast rms_contrast grayscale_brightness grayscale_brightness_std chan1_brightness chan1_brightness_std chan2_brightness chan2_brightness_std chan3_brightness chan3_brightness_std chan1_brightness_YUV chan1_brightness_std_YUV chan2_brightness_YUV chan2_brightness_std_YUV chan3_brightness_YUV chan3_brightness_std_YUV deconv_c0_mean deconv_c0_std deconv_c1_mean deconv_c1_std deconv_c2_mean deconv_c2_std pixels_to_use warnings +TCGA-EJ-5509-01A-01-BS1_ROI.svs (0, 0, 4092, 4092) 20.0 aperio 1 4092 4092 0.50149999999999995 0.50149999999999995 Aperio Fake |AppMag = 20|MPP = 0.5015 0.2487945556640625 0.0063374702931080495 0.006582309532083608 0 0 0 0.0 165 1.8909090909090909 15 -0.006420149391937802 1 10.0 10 0.00020446134658247406 1.0098250225817402 1.6726571860607253 0.2528666764200787 0.4252018475618517 0.9337698968400423 0.1142845916582232 0.8404324661847189 0.2749993733706908 0.897567529038673 0.18656097394720528 0.5999317010355991 0.4234307078840679 9.538838020360488e-06 0.9268259509452782 0.18247882870888657 0.9186934437180535 0.005377040601787484 235.76269760173108 1.3349606666359373 233.56590731502072 1.473921948906114 236.81282683176053 0.9182862023596723 3 1744.6666666666667 4915 0.10703695372093502 0.0 0.0 0.0005713989005378889 0.00210218250820665 0.00037149510237524463 0.0022534729516800454 2.4557276884185666 1.068922962978307 1.1239287111502159 0.3038952269197558 0.529846271258571 0.10999022358426115 0.08367593565359072 0.02877052817684292 0.28140090494640624 0.06700348012554541 0.05568572651126721 0.19289829129121416 0.0007475161992457143 0.9268259509452782 0.16339142134781473 0.48661575224918774 0.16339142134781473 164.30908049925569 34.5809331872731 109.32547807168213 45.571779037831845 152.00957288446125 33.74422284266695 0.5122806353183663 0.1569209423864019 0.04125629187087427 0.017144963702879386 0.11586162500617757 0.04980135461685613 0.03480984437371579 0.020603328666024174 0.02379815335291759 0.01340663580142446 0.03387246974681185 0.014160677538799784 43665 |TCGA-EJ-5509-01A-01-BS1_ROI.svs- saveMacro Can't Read 'macro' Image from Slide's Associated Images diff --git a/histoqc/tests/test_base_image.py b/histoqc/tests/test_base_image.py new file mode 100644 index 0000000..847a221 --- /dev/null +++ b/histoqc/tests/test_base_image.py @@ -0,0 +1,57 @@ +import unittest +from histoqc.BaseImage import BaseImage +from histoqc.BasicModule import getBasicStats, finalComputations + +file_name = './histoqc/tests/data/TCGA-EJ-5509-01A-01-BS1_ROI.svs' +fname_outdir = './histoqc/tests/new' +seed = 123 + + + + +class TestBaseImageModule(unittest.TestCase): + + def setUp(self): + self.base_image = BaseImage( + file_name, fname_outdir, seed, + dict( + image_work_size='1.25x', + in_memory_compression='True', + confirm_base_mag='False', + mask_statistics = 'relative2mask' + ) + ) + getBasicStats(self.base_image, dict(image_work_size = '1.25x')) + finalComputations(self.base_image, dict()) + self.base_info = {'filename':'TCGA-EJ-5509-01A-01-BS1_ROI.svs', + 'comments':' ', + 'image_bounding_box':(0, 0, 4092, 4092), + 'base_mag':20.0, + 'type':'aperio', + 'levels':'1', + 'height':'4092', + 'width':'4092', + 'mpp_x':'0.50149999999999995', + 'mpp_y':'0.50149999999999995', + 'comment':'Aperio Fake |AppMag = 20|MPP = 0.5015', + } + + self.pixels_to_use='65536' + + + def tearDown(self): + del self.base_image + + def test_get_basic_stats(self): + for field in self.base_info: + with self.subTest('BaseImage.getBasicStats', field=field): + self.assertEqual(self.base_image[field],self.base_info[field]) + print(f'{field} pass') + + def test_final_computations(self): + self.assertEqual(self.base_image['pixels_to_use'],self.pixels_to_use) + print(f'pixels_to_use pass') + + +if __name__ == '__main__': + unittest.main() diff --git a/histoqc/tests/test_images_tsv_results.py b/histoqc/tests/test_images_tsv_results.py new file mode 100644 index 0000000..db3af47 --- /dev/null +++ b/histoqc/tests/test_images_tsv_results.py @@ -0,0 +1,147 @@ +import os, shutil, unittest +import subprocess +import histoqc.tests.test_utils as tu + + + +# def setUpModule(): +# print('Running setUpModule') + + +# def tearDownModule(): +# print('Running tearDownModule') + +module_name = 'histoqc' +test_dir = './histoqc/tests' +config_file_path = './histoqc/config/config_v2.1.ini' +new_result_dir_path = 'new' +target_result_dir_path = 'target' +wsi_name = 'TCGA-EJ-5509-01A-01-BS1_ROI.svs' + +new_dir_full_path = os.path.join(test_dir, new_result_dir_path) +target_dir_full_path = os.path.join(test_dir, target_result_dir_path) +wsi_full_path = os.path.join(test_dir, 'data', wsi_name) + + +def setUpModule(): + print(f'Running the {module_name} module . . .') + print(config_file_path) + print(new_dir_full_path) + print(wsi_full_path) + + # Replace 'module_name' with the name of the Python module you want to run + + # Execute the Python module using the subprocess module + try: + subprocess.run(['python3.8', + '-m', module_name, + '-c', config_file_path, + '-o', new_dir_full_path, + '-s', "123", + wsi_full_path], check=True) + except subprocess.CalledProcessError as e: + print(f"Error running the {module_name} module: {e}") + + +def tearDownModule(): + print("tearDownModule") + + # Check if the dir exists + if os.path.exists(new_dir_full_path): + # Remove the dir and its contents + shutil.rmtree(new_dir_full_path) + print(f"Remove {new_dir_full_path} directory") + else: + print(f"The directory {new_dir_full_path} does not exist.") + +class TestTargetResultsModule(unittest.TestCase): + # @classmethod + # def setUpClass(cls): + # print('Running setUpClass') + + # @classmethod + # def tearDownClass(cls): + # print('Running tearDownClass') + + def setUp(self): + + # image suffixes + self.suffixes = [ + 'areathresh', + 'blurry', + 'bright', + 'dark', + 'deconv_c0', + 'deconv_c1', + 'deconv_c2', + 'equalized_thumb', + 'fatlike', + 'flat', + 'fuse', + 'hist', + 'mask_use', + 'small_fill', + 'small_remove', + 'spur', + 'thumb_small', + 'thumb' + ] + + # tsv + self.rs_name = "results.tsv" + # tsv_labels #### '#start_time:', + self.tsv_labels = ['#pipeline:', '#outdir:', '#command_line_args:', '#config_file:', '#dataset:'] + # tsv dataset fields + self.tsv_dataset_fields = ['filename','comments','image_bounding_box','base_mag','type','levels','height','width','mpp_x','mpp_y','comment','brightestPixels','dark','flat_areas','fatlike_tissue_removed_num_regions','fatlike_tissue_removed_mean_area','fatlike_tissue_removed_max_area','fatlike_tissue_removed_percent','small_tissue_filled_num_regions','small_tissue_filled_mean_area','small_tissue_filled_max_area','small_tissue_filled_percent','small_tissue_removed_num_regions','small_tissue_removed_mean_area','small_tissue_removed_max_area','small_tissue_removed_percent','background_contrast','background_contrast_std','background_dissimilarity','background_dissimilarity_std','background_homogeneity','background_homogeneity_std','background_ASM','background_ASM_std','background_energy','background_energy_std','background_correlation','background_correlation_std','background_tenenGrad_contrast','background_michelson_contrast','background_rms_contrast','background_grayscale_brightness','background_grayscale_brightness_std','background_chan1_brightness','background_chan1_brightness_std','background_chan2_brightness','background_chan2_brightness_std','background_chan3_brightness','background_chan3_brightness_std','blurry_removed_num_regions','blurry_removed_mean_area','blurry_removed_max_area','blurry_removed_percent','spur_pixels','areaThresh','template1_MSE_hist','template2_MSE_hist','template3_MSE_hist','template4_MSE_hist','final_contrast','final_contrast_std','final_dissimilarity','final_dissimilarity_std','final_homogeneity','final_homogeneity_std','final_ASM','final_ASM_std','final_energy','final_energy_std','final_correlation','final_correlation_std','tenenGrad_contrast','michelson_contrast','rms_contrast','grayscale_brightness','grayscale_brightness_std','chan1_brightness','chan1_brightness_std','chan2_brightness','chan2_brightness_std','chan3_brightness','chan3_brightness_std','chan1_brightness_YUV','chan1_brightness_std_YUV','chan2_brightness_YUV','chan2_brightness_std_YUV','chan3_brightness_YUV','chan3_brightness_std_YUV','deconv_c0_mean','deconv_c0_std','deconv_c1_mean','deconv_c1_std','deconv_c2_mean','deconv_c2_std','pixels_to_use','warnings'] + + def tearDown(self): + del self.suffixes + del self.rs_name + del self.tsv_labels + del self.tsv_dataset_fields + + def test_images(self): + # test all images + for suffix in self.suffixes: + with self.subTest('Test Generated Images', suffix=suffix): + slide_name = "TCGA-EJ-5509-01A-01-BS1_ROI.svs" + img_path1 = os.path.join(new_dir_full_path,slide_name,f"{slide_name}_{suffix}.png") + img_path2 = os.path.join(target_dir_full_path ,slide_name,f"{slide_name}_{suffix}.png") + self.assertTrue(tu.compare_images(img_path1, img_path2)) + print(f'{suffix} images comparison pass') + + + # test result files + def test_result_labels(self): + file_path1 = os.path.join(new_dir_full_path, self.rs_name) + file_path2 = os.path.join(target_dir_full_path, self.rs_name) + + with open(file_path1, 'r') as file1, open(file_path2, 'r') as file2: + content1 = file1.read() + content2 = file2.read() + + # comparing labels + for label in self.tsv_labels[:-1]: + + label_value1 = tu.parseLabel(label, content1) + label_value2 = tu.parseLabel(label, content2) + + self.assertEqual(label_value1, label_value2) + print(f'label {label} in tsv results comparison pass') + + # comparing dataset + dataset_label = self.tsv_labels[-1] + dataset1 = tu.parseDataset(dataset_label, content1) + dataset2 = tu.parseDataset(dataset_label, content2) + for field_name in self.tsv_dataset_fields: + + column1 = dataset1[field_name] + column2 = dataset2[field_name] + for idx in range(len(column1)): + self.assertAlmostEqual(column1[idx],column2[idx]) + print(f"datasets' {field_name} field in tsv results comparison pass") + +if __name__ == '__main__': + unittest.main() + + \ No newline at end of file diff --git a/histoqc/tests/test_utils.py b/histoqc/tests/test_utils.py new file mode 100644 index 0000000..5820a98 --- /dev/null +++ b/histoqc/tests/test_utils.py @@ -0,0 +1,34 @@ +import io, os, re, cv2 +import pandas as pd + +# the two images are equal if the histogram correlation coefficient is greater than or equal to 0.95. +def compare_images(image_path1, image_path2): + # Load the two images + image1 = cv2.imread(image_path1) + image2 = cv2.imread(image_path2) + # Check if the images have the same dimensions + if image1.shape == image2.shape: + # Compare the two images for exact equality + difference = cv2.subtract(image1, image2) + b, g, r = cv2.split(difference) + if cv2.countNonZero(b) == 0 and cv2.countNonZero(g) == 0 and cv2.countNonZero(r) == 0: + return True + else: + return False + else: + return False + # Compare the two images + +def parseLabel(label_name, content): + label_pattern = re.compile(fr'{label_name}?\s*([^\n]*)\n') + match = label_pattern.search(content) + if match: + return match.group(1) + else: + return None + +def parseDataset(name, content): + # print(f'name: {name}') + dataset_pattern = re.compile(fr'{name}\s?') + dataset_content = dataset_pattern.split(content, 1)[1] + return pd.read_csv(io.StringIO(dataset_content), sep='\t', header=0) \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 2397ab6..aa2d094 100644 --- a/requirements.txt +++ b/requirements.txt @@ -6,6 +6,7 @@ scipy==1.10.0 matplotlib~=3.7.1 dill==0.3.3 pytest~=7.1.3 +pytest-cov~=4.1.0 importlib-resources typing-extensions lazy-property