From b180d12032449a782feeb9babcc2e395ef40d6c1 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Tue, 9 Mar 2021 04:14:58 -0500 Subject: [PATCH 1/8] Update index.md --- docs/index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/index.md b/docs/index.md index d24b06a..8c28baa 100644 --- a/docs/index.md +++ b/docs/index.md @@ -1,4 +1,4 @@ -# Niffler: A DICOM Framework for Machine Learning Pipelines against Real-Time Radiology Images +# Niffler: A DICOM Framework for Machine Learning Pipelines and Processing Workflows Niffler is a research project for DICOM networking, supporting efficient DICOM retrievals and subsequent ML workflows on the images and metadata on a research environment. From 342f6887265aaafb59a368cce1d49887729abb0a Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 12 Mar 2021 09:55:37 -0500 Subject: [PATCH 2/8] Add 0x00251011 to the extracted tags --- modules/meta-extraction/conf/featureset1.txt | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/meta-extraction/conf/featureset1.txt b/modules/meta-extraction/conf/featureset1.txt index c99e0ef..5d18060 100644 --- a/modules/meta-extraction/conf/featureset1.txt +++ b/modules/meta-extraction/conf/featureset1.txt @@ -47,3 +47,4 @@ ManufacturerModelName 0x00090010 0x0019100B 0x0019105A +0x00251011 From 5f26fdf7b6f0f0968c1a2c451f7d1327f631385b Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 12 Mar 2021 13:17:41 -0500 Subject: [PATCH 3/8] Fix the pickle file creation --- modules/meta-extraction/MetadataExtractor.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/modules/meta-extraction/MetadataExtractor.py b/modules/meta-extraction/MetadataExtractor.py index 2e4dcee..45feb39 100644 --- a/modules/meta-extraction/MetadataExtractor.py +++ b/modules/meta-extraction/MetadataExtractor.py @@ -42,8 +42,8 @@ IS_DCM4CHE_NOT_RUNNING = True logging.basicConfig(level=logging.INFO) -FEATURES_FOLDER = "conf/" -PICKLE_FOLDER = "pickles/" +FEATURES_FOLDER = os.getcwd() + "/conf/" +PICKLE_FOLDER = os.getcwd() + "/pickles/" # Variables to track progress between iterations. @@ -258,6 +258,8 @@ def clear_storage(): def update_pickle(): global processed_series_but_yet_to_delete global processed_and_deleted_series + + os.chdir(PICKLE_FOLDER) # Pickle using the highest protocol available. with open(PICKLE_FOLDER + 'processed_series_but_yet_to_delete.pickle', 'wb') as f: From e204e5ae86eee479761e94e729d1110bb5e977d9 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 12 Mar 2021 14:26:08 -0500 Subject: [PATCH 4/8] Print the error log --- modules/meta-extraction/MetadataExtractor.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/meta-extraction/MetadataExtractor.py b/modules/meta-extraction/MetadataExtractor.py index 45feb39..fc384ee 100644 --- a/modules/meta-extraction/MetadataExtractor.py +++ b/modules/meta-extraction/MetadataExtractor.py @@ -110,8 +110,8 @@ def get_tuples(plan, features, outlist = None, key = ""): elif type(value1) is pydicom.uid.UID: value1 = str(value1) outlist.append((key + aa, value1)) # appends name, value pair for this file. these are later concatenated to the dataframe - except KeyError: - logging.debug("Key error encountered for %s", aa) + except KeyError as e: + logging.debug("Key error encountered for %s. %s", aa, e) return outlist From a4179f983b82f7d9b0f34f52fea3a0375e1b1283 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 12 Mar 2021 16:51:24 -0500 Subject: [PATCH 5/8] Store the variable even during the key exception --- modules/meta-extraction/MetadataExtractor.py | 25 +++++++++++++------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/modules/meta-extraction/MetadataExtractor.py b/modules/meta-extraction/MetadataExtractor.py index fc384ee..5432760 100644 --- a/modules/meta-extraction/MetadataExtractor.py +++ b/modules/meta-extraction/MetadataExtractor.py @@ -68,8 +68,6 @@ except: logging.info("Unable to load a valid pickle file. Initialized with empty value for processed_and_deleted_series") - - # Read the txt file which includes the features, then extract them os.chdir(FEATURES_FOLDER) txt_files = glob.glob('*.txt') @@ -84,7 +82,6 @@ feature_files.append(filename) - # Function for getting tuple for field, val pairs for this file # plan is instance of dicom class, the data for single mammo file def get_tuples(plan, features, outlist = None, key = ""): @@ -109,9 +106,23 @@ def get_tuples(plan, features, outlist = None, key = ""): value1 = tuple(value1) elif type(value1) is pydicom.uid.UID: value1 = str(value1) - outlist.append((key + aa, value1)) # appends name, value pair for this file. these are later concatenated to the dataframe + else: + if (value1): + value1 = str(value1) + else: + value1 = "" + logging.debug("Default case of extraction: %s", value1) + if key and aa: + key_ = key + aa + elif aa: + key_ = aa + elif key: + key_ = key + else: + logging.debug("key and aa are empty") + outlist.append((key_, value1)) # appends name, value pair for this file. these are later concatenated to the dataframe except KeyError as e: - logging.debug("Key error encountered for %s. %s", aa, e) + outlist.append((key_, value1)) # appends name, value pair for this file. these are later concatenated to the dataframe return outlist @@ -120,7 +131,6 @@ def get_dict_fields(bigdict, features): return {x: bigdict[x] for x in features if x in bigdict} - # The core method of extracting metadata def extract(): os.chdir(STORAGE_FOLDER) @@ -141,7 +151,6 @@ def extract(): logging.debug('There are no objects found. Waiting for new data to arrive.') - # Extract a list of one instance from each series def extract_metadata(): global processed_series_but_yet_to_delete @@ -154,7 +163,6 @@ def extract_metadata(): first_inst_of_series = list() headerlist = [] - if EXTRACTION_RUNNING: logging.info("Previous Extraction Thread Still Running. Skip this iteration.......................") else: @@ -173,7 +181,6 @@ def extract_metadata(): logging.info('Number of series to be processed: %s', len(series)) - for series_path in series: extracted_in_this_iteration += 1 logging.debug('Extracted: %s %s %s %s', str(extracted_in_this_iteration), ' out of ', str(len(series)), From ca115408830bbac7e56cba8e4b04e83f34a3afbf Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 12 Mar 2021 16:54:40 -0500 Subject: [PATCH 6/8] Remove too verbose debug log --- modules/meta-extraction/MetadataExtractor.py | 1 - 1 file changed, 1 deletion(-) diff --git a/modules/meta-extraction/MetadataExtractor.py b/modules/meta-extraction/MetadataExtractor.py index 5432760..94edde3 100644 --- a/modules/meta-extraction/MetadataExtractor.py +++ b/modules/meta-extraction/MetadataExtractor.py @@ -111,7 +111,6 @@ def get_tuples(plan, features, outlist = None, key = ""): value1 = str(value1) else: value1 = "" - logging.debug("Default case of extraction: %s", value1) if key and aa: key_ = key + aa elif aa: From 9c62400aecc9833a2b1d0a779d3793e4ad046e27 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 12 Mar 2021 17:06:27 -0500 Subject: [PATCH 7/8] Define the empty case for extraction --- modules/meta-extraction/MetadataExtractor.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/meta-extraction/MetadataExtractor.py b/modules/meta-extraction/MetadataExtractor.py index 94edde3..0f3d976 100644 --- a/modules/meta-extraction/MetadataExtractor.py +++ b/modules/meta-extraction/MetadataExtractor.py @@ -118,6 +118,7 @@ def get_tuples(plan, features, outlist = None, key = ""): elif key: key_ = key else: + key_ = "" logging.debug("key and aa are empty") outlist.append((key_, value1)) # appends name, value pair for this file. these are later concatenated to the dataframe except KeyError as e: From 31ea6a1906f1d737c57272e2c749b0cf485a9d4d Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 12 Mar 2021 17:42:28 -0500 Subject: [PATCH 8/8] Update for empty values --- modules/meta-extraction/MetadataExtractor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/modules/meta-extraction/MetadataExtractor.py b/modules/meta-extraction/MetadataExtractor.py index 0f3d976..9e9da01 100644 --- a/modules/meta-extraction/MetadataExtractor.py +++ b/modules/meta-extraction/MetadataExtractor.py @@ -122,7 +122,7 @@ def get_tuples(plan, features, outlist = None, key = ""): logging.debug("key and aa are empty") outlist.append((key_, value1)) # appends name, value pair for this file. these are later concatenated to the dataframe except KeyError as e: - outlist.append((key_, value1)) # appends name, value pair for this file. these are later concatenated to the dataframe + logging.debug("The value is empty, %s", e) return outlist