From 93b7a909b51e1c6708cc91ba6dd5e8e7c2c09254 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 14:18:58 +0530 Subject: [PATCH 01/18] Update nextflow.config adding parameters related to suvpar module --- modules/workflows/Nextflow_workflows/nextflow.config | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/modules/workflows/Nextflow_workflows/nextflow.config b/modules/workflows/Nextflow_workflows/nextflow.config index 966825f..3b584a1 100644 --- a/modules/workflows/Nextflow_workflows/nextflow.config +++ b/modules/workflows/Nextflow_workflows/nextflow.config @@ -38,4 +38,9 @@ params{ Featureset_File_for_png_extraction="" Featureset_File_for_suvpar="/path/to/featureset" + ScannerDetails="path/to/scannerDetails.txt" + ScannerFilter=false + IsStatistics=false + IsFinalCSV=false + IsAnonymized=false } From 41ac8a93e7793bfdce1379cba23640c696fec7b2 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 14:23:20 +0530 Subject: [PATCH 02/18] Update cold_extraction.py --- .../Nextflow_workflows/src/cold_extraction.py | 23 ++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/modules/workflows/Nextflow_workflows/src/cold_extraction.py b/modules/workflows/Nextflow_workflows/src/cold_extraction.py index 120bc59..2b57890 100644 --- a/modules/workflows/Nextflow_workflows/src/cold_extraction.py +++ b/modules/workflows/Nextflow_workflows/src/cold_extraction.py @@ -121,6 +121,27 @@ ColdDataRetriever.t_start = time.time() ColdDataRetriever.run_cold_extraction() - +ColdDataRetriever.read_csv() + # The thread scheduling +schedule.every(1).minutes.do(ColdDataRetriever.run_threaded, ColdDataRetriever.run_retrieval) +schedule.every(10).minutes.do(ColdDataRetriever.run_threaded, ColdDataRetriever.update_pickle) + +# Keep running in a loop. +while True: + try: + schedule.run_pending() + time.sleep(1) + except KeyboardInterrupt: + ColdDataRetriever.check_kill_process() + logging.shutdown() + print("hello10") + break + #sys.exit(0) + +for line in os.popen("ps -ax | grep storescp"): + fields = line.split() + pid = fields[0] + print(pid) + os.kill(int(pid), signal.SIGKILL) From 66590205ae53c3ac2860784cfdcf63253e3e5d8d Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 14:27:17 +0530 Subject: [PATCH 03/18] Update suvpar.py --- .../Nextflow_workflows/src/suvpar.py | 25 ++++++++++++++++--- 1 file changed, 22 insertions(+), 3 deletions(-) diff --git a/modules/workflows/Nextflow_workflows/src/suvpar.py b/modules/workflows/Nextflow_workflows/src/suvpar.py index f3421dd..467b4d3 100644 --- a/modules/workflows/Nextflow_workflows/src/suvpar.py +++ b/modules/workflows/Nextflow_workflows/src/suvpar.py @@ -12,15 +12,34 @@ ap.add_argument("--InputFile") ap.add_argument("--OutputFile") ap.add_argument("--FeaturesetFile") +ap.add_argument("--ScannerDetails") +ap.add_argument("--ScannerFilter") +ap.add_argument("--Statistics_File") +ap.add_argument("--IsStatistics") +ap.add_argument("--IsFinalCSV") +ap.add_argument("--IsAnonymized") + config = vars(ap.parse_args()) -global output_csv, df +logging.basicConfig(level=logging.INFO) +Suvpar.df = {} +Suvpar.sta = {} +Suvpar.statistics_csv = {} +Suvpar.output_csv = {} + + +global output_csv, df, device_SN, scanner_filter, statistics_csv, isStatistics, final_csv, isAnonymized Suvpar.feature_file = config['FeaturesetFile'] Suvpar.filename = config['InputFile'] Suvpar.output_csv = config['OutputFile'] - +Suvpar.scanner_file = config['ScannerDetails'] +Suvpar.scanner_filter = bool(config['ScannerFilter']) +Suvpar.statistics_csv = config['Statistics_File'] +Suvpar.isStatistics = bool(config['IsStatistics']) +Suvpar.final_csv = bool(config['IsFinalCSV']) +Suvpar.isAnonymized = bool(config['IsAnonymized']) Suvpar.text_file = open(Suvpar.feature_file, "r") Suvpar.feature_list = Suvpar.text_file.read().split('\n') Suvpar.df = pandas.read_csv(Suvpar.filename, usecols=lambda x: x in Suvpar.feature_list, sep=',') Suvpar.suvpar() -Suvpar.write() \ No newline at end of file +Suvpar.write() From f024df39c97c3c1ae907e3951b4393868b6b8fd0 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 14:42:35 +0530 Subject: [PATCH 04/18] Update mega_workflow.nf --- modules/workflows/Nextflow_workflows/mega_workflow.nf | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/modules/workflows/Nextflow_workflows/mega_workflow.nf b/modules/workflows/Nextflow_workflows/mega_workflow.nf index 8506290..16b6902 100644 --- a/modules/workflows/Nextflow_workflows/mega_workflow.nf +++ b/modules/workflows/Nextflow_workflows/mega_workflow.nf @@ -36,7 +36,7 @@ process cold_extraction{ params.workflow==1 || params.workflow==2 || params.workflow==3 || params.workflow==4 script: """ - python3 $pd/Modules/cold_extraction.py --StorageFolder $params.OutputDirectory/workflow_results/cold_extraction_results --FilePath $params.FilePath --CsvFile $params.CsvFile --NumberOfQueryAttributes $params.NumberOfQueryAttributes --FirstAttr $params.FirstAttr --FirstIndex $params.FirstIndex --SecondAttr $params.SecondAttr --SecondIndex $params.SecondIndex --ThirdAttr $params.ThirdAttr --ThirdIndex $params.ThirdIndex --DateFormat $params.DateFormat --SendEmail $params.SendEmail --YourEmail $params.YourEmail --DCM4CHEBin $params.DCM4CHEBin --SrcAet $params.SrcAet --QueryAet $params.QueryAet --DestAet $params.DestAet --NightlyOnly $params.NightlyOnly --StartHour $params.StartHour --EndHour $params.EndHour --NifflerID $params.NifflerID --MaxNifflerProcesses $params.MaxNifflerProcesses + python3 $pd/src/cold_extraction.py --StorageFolder $params.OutputDirectory/workflow_results/cold_extraction_results --FilePath $params.FilePath --CsvFile $params.CsvFile --NumberOfQueryAttributes $params.NumberOfQueryAttributes --FirstAttr $params.FirstAttr --FirstIndex $params.FirstIndex --SecondAttr $params.SecondAttr --SecondIndex $params.SecondIndex --ThirdAttr $params.ThirdAttr --ThirdIndex $params.ThirdIndex --DateFormat $params.DateFormat --SendEmail $params.SendEmail --YourEmail $params.YourEmail --DCM4CHEBin $params.DCM4CHEBin --SrcAet $params.SrcAet --QueryAet $params.QueryAet --DestAet $params.DestAet --NightlyOnly $params.NightlyOnly --StartHour $params.StartHour --EndHour $params.EndHour --NifflerID $params.NifflerID --MaxNifflerProcesses $params.MaxNifflerProcesses """ } @@ -50,7 +50,7 @@ if(params.workflow==1 || params.workflow==2 || params.workflow==3 || params.work val depth into png_ext_out script: """ - python3 $pd/Modules/ImageExtractor_nextflow.py --DICOMHome $DICOMHome --SplitIntoChunks $params.SplitIntoChunks --PrintImages $params.PrintImages --CommonHeadersOnly $params.CommonHeadersOnly --UseProcesses $params.UseProcesses --FlattenedToLevel $params.FlattenedToLevel --is16Bit $params.is16Bit --SendEmail $params.SendEmail --YourEmail $params.YourEmail --PublicHeadersOnly $params.PublicHeadersOnly --Depth $depth + python3 $pd/src/ImageExtractor_nextflow.py --DICOMHome $DICOMHome --SplitIntoChunks $params.SplitIntoChunks --PrintImages $params.PrintImages --CommonHeadersOnly $params.CommonHeadersOnly --UseProcesses $params.UseProcesses --FlattenedToLevel $params.FlattenedToLevel --is16Bit $params.is16Bit --SendEmail $params.SendEmail --YourEmail $params.YourEmail --PublicHeadersOnly $params.PublicHeadersOnly --Depth $depth """ } } @@ -62,7 +62,7 @@ else{ val depth into png_ext_out script: """ - python3 $pd/Modules/ImageExtractor_nextflow.py --DICOMHome $params.OutputDirectory/workflow_results/cold_extraction_results --OutputDirectory $params.OutputDirectory/workflow_results/png_extraction_results --SplitIntoChunks $params.SplitIntoChunks --PrintImages $params.PrintImages --CommonHeadersOnly $params.CommonHeadersOnly --UseProcesses $params.UseProcesses --FlattenedToLevel $params.FlattenedToLevel --is16Bit $params.is16Bit --SendEmail $params.SendEmail --YourEmail $params.YourEmail --PublicHeadersOnly $params.PublicHeadersOnly --Depth $depth + python3 $pd/src/suvpar.py --InputFile $params.OutputDirectory/workflow_results/png_extraction_results/metadata.csv --OutputFile $params.OutputDirectory/workflow_results/suvpar_resuts/output.csv --FeaturesetFile $params.Featureset_File_for_suvpar --ScannerDetails $params.ScannerDetails --ScannerFilter $params.ScannerFilter --Statistics_File $params.OutputDirectory/workflow_results/suvpar_resuts/statistics.csv --IsStatistics $params.IsStatistics --IsFinalCSV $params.IsFinalCSV --IsAnonymized $params.IsAnonymized """ } @@ -79,7 +79,7 @@ process suvpar{ """ - python3 $pd/Modules/suvpar.py --InputFile $params.OutputDirectory/workflow_results/png_extraction_results/metadata.csv --OutputFile $params.OutputDirectory/workflow_results/suvpar_resuts/output.csv --FeaturesetFile $params.Featureset_File_for_suvpar + python3 $pd/src/suvpar.py --InputFile $params.OutputDirectory/workflow_results/png_extraction_results/metadata.csv --OutputFile $params.OutputDirectory/workflow_results/suvpar_resuts/output.csv --FeaturesetFile $params.Featureset_File_for_suvpar """ @@ -102,7 +102,7 @@ process meta_anon{ params.workflow==1 || params.workflow==4 || params.workflow==5 || params.workflow==8 script: """ - python3 $pd/Modules/metadata_anonymization.py $params.OutputDirectory/workflow_results/png_extraction_results/metadata.csv $params.OutputDirectory/workflow_results/metaAnon_resuts/output.csv + python3 $pd/src/metadata_anonymization.py $params.OutputDirectory/workflow_results/png_extraction_results/metadata.csv $params.OutputDirectory/workflow_results/metaAnon_resuts/output.csv """ } From 5096593bca852c1c2603e6c074c6aff60e8d431f Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:00:53 +0530 Subject: [PATCH 05/18] Update cold_extraction.py --- modules/workflows/Nextflow_workflows/src/cold_extraction.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/modules/workflows/Nextflow_workflows/src/cold_extraction.py b/modules/workflows/Nextflow_workflows/src/cold_extraction.py index 2b57890..b7c90b2 100644 --- a/modules/workflows/Nextflow_workflows/src/cold_extraction.py +++ b/modules/workflows/Nextflow_workflows/src/cold_extraction.py @@ -3,6 +3,9 @@ import argparse import logging import time +import schedule +import signal + script_dir = os.path.dirname( __file__ ) module_dir=os.path.join(script_dir,"..","..","..","cold-extraction") print("script_dir is:",module_dir) @@ -54,6 +57,8 @@ ColdDataRetriever.date_format = valuesDict['DateFormat'] ColdDataRetriever.email = valuesDict['YourEmail'] ColdDataRetriever.send_email = bool(valuesDict['SendEmail']) +ColdDataRetriever.mod_csv_file = ColdDataRetriever.csv_file[:-4]+'_mod.csv' + ColdDataRetriever.DCM4CHE_BIN = valuesDict['DCM4CHEBin'] ColdDataRetriever.SRC_AET = valuesDict['SrcAet'] @@ -119,7 +124,6 @@ # record the start time ColdDataRetriever.t_start = time.time() -ColdDataRetriever.run_cold_extraction() ColdDataRetriever.read_csv() # The thread scheduling From 5f6365fe886f9efdfe5c210b696d6ddb4a550e25 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:02:28 +0530 Subject: [PATCH 06/18] Update mega_workflow.nf --- modules/workflows/Nextflow_workflows/mega_workflow.nf | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/workflows/Nextflow_workflows/mega_workflow.nf b/modules/workflows/Nextflow_workflows/mega_workflow.nf index 16b6902..2a8427f 100644 --- a/modules/workflows/Nextflow_workflows/mega_workflow.nf +++ b/modules/workflows/Nextflow_workflows/mega_workflow.nf @@ -50,7 +50,7 @@ if(params.workflow==1 || params.workflow==2 || params.workflow==3 || params.work val depth into png_ext_out script: """ - python3 $pd/src/ImageExtractor_nextflow.py --DICOMHome $DICOMHome --SplitIntoChunks $params.SplitIntoChunks --PrintImages $params.PrintImages --CommonHeadersOnly $params.CommonHeadersOnly --UseProcesses $params.UseProcesses --FlattenedToLevel $params.FlattenedToLevel --is16Bit $params.is16Bit --SendEmail $params.SendEmail --YourEmail $params.YourEmail --PublicHeadersOnly $params.PublicHeadersOnly --Depth $depth + python3 $pd/src/ImageExtractor_nextflow.py --DICOMHome $params.OutputDirectory/workflow_results/cold_extraction_results --OutputDirectory $params.OutputDirectory/workflow_results/png_extraction_results --SplitIntoChunks $params.SplitIntoChunks --PrintImages $params.PrintImages --CommonHeadersOnly $params.CommonHeadersOnly --UseProcesses $params.UseProcesses --FlattenedToLevel $params.FlattenedToLevel --is16Bit $params.is16Bit --SendEmail $params.SendEmail --YourEmail $params.YourEmail --PublicHeadersOnly $params.PublicHeadersOnly --Depth $depth """ } } @@ -62,7 +62,7 @@ else{ val depth into png_ext_out script: """ - python3 $pd/src/suvpar.py --InputFile $params.OutputDirectory/workflow_results/png_extraction_results/metadata.csv --OutputFile $params.OutputDirectory/workflow_results/suvpar_resuts/output.csv --FeaturesetFile $params.Featureset_File_for_suvpar --ScannerDetails $params.ScannerDetails --ScannerFilter $params.ScannerFilter --Statistics_File $params.OutputDirectory/workflow_results/suvpar_resuts/statistics.csv --IsStatistics $params.IsStatistics --IsFinalCSV $params.IsFinalCSV --IsAnonymized $params.IsAnonymized + python3 $pd/src/ImageExtractor_nextflow.py --DICOMHome $params.DICOMHome --OutputDirectory $params.OutputDirectory/workflow_results/png_extraction_results --SplitIntoChunks $params.SplitIntoChunks --PrintImages $params.PrintImages --CommonHeadersOnly $params.CommonHeadersOnly --UseProcesses $params.UseProcesses --FlattenedToLevel $params.FlattenedToLevel --is16Bit $params.is16Bit --SendEmail $params.SendEmail --YourEmail $params.YourEmail --PublicHeadersOnly $params.PublicHeadersOnly --Depth $depth """ } From 42588a253cdd05bfecda0cf6d636e02c0f13a390 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:06:57 +0530 Subject: [PATCH 07/18] Update suvpar.py --- modules/workflows/Nextflow_workflows/src/suvpar.py | 1 + 1 file changed, 1 insertion(+) diff --git a/modules/workflows/Nextflow_workflows/src/suvpar.py b/modules/workflows/Nextflow_workflows/src/suvpar.py index 467b4d3..56f3011 100644 --- a/modules/workflows/Nextflow_workflows/src/suvpar.py +++ b/modules/workflows/Nextflow_workflows/src/suvpar.py @@ -2,6 +2,7 @@ import sys import argparse import pandas +import logging script_dir = os.path.dirname( __file__ ) module_dir=os.path.join(script_dir,"..","..","..","suvpar") print("script_dir is:",module_dir) From b928562bda6f6100afa9beb30db80b421def90d6 Mon Sep 17 00:00:00 2001 From: Pavan Kumar Bellam <60264606+Pavan-Bellam@users.noreply.github.com> Date: Fri, 12 Aug 2022 15:14:35 +0530 Subject: [PATCH 08/18] Update cold_extraction.py --- modules/workflows/Nextflow_workflows/src/cold_extraction.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/modules/workflows/Nextflow_workflows/src/cold_extraction.py b/modules/workflows/Nextflow_workflows/src/cold_extraction.py index b7c90b2..4d1e05e 100644 --- a/modules/workflows/Nextflow_workflows/src/cold_extraction.py +++ b/modules/workflows/Nextflow_workflows/src/cold_extraction.py @@ -138,9 +138,8 @@ except KeyboardInterrupt: ColdDataRetriever.check_kill_process() logging.shutdown() - print("hello10") break - #sys.exit(0) + for line in os.popen("ps -ax | grep storescp"): fields = line.split() From 39e6e8fa21f1781a9edd9a04fef33ca4b7f75ab7 Mon Sep 17 00:00:00 2001 From: Nitesh639 Date: Sun, 14 Aug 2022 00:29:44 +0530 Subject: [PATCH 09/18] updated ImageExtractor file(common headers) --- modules/png-extraction/ImageExtractor.py | 59 ++++++++++++------------ 1 file changed, 30 insertions(+), 29 deletions(-) diff --git a/modules/png-extraction/ImageExtractor.py b/modules/png-extraction/ImageExtractor.py index 9a4a755..94b95af 100644 --- a/modules/png-extraction/ImageExtractor.py +++ b/modules/png-extraction/ImageExtractor.py @@ -469,38 +469,39 @@ def execute(pickle_file, dicom_home, output_directory, print_images, print_only_ metas = glob.glob("{}*.csv".format(meta_directory)) # for each meta file identify the columns that are not na's for at least 10% (metadata_col_freq_threshold) of data - for meta in metas: - m = pd.read_csv(meta, dtype='str') - d_len = m.shape[0] - total_length += d_len - - for e in m.columns: - col_pop = d_len - np.sum(m[e].isna()) # number of populated rows for this column in this metadata file - - if e in col_names: - col_names[e] += col_pop - else: - col_names[e] = col_pop + if print_only_common_headers: + for meta in metas: + m = pd.read_csv(meta, dtype='str') + d_len = m.shape[0] + total_length += d_len - # all_headers keeps track of number of appearances of each header. We later use this count to ensure that - # the headers we use are present in all metadata files. - if e in all_headers: - all_headers[e] += 1 - else: - all_headers[e] = 1 + for e in m.columns: + col_pop = d_len - np.sum(m[e].isna()) # number of populated rows for this column in this metadata file - loadable_names = list() - for k in col_names.keys(): - if k in all_headers and all_headers[k] >= no_splits: # no_splits == number of batches used - if col_names[k] >= metadata_col_freq_threshold * total_length: - loadable_names.append(k) # use header only if it's present in every metadata file + if e in col_names: + col_names[e] += col_pop + else: + col_names[e] = col_pop - # load every metadata file using only valid columns - meta_list = list() - for meta in metas: - m = pd.read_csv(meta, dtype='str', usecols=loadable_names) - meta_list.append(m) - merged_meta = pd.concat(meta_list, ignore_index=True) + # all_headers keeps track of number of appearances of each header. We later use this count to ensure that + # the headers we use are present in all metadata files. + if e in all_headers: + all_headers[e] += 1 + else: + all_headers[e] = 1 + + loadable_names = list() + for k in col_names.keys(): + if k in all_headers and all_headers[k] >= no_splits: # no_splits == number of batches used + if col_names[k] >= metadata_col_freq_threshold * total_length: + loadable_names.append(k) # use header only if it's present in every metadata file + + # load every metadata file using only valid columns + meta_list = list() + for meta in metas: + m = pd.read_csv(meta, dtype='str', usecols=loadable_names) + meta_list.append(m) + merged_meta = pd.concat(meta_list, ignore_index=True) # merging_meta merged_meta = pd.DataFrame() From 58eca893c2eca36d9161a53f8f9ed307cb1837a8 Mon Sep 17 00:00:00 2001 From: Nitesh639 Date: Thu, 18 Aug 2022 01:21:38 +0530 Subject: [PATCH 10/18] MergeWithRis --- modules/suvpar/Suvpar.py | 8 +++++++- modules/suvpar/config.json | 2 ++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/modules/suvpar/Suvpar.py b/modules/suvpar/Suvpar.py index a96c9d6..610caf1 100644 --- a/modules/suvpar/Suvpar.py +++ b/modules/suvpar/Suvpar.py @@ -12,7 +12,7 @@ def initialize(): - global output_csv, df, device_SN, scanner_filter, statistics_csv, isStatistics, final_csv, isAnonymized + global output_csv, df, device_SN, scanner_filter, statistics_csv, isStatistics, final_csv, isAnonymized, ris_df, is_merge_with_ris with open('config.json', 'r') as f: config = json.load(f) @@ -23,6 +23,8 @@ def initialize(): scanner_filter = bool(config['ScannerFilter']) statistics_csv = config['Statistics_File'] isStatistics = bool(config['IsStatistics']) + ris_csv = config['RIS_File'] + is_merge_with_ris = bool(config['IsMergeWithRis']) final_csv = bool(config['IsFinalCSV']) isAnonymized = bool(config['IsAnonymized']) text_file = open(feature_file, "r") @@ -31,6 +33,7 @@ def initialize(): scanner_file = open(scanner_file, "r") device_SN = scanner_file.read().split('\n') df = pandas.read_csv(filename, usecols=lambda x: x in feature_list, sep=',') + ris_df = pandas.read_csv(ris_csv) def suvpar(): @@ -362,6 +365,9 @@ def suvpar(): sta = df.describe() sta.loc['count', 'MultiStudyEncounter'] = df[df['MultiStudyEncounter'] == True]['InstanceNumber'].count() + if is_merge_with_ris: + df = pandas.merge(df, ris_df, on='PatientID') + def write(): global isStatistics diff --git a/modules/suvpar/config.json b/modules/suvpar/config.json index 1dae356..ac05aab 100644 --- a/modules/suvpar/config.json +++ b/modules/suvpar/config.json @@ -6,6 +6,8 @@ "FeaturesetFile": "featureset1.txt", "IsStatistics": false, "Statistics_File": "statistic.csv", + "RIS_File" : "ris.csv", + "IsMergeWithRis" : false, "IsFinalCSV": true, "IsAnonymized": true } From 2bcbc43356c70d6a42c6af17109af0d9f5e0c66d Mon Sep 17 00:00:00 2001 From: Nitesh Kumar Kushwaha Date: Thu, 18 Aug 2022 01:23:14 +0530 Subject: [PATCH 11/18] MergeWithRis --- modules/suvpar/ris.csv | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 modules/suvpar/ris.csv diff --git a/modules/suvpar/ris.csv b/modules/suvpar/ris.csv new file mode 100644 index 0000000..e8d8134 --- /dev/null +++ b/modules/suvpar/ris.csv @@ -0,0 +1,3 @@ +PatientID,Patient Age at Visit,Encounter,EEMR Accession Number,Exam Room,Encounter Type,Ordered Item,Metrics,Order Timestamp,Requested Start Timestamp,Ancillary Arrival Timestamp,Exam Start Timestamp,Exam Completion Timestamp +1111,44,234232323,23332221,ABC,OV,MRI Brain w/o Contrast,,5/31/20 4:51,5/31/20 4:51,5/31/20 4:51,5/31/20 10:03,5/31/20 10:22 +22222,55,23232332323,222222221,DEF,IN,MRI Brain w/+ w/o Contrast,,5/31/20 7:30,5/31/20 7:30,5/31/20 7:22,5/31/20 11:44,5/31/20 12:25 \ No newline at end of file From 0822a5555c5a800a208708347c5c228bf7a99b5e Mon Sep 17 00:00:00 2001 From: Ananth Reddy Date: Sat, 20 Aug 2022 13:28:44 -0400 Subject: [PATCH 12/18] Fixing the errors in Nextflow Workflow Module --- .../workflows/Nextflow_workflows/src/cold_extraction.py | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/modules/workflows/Nextflow_workflows/src/cold_extraction.py b/modules/workflows/Nextflow_workflows/src/cold_extraction.py index 4d1e05e..eefaffe 100644 --- a/modules/workflows/Nextflow_workflows/src/cold_extraction.py +++ b/modules/workflows/Nextflow_workflows/src/cold_extraction.py @@ -3,6 +3,7 @@ import argparse import logging import time +import shutil import schedule import signal @@ -39,11 +40,6 @@ ap.add_argument("--MaxNifflerProcesses", type=int) valuesDict = vars(ap.parse_args()) - - - - - ColdDataRetriever.storage_folder = valuesDict['StorageFolder'] ColdDataRetriever.file_path = valuesDict['FilePath'] ColdDataRetriever.csv_file = valuesDict['CsvFile'] @@ -58,7 +54,7 @@ ColdDataRetriever.email = valuesDict['YourEmail'] ColdDataRetriever.send_email = bool(valuesDict['SendEmail']) ColdDataRetriever.mod_csv_file = ColdDataRetriever.csv_file[:-4]+'_mod.csv' - +shutil.copyfile(ColdDataRetriever.csv_file, ColdDataRetriever.mod_csv_file) ColdDataRetriever.DCM4CHE_BIN = valuesDict['DCM4CHEBin'] ColdDataRetriever.SRC_AET = valuesDict['SrcAet'] @@ -147,4 +143,3 @@ print(pid) os.kill(int(pid), signal.SIGKILL) - From 4e40f7015eded76013392999013e4534150054fa Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Mon, 22 Aug 2022 18:11:26 -0400 Subject: [PATCH 13/18] Add ContentTime as an attribute to the output file --- modules/suvpar/featureset1.txt | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/suvpar/featureset1.txt b/modules/suvpar/featureset1.txt index 414e001..90aad70 100644 --- a/modules/suvpar/featureset1.txt +++ b/modules/suvpar/featureset1.txt @@ -23,4 +23,5 @@ Manufacturer ManufacturerModelName SeriesInstanceUID [SliceMeasurementDuration] -[Acquisition Duration] \ No newline at end of file +[Acquisition Duration] +ContentTime \ No newline at end of file From c84168708ed213f5db16eb8454eb2119879689ec Mon Sep 17 00:00:00 2001 From: Nitesh639 Date: Wed, 24 Aug 2022 21:58:56 +0530 Subject: [PATCH 14/18] ImageType Updated --- modules/suvpar/Suvpar.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/modules/suvpar/Suvpar.py b/modules/suvpar/Suvpar.py index 610caf1..e8f4c5c 100644 --- a/modules/suvpar/Suvpar.py +++ b/modules/suvpar/Suvpar.py @@ -55,8 +55,8 @@ def suvpar(): df.dropna(subset=["SeriesTime"], inplace=True) df.dropna(subset=["SeriesDate"], inplace=True) df.dropna(subset=["DeviceSerialNumber"], inplace=True) - # Consider only the ImageType that are ORIGINAL. - df = df[df['ImageType'].str.contains("ORIGINAL")] + # Remove only the ImageType that are NA or NPR. + df = df[(~df['ImageType'].str.contains('NA|NPR')) | (df['ImageType'].str.contains('ORIGINAL'))] # Consider only MR. Remove modalities such as PR and SR that are present in the original data. df = df[df.Modality == "MR"] # Dataset after removing unwanted Device Serial Number From fa98881b43b9592c99cedc07436516961e694ac7 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 24 Aug 2022 15:54:24 -0400 Subject: [PATCH 15/18] Update README.md --- modules/cold-extraction/README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/modules/cold-extraction/README.md b/modules/cold-extraction/README.md index 2bfaed1..619fb4d 100644 --- a/modules/cold-extraction/README.md +++ b/modules/cold-extraction/README.md @@ -208,7 +208,7 @@ To activate, use the below value, "FilePath": "CFIND-DETAILED", ``` -## Troubleshooting +# Troubleshooting If the process fails even when no one else's Niffler process is running, check your log file (UNIQUE-OUTPUT-FILE-FOR-YOUR-EXTRACTION.out) @@ -242,6 +242,7 @@ $ sudo ps -xa | grep storescp $ sudo kill 241720 ``` +## Testing your deployment with DCM4CHE Niffler strives to be stable for at least the latest stable releases. But since it is still an open-source research project by a university research group, it may have bugs at times - which we aim to fix as soon as we spot. But if your extraction fails for some reason, you could rule out whether the issue is really a Niffler bug or whether some other issue such as some problems in the PACS connection. From 3cd8710f34925b5bb74ae0a2b61c554b89aaff11 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Wed, 24 Aug 2022 15:59:11 -0400 Subject: [PATCH 16/18] Update README.md --- modules/cold-extraction/README.md | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/modules/cold-extraction/README.md b/modules/cold-extraction/README.md index 619fb4d..a0be45f 100644 --- a/modules/cold-extraction/README.md +++ b/modules/cold-extraction/README.md @@ -246,7 +246,7 @@ $ sudo kill 241720 Niffler strives to be stable for at least the latest stable releases. But since it is still an open-source research project by a university research group, it may have bugs at times - which we aim to fix as soon as we spot. But if your extraction fails for some reason, you could rule out whether the issue is really a Niffler bug or whether some other issue such as some problems in the PACS connection. -Simply start a storescp and movescu clients (in that order) of DCM4CHE from the server where you are attempting to run. If the below commands work, but Niffler still fails (after correctly following the README), it could indicate a Niffler bug. +Simply start a storescp and movescu clients (in that order) of DCM4CHE from the server where you are attempting to run Niffler. If the below commands work, but Niffler still fails (after correctly following the README), it could indicate a Niffler bug. The requests take the below format. @@ -275,3 +275,6 @@ nohup /opt/dcm4che-5.22.5/bin/storescp --accept-unknown --directory new-pydicom C-MOVE nohup /opt/dcm4che-5.22.5/bin/movescu -c "AE_ARCH2@xx.yy.ww.zz:104" -b "QBNIFFLER:4243" -M PatientRoot -m PatientID=12345678 --dest QBNIFFLER > movescu.out & ``` + +If the testing with DCM4CHE as above does not work, that is an issue likely with your PACS configuration to send DICOM data to your endpoint. Please get the above to work first in that case before attempting the execution with Niffler. + From 9ea19ab2d0e9982f852104e3a0b790a8bf6df5ec Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 26 Aug 2022 11:17:53 -0400 Subject: [PATCH 17/18] Create TestConnection.py as an echo method. --- modules/cold-extraction/TestConnection.py | 38 +++++++++++++++++++++++ 1 file changed, 38 insertions(+) create mode 100644 modules/cold-extraction/TestConnection.py diff --git a/modules/cold-extraction/TestConnection.py b/modules/cold-extraction/TestConnection.py new file mode 100644 index 0000000..b087425 --- /dev/null +++ b/modules/cold-extraction/TestConnection.py @@ -0,0 +1,38 @@ +from pynetdicom import AE +from pynetdicom.sop_class import VerificationSOPClass +import json + +# Reads the Niffler system configuration file, "system.json". +with open('system.json', 'r') as f: + niffler = json.load(f) + + +QUERY_AET = niffler['QueryAet'] +query = QUERY_AET.split(':') + +ae = AE() +ae.ae_title = query[0] +ae.port = query[1] + +ae.add_requested_context(VerificationSOPClass) + +SRC_AET = niffler['SrcAet'] +srct = SRC_AET.split(':') + +port = int(srct[1]) +src = srct[0].split('@') + +assoc = ae.associate(src[1], port, ae_title=src[0]) + +if assoc.is_established: + status = assoc.send_c_echo() + + if status: + # If successful, the outputstatus will be: 0x0000 + print('C-ECHO request status: 0x{0:04x}'.format(status.Status)) + else: + print('Connection timed out, was aborted or received invalid response') + + assoc.release() +else: + print('Association rejected, aborted or never connected') From 598f22a0207bffa5697829814b8985e701425628 Mon Sep 17 00:00:00 2001 From: Pradeeban Kathiravelu Date: Fri, 26 Aug 2022 11:29:00 -0400 Subject: [PATCH 18/18] Update with instructions for C-ECHO --- modules/cold-extraction/README.md | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/modules/cold-extraction/README.md b/modules/cold-extraction/README.md index a0be45f..b6c9194 100644 --- a/modules/cold-extraction/README.md +++ b/modules/cold-extraction/README.md @@ -241,6 +241,32 @@ $ sudo ps -xa | grep storescp $ sudo kill 241720 ``` +## Testing your deployment with Niffler C-ECHO Implementation + +Sometimes your connection may not succeed due to firewall issues or because the source PACS not recognizing your end as a valid AET. To confirm and rule out these issues, you can issue a C-ECHO command included with Niffler. + +**First, please make sure your system.json is updated with the correct values for "SrcAet" and "QueryAet"**. + +Then, run the below. + +```` +$ python3 TestConnection.py +```` + +The below output indicates the success. +```` +C-ECHO request status: 0x0000 +```` + +If you receive any other output such as the below, that indicates the connection was not successful. +```` +Association rejected, aborted or never connected +```` + +Please check again the "SrcAet" and "QueryAet" in system.json for correctness. + +If everything is correct in your/Niffler end, please consult your enterprise PACS deployment for configuration. Is it configured correctly to accept queries from your "QueryAet"? Is there a firewall? Is that firewall configured to accept queries **from** your QueryAet (host and port)? + ## Testing your deployment with DCM4CHE