From 87d894b681444d8120241a9a3e041040f1b6c476 Mon Sep 17 00:00:00 2001 From: Nishchal-007 Date: Tue, 30 Mar 2021 20:25:43 +0530 Subject: [PATCH 1/4] missing csv entries --- modules/cold-extraction/ColdDataRetriever.py | 60 ++++++++++---------- 1 file changed, 30 insertions(+), 30 deletions(-) diff --git a/modules/cold-extraction/ColdDataRetriever.py b/modules/cold-extraction/ColdDataRetriever.py index 24c863c..d358375 100644 --- a/modules/cold-extraction/ColdDataRetriever.py +++ b/modules/cold-extraction/ColdDataRetriever.py @@ -130,24 +130,40 @@ def initialize(): with open(csv_file, newline='') as f: reader = csv.reader(f) next(f) + + # Changed below part for finding missing csv entries and skipping them for row in reader: + row = [x.strip() for x in row] + #print(row) if (extraction_type == 'empi_date'): - patients.append(row[patient_index]) - temp_date = row[date_index] - dt_stamp = datetime.datetime.strptime(temp_date, date_format) - date_str = dt_stamp.strftime('%Y%m%d') - dates.append(date_str) - length = len(patients) + if set(row).pop()=='': + pass + else: + patients.append(row[patient_index]) + temp_date = row[date_index] + dt_stamp = datetime.datetime.strptime(temp_date, date_format) + date_str = dt_stamp.strftime('%Y%m%d') + dates.append(date_str) + length = len(patients) elif (extraction_type == 'empi'): - patients.append(row[patient_index]) - length = len(patients) + if set(row).pop()=='': + pass + else: + patients.append(row[patient_index]) + length = len(patients) elif (extraction_type == 'accession'): - accessions.append(row[accession_index]) - length = len(accessions) + if set(row).pop()=='': + pass + else: + accessions.append(row[accession_index]) + length = len(accessions) elif (extraction_type == 'empi_accession'): - patients.append(row[patient_index]) - accessions.append(row[accession_index]) - length = len(accessions) + if set(row).pop()=='': + pass + else: + patients.append(row[patient_index]) + accessions.append(row[accession_index]) + length = len(accessions) # Run the retrieval only once, when the extraction script starts, and keep it running in a separate thread. @@ -180,22 +196,6 @@ def retrieve(): subprocess.call("{0}/movescu -c {1} -b {2} -M PatientRoot -m PatientID={3} -m AccessionNumber={4} --dest {5}".format(DCM4CHE_BIN, SRC_AET, QUERY_AET, PatientID, Accession, DEST_AET), shell=True) extracted_ones.append(temp_id) - # For the cases that have the EMPI. - elif (extraction_type == 'empi'): - # Create our Identifier (query) dataset - for pid in range(0, len(patients)): - PatientID = patients[pid] - if NIGHTLY_ONLY: - if (datetime.datetime.now().hour >= END_HOUR and datetime.datetime.now().hour < START_HOUR): - # log once while sleeping - logging.info("Nightly mode. Niffler schedules the extraction to resume at start hour {0} and start within 30 minutes after that. It will then pause at the end hour {1}".format(START_HOUR, END_HOUR)) - while (datetime.datetime.now().hour >= END_HOUR and datetime.datetime.now().hour < START_HOUR): - # sleep for 5 minutes - time.sleep(300) - if ((not resume) or (resume and (PatientID not in extracted_ones))): - subprocess.call("{0}/movescu -c {1} -b {2} -M PatientRoot -m PatientID={3} --dest {4}".format(DCM4CHE_BIN, SRC_AET, QUERY_AET, PatientID, DEST_AET), shell=True) - extracted_ones.append(PatientID) - # For the cases that does not have the typical EMPI and Accession values together. elif (extraction_type == 'empi_date' or extraction_type == 'accession'): # Create our Identifier (query) dataset @@ -277,4 +277,4 @@ def run_threaded(job_func): time.sleep(1) except KeyboardInterrupt: check_kill_process() - sys.exit(0) \ No newline at end of file + sys.exit(0) From 45cf9afd004dbf9c08ff9589dc2f57aba3298c6f Mon Sep 17 00:00:00 2001 From: Nishchal Singi <71981858+Nishchal-007@users.noreply.github.com> Date: Tue, 30 Mar 2021 22:03:25 +0530 Subject: [PATCH 2/4] removed comment and added missing code --- modules/cold-extraction/ColdDataRetriever.py | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/modules/cold-extraction/ColdDataRetriever.py b/modules/cold-extraction/ColdDataRetriever.py index d358375..594a8eb 100644 --- a/modules/cold-extraction/ColdDataRetriever.py +++ b/modules/cold-extraction/ColdDataRetriever.py @@ -130,11 +130,10 @@ def initialize(): with open(csv_file, newline='') as f: reader = csv.reader(f) next(f) - + # Changed below part for finding missing csv entries and skipping them for row in reader: row = [x.strip() for x in row] - #print(row) if (extraction_type == 'empi_date'): if set(row).pop()=='': pass @@ -196,6 +195,22 @@ def retrieve(): subprocess.call("{0}/movescu -c {1} -b {2} -M PatientRoot -m PatientID={3} -m AccessionNumber={4} --dest {5}".format(DCM4CHE_BIN, SRC_AET, QUERY_AET, PatientID, Accession, DEST_AET), shell=True) extracted_ones.append(temp_id) + # For the cases that have the EMPI. + elif (extraction_type == 'empi'): + # Create our Identifier (query) dataset + for pid in range(0, len(patients)): + PatientID = patients[pid] + if NIGHTLY_ONLY: + if (datetime.datetime.now().hour >= END_HOUR and datetime.datetime.now().hour < START_HOUR): + # log once while sleeping + logging.info("Nightly mode. Niffler schedules the extraction to resume at start hour {0} and start within 30 minutes after that. It will then pause at the end hour {1}".format(START_HOUR, END_HOUR)) + while (datetime.datetime.now().hour >= END_HOUR and datetime.datetime.now().hour < START_HOUR): + # sleep for 5 minutes + time.sleep(300) + if ((not resume) or (resume and (PatientID not in extracted_ones))): + subprocess.call("{0}/movescu -c {1} -b {2} -M PatientRoot -m PatientID={3} --dest {4}".format(DCM4CHE_BIN, SRC_AET, QUERY_AET, PatientID, DEST_AET), shell=True) + extracted_ones.append(PatientID) + # For the cases that does not have the typical EMPI and Accession values together. elif (extraction_type == 'empi_date' or extraction_type == 'accession'): # Create our Identifier (query) dataset From 3ef6f100581c0fbf43a6152aa238427b71f5b3f7 Mon Sep 17 00:00:00 2001 From: Nishchal-007 Date: Thu, 1 Apr 2021 18:28:25 +0530 Subject: [PATCH 3/4] Updated ColdDataRetriever.py --- modules/cold-extraction/ColdDataRetriever.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/modules/cold-extraction/ColdDataRetriever.py b/modules/cold-extraction/ColdDataRetriever.py index 594a8eb..10c2176 100644 --- a/modules/cold-extraction/ColdDataRetriever.py +++ b/modules/cold-extraction/ColdDataRetriever.py @@ -135,7 +135,7 @@ def initialize(): for row in reader: row = [x.strip() for x in row] if (extraction_type == 'empi_date'): - if set(row).pop()=='': + if ((row[patient_index] == "") or (row[date_index] == "")): pass else: patients.append(row[patient_index]) @@ -145,19 +145,19 @@ def initialize(): dates.append(date_str) length = len(patients) elif (extraction_type == 'empi'): - if set(row).pop()=='': + if ((row[patient_index] == "")): pass else: patients.append(row[patient_index]) length = len(patients) elif (extraction_type == 'accession'): - if set(row).pop()=='': + if ((row[accession_index] == "")): pass else: accessions.append(row[accession_index]) length = len(accessions) elif (extraction_type == 'empi_accession'): - if set(row).pop()=='': + if ((row[patient_index] == "") or (row[accession_index] == "")): pass else: patients.append(row[patient_index]) From 4e1a602505684ce14c85f4f693e9b1e46399a02c Mon Sep 17 00:00:00 2001 From: Nishchal Singi <71981858+Nishchal-007@users.noreply.github.com> Date: Sat, 3 Apr 2021 19:33:51 +0530 Subject: [PATCH 4/4] Made required changes --- modules/cold-extraction/ColdDataRetriever.py | 18 ++++-------------- 1 file changed, 4 insertions(+), 14 deletions(-) diff --git a/modules/cold-extraction/ColdDataRetriever.py b/modules/cold-extraction/ColdDataRetriever.py index 10c2176..9f1e2d3 100644 --- a/modules/cold-extraction/ColdDataRetriever.py +++ b/modules/cold-extraction/ColdDataRetriever.py @@ -130,14 +130,10 @@ def initialize(): with open(csv_file, newline='') as f: reader = csv.reader(f) next(f) - - # Changed below part for finding missing csv entries and skipping them for row in reader: row = [x.strip() for x in row] if (extraction_type == 'empi_date'): - if ((row[patient_index] == "") or (row[date_index] == "")): - pass - else: + if not ((row[patient_index] == "") or (row[date_index] == "")): patients.append(row[patient_index]) temp_date = row[date_index] dt_stamp = datetime.datetime.strptime(temp_date, date_format) @@ -145,21 +141,15 @@ def initialize(): dates.append(date_str) length = len(patients) elif (extraction_type == 'empi'): - if ((row[patient_index] == "")): - pass - else: + if not ((row[patient_index] == "")): patients.append(row[patient_index]) length = len(patients) elif (extraction_type == 'accession'): - if ((row[accession_index] == "")): - pass - else: + if not ((row[accession_index] == "")): accessions.append(row[accession_index]) length = len(accessions) elif (extraction_type == 'empi_accession'): - if ((row[patient_index] == "") or (row[accession_index] == "")): - pass - else: + if not ((row[patient_index] == "") or (row[accession_index] == "")): patients.append(row[patient_index]) accessions.append(row[accession_index]) length = len(accessions)