From 0f04668acb6c51c30c6fb80371c1dcf7ce9b8e0c Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Tue, 12 Jan 2021 10:13:22 +0000 Subject: [PATCH 01/10] Stop runtime error for missing log file when calculating the preview --- .../monitor_normalisation/monitor_normalisation.py | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py b/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py index 2fa05ad3af6..e3e508e1a09 100644 --- a/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py +++ b/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py @@ -28,16 +28,22 @@ class MonitorNormalisation(BaseFilter): @staticmethod def filter_func(images: Images, cores=None, chunksize=None, progress=None) -> Images: counts = images.counts() - if counts is None: + if counts is None and images.num_projections > 1: raise RuntimeError("No loaded log values for this stack.") counts_val = counts.value / counts.value[0] div_partial = ptsm.create_partial(_divide_by_counts, fwd_function=ptsm.inplace) - images, _ = ptsm.execute(images.data, counts_val, div_partial, cores, chunksize, progress=progress) + images, _ = ptsm.execute(images.data, + counts_val, + div_partial, + cores, + chunksize, + progress=progress) return images @staticmethod - def register_gui(form: 'QFormLayout', on_change: Callable, view: 'BaseMainWindowView') -> Dict[str, 'QWidget']: + def register_gui(form: 'QFormLayout', on_change: Callable, + view: 'BaseMainWindowView') -> Dict[str, 'QWidget']: return {} @staticmethod From ba5bffc68abad02d86afc196be32e9fc89dea18f Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Tue, 12 Jan 2021 10:13:50 +0000 Subject: [PATCH 02/10] Refactored and can still read old log files --- mantidimaging/core/io/loader/loader.py | 14 ++- .../core/utility/imat_log_file_parser.py | 88 +++++++++++++------ .../utility/test/imat_log_file_parser_test.py | 39 ++++---- 3 files changed, 89 insertions(+), 52 deletions(-) diff --git a/mantidimaging/core/io/loader/loader.py b/mantidimaging/core/io/loader/loader.py index 8582492a13a..6642af724be 100644 --- a/mantidimaging/core/io/loader/loader.py +++ b/mantidimaging/core/io/loader/loader.py @@ -27,7 +27,8 @@ def _fitsread(filename): import astropy.io.fits as fits image = fits.open(filename) if len(image) < 1: - raise RuntimeError("Could not load at least one FITS image/table file from: {0}".format(filename)) + raise RuntimeError( + "Could not load at least one FITS image/table file from: {0}".format(filename)) # get the image data return image[0].data @@ -98,12 +99,8 @@ def read_in_file_information(input_path, def load_log(log_file: str) -> IMATLogFile: - data = [] with open(log_file, 'r') as f: - for line in f: - data.append(line.strip().split(" ")) - - return IMATLogFile(data, log_file) + return IMATLogFile(f.readlines(), log_file) def load_p(parameters: ImageParameters, dtype, progress) -> Images: @@ -169,8 +166,9 @@ def load(input_path=None, else: load_func = _imread - dataset = img_loader.execute(load_func, input_file_names, input_path_flat_before, input_path_flat_after, - input_path_dark_before, input_path_dark_after, in_format, dtype, indices, progress) + dataset = img_loader.execute(load_func, input_file_names, input_path_flat_before, + input_path_flat_after, input_path_dark_before, + input_path_dark_after, in_format, dtype, indices, progress) # Search for and load metadata file metadata_found_filenames = get_file_names(input_path, 'json', in_prefix, essential=False) diff --git a/mantidimaging/core/utility/imat_log_file_parser.py b/mantidimaging/core/utility/imat_log_file_parser.py index a6eafe20192..64bd0a587ce 100644 --- a/mantidimaging/core/utility/imat_log_file_parser.py +++ b/mantidimaging/core/utility/imat_log_file_parser.py @@ -9,10 +9,6 @@ from mantidimaging.core.utility.data_containers import ProjectionAngles, Counts -EXPECTED_HEADER_FOR_IMAT_LOG_FILE = [ - 'TIME STAMP IMAGE TYPE', 'IMAGE COUNTER', 'COUNTS BM3 before image', 'COUNTS BM3 after image' -] - class IMATLogColumn(Enum): TIMESTAMP = auto() @@ -23,39 +19,80 @@ class IMATLogColumn(Enum): COUNTS_AFTER = auto() -class IMATLogFile: - def __init__(self, data: List[List[str]], source_file: str): - self._source_file = source_file - self._data: Dict[IMATLogColumn, List] = { +class TextLogParser: + EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE = \ + ' TIME STAMP IMAGE TYPE IMAGE COUNTER COUNTS BM3 before image COUNTS BM3 after image\n' + + def __init__(self, data: List[str]) -> None: + self.data = [line.strip().split(" ") for line in data] + + def parse(self) -> Dict[IMATLogColumn, List]: + parsed_log: Dict[IMATLogColumn, List] = { IMATLogColumn.TIMESTAMP: [], IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER: [], IMATLogColumn.COUNTS_BEFORE: [], IMATLogColumn.COUNTS_AFTER: [] } - if EXPECTED_HEADER_FOR_IMAT_LOG_FILE != data[0]: - raise RuntimeError( - "The Log file found for this dataset does not seem to have the correct header for an IMAT log file.\n" - "The header is expected to contain the names of the columns:\n" - f"{str(EXPECTED_HEADER_FOR_IMAT_LOG_FILE)}") - # ignores the headers (index 0) as they're not the same as the data anyway # and index 1 is an empty line - for line in data[2:]: - self._data[IMATLogColumn.TIMESTAMP].append(line[0]) - self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER].append(line[1]) - self._data[IMATLogColumn.COUNTS_BEFORE].append(line[2]) - self._data[IMATLogColumn.COUNTS_AFTER].append(line[3]) + for line in self.data[2:]: + parsed_log[IMATLogColumn.TIMESTAMP].append(line[0]) + parsed_log[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER].append(line[1]) + parsed_log[IMATLogColumn.COUNTS_BEFORE].append(line[2]) + parsed_log[IMATLogColumn.COUNTS_AFTER].append(line[3]) + + return parsed_log + + @staticmethod + def validate(file_contents) -> bool: + + if TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE != file_contents[0]: + return False + return True + + +class CSVLogParser: + EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE = \ + "TIME STAMP,IMAGE TYPE,IMAGE COUNTER,COUNTS BM3 before image,COUNTS BM3 after image" + + def __init__(self, data: List[str]) -> None: + pass + + @staticmethod + def validate(file_contents) -> bool: + if CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE != file_contents[0]: + return False + return True + + +class IMATLogFile: + def __init__(self, data: List[str], source_file: str): + self._source_file = source_file + + self.parser = self.find_parser(data) + self._data = self.parser.parse() + + @staticmethod + def find_parser(data: List[str]): + if TextLogParser.validate(data): + return TextLogParser(data) + elif CSVLogParser.validate(data): + return CSVLogParser(data) + else: + raise RuntimeError("The format of the log file is not recognised.") @property def source_file(self) -> str: return self._source_file def projection_numbers(self): - proj_nums = numpy.zeros(len(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]), dtype=numpy.uint32) + proj_nums = numpy.zeros(len(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]), + dtype=numpy.uint32) for i, angle_str in enumerate(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]): if "angle:" not in angle_str: - raise ValueError("Projection angles loaded from logfile do not have the correct formatting!") + raise ValueError( + "Projection angles loaded from logfile do not have the correct formatting!") proj_nums[i] = int(angle_str[angle_str.find(": ") + 1:angle_str.find("a")]) return proj_nums @@ -64,16 +101,17 @@ def projection_angles(self) -> ProjectionAngles: angles = numpy.zeros(len(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER])) for i, angle_str in enumerate(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]): if "angle:" not in angle_str: - raise ValueError("Projection angles loaded from logfile do not have the correct formatting!") + raise ValueError( + "Projection angles loaded from logfile do not have the correct formatting!") angles[i] = float(angle_str[angle_str.rfind(": ") + 1:]) return ProjectionAngles(numpy.deg2rad(angles)) def counts(self) -> Counts: counts = numpy.zeros(len(self._data[IMATLogColumn.COUNTS_BEFORE])) - for i, [before, - after] in enumerate(zip(self._data[IMATLogColumn.COUNTS_BEFORE], - self._data[IMATLogColumn.COUNTS_AFTER])): + for i, [before, after] in enumerate( + zip(self._data[IMATLogColumn.COUNTS_BEFORE], + self._data[IMATLogColumn.COUNTS_AFTER])): # clips the string before the count number before = before[before.rfind(":") + 1:] after = after[after.rfind(":") + 1:] diff --git a/mantidimaging/core/utility/test/imat_log_file_parser_test.py b/mantidimaging/core/utility/test/imat_log_file_parser_test.py index 5a7dba275d1..b61c8472302 100644 --- a/mantidimaging/core/utility/test/imat_log_file_parser_test.py +++ b/mantidimaging/core/utility/test/imat_log_file_parser_test.py @@ -3,27 +3,28 @@ import numpy as np -from mantidimaging.core.utility.imat_log_file_parser import IMATLogFile, EXPECTED_HEADER_FOR_IMAT_LOG_FILE +from mantidimaging.core.utility.imat_log_file_parser import IMATLogFile, TextLogParser def test_parsing_log_file(): test_input = [ - EXPECTED_HEADER_FOR_IMAT_LOG_FILE, ["ignored line"], - ["timestamp", "Projection: 0 angle: 0.1", "counts before: 12345", "counts_after: 45678"] + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, "ignored line", + "timestamp Projection: 0 angle: 0.1 counts before: 12345 counts_after: 45678" ] logfile = IMATLogFile(test_input, "/tmp/fake") assert len(logfile.projection_angles().value) == 1 - assert logfile.projection_angles().value[0] == np.deg2rad(0.1), f"Got: {logfile.projection_angles().value[0]}" + assert logfile.projection_angles().value[0] == np.deg2rad( + 0.1), f"Got: {logfile.projection_angles().value[0]}" assert logfile.counts().value[0] == (45678 - 12345) def test_counts(): test_input = [ - EXPECTED_HEADER_FOR_IMAT_LOG_FILE, - ["ignored line"], - ["timestamp", "Projection: 0 angle: 0.0", "counts before: 12345", "counts_after: 45678"], - ["timestamp", "Projection: 1 angle: 0.1", "counts before: 45678", "counts_after: 84678"], - ["timestamp", "Projection: 2 angle: 0.2", "counts before: 84678", "counts_after: 124333"], + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, + "ignored line", + "timestamp Projection: 0 angle: 0.0 counts before: 12345 counts_after: 45678", + "timestamp Projection: 1 angle: 0.1 counts before: 45678 counts_after: 84678", + "timestamp Projection: 2 angle: 0.2 counts before: 84678 counts_after: 124333", ] logfile = IMATLogFile(test_input, "/tmp/fake") assert len(logfile.counts().value) == 3 @@ -44,11 +45,11 @@ def assert_raises(exc_type, callable, *args, **kwargs): def test_find_missing_projection_number(): test_input = [ - EXPECTED_HEADER_FOR_IMAT_LOG_FILE, - ["ignored line"], - ["timestamp", "Projection: 0 angle: 0.0", "counts before: 12345", "counts_after: 45678"], - ["timestamp", "Projection: 1 angle: 0.1", "counts before: 12345", "counts_after: 45678"], - ["timestamp", "Projection: 2 angle: 0.2", "counts before: 12345", "counts_after: 45678"], + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, + "ignored line", + "timestamp Projection: 0 angle: 0.0 counts before: 12345 counts_after: 45678", + "timestamp Projection: 1 angle: 0.1 counts before: 12345 counts_after: 45678", + "timestamp Projection: 2 angle: 0.2 counts before: 12345 counts_after: 45678", ] logfile = IMATLogFile(test_input, "/tmp/fake") assert len(logfile.projection_numbers()) == 3 @@ -64,11 +65,11 @@ def test_find_missing_projection_number(): def test_source_file(): test_input = [ - EXPECTED_HEADER_FOR_IMAT_LOG_FILE, - ["ignored line"], - ["timestamp", "Projection: 0 angle: 0.0", "counts before: 12345", "counts_after: 45678"], - ["timestamp", "Projection: 1 angle: 0.1", "counts before: 12345", "counts_after: 45678"], - ["timestamp", "Projection: 2 angle: 0.2", "counts before: 12345", "counts_after: 45678"], + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, + "ignored line", + "timestamp Projection: 0 angle: 0.0 counts before: 12345 counts_after: 45678", + "timestamp Projection: 1 angle: 0.1 counts before: 12345 counts_after: 45678", + "timestamp Projection: 2 angle: 0.2 counts before: 12345 counts_after: 45678", ] logfile = IMATLogFile(test_input, "/tmp/fake") assert logfile.source_file == "/tmp/fake" From 6a452b8ae2e8792eba3e39a48470a4a82973bc47 Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Tue, 12 Jan 2021 15:48:06 +0000 Subject: [PATCH 03/10] Refactors log parser to move parsing into child classes Tests didn't need updating as this is an internal change --- .../core/utility/imat_log_file_parser.py | 72 +++++++++++++------ mantidimaging/gui/windows/load_dialog/view.py | 33 ++++++--- 2 files changed, 74 insertions(+), 31 deletions(-) diff --git a/mantidimaging/core/utility/imat_log_file_parser.py b/mantidimaging/core/utility/imat_log_file_parser.py index 64bd0a587ce..1af7653b9ef 100644 --- a/mantidimaging/core/utility/imat_log_file_parser.py +++ b/mantidimaging/core/utility/imat_log_file_parser.py @@ -1,13 +1,22 @@ # Copyright (C) 2020 ISIS Rutherford Appleton Laboratory UKRI # SPDX - License - Identifier: GPL-3.0-or-later +import csv from enum import Enum, auto from itertools import zip_longest -from typing import List, Dict +from typing import Dict, List import numpy -from mantidimaging.core.utility.data_containers import ProjectionAngles, Counts +from mantidimaging.core.utility.data_containers import Counts, ProjectionAngles + + +def _get_projection_number(s: str) -> int: + return int(s[s.find(": ") + 1:s.find("a")]) + + +def _get_angle(s: str) -> float: + return float(s[s.rfind(": ") + 1:]) class IMATLogColumn(Enum): @@ -15,6 +24,8 @@ class IMATLogColumn(Enum): # currently these 2 are the same column because # the log file formatting is inconsistent IMAGE_TYPE_IMAGE_COUNTER = auto() + PROJECTION_NUMBER = auto() + PROJECTION_ANGLE = auto() COUNTS_BEFORE = auto() COUNTS_AFTER = auto() @@ -29,16 +40,17 @@ def __init__(self, data: List[str]) -> None: def parse(self) -> Dict[IMATLogColumn, List]: parsed_log: Dict[IMATLogColumn, List] = { IMATLogColumn.TIMESTAMP: [], - IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER: [], + IMATLogColumn.PROJECTION_NUMBER: [], + IMATLogColumn.PROJECTION_ANGLE: [], IMATLogColumn.COUNTS_BEFORE: [], IMATLogColumn.COUNTS_AFTER: [] } - # ignores the headers (index 0) as they're not the same as the data anyway # and index 1 is an empty line for line in self.data[2:]: parsed_log[IMATLogColumn.TIMESTAMP].append(line[0]) - parsed_log[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER].append(line[1]) + parsed_log[IMATLogColumn.PROJECTION_NUMBER].append(_get_projection_number(line[1])) + parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(_get_angle(line[1])) parsed_log[IMATLogColumn.COUNTS_BEFORE].append(line[2]) parsed_log[IMATLogColumn.COUNTS_AFTER].append(line[3]) @@ -54,10 +66,38 @@ def validate(file_contents) -> bool: class CSVLogParser: EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE = \ - "TIME STAMP,IMAGE TYPE,IMAGE COUNTER,COUNTS BM3 before image,COUNTS BM3 after image" + "TIME STAMP,IMAGE TYPE,IMAGE COUNTER,COUNTS BM3 before image,COUNTS BM3 after image\n" def __init__(self, data: List[str]) -> None: - pass + self.data = data + + def parse(self) -> Dict[IMATLogColumn, List]: + parsed_log: Dict[IMATLogColumn, List] = { + IMATLogColumn.TIMESTAMP: [], + IMATLogColumn.PROJECTION_NUMBER: [], + IMATLogColumn.PROJECTION_ANGLE: [], + IMATLogColumn.COUNTS_BEFORE: [], + IMATLogColumn.COUNTS_AFTER: [] + } + + reader = csv.reader(self.data) + + # skip headings + next(reader) + + for row in reader: + parsed_log[IMATLogColumn.TIMESTAMP].append(row[0]) + parsed_log[IMATLogColumn.PROJECTION_NUMBER].append(row[2]) + angle_raw = row[3] + parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(_get_angle(angle_raw)) + + counts_before_raw = row[4] + parsed_log[IMATLogColumn.COUNTS_BEFORE].append(_get_angle(counts_before_raw)) + + counts_after_raw = row[5] + parsed_log[IMATLogColumn.COUNTS_AFTER].append(_get_angle(counts_after_raw)) + + return parsed_log @staticmethod def validate(file_contents) -> bool: @@ -87,24 +127,14 @@ def source_file(self) -> str: return self._source_file def projection_numbers(self): - proj_nums = numpy.zeros(len(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]), + proj_nums = numpy.zeros(len(self._data[IMATLogColumn.PROJECTION_NUMBER]), dtype=numpy.uint32) - for i, angle_str in enumerate(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]): - if "angle:" not in angle_str: - raise ValueError( - "Projection angles loaded from logfile do not have the correct formatting!") - proj_nums[i] = int(angle_str[angle_str.find(": ") + 1:angle_str.find("a")]) - + proj_nums[:] = self._data[IMATLogColumn.PROJECTION_NUMBER] return proj_nums def projection_angles(self) -> ProjectionAngles: - angles = numpy.zeros(len(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER])) - for i, angle_str in enumerate(self._data[IMATLogColumn.IMAGE_TYPE_IMAGE_COUNTER]): - if "angle:" not in angle_str: - raise ValueError( - "Projection angles loaded from logfile do not have the correct formatting!") - angles[i] = float(angle_str[angle_str.rfind(": ") + 1:]) - + angles = numpy.zeros(len(self._data[IMATLogColumn.PROJECTION_ANGLE])) + angles[:] = self._data[IMATLogColumn.PROJECTION_ANGLE] return ProjectionAngles(numpy.deg2rad(angles)) def counts(self) -> Counts: diff --git a/mantidimaging/gui/windows/load_dialog/view.py b/mantidimaging/gui/windows/load_dialog/view.py index c49d45aabee..78c4a5caca1 100644 --- a/mantidimaging/gui/windows/load_dialog/view.py +++ b/mantidimaging/gui/windows/load_dialog/view.py @@ -45,7 +45,8 @@ def __init__(self, parent): self.tree.setTabKeyNavigation(True) self.sample, self.select_sample = self.create_file_input(0) - self.select_sample.clicked.connect(lambda: self.presenter.notify(Notification.UPDATE_ALL_FIELDS)) + self.select_sample.clicked.connect( + lambda: self.presenter.notify(Notification.UPDATE_ALL_FIELDS)) self.flat_before, self.select_flat_before = self.create_file_input(1) self.select_flat_before.clicked.connect(lambda: self.presenter.notify( @@ -64,20 +65,32 @@ def __init__(self, parent): Notification.UPDATE_FLAT_OR_DARK, field=self.dark_after, name="Dark", suffix="After")) self.proj_180deg, self.select_proj_180deg = self.create_file_input(5) - self.select_proj_180deg.clicked.connect(lambda: self.presenter.notify( - Notification.UPDATE_SINGLE_FILE, field=self.proj_180deg, name="180 degree", is_image_file=True)) + self.select_proj_180deg.clicked.connect( + lambda: self.presenter.notify(Notification.UPDATE_SINGLE_FILE, + field=self.proj_180deg, + name="180 degree", + is_image_file=True)) self.sample_log, self.select_sample_log = self.create_file_input(6) - self.select_sample_log.clicked.connect(lambda: self.presenter.notify( - Notification.UPDATE_SAMPLE_LOG, field=self.sample_log, name="Sample Log", is_image_file=False)) + self.select_sample_log.clicked.connect( + lambda: self.presenter.notify(Notification.UPDATE_SAMPLE_LOG, + field=self.sample_log, + name="Sample Log", + is_image_file=False)) self.flat_before_log, self.select_flat_before_log = self.create_file_input(7) - self.select_flat_before_log.clicked.connect(lambda: self.presenter.notify( - Notification.UPDATE_SINGLE_FILE, field=self.flat_before_log, name="Flat Before Log", image_file=False)) + self.select_flat_before_log.clicked.connect( + lambda: self.presenter.notify(Notification.UPDATE_SINGLE_FILE, + field=self.flat_before_log, + name="Flat Before Log", + image_file=False)) self.flat_after_log, self.select_flat_after_log = self.create_file_input(8) - self.select_flat_after_log.clicked.connect(lambda: self.presenter.notify( - Notification.UPDATE_SINGLE_FILE, field=self.flat_after_log, name="Flat After Log", image_file=False)) + self.select_flat_after_log.clicked.connect( + lambda: self.presenter.notify(Notification.UPDATE_SINGLE_FILE, + field=self.flat_after_log, + name="Flat After Log", + image_file=False)) self.step_all.clicked.connect(self._set_all_step) self.step_preview.clicked.connect(self._set_preview_step) @@ -115,7 +128,7 @@ def select_file(caption: str, image_file=True) -> Optional[str]: file_filter = "Images (*.png *.jpg *.tif *.tiff *.fit *.fits)" else: # Assume text file - file_filter = "Log File (*.txt *.log)" + file_filter = "Log File (*.txt *.log *.csv)" selected_file, _ = Qt.QFileDialog.getOpenFileName(caption=caption, filter=f"{file_filter};;All (*.*)", initialFilter=file_filter) From 25d791dc199733f981f96fb8dfd8737aa2a982ed Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Tue, 12 Jan 2021 16:24:40 +0000 Subject: [PATCH 04/10] Adds CSV parsing --- .../core/utility/imat_log_file_parser.py | 23 ++++++++----------- 1 file changed, 10 insertions(+), 13 deletions(-) diff --git a/mantidimaging/core/utility/imat_log_file_parser.py b/mantidimaging/core/utility/imat_log_file_parser.py index 1af7653b9ef..84451bb8f84 100644 --- a/mantidimaging/core/utility/imat_log_file_parser.py +++ b/mantidimaging/core/utility/imat_log_file_parser.py @@ -12,11 +12,11 @@ def _get_projection_number(s: str) -> int: - return int(s[s.find(": ") + 1:s.find("a")]) + return int(s[s.find(":") + 1:s.find("a")].strip()) -def _get_angle(s: str) -> float: - return float(s[s.rfind(": ") + 1:]) +def _get_angle(s: str) -> str: + return s[s.rfind(":") + 1:].strip() class IMATLogColumn(Enum): @@ -50,15 +50,14 @@ def parse(self) -> Dict[IMATLogColumn, List]: for line in self.data[2:]: parsed_log[IMATLogColumn.TIMESTAMP].append(line[0]) parsed_log[IMATLogColumn.PROJECTION_NUMBER].append(_get_projection_number(line[1])) - parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(_get_angle(line[1])) - parsed_log[IMATLogColumn.COUNTS_BEFORE].append(line[2]) - parsed_log[IMATLogColumn.COUNTS_AFTER].append(line[3]) + parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(float(_get_angle(line[1]))) + parsed_log[IMATLogColumn.COUNTS_BEFORE].append(int(_get_angle(line[2]))) + parsed_log[IMATLogColumn.COUNTS_AFTER].append(int(_get_angle(line[3]))) return parsed_log @staticmethod def validate(file_contents) -> bool: - if TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE != file_contents[0]: return False return True @@ -87,15 +86,15 @@ def parse(self) -> Dict[IMATLogColumn, List]: for row in reader: parsed_log[IMATLogColumn.TIMESTAMP].append(row[0]) - parsed_log[IMATLogColumn.PROJECTION_NUMBER].append(row[2]) + parsed_log[IMATLogColumn.PROJECTION_NUMBER].append(int(row[2])) angle_raw = row[3] - parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(_get_angle(angle_raw)) + parsed_log[IMATLogColumn.PROJECTION_ANGLE].append(float(_get_angle(angle_raw))) counts_before_raw = row[4] - parsed_log[IMATLogColumn.COUNTS_BEFORE].append(_get_angle(counts_before_raw)) + parsed_log[IMATLogColumn.COUNTS_BEFORE].append(int(_get_angle(counts_before_raw))) counts_after_raw = row[5] - parsed_log[IMATLogColumn.COUNTS_AFTER].append(_get_angle(counts_after_raw)) + parsed_log[IMATLogColumn.COUNTS_AFTER].append(int(_get_angle(counts_after_raw))) return parsed_log @@ -143,8 +142,6 @@ def counts(self) -> Counts: zip(self._data[IMATLogColumn.COUNTS_BEFORE], self._data[IMATLogColumn.COUNTS_AFTER])): # clips the string before the count number - before = before[before.rfind(":") + 1:] - after = after[after.rfind(":") + 1:] counts[i] = float(after) - float(before) return Counts(counts) From d3262bb1a377a14a80030d8b3f9aad0cebc7c061 Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Tue, 12 Jan 2021 16:24:45 +0000 Subject: [PATCH 05/10] Adds tests for CSV parsing --- .../utility/test/imat_log_file_parser_test.py | 69 +++++++++++-------- 1 file changed, 41 insertions(+), 28 deletions(-) diff --git a/mantidimaging/core/utility/test/imat_log_file_parser_test.py b/mantidimaging/core/utility/test/imat_log_file_parser_test.py index b61c8472302..d8625a32986 100644 --- a/mantidimaging/core/utility/test/imat_log_file_parser_test.py +++ b/mantidimaging/core/utility/test/imat_log_file_parser_test.py @@ -2,15 +2,21 @@ # SPDX - License - Identifier: GPL-3.0-or-later import numpy as np +import pytest -from mantidimaging.core.utility.imat_log_file_parser import IMATLogFile, TextLogParser +from mantidimaging.core.utility.imat_log_file_parser import CSVLogParser, IMATLogFile, TextLogParser -def test_parsing_log_file(): - test_input = [ +@pytest.mark.parametrize( + 'test_input', [[ TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, "ignored line", "timestamp Projection: 0 angle: 0.1 counts before: 12345 counts_after: 45678" - ] + ], + [ + CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE, + "timestamp,Projection,0,angle:0.1,counts before: 12345,counts_after: 45678" + ]]) +def test_parsing_log_file(test_input): logfile = IMATLogFile(test_input, "/tmp/fake") assert len(logfile.projection_angles().value) == 1 assert logfile.projection_angles().value[0] == np.deg2rad( @@ -18,14 +24,23 @@ def test_parsing_log_file(): assert logfile.counts().value[0] == (45678 - 12345) -def test_counts(): - test_input = [ - TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, - "ignored line", - "timestamp Projection: 0 angle: 0.0 counts before: 12345 counts_after: 45678", - "timestamp Projection: 1 angle: 0.1 counts before: 45678 counts_after: 84678", - "timestamp Projection: 2 angle: 0.2 counts before: 84678 counts_after: 124333", - ] +TXT_LOG_FILE = [ + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, + "ignored line", + "timestamp Projection: 0 angle: 0.0 counts before: 12345 counts_after: 45678", + "timestamp Projection: 1 angle: 0.1 counts before: 45678 counts_after: 84678", + "timestamp Projection: 2 angle: 0.2 counts before: 84678 counts_after: 124333", +] +CSV_LOG_FILE = [ + CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE, + "timestamp,Projection,0,angle:0.0,counts before: 12345,counts_after: 45678", + "timestamp,Projection,1,angle:0.1,counts before: 45678,counts_after: 84678", + "timestamp,Projection,2,angle:0.2,counts before: 84678,counts_after: 124333", +] + + +@pytest.mark.parametrize('test_input', [TXT_LOG_FILE, CSV_LOG_FILE]) +def test_counts(test_input): logfile = IMATLogFile(test_input, "/tmp/fake") assert len(logfile.counts().value) == 3 assert logfile.counts().value[0] == 45678 - 12345 @@ -33,6 +48,16 @@ def test_counts(): assert logfile.counts().value[2] == 124333 - 84678 +def test_counts_compare(): + logfile = IMATLogFile(TXT_LOG_FILE, "/tmp/fake") + logfile_from_csv = IMATLogFile(CSV_LOG_FILE, "/tmp/fake") + + assert len(logfile.counts().value) == len(logfile_from_csv.counts().value) + assert logfile.counts().value[0] == logfile_from_csv.counts().value[0] + assert logfile.counts().value[1] == logfile_from_csv.counts().value[1] + assert logfile.counts().value[2] == logfile_from_csv.counts().value[2] + + def assert_raises(exc_type, callable, *args, **kwargs): try: callable(*args, **kwargs) @@ -43,14 +68,8 @@ def assert_raises(exc_type, callable, *args, **kwargs): assert False, "Did not raise expected exception." -def test_find_missing_projection_number(): - test_input = [ - TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, - "ignored line", - "timestamp Projection: 0 angle: 0.0 counts before: 12345 counts_after: 45678", - "timestamp Projection: 1 angle: 0.1 counts before: 12345 counts_after: 45678", - "timestamp Projection: 2 angle: 0.2 counts before: 12345 counts_after: 45678", - ] +@pytest.mark.parametrize('test_input', [TXT_LOG_FILE, CSV_LOG_FILE]) +def test_find_missing_projection_number(test_input): logfile = IMATLogFile(test_input, "/tmp/fake") assert len(logfile.projection_numbers()) == 3 # nothing missing @@ -63,13 +82,7 @@ def test_find_missing_projection_number(): ["file_000.tif", "file_001.tif", "file_002.tif", "file_003.tif"]) -def test_source_file(): - test_input = [ - TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, - "ignored line", - "timestamp Projection: 0 angle: 0.0 counts before: 12345 counts_after: 45678", - "timestamp Projection: 1 angle: 0.1 counts before: 12345 counts_after: 45678", - "timestamp Projection: 2 angle: 0.2 counts before: 12345 counts_after: 45678", - ] +@pytest.mark.parametrize('test_input', [TXT_LOG_FILE, CSV_LOG_FILE]) +def test_source_file(test_input): logfile = IMATLogFile(test_input, "/tmp/fake") assert logfile.source_file == "/tmp/fake" From 63713bce6037847ede22aff11a4560b0255399c0 Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Tue, 12 Jan 2021 16:42:23 +0000 Subject: [PATCH 06/10] Fixes showing error when calculating the preview for monitor normalisation --- .../monitor_normalisation/monitor_normalisation.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py b/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py index e3e508e1a09..1f34c5ab9b8 100644 --- a/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py +++ b/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py @@ -27,8 +27,13 @@ class MonitorNormalisation(BaseFilter): @staticmethod def filter_func(images: Images, cores=None, chunksize=None, progress=None) -> Images: + if images.num_projections == 1: + # we can't really compute the preview as the image stack copy + # passed in doesn't have the logfile in it + return images counts = images.counts() - if counts is None and images.num_projections > 1: + + if counts is None: raise RuntimeError("No loaded log values for this stack.") counts_val = counts.value / counts.value[0] From 94c1cc92af44cf57913d111c9857c811912a551b Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Wed, 13 Jan 2021 09:32:28 +0000 Subject: [PATCH 07/10] Formatting changes --- mantidimaging/core/io/loader/loader.py | 8 ++--- .../monitor_normalisation.py | 10 ++---- mantidimaging/gui/windows/load_dialog/view.py | 31 ++++++------------- 3 files changed, 14 insertions(+), 35 deletions(-) diff --git a/mantidimaging/core/io/loader/loader.py b/mantidimaging/core/io/loader/loader.py index 6642af724be..7745a057a1a 100644 --- a/mantidimaging/core/io/loader/loader.py +++ b/mantidimaging/core/io/loader/loader.py @@ -27,8 +27,7 @@ def _fitsread(filename): import astropy.io.fits as fits image = fits.open(filename) if len(image) < 1: - raise RuntimeError( - "Could not load at least one FITS image/table file from: {0}".format(filename)) + raise RuntimeError("Could not load at least one FITS image/table file from: {0}".format(filename)) # get the image data return image[0].data @@ -166,9 +165,8 @@ def load(input_path=None, else: load_func = _imread - dataset = img_loader.execute(load_func, input_file_names, input_path_flat_before, - input_path_flat_after, input_path_dark_before, - input_path_dark_after, in_format, dtype, indices, progress) + dataset = img_loader.execute(load_func, input_file_names, input_path_flat_before, input_path_flat_after, + input_path_dark_before, input_path_dark_after, in_format, dtype, indices, progress) # Search for and load metadata file metadata_found_filenames = get_file_names(input_path, 'json', in_prefix, essential=False) diff --git a/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py b/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py index 1f34c5ab9b8..b9d08396de3 100644 --- a/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py +++ b/mantidimaging/core/operations/monitor_normalisation/monitor_normalisation.py @@ -38,17 +38,11 @@ def filter_func(images: Images, cores=None, chunksize=None, progress=None) -> Im counts_val = counts.value / counts.value[0] div_partial = ptsm.create_partial(_divide_by_counts, fwd_function=ptsm.inplace) - images, _ = ptsm.execute(images.data, - counts_val, - div_partial, - cores, - chunksize, - progress=progress) + images, _ = ptsm.execute(images.data, counts_val, div_partial, cores, chunksize, progress=progress) return images @staticmethod - def register_gui(form: 'QFormLayout', on_change: Callable, - view: 'BaseMainWindowView') -> Dict[str, 'QWidget']: + def register_gui(form: 'QFormLayout', on_change: Callable, view: 'BaseMainWindowView') -> Dict[str, 'QWidget']: return {} @staticmethod diff --git a/mantidimaging/gui/windows/load_dialog/view.py b/mantidimaging/gui/windows/load_dialog/view.py index 78c4a5caca1..ce3877ca660 100644 --- a/mantidimaging/gui/windows/load_dialog/view.py +++ b/mantidimaging/gui/windows/load_dialog/view.py @@ -45,8 +45,7 @@ def __init__(self, parent): self.tree.setTabKeyNavigation(True) self.sample, self.select_sample = self.create_file_input(0) - self.select_sample.clicked.connect( - lambda: self.presenter.notify(Notification.UPDATE_ALL_FIELDS)) + self.select_sample.clicked.connect(lambda: self.presenter.notify(Notification.UPDATE_ALL_FIELDS)) self.flat_before, self.select_flat_before = self.create_file_input(1) self.select_flat_before.clicked.connect(lambda: self.presenter.notify( @@ -65,32 +64,20 @@ def __init__(self, parent): Notification.UPDATE_FLAT_OR_DARK, field=self.dark_after, name="Dark", suffix="After")) self.proj_180deg, self.select_proj_180deg = self.create_file_input(5) - self.select_proj_180deg.clicked.connect( - lambda: self.presenter.notify(Notification.UPDATE_SINGLE_FILE, - field=self.proj_180deg, - name="180 degree", - is_image_file=True)) + self.select_proj_180deg.clicked.connect(lambda: self.presenter.notify( + Notification.UPDATE_SINGLE_FILE, field=self.proj_180deg, name="180 degree", is_image_file=True)) self.sample_log, self.select_sample_log = self.create_file_input(6) - self.select_sample_log.clicked.connect( - lambda: self.presenter.notify(Notification.UPDATE_SAMPLE_LOG, - field=self.sample_log, - name="Sample Log", - is_image_file=False)) + self.select_sample_log.clicked.connect(lambda: self.presenter.notify( + Notification.UPDATE_SAMPLE_LOG, field=self.sample_log, name="Sample Log", is_image_file=False)) self.flat_before_log, self.select_flat_before_log = self.create_file_input(7) - self.select_flat_before_log.clicked.connect( - lambda: self.presenter.notify(Notification.UPDATE_SINGLE_FILE, - field=self.flat_before_log, - name="Flat Before Log", - image_file=False)) + self.select_flat_before_log.clicked.connect(lambda: self.presenter.notify( + Notification.UPDATE_SINGLE_FILE, field=self.flat_before_log, name="Flat Before Log", image_file=False)) self.flat_after_log, self.select_flat_after_log = self.create_file_input(8) - self.select_flat_after_log.clicked.connect( - lambda: self.presenter.notify(Notification.UPDATE_SINGLE_FILE, - field=self.flat_after_log, - name="Flat After Log", - image_file=False)) + self.select_flat_after_log.clicked.connect(lambda: self.presenter.notify( + Notification.UPDATE_SINGLE_FILE, field=self.flat_after_log, name="Flat After Log", image_file=False)) self.step_all.clicked.connect(self._set_all_step) self.step_preview.clicked.connect(self._set_preview_step) From f178437464af0da768867686bbfd991afc7e745e Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Wed, 13 Jan 2021 09:33:09 +0000 Subject: [PATCH 08/10] Adds some more tests for CSV parsing --- mantidimaging/core/data/test/fake_logfile.py | 77 ++++++++----------- mantidimaging/core/data/test/images_test.py | 25 ++++-- .../core/utility/imat_log_file_parser.py | 9 +-- .../utility/test/imat_log_file_parser_test.py | 20 +++-- 4 files changed, 65 insertions(+), 66 deletions(-) diff --git a/mantidimaging/core/data/test/fake_logfile.py b/mantidimaging/core/data/test/fake_logfile.py index 7d0e06a8aad..d879b783779 100644 --- a/mantidimaging/core/data/test/fake_logfile.py +++ b/mantidimaging/core/data/test/fake_logfile.py @@ -1,53 +1,42 @@ # Copyright (C) 2020 ISIS Rutherford Appleton Laboratory UKRI # SPDX - License - Identifier: GPL-3.0-or-later -from mantidimaging.core.utility.imat_log_file_parser import IMATLogFile, EXPECTED_HEADER_FOR_IMAT_LOG_FILE +from mantidimaging.core.utility.imat_log_file_parser import CSVLogParser, IMATLogFile, TextLogParser -def generate_logfile() -> IMATLogFile: +def generate_txt_logfile() -> IMATLogFile: data = [ - EXPECTED_HEADER_FOR_IMAT_LOG_FILE, # checked if exists, but skipped - [""], # skipped when parsing + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, # checked if exists, but skipped + "", # skipped when parsing # for each row a list with 4 entries is currently expected - [ - "Sun Feb 10 00:22:04 2019", "Projection: 0 angle: 0.0", "Monitor 3 before: 4577907", - "Monitor 3 after: 4720271" - ], - [ - "Sun Feb 10 00:22:37 2019", "Projection: 1 angle: 0.3152", "Monitor 3 before: 4729337", - "Monitor 3 after: 4871319" - ], - [ - "Sun Feb 10 00:23:10 2019", "Projection: 2 angle: 0.6304", "Monitor 3 before: 4879923", - "Monitor 3 after: 5022689" - ], - [ - "Sun Feb 10 00:23:43 2019", "Projection: 3 angle: 0.9456", "Monitor 3 before: 5031423", - "Monitor 3 after: 5172216" - ], - [ - "Sun Feb 10 00:24:16 2019", "Projection: 4 angle: 1.2608", "Monitor 3 before: 5180904", - "Monitor 3 after: 5322691" - ], - [ - "Sun Feb 10 00:24:49 2019", "Projection: 5 angle: 1.576", "Monitor 3 before: 5334225", - "Monitor 3 after: 5475239" - ], - [ - "Sun Feb 10 00:25:22 2019", "Projection: 6 angle: 1.8912", "Monitor 3 before: 5483964", - "Monitor 3 after: 5626608" - ], - [ - "Sun Feb 10 00:25:55 2019", "Projection: 7 angle: 2.2064", "Monitor 3 before: 5635673", - "Monitor 3 after: 5777316" - ], - [ - "Sun Feb 10 00:26:29 2019", "Projection: 8 angle: 2.5216", "Monitor 3 before: 5786535", - "Monitor 3 after: 5929002" - ], - [ - "Sun Feb 10 00:27:02 2019", "Projection: 9 angle: 2.8368", "Monitor 3 before: 5938142", - "Monitor 3 after: 6078866" - ] + "Sun Feb 10 00:22:04 2019 Projection: 0 angle: 0.0 Monitor 3 before: 4577907 Monitor 3 after: 4720271", # noqa: E501 + "Sun Feb 10 00:22:37 2019 Projection: 1 angle: 0.3152 Monitor 3 before: 4729337 Monitor 3 after: 4871319", # noqa: E501 + "Sun Feb 10 00:23:10 2019 Projection: 2 angle: 0.6304 Monitor 3 before: 4879923 Monitor 3 after: 5022689", # noqa: E501 + "Sun Feb 10 00:23:43 2019 Projection: 3 angle: 0.9456 Monitor 3 before: 5031423 Monitor 3 after: 5172216", # noqa: E501 + "Sun Feb 10 00:24:16 2019 Projection: 4 angle: 1.2608 Monitor 3 before: 5180904 Monitor 3 after: 5322691", # noqa: E501 + "Sun Feb 10 00:24:49 2019 Projection: 5 angle: 1.576 Monitor 3 before: 5334225 Monitor 3 after: 5475239", # noqa: E501 + "Sun Feb 10 00:25:22 2019 Projection: 6 angle: 1.8912 Monitor 3 before: 5483964 Monitor 3 after: 5626608", # noqa: E501 + "Sun Feb 10 00:25:55 2019 Projection: 7 angle: 2.2064 Monitor 3 before: 5635673 Monitor 3 after: 5777316", # noqa: E501 + "Sun Feb 10 00:26:29 2019 Projection: 8 angle: 2.5216 Monitor 3 before: 5786535 Monitor 3 after: 5929002", # noqa: E501 + "Sun Feb 10 00:27:02 2019 Projection: 9 angle: 2.8368 Monitor 3 before: 5938142 Monitor 3 after: 6078866", # noqa: E501 + ] + return IMATLogFile(data, "/tmp/fake") + + +def generate_csv_logfile() -> IMATLogFile: + data = [ + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, # checked if exists, but skipped + "", # skipped when parsing + # for each row a list with 4 entries is currently expected + "Sun Feb 10 00:22:04 2019,Projection,0,angle: 0.0,Monitor 3 before: 4577907,Monitor 3 after: 4720271", + "Sun Feb 10 00:22:37 2019,Projection,1,angle: 0.3152,Monitor 3 before: 4729337,Monitor 3 after: 4871319", + "Sun Feb 10 00:23:10 2019,Projection,2,angle: 0.6304,Monitor 3 before: 4879923,Monitor 3 after: 5022689", + "Sun Feb 10 00:23:43 2019,Projection,3,angle: 0.9456,Monitor 3 before: 5031423,Monitor 3 after: 5172216", + "Sun Feb 10 00:24:16 2019,Projection,4,angle: 1.2608,Monitor 3 before: 5180904,Monitor 3 after: 5322691", + "Sun Feb 10 00:24:49 2019,Projection,5,angle: 1.576,Monitor 3 before: 5334225,Monitor 3 after: 5475239", + "Sun Feb 10 00:25:22 2019,Projection,6,angle: 1.8912,Monitor 3 before: 5483964,Monitor 3 after: 5626608", + "Sun Feb 10 00:25:55 2019,Projection,7,angle: 2.2064,Monitor 3 before: 5635673,Monitor 3 after: 5777316", + "Sun Feb 10 00:26:29 2019,Projection,8,angle: 2.5216,Monitor 3 before: 5786535,Monitor 3 after: 5929002", + "Sun Feb 10 00:27:02 2019,Projection,9,angle: 2.8368,Monitor 3 before: 5938142,Monitor 3 after: 6078866", ] return IMATLogFile(data, "/tmp/fake") diff --git a/mantidimaging/core/data/test/images_test.py b/mantidimaging/core/data/test/images_test.py index 4a156f131de..58048ab6cf5 100644 --- a/mantidimaging/core/data/test/images_test.py +++ b/mantidimaging/core/data/test/images_test.py @@ -9,7 +9,7 @@ from six import StringIO from mantidimaging.core.data import Images -from mantidimaging.core.data.test.fake_logfile import generate_logfile +from mantidimaging.core.data.test.fake_logfile import generate_csv_logfile, generate_txt_logfile from mantidimaging.core.operations.crop_coords import CropCoordinatesFilter from mantidimaging.core.operation_history import const from mantidimaging.core.utility.sensible_roi import SensibleROI @@ -101,8 +101,9 @@ def test_copy_roi(self): self.assertEqual(cropped_copy, images.data[:, 0:5, 0:5]) self.assertEqual(len(cropped_copy.metadata[const.OPERATION_HISTORY]), 2) - self.assertEqual(cropped_copy.metadata[const.OPERATION_HISTORY][-1][const.OPERATION_DISPLAY_NAME], - CropCoordinatesFilter.filter_name) + self.assertEqual( + cropped_copy.metadata[const.OPERATION_HISTORY][-1][const.OPERATION_DISPLAY_NAME], + CropCoordinatesFilter.filter_name) # remove the extra crop operation cropped_copy.metadata[const.OPERATION_HISTORY].pop(-1) @@ -184,8 +185,20 @@ def test_create_empty_images(self): def test_get_projection_angles_from_logfile(self): images = generate_images() - images.log_file = generate_logfile() - expected = np.deg2rad(np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, 2.8368])) + images.log_file = generate_txt_logfile() + expected = np.deg2rad( + np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, + 2.8368])) + actual = images.projection_angles(360.0) + self.assertEqual(len(actual.value), len(expected)) + np.testing.assert_equal(actual.value, expected) + + def test_get_projection_angles_from_logfile_csv(self): + images = generate_images() + images.log_file = generate_csv_logfile() + expected = np.deg2rad( + np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, + 2.8368])) actual = images.projection_angles(360.0) self.assertEqual(len(actual.value), len(expected)) np.testing.assert_equal(actual.value, expected) @@ -202,7 +215,7 @@ def test_get_projection_angles_no_logfile(self): def test_metadata_gets_updated_with_logfile(self): images = generate_images() - images.log_file = generate_logfile() + images.log_file = generate_txt_logfile() self.assertEqual(images.log_file.source_file, images.metadata[const.LOG_FILE]) def test_set_projection_angles(self): diff --git a/mantidimaging/core/utility/imat_log_file_parser.py b/mantidimaging/core/utility/imat_log_file_parser.py index 84451bb8f84..4fccd08bddf 100644 --- a/mantidimaging/core/utility/imat_log_file_parser.py +++ b/mantidimaging/core/utility/imat_log_file_parser.py @@ -126,8 +126,7 @@ def source_file(self) -> str: return self._source_file def projection_numbers(self): - proj_nums = numpy.zeros(len(self._data[IMATLogColumn.PROJECTION_NUMBER]), - dtype=numpy.uint32) + proj_nums = numpy.zeros(len(self._data[IMATLogColumn.PROJECTION_NUMBER]), dtype=numpy.uint32) proj_nums[:] = self._data[IMATLogColumn.PROJECTION_NUMBER] return proj_nums @@ -138,9 +137,9 @@ def projection_angles(self) -> ProjectionAngles: def counts(self) -> Counts: counts = numpy.zeros(len(self._data[IMATLogColumn.COUNTS_BEFORE])) - for i, [before, after] in enumerate( - zip(self._data[IMATLogColumn.COUNTS_BEFORE], - self._data[IMATLogColumn.COUNTS_AFTER])): + for i, [before, + after] in enumerate(zip(self._data[IMATLogColumn.COUNTS_BEFORE], + self._data[IMATLogColumn.COUNTS_AFTER])): # clips the string before the count number counts[i] = float(after) - float(before) diff --git a/mantidimaging/core/utility/test/imat_log_file_parser_test.py b/mantidimaging/core/utility/test/imat_log_file_parser_test.py index d8625a32986..d4f18221a55 100644 --- a/mantidimaging/core/utility/test/imat_log_file_parser_test.py +++ b/mantidimaging/core/utility/test/imat_log_file_parser_test.py @@ -7,20 +7,18 @@ from mantidimaging.core.utility.imat_log_file_parser import CSVLogParser, IMATLogFile, TextLogParser -@pytest.mark.parametrize( - 'test_input', [[ - TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, "ignored line", - "timestamp Projection: 0 angle: 0.1 counts before: 12345 counts_after: 45678" - ], - [ - CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE, - "timestamp,Projection,0,angle:0.1,counts before: 12345,counts_after: 45678" - ]]) +@pytest.mark.parametrize('test_input', [[ + TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, "ignored line", + "timestamp Projection: 0 angle: 0.1 counts before: 12345 counts_after: 45678" +], + [ + CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE, + "timestamp,Projection,0,angle:0.1,counts before: 12345,counts_after: 45678" + ]]) def test_parsing_log_file(test_input): logfile = IMATLogFile(test_input, "/tmp/fake") assert len(logfile.projection_angles().value) == 1 - assert logfile.projection_angles().value[0] == np.deg2rad( - 0.1), f"Got: {logfile.projection_angles().value[0]}" + assert logfile.projection_angles().value[0] == np.deg2rad(0.1), f"Got: {logfile.projection_angles().value[0]}" assert logfile.counts().value[0] == (45678 - 12345) From 92bbd9a3af15424934d793878e715ebe0d911393 Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Wed, 13 Jan 2021 15:26:04 +0000 Subject: [PATCH 09/10] YAPF and a mypy hint improvement --- mantidimaging/core/data/test/fake_logfile.py | 4 +--- mantidimaging/core/data/test/images_test.py | 13 ++++--------- mantidimaging/gui/dialogs/cor_inspection/view.py | 4 +++- 3 files changed, 8 insertions(+), 13 deletions(-) diff --git a/mantidimaging/core/data/test/fake_logfile.py b/mantidimaging/core/data/test/fake_logfile.py index d879b783779..f0f393a0adc 100644 --- a/mantidimaging/core/data/test/fake_logfile.py +++ b/mantidimaging/core/data/test/fake_logfile.py @@ -25,9 +25,7 @@ def generate_txt_logfile() -> IMATLogFile: def generate_csv_logfile() -> IMATLogFile: data = [ - TextLogParser.EXPECTED_HEADER_FOR_IMAT_TEXT_LOG_FILE, # checked if exists, but skipped - "", # skipped when parsing - # for each row a list with 4 entries is currently expected + CSVLogParser.EXPECTED_HEADER_FOR_IMAT_CSV_LOG_FILE, "Sun Feb 10 00:22:04 2019,Projection,0,angle: 0.0,Monitor 3 before: 4577907,Monitor 3 after: 4720271", "Sun Feb 10 00:22:37 2019,Projection,1,angle: 0.3152,Monitor 3 before: 4729337,Monitor 3 after: 4871319", "Sun Feb 10 00:23:10 2019,Projection,2,angle: 0.6304,Monitor 3 before: 4879923,Monitor 3 after: 5022689", diff --git a/mantidimaging/core/data/test/images_test.py b/mantidimaging/core/data/test/images_test.py index 58048ab6cf5..ec4a3b4b541 100644 --- a/mantidimaging/core/data/test/images_test.py +++ b/mantidimaging/core/data/test/images_test.py @@ -101,9 +101,8 @@ def test_copy_roi(self): self.assertEqual(cropped_copy, images.data[:, 0:5, 0:5]) self.assertEqual(len(cropped_copy.metadata[const.OPERATION_HISTORY]), 2) - self.assertEqual( - cropped_copy.metadata[const.OPERATION_HISTORY][-1][const.OPERATION_DISPLAY_NAME], - CropCoordinatesFilter.filter_name) + self.assertEqual(cropped_copy.metadata[const.OPERATION_HISTORY][-1][const.OPERATION_DISPLAY_NAME], + CropCoordinatesFilter.filter_name) # remove the extra crop operation cropped_copy.metadata[const.OPERATION_HISTORY].pop(-1) @@ -186,9 +185,7 @@ def test_create_empty_images(self): def test_get_projection_angles_from_logfile(self): images = generate_images() images.log_file = generate_txt_logfile() - expected = np.deg2rad( - np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, - 2.8368])) + expected = np.deg2rad(np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, 2.8368])) actual = images.projection_angles(360.0) self.assertEqual(len(actual.value), len(expected)) np.testing.assert_equal(actual.value, expected) @@ -196,9 +193,7 @@ def test_get_projection_angles_from_logfile(self): def test_get_projection_angles_from_logfile_csv(self): images = generate_images() images.log_file = generate_csv_logfile() - expected = np.deg2rad( - np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, - 2.8368])) + expected = np.deg2rad(np.asarray([0.0, 0.3152, 0.6304, 0.9456, 1.2608, 1.576, 1.8912, 2.2064, 2.5216, 2.8368])) actual = images.projection_angles(360.0) self.assertEqual(len(actual.value), len(expected)) np.testing.assert_equal(actual.value, expected) diff --git a/mantidimaging/gui/dialogs/cor_inspection/view.py b/mantidimaging/gui/dialogs/cor_inspection/view.py index ea08fd6c47c..9940b2cc3f9 100644 --- a/mantidimaging/gui/dialogs/cor_inspection/view.py +++ b/mantidimaging/gui/dialogs/cor_inspection/view.py @@ -1,7 +1,7 @@ # Copyright (C) 2020 ISIS Rutherford Appleton Laboratory UKRI # SPDX - License - Identifier: GPL-3.0-or-later -from typing import List +from typing import List, Union import numpy as np from PyQt5.QtWidgets import QPushButton, QDoubleSpinBox, QSpinBox, QStackedWidget @@ -24,6 +24,8 @@ class CORInspectionDialogView(BaseDialogView): stepStackedWidget: QStackedWidget instructionStackedWidget: QStackedWidget + spin_box: Union[QSpinBox, QDoubleSpinBox] + def __init__(self, parent, images: Images, slice_index: int, initial_cor: ScalarCoR, recon_params: ReconstructionParameters, iters_mode: bool): super().__init__(parent, 'gui/ui/cor_inspection_dialog.ui') From 7270a24ba9615f03394c4a5a39aed316217c3457 Mon Sep 17 00:00:00 2001 From: Dimitar Tasev Date: Fri, 15 Jan 2021 09:27:46 +0000 Subject: [PATCH 10/10] Add CSV to log loading dialog file filter --- mantidimaging/gui/windows/main/view.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mantidimaging/gui/windows/main/view.py b/mantidimaging/gui/windows/main/view.py index 53628e949d0..d422f6bf961 100644 --- a/mantidimaging/gui/windows/main/view.py +++ b/mantidimaging/gui/windows/main/view.py @@ -171,7 +171,7 @@ def load_sample_log_dialog(self): stack_to_add_log_to = stack_selector.selected_stack # Open file dialog - file_filter = "Log File (*.txt *.log)" + file_filter = "Log File (*.txt *.log *.csv)" selected_file, _ = QFileDialog.getOpenFileName(caption="Log to be loaded", filter=f"{file_filter};;All (*.*)", initialFilter=file_filter)