From 529e7f06f15d6018ac8ed053a86a19696847edab Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:08:34 -0800 Subject: [PATCH 01/12] Add the ability to remove the flyback column in from_hdf5 function. --- python/stempy/io/sparse_array.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index 70e83e18..b80fa3a6 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -170,11 +170,13 @@ def _validate(self): raise Exception(msg) @classmethod - def from_hdf5(cls, filepath, **init_kwargs): + def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): """Create a SparseArray from a stempy HDF5 file :param filepath: the path to the HDF5 file :type filepath: str + :param keep_flyback: option to crop the flyback column during loading + :type keep_flyback: bool :param init_kwargs: any kwargs to forward to SparseArray.__init__() :type init_kwargs: dict @@ -188,12 +190,18 @@ def from_hdf5(cls, filepath, **init_kwargs): frames = f['electron_events/frames'] scan_positions_group = f['electron_events/scan_positions'] - - data = frames[()] scan_shape = [scan_positions_group.attrs[x] for x in ['Nx', 'Ny']] frame_shape = [frames.attrs[x] for x in ['Nx', 'Ny']] - - scan_positions = scan_positions_group[()] + + if keep_flyback: + data = frames[()] + else: + orig_indices = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) + crop_indices = np.delete(orig_indices, orig_indices[::scan_shape[1]]) + data = frames[crop_indices] + scan_shape[1] = scan_shape[1] - 1 + + # Load any metadata metadata = {} if 'metadata' in f: From 89f540f67d12c812c80bb7ecc031912161122b3e Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:15:56 -0800 Subject: [PATCH 02/12] Add commenting --- python/stempy/io/sparse_array.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index b80fa3a6..95d03888 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -194,13 +194,19 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): frame_shape = [frames.attrs[x] for x in ['Nx', 'Ny']] if keep_flyback: - data = frames[()] + data = frames[()] # load the full data set + scan_positions = scan_positions_group[()] else: + # Generate the original scan indices from the scan_shape orig_indices = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) + # Remove the indices of the last column crop_indices = np.delete(orig_indices, orig_indices[::scan_shape[1]]) + # Load only the data needed data = frames[crop_indices] + # Reduce the column shape by 1 scan_shape[1] = scan_shape[1] - 1 - + # Create the proper scan_positions without the flyback column + scan_positions = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) # Load any metadata metadata = {} From e77d92424d0dfaa8b4f2276f66bbd56dd02542f6 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:16:29 -0800 Subject: [PATCH 03/12] Test for removing flyback --- tests/test_sparse_array.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index 98cdf5e9..d387c3df 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -710,6 +710,12 @@ def compare_with_sparse(full, sparse): assert np.array_equal(m_array[[False, True], 0][0], position_one) +def test_keep_flyback(sparse_array_small): + flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + assert flyback.scan_shape[1] == 50 + no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + assert flyback.scan_shape[1] == 49 + # Test binning until this number TEST_BINNING_UNTIL = 33 From 0cc0b2796f65ee384b7010e5d5a7693f84f984f0 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:38:36 -0800 Subject: [PATCH 04/12] remove test for initial build --- tests/test_sparse_array.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index d387c3df..aad60778 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -711,10 +711,11 @@ def compare_with_sparse(full, sparse): def test_keep_flyback(sparse_array_small): - flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) - assert flyback.scan_shape[1] == 50 - no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) - assert flyback.scan_shape[1] == 49 + pass + #flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + #assert flyback.scan_shape[1] == 50 + #no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + #assert flyback.scan_shape[1] == 49 # Test binning until this number TEST_BINNING_UNTIL = 33 From 30aa5652e1aad9bc5349945fa8c2df39a84a96e7 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:08:34 -0800 Subject: [PATCH 05/12] Add the ability to remove the flyback column in from_hdf5 function. --- python/stempy/io/sparse_array.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index 70e83e18..b80fa3a6 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -170,11 +170,13 @@ def _validate(self): raise Exception(msg) @classmethod - def from_hdf5(cls, filepath, **init_kwargs): + def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): """Create a SparseArray from a stempy HDF5 file :param filepath: the path to the HDF5 file :type filepath: str + :param keep_flyback: option to crop the flyback column during loading + :type keep_flyback: bool :param init_kwargs: any kwargs to forward to SparseArray.__init__() :type init_kwargs: dict @@ -188,12 +190,18 @@ def from_hdf5(cls, filepath, **init_kwargs): frames = f['electron_events/frames'] scan_positions_group = f['electron_events/scan_positions'] - - data = frames[()] scan_shape = [scan_positions_group.attrs[x] for x in ['Nx', 'Ny']] frame_shape = [frames.attrs[x] for x in ['Nx', 'Ny']] - - scan_positions = scan_positions_group[()] + + if keep_flyback: + data = frames[()] + else: + orig_indices = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) + crop_indices = np.delete(orig_indices, orig_indices[::scan_shape[1]]) + data = frames[crop_indices] + scan_shape[1] = scan_shape[1] - 1 + + # Load any metadata metadata = {} if 'metadata' in f: From ced833633aabaea1c205322fa1380e046d5e1510 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:15:56 -0800 Subject: [PATCH 06/12] Add commenting --- python/stempy/io/sparse_array.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index b80fa3a6..95d03888 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -194,13 +194,19 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): frame_shape = [frames.attrs[x] for x in ['Nx', 'Ny']] if keep_flyback: - data = frames[()] + data = frames[()] # load the full data set + scan_positions = scan_positions_group[()] else: + # Generate the original scan indices from the scan_shape orig_indices = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) + # Remove the indices of the last column crop_indices = np.delete(orig_indices, orig_indices[::scan_shape[1]]) + # Load only the data needed data = frames[crop_indices] + # Reduce the column shape by 1 scan_shape[1] = scan_shape[1] - 1 - + # Create the proper scan_positions without the flyback column + scan_positions = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) # Load any metadata metadata = {} From 2db681c65fb9934eb55664c6c5a115be8213e8f3 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:16:29 -0800 Subject: [PATCH 07/12] Test for removing flyback --- tests/test_sparse_array.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index 98cdf5e9..d387c3df 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -710,6 +710,12 @@ def compare_with_sparse(full, sparse): assert np.array_equal(m_array[[False, True], 0][0], position_one) +def test_keep_flyback(sparse_array_small): + flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + assert flyback.scan_shape[1] == 50 + no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + assert flyback.scan_shape[1] == 49 + # Test binning until this number TEST_BINNING_UNTIL = 33 From cd5718c420cf1b564ba6f52d3d1d2908f2c2fe40 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 13:38:36 -0800 Subject: [PATCH 08/12] remove test for initial build --- tests/test_sparse_array.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index d387c3df..aad60778 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -711,10 +711,11 @@ def compare_with_sparse(full, sparse): def test_keep_flyback(sparse_array_small): - flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) - assert flyback.scan_shape[1] == 50 - no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) - assert flyback.scan_shape[1] == 49 + pass + #flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + #assert flyback.scan_shape[1] == 50 + #no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + #assert flyback.scan_shape[1] == 49 # Test binning until this number TEST_BINNING_UNTIL = 33 From 77d0eada1b124d00f47f15ee5a3fee30ab85957c Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 15:08:52 -0800 Subject: [PATCH 09/12] scan shape values were reversed. Fix that and delete the correct values before loading data. --- python/stempy/io/sparse_array.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index 95d03888..48c6e4e5 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -197,24 +197,32 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): data = frames[()] # load the full data set scan_positions = scan_positions_group[()] else: + print(scan_shape) # Generate the original scan indices from the scan_shape orig_indices = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) + print(orig_indices.shape) + print(orig_indices) # Remove the indices of the last column - crop_indices = np.delete(orig_indices, orig_indices[::scan_shape[1]]) + crop_indices = np.delete(orig_indices, orig_indices[scan_shape[0]-1::scan_shape[0]]) + print(crop_indices.shape) + print(crop_indices) # Load only the data needed data = frames[crop_indices] + print(data.shape) # Reduce the column shape by 1 - scan_shape[1] = scan_shape[1] - 1 + scan_shape[0] = scan_shape[0] - 1 # Create the proper scan_positions without the flyback column scan_positions = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) - + # Load any metadata metadata = {} if 'metadata' in f: load_h5_to_dict(f['metadata'], metadata) scan_shape = scan_shape[::-1] - + + print(scan_shape) + if version >= 3: # Convert to int to avoid integer division that results in # a float From 7b867402b23aeb2fea649b84484a7c980f5bb2f5 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 17:15:55 -0800 Subject: [PATCH 10/12] add proper keyword to load_electron_counts in io.__init__ --- python/stempy/io/__init__.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/python/stempy/io/__init__.py b/python/stempy/io/__init__.py index d9b9efd3..22ffc9a3 100644 --- a/python/stempy/io/__init__.py +++ b/python/stempy/io/__init__.py @@ -259,16 +259,17 @@ def save_electron_counts(path, array): """ array.write_to_hdf5(path) -def load_electron_counts(path): +def load_electron_counts(path, keep_flyback=True): """Load electron counted data from an HDF5 file. :param path: path to the HDF5 file. :type path: str - + :param keep_flyback: option to crop the flyback column during loading + :type keep_flyback: bool :return: a SparseArray containing the electron counted data :rtype: SparseArray """ - return SparseArray.from_hdf5(path) + return SparseArray.from_hdf5(path, keep_flyback=keep_flyback) def save_stem_images(outputFile, images, names): """Save STEM images to an HDF5 file. From e3b5dff63d3d3f144bbfde32ede02cbd3604e0e1 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Mon, 6 Nov 2023 17:22:20 -0800 Subject: [PATCH 11/12] remove printing lines. --- python/stempy/io/sparse_array.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/python/stempy/io/sparse_array.py b/python/stempy/io/sparse_array.py index 48c6e4e5..cec4a04b 100644 --- a/python/stempy/io/sparse_array.py +++ b/python/stempy/io/sparse_array.py @@ -197,18 +197,12 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): data = frames[()] # load the full data set scan_positions = scan_positions_group[()] else: - print(scan_shape) # Generate the original scan indices from the scan_shape orig_indices = np.ravel_multi_index([ii.ravel() for ii in np.indices(scan_shape)],scan_shape) - print(orig_indices.shape) - print(orig_indices) # Remove the indices of the last column crop_indices = np.delete(orig_indices, orig_indices[scan_shape[0]-1::scan_shape[0]]) - print(crop_indices.shape) - print(crop_indices) # Load only the data needed data = frames[crop_indices] - print(data.shape) # Reduce the column shape by 1 scan_shape[0] = scan_shape[0] - 1 # Create the proper scan_positions without the flyback column @@ -221,8 +215,6 @@ def from_hdf5(cls, filepath, keep_flyback=True, **init_kwargs): scan_shape = scan_shape[::-1] - print(scan_shape) - if version >= 3: # Convert to int to avoid integer division that results in # a float From 1bd57a6b7b459b21c00ff5a8fdb4d86130c5dc11 Mon Sep 17 00:00:00 2001 From: Peter Ercius ncem-gauss jupyter Date: Wed, 27 Mar 2024 08:50:53 -0700 Subject: [PATCH 12/12] add test for removing flyback when loading from HDF5 --- tests/test_sparse_array.py | 11 +++++------ 1 file changed, 5 insertions(+), 6 deletions(-) diff --git a/tests/test_sparse_array.py b/tests/test_sparse_array.py index aad60778..e088884c 100644 --- a/tests/test_sparse_array.py +++ b/tests/test_sparse_array.py @@ -710,12 +710,11 @@ def compare_with_sparse(full, sparse): assert np.array_equal(m_array[[False, True], 0][0], position_one) -def test_keep_flyback(sparse_array_small): - pass - #flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) - #assert flyback.scan_shape[1] == 50 - #no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) - #assert flyback.scan_shape[1] == 49 +def test_keep_flyback(electron_data_small): + flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=True) + assert flyback.scan_shape[1] == 50 + no_flyback = SparseArray.from_hdf5(electron_data_small, keep_flyback=False) + assert no_flyback.scan_shape[1] == 49 # Test binning until this number TEST_BINNING_UNTIL = 33