From 3b81c3fb8e39155493a68c799628cf44dd6d539e Mon Sep 17 00:00:00 2001 From: Aleksandar Topic Date: Fri, 25 Oct 2024 13:11:21 +0200 Subject: [PATCH 1/3] added novisim as metadata flag for h5 files For h5 files that are not related to novisim, the attribute "novisim" will have value 0, or otherwise 1. This avoids checks being hardcoding during the processing steps. There also seems to be an error in the string recieved for meta["ByteOrder"] which contains a whitespace and a semicolon. For now the quick fix was to strip the string and keep only the first part. --- src/lib/py/esrf_read.py | 3 ++- src/processing_steps/0200_generate_byte_hdf5.py | 2 ++ src/processing_steps/0225_generate_hdf5_novisim.py | 1 + src/processing_steps/0600_segment_implant_cc.py | 4 +++- 4 files changed, 8 insertions(+), 2 deletions(-) diff --git a/src/lib/py/esrf_read.py b/src/lib/py/esrf_read.py index cb6edee..d26746d 100644 --- a/src/lib/py/esrf_read.py +++ b/src/lib/py/esrf_read.py @@ -36,7 +36,8 @@ def esrf_edf_metadata(filename): if (len(kv) >= 2): meta[kv[0].strip()] = kv[1].strip() - assert meta["ByteOrder"] == "LowByteFirst" + # removing " ;" + assert meta["ByteOrder"].split()[0] == "LowByteFirst" if (meta["DataType"] == "UnsignedShort"): meta["NumpyType"] = np.uint16 diff --git a/src/processing_steps/0200_generate_byte_hdf5.py b/src/processing_steps/0200_generate_byte_hdf5.py index 9230beb..8ef7eaf 100755 --- a/src/processing_steps/0200_generate_byte_hdf5.py +++ b/src/processing_steps/0200_generate_byte_hdf5.py @@ -71,6 +71,8 @@ # Store metadata in both files for each subvolume scan for h5file in [h5file_msb,h5file_lsb]: + # never a novisim tomogram at this point, but added to all *.h5 files for later checks + h5file.attrs["novisim"] = 0 grp_meta = h5file.create_group("metadata") for i in range(len(subvolume_metadata)): subvolume_info = subvolume_metadata[i] diff --git a/src/processing_steps/0225_generate_hdf5_novisim.py b/src/processing_steps/0225_generate_hdf5_novisim.py index 0ffb4a1..30a8bbe 100644 --- a/src/processing_steps/0225_generate_hdf5_novisim.py +++ b/src/processing_steps/0225_generate_hdf5_novisim.py @@ -112,6 +112,7 @@ h5_msb = h5py.File(f'{output_dir}/msb/{args.sample}.h5', 'w') for h5, tomo in [(h5_lsb, tomo_lsb), (h5_msb, tomo_msb)]: + h5.attrs["novisim"] = 1 # set flag to mark file as novisim tomogram h5.create_dataset('subvolume_dimensions', (1,3), data=[[nz, ny, nx]], dtype=np.uint16) h5.create_dataset('subvolume_range', (1,2,), dtype=np.float32, data=np.array([0, 65535])) h5_tomo = h5.create_dataset('voxels', (nz, ny, nx), dtype=np.uint8, data=tomo) diff --git a/src/processing_steps/0600_segment_implant_cc.py b/src/processing_steps/0600_segment_implant_cc.py index 20e5d1f..2e7ba03 100644 --- a/src/processing_steps/0600_segment_implant_cc.py +++ b/src/processing_steps/0600_segment_implant_cc.py @@ -42,7 +42,9 @@ global_vmax = np.max(h5meta['subvolume_range'][:,1]) values = np.linspace(global_vmin,global_vmax,2**16) implant_threshold_u16 = np.argmin(np.abs(values-implant_threshold)) - if 'novisim' in args.sample: + + # check if chosen sample has novisim flag set in metadata + if h5meta["novisim"]: implant_threshold_u16 = implant_threshold_u16_novisim if args.verbose >= 2: print(f""" From 2f1aa0f7555b33ad574e62d4c2ae60d5f69d3416 Mon Sep 17 00:00:00 2001 From: Aleksandar Topic Date: Fri, 25 Oct 2024 13:43:18 +0200 Subject: [PATCH 2/3] found 2 other occurances og hardcoded check --- src/processing_steps/0750_bone_region.py | 3 ++- src/processing_steps/1000_compute_histograms.py | 3 ++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/processing_steps/0750_bone_region.py b/src/processing_steps/0750_bone_region.py index 3ffd68d..7babef9 100644 --- a/src/processing_steps/0750_bone_region.py +++ b/src/processing_steps/0750_bone_region.py @@ -33,6 +33,7 @@ implant_file = h5py.File(f"{hdf5_root}/masks/{args.sample_scale}x/{args.sample}.h5",'r') implant = implant_file["implant/mask"][:].astype(np.uint8) voxel_size = implant_file["implant"].attrs["voxel_size"] + novisimflag = implant_file.attrs["novisim"] implant_file.close() nz, ny, nx = implant.shape @@ -132,7 +133,7 @@ del front_part if args.plotting: plot_middle_planes(bone_mask1, plotting_dir, 'implant-bone1-sanity', verbose=args.verbose) - if 'novisim' in args.sample: + if novisimflag: closing_diameter, opening_diameter, implant_dilate_diameter = 400, 300, 15 # micrometers else: closing_diameter, opening_diameter, implant_dilate_diameter = 400, 300, 5 # micrometers diff --git a/src/processing_steps/1000_compute_histograms.py b/src/processing_steps/1000_compute_histograms.py index fc82698..e72090d 100755 --- a/src/processing_steps/1000_compute_histograms.py +++ b/src/processing_steps/1000_compute_histograms.py @@ -532,7 +532,8 @@ def run_out_of_core(sample, scale=1, chunk_size=128, z_offset=0, n_chunks=0, if args.verbose >= 1: print(f"Loaded {gb:.02f} GB in {end-start} ({gb / (end-start).total_seconds()} GB/s)") verify_and_benchmark(voxels, field, plotting_dir, args.voxel_bins // args.sample_scale, args.benchmark_runs, args.verbose) else: - if 'novisim' in args.sample: + h5meta = h5py.File(f'{hdf5_root}/hdf5-byte/msb/{args.sample}.h5', 'r') + if h5meta["novisim"]: implant_threshold = 40000 else: implant_threshold = implant_threshold_u16 From 4abddcf75458a85fe8d968ddbfec57959d3e561b Mon Sep 17 00:00:00 2001 From: Aleksandar Topic Date: Fri, 25 Oct 2024 13:45:26 +0200 Subject: [PATCH 3/3] rephrased comment --- src/processing_steps/0200_generate_byte_hdf5.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/processing_steps/0200_generate_byte_hdf5.py b/src/processing_steps/0200_generate_byte_hdf5.py index 8ef7eaf..8cdc3df 100755 --- a/src/processing_steps/0200_generate_byte_hdf5.py +++ b/src/processing_steps/0200_generate_byte_hdf5.py @@ -71,7 +71,7 @@ # Store metadata in both files for each subvolume scan for h5file in [h5file_msb,h5file_lsb]: - # never a novisim tomogram at this point, but added to all *.h5 files for later checks + # h5 file is marked as not being related to novisim. This is used in later checks. h5file.attrs["novisim"] = 0 grp_meta = h5file.create_group("metadata") for i in range(len(subvolume_metadata)):