From 742d55554497b13954c4c2e9112759c2ce85319c Mon Sep 17 00:00:00 2001 From: Shane Maloney Date: Thu, 15 Feb 2024 15:53:38 +0000 Subject: [PATCH 1/2] Fix bug processing new scaled trigger data. * Allow for QL file to unscale by batch * Fix bug the replicate extra triggers in Spectrogram --- stixcore/products/common.py | 11 +++-- stixcore/products/level0/quicklookL0.py | 65 ++++++++++++++++--------- stixcore/products/level0/scienceL0.py | 24 ++++----- 3 files changed, 59 insertions(+), 41 deletions(-) diff --git a/stixcore/products/common.py b/stixcore/products/common.py index 31c1c60f..139cba6f 100644 --- a/stixcore/products/common.py +++ b/stixcore/products/common.py @@ -35,11 +35,12 @@ def _get_compression_scheme(packets, nix): S,K,M compression scheme parameters and names """ param = packets.get(nix) - skm = param[0].skm - values = np.array((skm[0].value, skm[1].value, skm[2].value), np.ubyte).reshape(1, -1) + skms = [p.skm for p in param] + comp_scheme = np.array([[skm[0].value, skm[1].value, skm[2].value] for skm in skms], + dtype=np.ubyte) - return values, {'NIXS': [skm[0].name, skm[1].name, skm[2].name], - 'PCF_CURTX': [p.idb_info.PCF_CURTX for p in skm]} + return comp_scheme, {'NIXS': [skms[0][0].name, skms[0][1].name, skms[0][2].name], + 'PCF_CURTX': [p.idb_info.PCF_CURTX for p in skms[0]]} def _get_energy_bins(packets, nixlower, nixuppper): @@ -324,7 +325,7 @@ def unscale_triggers(scaled_triggers, *, integration, detector_masks, ssid, fact # BSD SPEC data are summed to total trigger (1 trigger value) if ssid in {30, 31, 32, 24}: n_group = active_trigger_groups.astype(int).sum() - n_int = integration.as_float().to_value(u.ds).reshape(-1, 1) # units of 0.1s + n_int = integration.as_float().to_value(u.ds).flatten() # units of 0.1s # BSD pixel/vis data not summed (16 trigger values) elif ssid in {21, 22, 23}: n_group = active_trigger_groups.astype(int) diff --git a/stixcore/products/level0/quicklookL0.py b/stixcore/products/level0/quicklookL0.py index edb23db7..43a7ffc5 100644 --- a/stixcore/products/level0/quicklookL0.py +++ b/stixcore/products/level0/quicklookL0.py @@ -150,13 +150,19 @@ def from_levelb(cls, levelb, parent=''): control.add_data('compression_scheme_triggers_skm', _get_compression_scheme(packets, 'NIX00274')) - triggers = packets.get_value('NIX00274').T - triggers_var = packets.get_value('NIX00274', attr="error").T - if control['compression_scheme_triggers_skm'].tolist() == [[0, 0, 7]]: - logger.debug('Unscaling trigger ') - triggers, triggers_var = unscale_triggers( - triggers, integration=duration, - detector_masks=control['detector_mask'], ssid=levelb.ssid) + triggers = packets.get_value('NIX00274').flatten() + triggers_var = packets.get_value('NIX00274', attr="error").flatten() + + cur_index = 0 + for i, (num, skm) in enumerate(zip(control['num_samples'], + control['compression_scheme_triggers_skm'])): + if skm.tolist() == [0, 0, 7]: + cur_slice = slice(cur_index, cur_index+num) + logger.debug('Unscaling triggers') + triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers( + triggers[cur_slice], integration=duration[cur_slice], + detector_masks=control['detector_mask'][i:i+1], ssid=levelb.ssid) + cur_index += num data = Data() data['control_index'] = control_indices @@ -241,17 +247,23 @@ def from_levelb(cls, levelb, parent=''): control.add_data('compression_scheme_triggers_skm', _get_compression_scheme(packets, 'NIX00274')) - triggers = packets.get_value('NIX00274').T - triggers_var = packets.get_value('NIX00274', attr="error").T + triggers = packets.get_value('NIX00274').flatten() + triggers_var = packets.get_value('NIX00274', attr="error").flatten() # As fixed not sent in TM so hard code here BKC detector is index 9 dmask = np.zeros(shape=(1, 32), dtype=int) dmask[0, 9] = 1 - if control['compression_scheme_triggers_skm'].tolist() == [[0, 0, 7]]: - logger.debug('Unscaling trigger ') - triggers, triggers_var = unscale_triggers( - triggers, integration=duration, detector_masks=dmask, - ssid=levelb.ssid) + + cur_index = 0 + for i, (num, skm) in enumerate(zip(control['num_samples'], + control['compression_scheme_triggers_skm'])): + if skm.tolist() == [0, 0, 7]: + cur_slice = slice(cur_index, cur_index+num) + logger.debug('Unscaling triggers') + triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers( + triggers[cur_slice], integration=duration[cur_slice], + detector_masks=dmask, ssid=levelb.ssid) + cur_index += num data = Data() data['control_index'] = control_indices @@ -334,21 +346,26 @@ def from_levelb(cls, levelb, parent=''): counts_var = np.vstack(counts_var).T counts = np.pad(counts, ((pad_before, pad_after), (0, 0)), constant_values=0) counts_var = np.pad(counts_var, ((pad_before, pad_after), (0, 0)), constant_values=0) - triggers = packets.get_value('NIX00484').T - triggers_var = packets.get_value('NIX00484', attr='error').T + triggers = packets.get_value('NIX00484').flatten() + triggers_var = packets.get_value('NIX00484', attr='error').flatten() + + triggers = np.pad(triggers, (pad_before, pad_after), mode='edge') + triggers_var = np.pad(triggers_var, (pad_before, pad_after), mode='edge') # These are per detector spectra so n_acc is 1 by design and not in TM so hard code here dmask = np.zeros(shape=(1, 32), dtype=int) dmask[0, 9] = 1 - if control['compression_scheme_triggers_skm'].tolist() == [[0, 0, 7]]: - logger.debug('Unscaling trigger ') - triggers, triggers_var = unscale_triggers( - triggers, integration=duration, detector_masks=dmask, - ssid=levelb.ssid) - - triggers = np.pad(triggers, ((pad_before, pad_after), (0, 0)), mode='edge') - triggers_var = np.pad(triggers_var, ((pad_before, pad_after), (0, 0)), mode='edge') + cur_index = 0 + for i, (num, skm) in enumerate(zip(control['num_samples'], + control['compression_scheme_triggers_skm'])): + if skm.tolist() == [0, 0, 7]: + cur_slice = slice(cur_index, cur_index+num) + logger.debug('Unscaling triggers') + triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers( + triggers[cur_slice], integration=duration[i][0], + detector_masks=dmask, ssid=levelb.ssid) + cur_index += num detector_index = np.pad(np.array(did, np.int16), (pad_before, pad_after), mode='edge') num_integrations = np.pad(np.array(packets.get_value('NIX00485'), np.uint16), diff --git a/stixcore/products/level0/scienceL0.py b/stixcore/products/level0/scienceL0.py index ff105d23..eaaee22e 100644 --- a/stixcore/products/level0/scienceL0.py +++ b/stixcore/products/level0/scienceL0.py @@ -297,11 +297,11 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, def from_levelb(cls, levelb, parent=''): packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent) - control.add_data('compression_scheme_counts_skm', - _get_compression_scheme(packets, 'NIX00260')) + c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00260') + control.add_data('compression_scheme_counts_skm', (c_skm[0].reshape(1, 3), c_skm_meta)) - control.add_data('compression_scheme_triggers_skm', - _get_compression_scheme(packets, 'NIX00242')) + t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00242') + control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta)) data = Data() try: @@ -557,11 +557,11 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, def from_levelb(cls, levelb, parent=''): packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent) - control.add_data('compression_scheme_counts_skm', - _get_compression_scheme(packets, 'NIX00263')) + c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00263') + control.add_data('compression_scheme_counts_skm', (c_skm[0].reshape(1, 3), c_skm_meta)) - control.add_data('compression_scheme_triggers_skm', - _get_compression_scheme(packets, 'NIX00242')) + t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00242') + control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta)) data = Data() try: @@ -671,11 +671,11 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, def from_levelb(cls, levelb, parent=''): packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent) - control.add_data('compression_scheme_counts_skm', - _get_compression_scheme(packets, 'NIX00268')) + c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00268') + control.add_data('compression_scheme_counts_skm', (c_skm[0].reshape(1, 3), c_skm_meta)) - control.add_data('compression_scheme_triggers_skm', - _get_compression_scheme(packets, 'NIX00267')) + t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00267') + control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta)) control['detector_masks'] = np.unique(_get_detector_mask(packets)[0], axis=0) control['detector_masks'] = fix_detector_mask(control, control['detector_masks']) From 604179949c79b8866496e8c72b8751c8c4ef8192 Mon Sep 17 00:00:00 2001 From: Shane Maloney Date: Tue, 5 Mar 2024 12:41:51 +0000 Subject: [PATCH 2/2] Refactor --- stixcore/products/level0/quicklookL0.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/stixcore/products/level0/quicklookL0.py b/stixcore/products/level0/quicklookL0.py index 43a7ffc5..d27670ce 100644 --- a/stixcore/products/level0/quicklookL0.py +++ b/stixcore/products/level0/quicklookL0.py @@ -24,6 +24,9 @@ __all__ = ['QLProduct', 'LightCurve', 'Background', 'Spectra'] +# SKM for scaling +_SKM_SCALING = [0, 0, 7] + logger = get_logger(__name__) QLNIX00405_off = 0.1 @@ -156,7 +159,7 @@ def from_levelb(cls, levelb, parent=''): cur_index = 0 for i, (num, skm) in enumerate(zip(control['num_samples'], control['compression_scheme_triggers_skm'])): - if skm.tolist() == [0, 0, 7]: + if skm.tolist() == _SKM_SCALING: cur_slice = slice(cur_index, cur_index+num) logger.debug('Unscaling triggers') triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers( @@ -257,7 +260,7 @@ def from_levelb(cls, levelb, parent=''): cur_index = 0 for i, (num, skm) in enumerate(zip(control['num_samples'], control['compression_scheme_triggers_skm'])): - if skm.tolist() == [0, 0, 7]: + if skm.tolist() == _SKM_SCALING: cur_slice = slice(cur_index, cur_index+num) logger.debug('Unscaling triggers') triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers( @@ -359,7 +362,7 @@ def from_levelb(cls, levelb, parent=''): cur_index = 0 for i, (num, skm) in enumerate(zip(control['num_samples'], control['compression_scheme_triggers_skm'])): - if skm.tolist() == [0, 0, 7]: + if skm.tolist() == _SKM_SCALING: cur_slice = slice(cur_index, cur_index+num) logger.debug('Unscaling triggers') triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers(