Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Fix bugs found when processing new scaled trigger data. #382

Merged
merged 2 commits into from
Mar 21, 2024
Merged

Fix bugs found when processing new scaled trigger data. #382

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
742d555
Fix bug processing new scaled trigger data.
samaloney Feb 15, 2024
6041799
Refactor
samaloney Mar 5, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 6 additions & 5 deletions stixcore/products/common.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -35,11 +35,12 @@ def _get_compression_scheme(packets, nix):
S,K,M compression scheme parameters and names
"""
param = packets.get(nix)
skm = param[0].skm
values = np.array((skm[0].value, skm[1].value, skm[2].value), np.ubyte).reshape(1, -1)
skms = [p.skm for p in param]
comp_scheme = np.array([[skm[0].value, skm[1].value, skm[2].value] for skm in skms],
dtype=np.ubyte)

return values, {'NIXS': [skm[0].name, skm[1].name, skm[2].name],
'PCF_CURTX': [p.idb_info.PCF_CURTX for p in skm]}
return comp_scheme, {'NIXS': [skms[0][0].name, skms[0][1].name, skms[0][2].name],
'PCF_CURTX': [p.idb_info.PCF_CURTX for p in skms[0]]}


def _get_energy_bins(packets, nixlower, nixuppper):
Expand Down Expand Up @@ -324,7 +325,7 @@ def unscale_triggers(scaled_triggers, *, integration, detector_masks, ssid, fact
# BSD SPEC data are summed to total trigger (1 trigger value)
if ssid in {30, 31, 32, 24}:
n_group = active_trigger_groups.astype(int).sum()
n_int = integration.as_float().to_value(u.ds).reshape(-1, 1) # units of 0.1s
n_int = integration.as_float().to_value(u.ds).flatten() # units of 0.1s
# BSD pixel/vis data not summed (16 trigger values)
elif ssid in {21, 22, 23}:
n_group = active_trigger_groups.astype(int)
Expand Down
68 changes: 44 additions & 24 deletions stixcore/products/level0/quicklookL0.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,9 @@

__all__ = ['QLProduct', 'LightCurve', 'Background', 'Spectra']

# SKM for scaling
_SKM_SCALING = [0, 0, 7]

logger = get_logger(__name__)

QLNIX00405_off = 0.1
Expand Down Expand Up @@ -150,13 +153,19 @@ def from_levelb(cls, levelb, parent=''):
control.add_data('compression_scheme_triggers_skm',
_get_compression_scheme(packets, 'NIX00274'))

triggers = packets.get_value('NIX00274').T
triggers_var = packets.get_value('NIX00274', attr="error").T
if control['compression_scheme_triggers_skm'].tolist() == [[0, 0, 7]]:
logger.debug('Unscaling trigger ')
triggers, triggers_var = unscale_triggers(
triggers, integration=duration,
detector_masks=control['detector_mask'], ssid=levelb.ssid)
triggers = packets.get_value('NIX00274').flatten()
triggers_var = packets.get_value('NIX00274', attr="error").flatten()

cur_index = 0
for i, (num, skm) in enumerate(zip(control['num_samples'],
control['compression_scheme_triggers_skm'])):
if skm.tolist() == _SKM_SCALING:
cur_slice = slice(cur_index, cur_index+num)
logger.debug('Unscaling triggers')
triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers(
triggers[cur_slice], integration=duration[cur_slice],
detector_masks=control['detector_mask'][i:i+1], ssid=levelb.ssid)
cur_index += num
Comment on lines +160 to +168
Copy link
Collaborator Author

@samaloney samaloney Mar 21, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Possible optimisation and could be extended for the other science data is rather than looping over all packets loop over data where skm apply as usually only 1 skm and in the vast majority of cases the number of different skm << num of packets.


data = Data()
data['control_index'] = control_indices
Expand Down Expand Up @@ -241,17 +250,23 @@ def from_levelb(cls, levelb, parent=''):
control.add_data('compression_scheme_triggers_skm',
_get_compression_scheme(packets, 'NIX00274'))

triggers = packets.get_value('NIX00274').T
triggers_var = packets.get_value('NIX00274', attr="error").T
triggers = packets.get_value('NIX00274').flatten()
triggers_var = packets.get_value('NIX00274', attr="error").flatten()

# As fixed not sent in TM so hard code here BKC detector is index 9
dmask = np.zeros(shape=(1, 32), dtype=int)
dmask[0, 9] = 1
if control['compression_scheme_triggers_skm'].tolist() == [[0, 0, 7]]:
logger.debug('Unscaling trigger ')
triggers, triggers_var = unscale_triggers(
triggers, integration=duration, detector_masks=dmask,
ssid=levelb.ssid)

cur_index = 0
for i, (num, skm) in enumerate(zip(control['num_samples'],
control['compression_scheme_triggers_skm'])):
if skm.tolist() == _SKM_SCALING:
cur_slice = slice(cur_index, cur_index+num)
logger.debug('Unscaling triggers')
triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers(
triggers[cur_slice], integration=duration[cur_slice],
detector_masks=dmask, ssid=levelb.ssid)
cur_index += num

data = Data()
data['control_index'] = control_indices
Expand Down Expand Up @@ -334,21 +349,26 @@ def from_levelb(cls, levelb, parent=''):
counts_var = np.vstack(counts_var).T
counts = np.pad(counts, ((pad_before, pad_after), (0, 0)), constant_values=0)
counts_var = np.pad(counts_var, ((pad_before, pad_after), (0, 0)), constant_values=0)
triggers = packets.get_value('NIX00484').T
triggers_var = packets.get_value('NIX00484', attr='error').T
triggers = packets.get_value('NIX00484').flatten()
triggers_var = packets.get_value('NIX00484', attr='error').flatten()

triggers = np.pad(triggers, (pad_before, pad_after), mode='edge')
triggers_var = np.pad(triggers_var, (pad_before, pad_after), mode='edge')

# These are per detector spectra so n_acc is 1 by design and not in TM so hard code here
dmask = np.zeros(shape=(1, 32), dtype=int)
dmask[0, 9] = 1

if control['compression_scheme_triggers_skm'].tolist() == [[0, 0, 7]]:
logger.debug('Unscaling trigger ')
triggers, triggers_var = unscale_triggers(
triggers, integration=duration, detector_masks=dmask,
ssid=levelb.ssid)

triggers = np.pad(triggers, ((pad_before, pad_after), (0, 0)), mode='edge')
triggers_var = np.pad(triggers_var, ((pad_before, pad_after), (0, 0)), mode='edge')
cur_index = 0
for i, (num, skm) in enumerate(zip(control['num_samples'],
control['compression_scheme_triggers_skm'])):
if skm.tolist() == _SKM_SCALING:
cur_slice = slice(cur_index, cur_index+num)
logger.debug('Unscaling triggers')
triggers[cur_slice], triggers_var[cur_slice] = unscale_triggers(
triggers[cur_slice], integration=duration[i][0],
detector_masks=dmask, ssid=levelb.ssid)
cur_index += num

detector_index = np.pad(np.array(did, np.int16), (pad_before, pad_after), mode='edge')
num_integrations = np.pad(np.array(packets.get_value('NIX00485'), np.uint16),
Expand Down
24 changes: 12 additions & 12 deletions stixcore/products/level0/scienceL0.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -297,11 +297,11 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data,
def from_levelb(cls, levelb, parent=''):
packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent)

control.add_data('compression_scheme_counts_skm',
_get_compression_scheme(packets, 'NIX00260'))
c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00260')
control.add_data('compression_scheme_counts_skm', (c_skm[0].reshape(1, 3), c_skm_meta))

control.add_data('compression_scheme_triggers_skm',
_get_compression_scheme(packets, 'NIX00242'))
t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00242')
control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta))

data = Data()
try:
Expand Down Expand Up @@ -557,11 +557,11 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data,
def from_levelb(cls, levelb, parent=''):
packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent)

control.add_data('compression_scheme_counts_skm',
_get_compression_scheme(packets, 'NIX00263'))
c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00263')
control.add_data('compression_scheme_counts_skm', (c_skm[0].reshape(1, 3), c_skm_meta))

control.add_data('compression_scheme_triggers_skm',
_get_compression_scheme(packets, 'NIX00242'))
t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00242')
control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta))

data = Data()
try:
Expand Down Expand Up @@ -671,11 +671,11 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data,
def from_levelb(cls, levelb, parent=''):
packets, idb_versions, control = ScienceProduct.from_levelb(levelb, parent=parent)

control.add_data('compression_scheme_counts_skm',
_get_compression_scheme(packets, 'NIX00268'))
c_skm, c_skm_meta = _get_compression_scheme(packets, 'NIX00268')
control.add_data('compression_scheme_counts_skm', (c_skm[0].reshape(1, 3), c_skm_meta))

control.add_data('compression_scheme_triggers_skm',
_get_compression_scheme(packets, 'NIX00267'))
t_skm, t_skm_meta = _get_compression_scheme(packets, 'NIX00267')
control.add_data('compression_scheme_triggers_skm', (t_skm[0].reshape(1, 3), t_skm_meta))

control['detector_masks'] = np.unique(_get_detector_mask(packets)[0], axis=0)
control['detector_masks'] = fix_detector_mask(control, control['detector_masks'])
Expand Down
Loading