From d9c5387a48e13c720c149347b3a2c66c6ee258a3 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Tue, 19 Dec 2023 09:38:39 +0100 Subject: [PATCH 1/7] bump all product versions to 2 --- stixcore/products/product.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 23fead36..b0183de1 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -117,7 +117,7 @@ class BaseProduct: Base TMProduct that all other product inherit from contains the registry for the factory pattern """ - PRODUCT_PROCESSING_VERSION = 1 + PRODUCT_PROCESSING_VERSION = 2 _registry = {} From 5d6b6bd496e835e69127b3d1ba95f8ce3be9f513 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 13 May 2024 16:57:18 +0200 Subject: [PATCH 2/7] add properties for dedicated fits header keywords with class inhe. / override --- stixcore/io/fits/processors.py | 34 ++++--------- stixcore/processing/find.py | 2 +- stixcore/products/level0/quicklookL0.py | 59 +++++++++++++++++++++- stixcore/products/level0/scienceL0.py | 36 ++++++++++++- stixcore/products/level1/quicklookL1.py | 34 +++++++++++++ stixcore/products/level1/scienceL1.py | 33 ++++++++++++ stixcore/products/level2/housekeepingL2.py | 12 +++++ stixcore/products/product.py | 38 ++++++++++++++ 8 files changed, 221 insertions(+), 27 deletions(-) diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index cfb433e5..2e803bc5 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -695,13 +695,6 @@ def generate_primary_header(cls, filename, product, *, version=0): # raise ValueError(f"Try to crate FITS file L0 for {product.level} data product") # if not isinstance(product.obt_beg, SCETime): # raise ValueError("Expected SCETime as time format") - dmin = 0.0 - dmax = 0.0 - bunit = ' ' - if 'counts' in product.data.colnames: - dmax = product.data['counts'].max().value - dmin = product.data['counts'].min().value - bunit = 'counts' headers = FitsProcessor.generate_common_header(filename, product, version=version) + ( # Name, Value, Comment @@ -716,9 +709,9 @@ def generate_primary_header(cls, filename, product, *, version=0): ('DATE-BEG', product.scet_timerange.start.to_string(), 'Start time of observation'), ('DATE-AVG', product.scet_timerange.avg.to_string(), 'Average time of observation'), ('DATE-END', product.scet_timerange.end.to_string(), 'End time of observation'), - ('DATAMIN', dmin, 'Minimum valid physical value'), - ('DATAMAX', dmax, 'Maximum valid physical value'), - ('BUNIT', bunit, 'Units of physical value, after application of BSCALE, BZERO') + ('DATAMIN', product.dmin, 'Minimum valid physical value'), + ('DATAMAX', product.dmax, 'Maximum valid physical value'), + ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO') ) return headers @@ -745,20 +738,11 @@ def generate_primary_header(self, filename, product, *, version=0): headers = FitsProcessor.generate_common_header(filename, product, version=version) - dmin = 0.0 - dmax = 0.0 - bunit = ' ' - exposure = 0.0 - if 'counts' in product.data.colnames: - dmax = product.data['counts'].max().value - dmin = product.data['counts'].min().value - bunit = 'counts' - exposure = product.data['timedel'].as_float().min().to_value('s') data_headers = ( - ('DATAMIN', dmin, 'Minimum valid physical value'), - ('DATAMAX', dmax, 'Maximum valid physical value'), - ('BUNIT', bunit, 'Units of physical value, after application of BSCALE, BZERO'), - ('XPOSURE', exposure, '[s] shortest exposure time') + ('DATAMIN', product.dmin, 'Minimum valid physical value'), + ('DATAMAX', product.dmax, 'Maximum valid physical value'), + ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), + ('XPOSURE', product.exposure, '[s] shortest exposure time') ) soop_keywords = SOOPManager.instance.get_keywords(start=product.utc_timerange.start, @@ -972,6 +956,10 @@ def generate_primary_header(self, filename, product, *, version=0): ('VERS_CFG', str(stixcore.__version_conf__), 'Version of the common instrument configuration package'), ('HISTORY', 'Processed by STIXCore L2'), + ('DATAMIN', product.dmin, 'Minimum valid physical value'), + ('DATAMAX', product.dmax, 'Maximum valid physical value'), + ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), + ('XPOSURE', product.exposure, '[s] shortest exposure time') ) return L1headers, L2headers diff --git a/stixcore/processing/find.py b/stixcore/processing/find.py index 63941751..1ecebba4 100644 --- a/stixcore/processing/find.py +++ b/stixcore/processing/find.py @@ -171,7 +171,7 @@ def find_fits(args): logger.info(f'#candidates: {n_candidates}') logger.info(f'#found: {len(found)}') - for f in found: + for f in sorted(found): print(str(f)) if args.copy_dest: fits_parent_path = f.relative_to(fits_dir) diff --git a/stixcore/products/level0/quicklookL0.py b/stixcore/products/level0/quicklookL0.py index d27670ce..9166951d 100644 --- a/stixcore/products/level0/quicklookL0.py +++ b/stixcore/products/level0/quicklookL0.py @@ -18,7 +18,13 @@ rebin_proportional, unscale_triggers, ) -from stixcore.products.product import Control, Data, EnergyChannelsMixin, GenericProduct +from stixcore.products.product import ( + Control, + CountDataMixin, + Data, + EnergyChannelsMixin, + GenericProduct, +) from stixcore.time import SCETime, SCETimeDelta, SCETimeRange from stixcore.util.logging import get_logger @@ -32,7 +38,7 @@ QLNIX00405_off = 0.1 -class QLProduct(GenericProduct, EnergyChannelsMixin): +class QLProduct(CountDataMixin, GenericProduct, EnergyChannelsMixin): """Generic QL product class composed of control and data.""" def __init__(self, *, service_type, service_subtype, ssid, control, data, idb_versions=defaultdict(SCETimeRange), **kwargs): @@ -400,6 +406,14 @@ def from_levelb(cls, levelb, parent=''): idb_versions=idb_versions, packets=packets) + @property + def dmin(self): + return self.data['spectra'].min().value + + @property + def dmax(self): + return self.data['spectra'].max().value + @classmethod def _get_time(cls, control, num_energies, packets, pad_before, pad_after): times = [] @@ -494,6 +508,19 @@ def from_levelb(cls, levelb, parent=''): idb_versions=idb_versions, packets=packets) + @property + def dmin(self): + return self.data['variance'].min() + + @property + def dmax(self): + return self.data['variance'].max() + + @property + def bunit(self): + # TODO define + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L0' and service_type == 21 @@ -554,6 +581,19 @@ def from_levelb(cls, levelb, parent=''): idb_versions=idb_versions, packets=packets) + @property + def dmin(self): + return min([self.data['loc_y'].min(), self.data['loc_z'].min()]) + + @property + def dmax(self): + return max([self.data['loc_y'].max(), self.data['loc_z'].max()]) + + @property + def bunit(self): + # TODO define + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L0' and service_type == 21 @@ -784,6 +824,21 @@ def from_levelb(cls, levelb, parent=''): idb_versions=idb_versions, packets=packets) + @property + def dmin(self): + # TODO define + return 0.0 + + @property + def dmax(self): + # TODO define + return 0.0 + + @property + def bunit(self): + # TODO define + return '' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L0' and service_type == 21 diff --git a/stixcore/products/level0/scienceL0.py b/stixcore/products/level0/scienceL0.py index 82370326..b7bd598b 100644 --- a/stixcore/products/level0/scienceL0.py +++ b/stixcore/products/level0/scienceL0.py @@ -16,6 +16,7 @@ ) from stixcore.products.product import ( ControlSci, + CountDataMixin, Data, EnergyChannelsMixin, FitsHeaderMixin, @@ -71,7 +72,7 @@ class NotCombineException(Exception): pass -class ScienceProduct(GenericProduct, EnergyChannelsMixin, FitsHeaderMixin): +class ScienceProduct(CountDataMixin, GenericProduct, EnergyChannelsMixin, FitsHeaderMixin): """Generic science data product class composed of control and data.""" def __init__(self, *, service_type, service_subtype, ssid, control, data, **kwargs): """Create a generic science data product composed of control and data. @@ -682,6 +683,21 @@ def from_levelb(cls, levelb, parent=''): prod.add_additional_header_keywords(additional_header_keywords) return prod + @property + def dmin(self): + # TODO define columns for dmin/max + return 0.0 + + @property + def dmax(self): + # TODO define columns for dmin/max + return 0.0 + + @property + def bunit(self): + # TODO define columns for dmin/max + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L0' and service_type == 21 @@ -938,6 +954,24 @@ def from_levelb(cls, levelb, parent=''): idb_versions=idb_versions, packets=packets) + @property + def dmin(self): + return min([self.data['cha_diode0'].min(), + self.data['cha_diode1'].min(), + self.data['chb_diode0'].min(), + self.data['chb_diode1'].min()]) + + @property + def dmax(self): + return max([self.data['cha_diode0'].max(), + self.data['cha_diode1'].max(), + self.data['chb_diode0'].max(), + self.data['chb_diode1'].max()]) + + @property + def bunit(self): + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L0' and service_type == 21 diff --git a/stixcore/products/level1/quicklookL1.py b/stixcore/products/level1/quicklookL1.py index deba16a9..df4564c7 100644 --- a/stixcore/products/level1/quicklookL1.py +++ b/stixcore/products/level1/quicklookL1.py @@ -69,6 +69,14 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, self.name = 'spectra' self.level = 'L1' + @property + def dmin(self): + return self.data['spectra'].min().value + + @property + def dmax(self): + return self.data['spectra'].max().value + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L1' and service_type == 21 @@ -89,6 +97,19 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, self.name = 'variance' self.level = 'L1' + @property + def dmin(self): + return self.data['variance'].min() + + @property + def dmax(self): + return self.data['variance'].max() + + @property + def bunit(self): + # TODO define + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L1' and service_type == 21 @@ -109,6 +130,19 @@ def __init__(self, *, service_type, service_subtype, ssid, control, data, self.name = 'flareflag' self.level = 'L1' + @property + def dmin(self): + return min([self.data['loc_y'].min(), self.data['loc_z'].min()]) + + @property + def dmax(self): + return max([self.data['loc_y'].max(), self.data['loc_z'].max()]) + + @property + def bunit(self): + # TODO define + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L1' and service_type == 21 diff --git a/stixcore/products/level1/scienceL1.py b/stixcore/products/level1/scienceL1.py index 1f128680..dc7c529b 100644 --- a/stixcore/products/level1/scienceL1.py +++ b/stixcore/products/level1/scienceL1.py @@ -75,6 +75,21 @@ def __init__(self, *, service_type, service_subtype, ssid, control, self.name = 'xray-vis' self.level = 'L1' + @property + def dmin(self): + # TODO define columns for dmin/max + return 0.0 + + @property + def dmax(self): + # TODO define columns for dmin/max + return 0.0 + + @property + def bunit(self): + # TODO define columns for dmin/max + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L1' and service_type == 21 @@ -113,6 +128,24 @@ def __init__(self, *, service_type, service_subtype, ssid, control, self.name = 'aspect-burst' self.level = 'L1' + @property + def dmin(self): + return min([self.data['cha_diode0'].min(), + self.data['cha_diode1'].min(), + self.data['chb_diode0'].min(), + self.data['chb_diode1'].min()]) + + @property + def dmax(self): + return max([self.data['cha_diode0'].max(), + self.data['cha_diode1'].max(), + self.data['chb_diode0'].max(), + self.data['chb_diode1'].max()]) + + @property + def bunit(self): + return ' ' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L1' and service_type == 21 diff --git a/stixcore/products/level2/housekeepingL2.py b/stixcore/products/level2/housekeepingL2.py index a993f4a4..0cbc9126 100644 --- a/stixcore/products/level2/housekeepingL2.py +++ b/stixcore/products/level2/housekeepingL2.py @@ -342,6 +342,18 @@ def __init__(self, *, service_type=0, service_subtype=0, ssid=1, control, data, self.service_subtype = 0 self.service_type = 0 + @property + def dmin(self): + return min([self.data['y_srf'].min().value, self.data['z_srf'].min().value]) + + @property + def dmax(self): + return max([self.data['y_srf'].max().value, self.data['z_srf'].max().value]) + + @property + def bunit(self): + return 'arcsec' + @classmethod def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs): return (kwargs['level'] == 'L2' and service_type == 0 diff --git a/stixcore/products/product.py b/stixcore/products/product.py index 01185aa6..ead4bdcf 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -497,6 +497,26 @@ def raw(self): def parent(self): return np.unique(self.control['parent']).tolist() + @property + def dmin(self): + # default for FITS HEADER + return 0.0 + + @property + def dmax(self): + # default for FITS HEADER + return 0.0 + + @property + def bunit(self): + # default for FITS HEADER + return ' ' + + @property + def exposure(self): + # default for FITS HEADER + return 0.0 + def find_parent_products(self, root): """ Conveniant way to get access to the parent products. @@ -743,6 +763,24 @@ def fits_daily_file(self): return False +class CountDataMixin: + @property + def dmin(self): + return self.data['counts'].min().value + + @property + def dmax(self): + return self.data['counts'].max().value + + @property + def bunit(self): + return 'counts' + + @property + def exposure(self): + return self.data['timedel'].as_float().min().to_value('s') + + class EnergyChannelsMixin: def get_energies(self): """Return energy channels for this TM data. Or default ones. From 11f80d7a6b68beef50a9df7cbfbb5cfb764ac6f4 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Mon, 30 Sep 2024 17:21:23 +0200 Subject: [PATCH 3/7] add test and minor fixes after review --- stixcore/data/test/publish/rid_lut.csv | 4 ++-- stixcore/io/fits/processors.py | 6 ++++- stixcore/io/fits/tests/test_processors.py | 27 ++++++++++++++++++++++ stixcore/products/level2/housekeepingL2.py | 6 +++-- stixcore/products/product.py | 9 ++++++++ 5 files changed, 47 insertions(+), 5 deletions(-) diff --git a/stixcore/data/test/publish/rid_lut.csv b/stixcore/data/test/publish/rid_lut.csv index 7e23869c..a6f03173 100644 --- a/stixcore/data/test/publish/rid_lut.csv +++ b/stixcore/data/test/publish/rid_lut.csv @@ -1,3 +1,3 @@ version https://git-lfs.github.com/spec/v1 -oid sha256:a04105a1f367e11ec1c68a0d1271832b83060bdd7dfb02af46da148b7dd197bd -size 1009 +oid sha256:2ba33aa45a7d045dcbc08d1ddddf8bdf8d01dde93e01fdf4db0c12e1dfb0b4bf +size 1005 diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index 2e803bc5..1ac7b36a 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -711,7 +711,9 @@ def generate_primary_header(cls, filename, product, *, version=0): ('DATE-END', product.scet_timerange.end.to_string(), 'End time of observation'), ('DATAMIN', product.dmin, 'Minimum valid physical value'), ('DATAMAX', product.dmax, 'Maximum valid physical value'), - ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO') + ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), + ('XPOSURE', product.exposure, '[s] shortest exposure time'), + ('XPOMAX', product.max_exposure, '[s] maximum exposure time') ) return headers @@ -743,6 +745,7 @@ def generate_primary_header(self, filename, product, *, version=0): ('DATAMAX', product.dmax, 'Maximum valid physical value'), ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), ('XPOSURE', product.exposure, '[s] shortest exposure time') + ('XPOMAX', product.max_exposure, '[s] maximum exposure time') ) soop_keywords = SOOPManager.instance.get_keywords(start=product.utc_timerange.start, @@ -960,6 +963,7 @@ def generate_primary_header(self, filename, product, *, version=0): ('DATAMAX', product.dmax, 'Maximum valid physical value'), ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), ('XPOSURE', product.exposure, '[s] shortest exposure time') + ('XPOMAX', product.max_exposure, '[s] maximum exposure time') ) return L1headers, L2headers diff --git a/stixcore/io/fits/tests/test_processors.py b/stixcore/io/fits/tests/test_processors.py index b41775b5..fb83f7b4 100644 --- a/stixcore/io/fits/tests/test_processors.py +++ b/stixcore/io/fits/tests/test_processors.py @@ -7,6 +7,7 @@ from stixcore.data.test import test_data from stixcore.io.fits.processors import FitsL0Processor, FitsL1Processor, FitsLBProcessor +from stixcore.products.product import Product from stixcore.soop.manager import SOOPManager from stixcore.time import SCETime, SCETimeRange @@ -145,6 +146,7 @@ def test_level0_processor_generate_primary_header(datetime, product): product.obs_beg = SCETime(coarse=0, fine=0) product.obs_avg = SCETime(coarse=0, fine=2 ** 15) product.obs_end = SCETime(coarse=1, fine=2 ** 15) + product.scet_timerange = SCETimeRange(start=product.obs_beg, end=product.obs_end) product.raw = ['packet1.xml', 'packet2.xml'] product.parent = ['lb1.fits', 'lb2.fts'] @@ -177,6 +179,31 @@ def test_level0_processor_generate_primary_header(datetime, product): assert value == test_data[name] +@pytest.mark.parametrize('p_file', [test_data.products.L0_LightCurve_fits[0], + test_data.products.L1_LightCurve_fits[0]], + ids=["ql_l0", "ql_l1"]) +def test_count_data_mixin(p_file): + processor = FitsL0Processor('some/path') + p = Product(p_file) + assert p.dmin == p.data["counts"].min().value + assert p.dmax == p.data["counts"].max().value + assert p.exposure == p.data["timedel"].min().as_float().to_value() + assert p.max_exposure == p.data["timedel"].max().as_float().to_value() + + test_data = { + "DATAMAX": p.dmax, + "DATAMIN": p.dmin, + "XPOSURE": p.exposure, + "XPOMAX": p.max_exposure, + "BUNIT": "counts" + } + + header = processor.generate_primary_header('a_filename.fits', p) + for name, value, *comment in header: + if name in test_data.keys(): + assert value == test_data[name] + + def test_level1_processor_init(): pro = FitsL1Processor('some/path') assert pro.archive_path == 'some/path' diff --git a/stixcore/products/level2/housekeepingL2.py b/stixcore/products/level2/housekeepingL2.py index 0cbc9126..360aae53 100644 --- a/stixcore/products/level2/housekeepingL2.py +++ b/stixcore/products/level2/housekeepingL2.py @@ -344,11 +344,13 @@ def __init__(self, *, service_type=0, service_subtype=0, ssid=1, control, data, @property def dmin(self): - return min([self.data['y_srf'].min().value, self.data['z_srf'].min().value]) + return min([self.data['y_srf'].min().to_value('arcsec'), + self.data['z_srf'].min().to_value('arcsec')]) @property def dmax(self): - return max([self.data['y_srf'].max().value, self.data['z_srf'].max().value]) + return max([self.data['y_srf'].max().to_value('arcsec'), + self.data['z_srf'].max().to_value('arcsec')]) @property def bunit(self): diff --git a/stixcore/products/product.py b/stixcore/products/product.py index ead4bdcf..b3730457 100644 --- a/stixcore/products/product.py +++ b/stixcore/products/product.py @@ -517,6 +517,11 @@ def exposure(self): # default for FITS HEADER return 0.0 + @property + def max_exposure(self): + # default for FITS HEADER + return 0.0 + def find_parent_products(self, root): """ Conveniant way to get access to the parent products. @@ -780,6 +785,10 @@ def bunit(self): def exposure(self): return self.data['timedel'].as_float().min().to_value('s') + @property + def max_exposure(self): + return self.data['timedel'].as_float().max().to_value('s') + class EnergyChannelsMixin: def get_energies(self): From 8f14db850eefa7f90543099b4add303c2ae6f6a4 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Thu, 10 Oct 2024 16:57:42 +0200 Subject: [PATCH 4/7] fix XPOSURE/XPOMAX header error --- stixcore/config/data/common | 2 +- stixcore/io/fits/processors.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/stixcore/config/data/common b/stixcore/config/data/common index f9d8393f..7ff26901 160000 --- a/stixcore/config/data/common +++ b/stixcore/config/data/common @@ -1 +1 @@ -Subproject commit f9d8393f9fac90324719d56450191a98970b0acb +Subproject commit 7ff26901001a8730939333ab252f14dd583bec96 diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py index 1ac7b36a..e4524b31 100644 --- a/stixcore/io/fits/processors.py +++ b/stixcore/io/fits/processors.py @@ -744,7 +744,7 @@ def generate_primary_header(self, filename, product, *, version=0): ('DATAMIN', product.dmin, 'Minimum valid physical value'), ('DATAMAX', product.dmax, 'Maximum valid physical value'), ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), - ('XPOSURE', product.exposure, '[s] shortest exposure time') + ('XPOSURE', product.exposure, '[s] shortest exposure time'), ('XPOMAX', product.max_exposure, '[s] maximum exposure time') ) @@ -962,7 +962,7 @@ def generate_primary_header(self, filename, product, *, version=0): ('DATAMIN', product.dmin, 'Minimum valid physical value'), ('DATAMAX', product.dmax, 'Maximum valid physical value'), ('BUNIT', product.bunit, 'Units of physical value, after application of BSCALE, BZERO'), - ('XPOSURE', product.exposure, '[s] shortest exposure time') + ('XPOSURE', product.exposure, '[s] shortest exposure time'), ('XPOMAX', product.max_exposure, '[s] maximum exposure time') ) From 4392bee27fd81ae2e47e92f01a441323fd6349ea Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Thu, 10 Oct 2024 18:13:07 +0200 Subject: [PATCH 5/7] fix tests for dmax... --- stixcore/processing/tests/test_publish.py | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/stixcore/processing/tests/test_publish.py b/stixcore/processing/tests/test_publish.py index 9e4bee7d..8ebdabce 100644 --- a/stixcore/processing/tests/test_publish.py +++ b/stixcore/processing/tests/test_publish.py @@ -6,6 +6,7 @@ import numpy as np import pytest +import astropy.units as u from astropy.io import fits from astropy.table import QTable @@ -138,6 +139,11 @@ def test_publish_fits_to_esa_incomplete(product, out_dir): product.date_obs = beg product.date_beg = beg product.date_end = end + product.exposure = 2 + product.max_exposure = 3 + product.dmin = 2 + product.dmax = 3 + product.bunit = "count" product.split_to_files.return_value = [product] product.get_energies = False product.get_processing_version.return_value = 2 @@ -203,6 +209,7 @@ def test_fits_incomplete_switch_over(out_dir): product.data = QTable({"time": t, "timedel": t-beg, "fcounts": np.array([1, 2]), + "counts": np.array([1, 2]) * u.deg_C, "control_index": [1, 1]}) product.raw = ['packet1.xml', 'packet2.xml'] product.parent = ['packet1.xml', 'packet2.xml'] @@ -218,6 +225,11 @@ def test_fits_incomplete_switch_over(out_dir): product.date_obs = beg product.date_beg = beg product.date_end = end + product.exposure = 2 + product.max_exposure = 3 + product.dmin = 2 + product.dmax = 3 + product.bunit = "counts" product.split_to_files.return_value = [product] product.get_energies = False product.get_processing_version.return_value = 2 @@ -256,6 +268,7 @@ def test_fits_incomplete_switch_over(out_dir): p = Product(get_complete_file_name_and_path(f)) # old data.fcounts = [t1: 1, t2: 2] p.data['fcounts'][0] = 3 + p.data['counts'] = 3 * u.deg_C # remove the second time stamp p.data.remove_row(1) # new data.fcounts = [t1: 3] @@ -339,6 +352,11 @@ def test_publish_fits_to_esa(product, out_dir): product.date_obs = beg product.date_beg = beg product.date_end = end + product.exposure = 2 + product.max_exposure = 3 + product.dmin = 2 + product.dmax = 3 + product.bunit = "count" product.split_to_files.return_value = [product] product.get_processing_version.return_value = 2 product.get_energies = False From e243e9964df478a65e23a1823658c97aa806c61b Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Fri, 11 Oct 2024 13:21:48 +0200 Subject: [PATCH 6/7] relax end2end fitsdiff --- stixcore/processing/tests/test_end2end.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/stixcore/processing/tests/test_end2end.py b/stixcore/processing/tests/test_end2end.py index 642f09b6..ddec341b 100644 --- a/stixcore/processing/tests/test_end2end.py +++ b/stixcore/processing/tests/test_end2end.py @@ -79,8 +79,9 @@ def test_identical(orig_fits, current_fits): error_c += 1 warnings.warn(f"no corresponding file found for {cfits} in the original fits files") continue - diff = FITSDiff(ofits, cfits, - ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE', 'VERS_SW', 'HISTORY']) + diff = FITSDiff(ofits, cfits, atol=0.0001, rtol=0.0001, + ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE', + 'VERS_SW', 'VERS_CFG', 'HISTORY']) if not diff.identical: error_c += 1 warnings.warn(diff.report()) From a4651d99688cf1e0bc2480dfe77ae22efe3d5849 Mon Sep 17 00:00:00 2001 From: Nicky Hochmuth Date: Fri, 11 Oct 2024 16:47:15 +0200 Subject: [PATCH 7/7] less tolerance in end2end test --- stixcore/processing/tests/test_end2end.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/stixcore/processing/tests/test_end2end.py b/stixcore/processing/tests/test_end2end.py index ddec341b..e26d3133 100644 --- a/stixcore/processing/tests/test_end2end.py +++ b/stixcore/processing/tests/test_end2end.py @@ -79,7 +79,7 @@ def test_identical(orig_fits, current_fits): error_c += 1 warnings.warn(f"no corresponding file found for {cfits} in the original fits files") continue - diff = FITSDiff(ofits, cfits, atol=0.0001, rtol=0.0001, + diff = FITSDiff(ofits, cfits, atol=0.00001, rtol=0.00001, ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE', 'VERS_SW', 'VERS_CFG', 'HISTORY']) if not diff.identical: