ddpd (i4Ds#221)

* add first version of ddpd generator based on html * use new fits table read method * add intro block for each product * add first version of ddpd generator based on html * use new fits table read method * add intro block for each product * add description from lookup files * add first version of ddpd generator based on html * add low latency to dpdd * dimension format change * fix download path * update script documentation * add ref to issue i4Ds#224
nicHoch · Feb 21, 2023 · 95519b5 · 95519b5
1 parent ae6dd3e
commit 95519b5
Show file tree

Hide file tree

Showing 12 changed files with 562 additions and 17 deletions.
diff --git a/.github/workflows/end2end_build.yml b/.github/workflows/end2end_build.yml
@@ -1,4 +1,4 @@
-# This workflow triggers the rebuild the end2end test data after each merge on master on pup099 via a web hook
+# This is a basic workflow to help you get started with Actions
 
 name: EndToEndRebuildData
 
@@ -11,13 +11,18 @@ on:
   # Allows you to run this workflow manually from the Actions tab
   workflow_dispatch:
 
+# A workflow run is made up of one or more jobs that can run sequentially or in parallel
 jobs:
+  # This workflow contains a single job called "build"
   trigger_testdata_rebuild:
+    # The type of runner that the job will run on
     runs-on: ubuntu-latest
 
+    # Steps represent a sequence of tasks that will be executed as part of the job
     steps:
+      # Runs a set of commands using the runners shell
       - name: update end2end test data
         run: |
-          wget -qO- http://pub099.cs.technik.fhnw.ch/data/end2end/rebuild_hook.cgi
+          wget -qO- http://pub099.cs.technik.fhnw.ch/data/end2end/index.py
           echo "git action done"
-        if: github.repository == 'i4Ds/STIXCore'
+        if: github.repository == 'nicHoch2/STIXCore'
diff --git a/.gitignore b/.gitignore
@@ -159,3 +159,5 @@ docs/whatsnew/latest_changelog.txt
 stixcore/version.py
 htmlcov/
 *.prof
+pdpp.html
+stixcore/util/scripts/ddpd.html
diff --git a/stixcore/data/idb/idbVersionHistory.json b/stixcore/data/idb/idbVersionHistory.json
@@ -31,16 +31,15 @@
                 "version":"2.26.35",
                 "aswVersion":181,
                 "validityPeriodUTC":["2021-06-23T14:38:41.609", "2021-012-010T00:00:00.000"],
-                "validityPeriodOBT":[{"coarse":677774250, "fine":0}, {"coarse":692380351, "fine":0}]
-
+                "validityPeriodOBT":[{"coarse":677774250, "fine":0}, {"coarse":692409503, "fine":0}]
 
         },
         {
                 "version":"2.26.36",
                 "aswVersion":183,
                 "comment": "return this for the next future",
                 "validityPeriodUTC":["2021-12-10T00:00:00.000", "2136-02-07T08:11:44.778"],
-                "validityPeriodOBT":[{"coarse":692380351, "fine":0}, {"coarse":4294967295, "fine":0}]
+                "validityPeriodOBT":[{"coarse":692409503, "fine":0}, {"coarse":4294967295, "fine":0}]
 
         }
 ]
diff --git a/stixcore/data/test/ddpd.in.csv b/stixcore/data/test/ddpd.in.csv
@@ -0,0 +1,97 @@
+counter	name	description
+8	control_index	"join index to the control table"
+8	time	"unique time index: center of bin relative to DATE_BEG"
+8	timedel	"duration of the time bin"
+8	index	"index column"
+8	integration_time	"duration of the (aggregated) time bin"
+8	packet	"packet ID from XML TM file"
+8	parent	"the parent fits file(s) of this data entry"
+8	raw_file	"the RAW TM file from SOC of this data entry"
+8	scet_coarse	"coarse part (s) of the spacecraft elapsed time (OBT)"
+8	scet_fine	"fine part (bin s) of the spacecraft elapsed time (OBT)"
+8	obt_end	"Start of the IDB period in OBT"
+8	obt_start	"End of the IDB period in OBT"
+8	version	"valid IDB version for the time period"
+6	channel	"Energy Band Index"
+6	e_high	"end of the energy band"
+6	e_low	"beginning of the energy band"
+8	triggers	"trigger counts for the time period"
+7	counts	"pixel counts for the time period"
+7	triggers_err	"estimated error due to compression alone for the trigger values"
+7	variance_err	"estimated error due to compression alone for the variance values"
+7	real_err	"estimated error due to compression alone for the real part values"
+7	imaginary_err	"estimated error due to compression alone for the imaginary part values"
+7	compression_scheme_counts_skm	"SKM compression schema parameter used for count values"
+7	compression_scheme_triggers_skm	"SKM compression schema parameter used for tigger values"
+7	compression_scheme_variance_skm	"SKM compression schema parameter used for variance values"
+7	compression_scheme_spectra_skm	"SKM compression schema parameter used for spectra values"
+6	rcr	"current rate control regime of the instrument"
+6	energy_bin_mask	"mask for all 32 energy bands used for the combined data"
+6	request_id	"unique request ID for the BSD processing"
+6	counts_err	"estimated error due to compression alone for the pixel count values"
+6	num_samples	"number of data samples"
+5	num_substructures	"number of science sub packets"
+5	time_stamp	"OBT time stamp"
+5	pixel_mask	"mask for all 12 detector pixels used for the combined data"
+4	detector_masks	"mask for all 32 detectors used for the combined data"
+4	detector_mask	"mask for all 32 detectors used for the combined data"
+4	num_energies	"number of energy bins"
+3	pixel_masks	"mask for all 12 detector pixels used for the combined data"
+12	num_samples	"number of data samples"
+10	tc_packet_id_ref	"reference to the TC type that triggered this TM"
+10	tc_packet_seq_control	"refernece to the TC (counter) that triggered this TM"
+4	num_energy_groups	"number of energy bands"
+4	num_pixel_sets	"number of pixelsets in the count structure"
+4	energy_bin_edge_mask	"mask of 33 bits - setting start and closing points of enery ranges"
+1	cha_diode0	"Photodiode A counts|Photodiode A voltage"
+1	cha_diode1	"Photodiode B counts|Photodiode B voltage"
+1	chb_diode0	"Photodiode C counts|Photodiode C voltage"
+1	chb_diode1	"Photodiode D counts|Photodiode D voltage"
+1	spectra	"spectra count values"
+1	spectra_err	"estimated error due to compression alone for the spectra values"
+1	detector_id	"ID of the detector"
+1	num_channels	"number of channels"
+1	subspec_lowest_channel	"lowest channel of the subspectra"
+1	subspec_num_summed_channel	"number of summed channel in spectral point"
+1	subspec_num_points	"number of spectral points"
+1	average_temperature	"average temperature"
+1	quiet_time	"quiet time"
+1	det_q1_t	"Detector Temperature Quarter 1"
+1	det_q2_t	"Detector Temperature Quarter 2"
+1	det_q3_t	"Detector Temperature Quarter 3"
+1	det_q4_t	"Detector Temperature Quarter 4"
+1	hk_asp_photoa0_v	"Aspect system Photodiode A0 voltage"
+1	hk_asp_photoa1_v	"Aspect system Photodiode A1 voltage"
+1	hk_asp_photob0_v	"Aspect system Photodiode B0 voltage"
+1	hk_asp_photob1_v	"Aspect system Photodiode B1 voltage"
+1	hk_asp_ref_2v5a_v	"Aspect System A 2.5V reference"
+1	hk_asp_ref_2v5b_v	"Aspect System B 2.5V reference"
+1	hk_asp_tim01_t	"Aspect System Temperature TIM01"
+1	hk_asp_tim02_t	"Aspect System Temperature TIM02"
+1	hk_asp_tim03_t	"Aspect System Temperature TIM03"
+1	hk_asp_tim04_t	"Aspect System Temperature TIM04"
+1	hk_asp_tim05_t	"Aspect System Temperature TIM05"
+1	hk_asp_tim06_t	"Aspect System Temperature TIM06"
+1	hk_asp_tim07_t	"Aspect System Temperature TIM07"
+1	hk_asp_tim08_t	"Aspect System Temperature TIM08"
+1	hk_asp_vsensa_v	"Aspect System A supply voltage"
+1	hk_asp_vsensb_v	"Aspect System B supply voltage"
+1	hk_att_c	"Attenuator current"
+1	hk_att_t	"Attenuator temperature"
+1	hk_att_v	"Attenuator voltage"
+1	hk_det_c	"Detector current"
+1	hk_dpu_1v5_c	"IDPU FPGA 1.5V Current"
+1	hk_dpu_1v5_v	"IDPU FPGA 1.5V"
+1	hk_dpu_2v5_c	"IDPU FPGA 2.5V Current"
+1	hk_dpu_2v9_v	"IDPU 2.9V"
+1	hk_dpu_3v3_c	"IDPU 3.3V Current"
+1	hk_dpu_fpga_t	"IDPU FPGA Temperature"
+1	hk_dpu_pcb_t	"IDPU PCB Temperature"
+1	hk_dpu_spw0_v	"SpaceWire Main Supply Voltage"
+1	hk_dpu_spw1_v	"SpaceWire Redundant Supply Voltage"
+1	hk_dpu_spw_c	"IDPU SpaceWire Current"
+1	hk_hv_01_16_v	"High Voltage supply for Detectors 1-16"
+1	hk_hv_17_32_v	"High Voltage supply for Detectors 17-32"
+1	hk_psu_temp_t	"PSU Temperature"
+1	hk_ref_2v5_v	"IDPU 2.5V Analog Reference"
+1	live_time	"live time"
diff --git a/stixcore/ephemeris/manager.py b/stixcore/ephemeris/manager.py
@@ -141,6 +141,7 @@ def __init__(self, meta_kernel_path):
 
         curdir = os.getcwd()
         try:
+            print(f"LOAD NEW META KERNEL: {self.meta_kernel_path}")
             # change to the 'kernels' dir
             os.chdir(self.meta_kernel_path.parent)
             # unload all old kernels
@@ -427,7 +428,7 @@ def get_fits_headers(self, *, start_time, average_time):
             ('DATE_EAR', (start_time + np.around((sun_earth_lt - sun_solo_lt), precision)*u.s).fits,
              'Start time of observation, corrected to Earth'),
             ('DATE_SUN', (start_time - np.around(sun_solo_lt, precision)*u.s).fits,
-             'Start time of observation, corrected to Su'),
+             'Start time of observation, corrected to Sun'),
         )
 
         return headers

diff --git a/stixcore/io/fits/processors.py b/stixcore/io/fits/processors.py
@@ -686,7 +686,7 @@ def generate_primary_header(self, filename, product):
             ('DATAMIN', dmin, 'Minimum valid physical value'),
             ('DATAMAX', dmax, 'Maximum valid physical value'),
             ('BUNIT', bunit, 'Units of physical value, after application of BSCALE, BZERO'),
-            ('XPOSURE', exposure, '[s] Total effective exposure time')
+            ('XPOSURE', exposure, '[s] shortest exposure time')
         )
 
         soop_keywords = self.soop_manager.get_keywords(start=product.utc_timerange.start,

diff --git a/stixcore/products/__init__.py b/stixcore/products/__init__.py
@@ -5,4 +5,5 @@
 from stixcore.products.level1.quicklookL1 import *
 from stixcore.products.level1.scienceL1 import *
 from stixcore.products.levelb.binary import LevelB
+from stixcore.products.lowlatency.quicklookLL import *
 from stixcore.products.product import Product
diff --git a/stixcore/products/level0/scienceL0.py b/stixcore/products/level0/scienceL0.py
@@ -462,7 +462,7 @@ def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs):
 
 class Visibility(ScienceProduct):
     """
-    X-ray Visibilities or compression Level 3 data
+    X-ray Visibilities or compression Level 3 data.
 
     In level 0 format.
     """

diff --git a/stixcore/products/lowlatency/quicklookLL.py b/stixcore/products/lowlatency/quicklookLL.py
@@ -0,0 +1,42 @@
+from collections import defaultdict
+
+from stixcore.products.product import GenericProduct
+from stixcore.time.datetime import SCETimeRange
+
+
+class LightCurve(GenericProduct):
+    """"Low Latency Quick Look Light Curve data product.
+
+    for Low Latency Processing
+    """
+    def __init__(self, *, service_type, service_subtype, ssid, control, data,
+                 idb_versions=defaultdict(SCETimeRange), **kwargs):
+        super().__init__(service_type=service_type, service_subtype=service_subtype,
+                         ssid=ssid, control=control, data=data, idb_versions=idb_versions, **kwargs)
+        self.name = 'lightcurve'
+        self.level = kwargs.get('level', 'LL01')
+        self.type = 'ql'
+
+    @classmethod
+    def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs):
+        return (kwargs['level'] == 'LL01' and service_type == 21
+                and service_subtype == 6 and ssid == 30)
+
+
+class FlareFlag(GenericProduct):
+    """Low Latency Quick Look Flare Flag and Location data product.
+
+    for Low Latency Processing
+    """
+    def __init__(self, *, service_type, service_subtype, ssid, control, data,
+                 idb_versions=defaultdict(SCETimeRange), **kwargs):
+        super().__init__(service_type=service_type, service_subtype=service_subtype,
+                         ssid=ssid, control=control, data=data, idb_versions=idb_versions, **kwargs)
+        self.name = 'flareflag'
+        self.level = kwargs.get('level', 'LL01')
+        self.type = 'ql'
+
+    @classmethod
+    def is_datasource_for(cls, *, service_type, service_subtype, ssid, **kwargs):
+        return (kwargs['level'] == 'LL01' and service_type == 21
+                and service_subtype == 6 and ssid == 34)
diff --git a/stixcore/products/product.py b/stixcore/products/product.py
@@ -21,7 +21,7 @@
 from stixcore.tmtc.packets import GenericPacket, PacketSequence
 
 __all__ = ['GenericProduct', 'ProductFactory', 'Product', 'ControlSci',
-           'Control', 'Data', 'L1Mixin', 'EnergyChannelsMixin']
+           'Control', 'Data', 'L1Mixin', 'EnergyChannelsMixin', 'read_qtable']
 
 from collections import defaultdict
 
@@ -129,13 +129,14 @@ def __call__(self, *args, **kwargs):
             if isinstance(args[0], (str, Path)):
                 file_path = Path(args[0])
                 header = fits.getheader(file_path)
-                service_type = int(header.get('stype'))
-                service_subtype = int(header.get('sstype'))
-                parent = header.get('parent').split(';')
-                raw = header.get('raw_file').split(';')
+
+                service_type = int(header.get('stype')) if 'stype' in header else 0
+                service_subtype = int(header.get('sstype')) if 'sstype' in header else 0
+                parent = header.get('parent').split(';') if 'parent' in header else ''
+                raw = header.get('raw_file').split(';') if 'raw_file' in header else ''
                 try:
                     ssid = int(header.get('ssid'))
-                except ValueError:
+                except (ValueError, TypeError):
                     ssid = None
                 level = header.get('Level')
                 header.get('TIMESYS')
@@ -151,7 +152,19 @@ def __call__(self, *args, **kwargs):
 
                 data = read_qtable(file_path, hdu='DATA', hdul=hdul)
 
-                if level != 'LB':
+                if level == 'LL01':
+                    # TODO remova that hack in favor for a proper header information after
+                    # https://github.com/i4Ds/STIXCore/issues/224 is solved
+                    if "lightcurve" in args[0]:
+                        service_type = 21
+                        service_subtype = 6
+                        ssid = 30
+                    if "flareflag" in args[0]:
+                        service_type = 21
+                        service_subtype = 6
+                        ssid = 34
+
+                if level not in ['LB', 'LL01']:
                     data['timedel'] = SCETimeDelta(data['timedel'])
                     offset = SCETime.from_float(header['OBT_BEG']*u.s)
 
@@ -548,7 +561,8 @@ class DefaultProduct(GenericProduct, L1Mixin):
         236: 'config',
         237: 'params',
         238: 'archive',
-        239: 'diagnostics'
+        239: 'diagnostics',
+        300: 'low-latency'
     }
 
     def __init__(self, *, service_type, service_subtype, ssid, control, data,

diff --git a/stixcore/testEnd2End.py b/stixcore/testEnd2End.py
@@ -0,0 +1,52 @@
+import tempfile
+from pathlib import Path
+from collections import defaultdict
+
+from astropy.io import fits
+
+from stixcore.data.test import test_data
+from stixcore.idb.manager import IDBManager
+from stixcore.products.product import Product
+
+name_counter = defaultdict(int)
+
+IDB = IDBManager(test_data.idb.DIR).get_idb("2.26.34")
+
+collector = defaultdict(lambda: defaultdict(list))
+files = [
+      # L0
+           # science
+    "L0/21/6/20/solo_L0_stix-sci-xray-rpd_0678187309f00000-0678187429f00000_V01_2106280011-54760.fits", # noqa
+    "L0/21/6/21/solo_L0_stix-sci-xray-cpd_0688454771f32768-0688455372f13108_V01_2110250007-65280.fits", # noqa
+    "L0/21/6/22/solo_L0_stix-sci-xray-spd_0678187309f00000-0678187429f58982_V01_2106280006-54720.fits", # noqa
+    "L0/21/6/23/solo_L0_stix-sci-xray-vis_0678187308f65535-0678187429f58982_V01_2106280004-54716.fits", # noqa
+    "L0/21/6/42/solo_L0_stix-sci-aspect-burst_0687412111f52953-0687412634f03145_V01_0.fits", # noqa
+    "L0/21/6/24/solo_L0_stix-sci-xray-spec_0689786926f13107-0689801914f19660_V01_2111090002-50819.fits", # noqa
+    # QL
+    "L0/21/6/31/solo_L0_stix-ql-background_0668822400_V01.fits",
+    "L0/21/6/34/solo_L0_stix-ql-flareflag_0684547200_V01.fits",
+    "L0/21/6/30/solo_L0_stix-ql-lightcurve_0684892800_V01.fits",
+    "L0/21/6/33/solo_L0_stix-ql-variance_0687484800_V01.fits",
+    "L0/21/6/32/solo_L0_stix-ql-spectra_0680400000_V01.fits",
+    "L0/21/6/41/solo_L0_stix-cal-energy_0683856000_V01.fits",
+    # HK
+    "L0/3/25/2/solo_L0_stix-hk-maxi_0647913600_V01.fits",
+    "L0/3/25/1/solo_L0_stix-hk-mini_0643507200_V01.fits"]
+
+remote = ["http://pub099.cs.technik.fhnw.ch/data/fits_test/" + x for x in files]
+# files = ["/home/shane/fits_test/" + x for x in files]
+files = [("/home/shane/fits_test_latest/" + x, remote[i]) for i, x in enumerate(files)]
+
+with tempfile.TemporaryDirectory() as tempdir:
+    temppath = Path(tempdir)
+    for f, rm in files:
+        try:
+            pr = Product(f)
+            prLB = Product(pr.parent)
+            prlb = Product(Path("/home/shane/fits_test_latest/LB/21/6/20/"))
+            # / header.get("parent"))
+
+            header = fits.getheader(f)
+            collector[pr.level][pr.type].append((pr, header))
+        except Exception as e:
+            print(e)