E2e fix sym add (#412)

* bump all product versions to 2

* dep np.bool8

* disable test

stixcore/processing/tests/test_end2end.py

@@ -8,7 +8,15 @@
 
 from astropy.io.fits.diff import FITSDiff
 
+from stixcore.config.config import CONFIG
+from stixcore.data.test import test_data
+from stixcore.ephemeris.manager import Spice, SpiceKernelManager
+from stixcore.idb.manager import IDBManager
+from stixcore.io.RidLutManager import RidLutManager
+from stixcore.processing.LBtoL0 import Level0
+from stixcore.processing.pipeline import PipelineStatus
 from stixcore.products.product import Product
+from stixcore.soop.manager import SOOPManager
 from stixcore.util.logging import get_logger
 from stixcore.util.scripts.end2end_testing import end2end_pipeline
 
@@ -90,3 +98,47 @@ def test_identical(orig_fits, current_fits):
     if error_c > 0:
         raise ValueError(f"{error_c} errors out of {len(current_fits)}\n"
                          f"there are differences in FITS files\n {pformat(error_files)}")
+
+
+@pytest.mark.skip(reason="used as a local test at the moment")
+def test_e2e_21_6_32(out_dir):
+    _spm = SpiceKernelManager(test_data.ephemeris.KERNELS_DIR)
+    Spice.instance = Spice(_spm.get_latest_mk())
+
+    # pinpoint the api files location
+    CONFIG.set('SOOP', 'soop_files_download', str(test_data.soop.DIR))
+
+    SOOPManager.instance = SOOPManager(test_data.soop.DIR, mock_api=True)
+
+    idbpath = Path(__file__).parent.parent.parent / "data" / "idb"
+    IDBManager.instance = IDBManager(idbpath)  # force_version="2.26.35")
+
+    RidLutManager.instance = RidLutManager(Path(CONFIG.get('Publish', 'rid_lut_file')), update=True)
+
+    PipelineStatus.log_setup()
+
+    f1 = Path("/data/stix/out/test/e2e_21_6_32/solo_LB_stix-21-6-32_0678153600_V02.fits")
+    f2 = Path("/data/stix/out/test/e2e_21_6_32/solo_LB_stix-21-6-32_0678240000_V02.fits")
+
+    out_dir_o1 = out_dir / "o1"
+    out_dir_o2 = out_dir / "o2"
+
+    l0_proc_o1 = Level0(out_dir_o1, out_dir_o1)
+    l0_files_o1 = l0_proc_o1.process_fits_files(files=[f2, f1])
+
+    l0_proc_o2 = Level0(out_dir_o2, out_dir_o2)
+    l0_files_o2 = l0_proc_o2.process_fits_files(files=[f1, f2])
+
+    n_errors = 0
+
+    for fits_1 in l0_files_o1:
+        # find corresponding original file
+        fits_2 = next(ofits for ofits in l0_files_o2 if ofits.name == fits_1.name)
+        diff = FITSDiff(fits_1, fits_2, atol=0.00001, rtol=0.00001,
+                        ignore_keywords=['CHECKSUM', 'DATASUM', 'DATE',
+                                         'VERS_SW', 'VERS_CFG', 'HISTORY'])
+        if not diff.identical:
+            print(diff.report())
+            n_errors += 1
+
+    assert n_errors == 0

stixcore/products/common.py

@@ -62,7 +62,7 @@ def _get_energy_bins(packets, nixlower, nixuppper):
         Full energy mask of len 33
     """
     energy_bin_mask = np.array(packets.get_value(nixlower), np.uint32)
-    energy_bin_mask_upper = np.array(packets.get_value(nixuppper), np.bool8)
+    energy_bin_mask_upper = np.array(packets.get_value(nixuppper), np.bool_)
     full_energy_mask = [format(mask, 'b').zfill(32)[::-1] + format(upper, 'b') for mask, upper in
                         zip(energy_bin_mask, energy_bin_mask_upper)]
     full_energy_mask = [list(map(int, m)) for m in full_energy_mask]

stixcore/util/scripts/end2end_testing.py

@@ -152,10 +152,15 @@ def end2end_pipeline(indir, fitsdir):
 
     soc = SOCManager(indir)
     lb_files = process_tmtc_to_levelbinary(soc.get_files(TMTC.TM), archive_path=fitsdir)
+    lb_files = sorted(lb_files)
+
     l0_proc = Level0(indir, fitsdir)
     l0_files = l0_proc.process_fits_files(files=lb_files)
+    l0_files = sorted(l0_files)
+
     l1_proc = Level1(indir, fitsdir)
     l1_files = l1_proc.process_fits_files(files=l0_files)
+    l1_files = sorted(l1_files)
 
     allfiles = list(lb_files)
     allfiles.extend(l0_files)