Merge pull request #233 from DavidT3/feat/eRASS1MissionClass

Feat/e rass1 mission class

daxa/config.py

@@ -1,5 +1,5 @@
 #  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-#  Last modified by David J Turner ([email protected]) 18/01/2023, 15:25. Copyright (c) The Contributors
+#  Last modified by David J Turner ([email protected]) 31/01/2024, 11:35. Copyright (c) The Contributors
 
 import os
 from configparser import ConfigParser
@@ -71,5 +71,14 @@
 SASERROR_LIST = errors["ErrName"].values
 SASWARNING_LIST = warnings["WarnName"].values
 
-# Reading in the data available in the Cal-PV release
-CALPV_INFO = pd.read_csv(pkg_resources.resource_filename(__name__, "files/CALPV_INFO.csv"), header="infer",  dtype={'ObsID': str})
+# Reading in the file with information on the eROSITA observations that were made available in the
+#  eROSITA CalPV release
+EROSITA_CALPV_INFO = pd.read_csv(pkg_resources.resource_filename(__name__, "files/erosita_calpv_info.csv"),
+                                 header="infer", dtype={'ObsID': str})
+# TODO This may end up changing when we get access to the DR1 release - it could be in a format that makes this
+#  a bad way of doing it
+# Then doing the same thing, but for the German eRASS:1 release
+ERASS_DE_DR1_INFO = pd.read_csv(pkg_resources.resource_filename(__name__, "files/erass_de_dr1_info.csv"),
+                                header="infer", dtype={'ObsID': str, 'FIELD1': str, 'FIELD2': str, 'FIELD3': str,
+                                                       'FIELD4': str, 'FIELD5': str, 'FIELD6': str, 'FIELD7': str,
+                                                       'FIELD8': str, 'FIELD9': str})

daxa/mission/__init__.py

@@ -1,10 +1,10 @@
 #  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-#  Last modified by David J Turner ([email protected]) 09/10/2023, 23:36. Copyright (c) The Contributors
+#  Last modified by David J Turner ([email protected]) 30/01/2024, 16:13. Copyright (c) The Contributors
 
 from .asca import ASCA
 from .base import BaseMission
 from .chandra import Chandra
-from .erosita import eROSITACalPV
+from .erosita import eROSITACalPV, eRASS1DE
 from .integral import INTEGRALPointed
 from .nustar import NuSTARPointed
 from .rosat import ROSATAllSky, ROSATPointed
@@ -14,8 +14,9 @@
 
 # This just links the internal DAXA names of missions to their class
 MISS_INDEX = {'xmm_pointed': XMMPointed, 'nustar_pointed': NuSTARPointed, 'erosita_calpv': eROSITACalPV,
-              'chandra': Chandra, 'rosat_all_sky': ROSATAllSky, 'rosat_pointed': ROSATPointed, 'swift': Swift,
-              'suzaku': Suzaku, 'asca': ASCA, 'integral_pointed': INTEGRALPointed}
+              'erosita_all_sky_de_dr1': eRASS1DE, 'chandra': Chandra, 'rosat_all_sky': ROSATAllSky,
+              'rosat_pointed': ROSATPointed, 'swift': Swift, 'suzaku': Suzaku, 'asca': ASCA,
+              'integral_pointed': INTEGRALPointed}
 
 
 

daxa/mission/base.py

@@ -1,5 +1,5 @@
 #  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-#  Last modified by David J Turner ([email protected]) 29/01/2024, 15:50. Copyright (c) The Contributors
+#  Last modified by David J Turner ([email protected]) 30/01/2024, 14:18. Copyright (c) The Contributors
 
 import os.path
 import re
@@ -539,7 +539,8 @@ def _check_chos_insts(self, insts: Union[List[str], str]):
         """
         An internal function to perform some checks on the validity of chosen instrument names for a given mission.
 
-        :param List[str]/str insts:
+        :param List[str]/str insts: Instrument names that are to be checked for the current mission, either a single
+            name or a list of names.
         :return: The list of instruments (possibly altered to match formats expected by this module).
         :rtype: List
         """

daxa/mission/suzaku.py

@@ -1,5 +1,5 @@
 #  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-#  Last modified by David J Turner ([email protected]) 09/10/2023, 20:48. Copyright (c) The Contributors
+#  Last modified by David J Turner ([email protected]) 31/01/2024, 12:26. Copyright (c) The Contributors
 
 import gzip
 import io
@@ -501,7 +501,7 @@ def err_callback(err):
             self._download_done = True
 
         else:
-            warn("The raw data for this mission have already been downloaded.")
+            warn("The raw data for this mission have already been downloaded.", stacklevel=2)
 
     def assess_process_obs(self, obs_info: dict):
         """

daxa/process/erosita/__init__.py

@@ -1,5 +1,5 @@
 #  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-#  Last modified by Jessica Pilling ([email protected]) 21/07/2023, 15:38. Copyright (c) The Contributors
+#  Last modified by David J Turner ([email protected]) 01/02/2024, 11:15. Copyright (c) The Contributors
 
 from .clean import flaregti
-from .setup import _prepare_erositacalpv_info
+from .setup import _prepare_erosita_info

daxa/process/erosita/_common.py

@@ -1,27 +1,25 @@
-# This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-# Last modified by David J Turner ([email protected]) Thu Apr 13 2023, 15:16. Copyright (c) The Contributors
+#  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
+#  Last modified by David J Turner ([email protected]) 01/02/2024, 11:15. Copyright (c) The Contributors
 import glob
-from typing import Tuple, List
-from warnings import warn
 import os.path
-from subprocess import Popen, PIPE
+from enum import Flag
 from functools import wraps
 from multiprocessing.dummy import Pool
-from enum import Flag
+from subprocess import Popen, PIPE
+from typing import Tuple, List
+from warnings import warn
 
-import itertools
 from astropy.units import UnitConversionError
-from tqdm import tqdm
 from exceptiongroup import ExceptionGroup
+from tqdm import tqdm
 
 from daxa.archive.base import Archive
 from daxa.exceptions import NoEROSITAMissionsError
 from daxa.process._backend_check import find_esass
 
-from daxa.process.erosita.setup import _prepare_erositacalpv_info
-
-ALLOWED_EROSITA_MISSIONS = ['erosita_calpv']
+ALLOWED_EROSITA_MISSIONS = ['erosita_calpv', 'erosita_all_sky_de_dr1']
 
+# TODO Make this compliant with how I normally do docstrings
 class _eSASS_Flag(Flag):
     """
     This class was written by Toby Wallage found on Github @TobyWallage.
@@ -218,7 +216,7 @@ def esass_call(esass_func):
     @wraps(esass_func)
     def wrapper(*args, **kwargs):
         # This is here to avoid a circular import issue
-        from daxa.process.erosita.setup import _prepare_erositacalpv_info
+        from daxa.process.erosita.setup import _prepare_erosita_info
 
         # The first argument of all the eSASS processing functions will be an archive instance, and pulling
         #  that out of the arguments will be useful later
@@ -236,7 +234,7 @@ def wrapper(*args, **kwargs):
                 #   processing functions. It will also populate the _process_extra_info dictionary for the archive
                 #   with top level keys of the erositacalpv mission and lower level keys of obs_ids with lower level keys
                 #   of 'path', which will store the raw data path for that obs id.
-                _prepare_erositacalpv_info(obs_archive, miss)
+                _prepare_erosita_info(obs_archive, miss)
 
         # This is the output from whatever function this is a decorator for
         miss_cmds, miss_final_paths, miss_extras, process_message, cores, disable, timeout, esass_in_docker = esass_func(*args, **kwargs)

daxa/process/erosita/assemble.py

@@ -1,21 +1,24 @@
-# This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-# Last modified by Jessica Pilling ([email protected]) Wed Jul 19 2023, 13:52. Copyright (c) The Contributors
-from random import randint
+#  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
+#  Last modified by David J Turner ([email protected]) 01/02/2024, 09:04. Copyright (c) The Contributors
+
 import os.path
+from random import randint
 
-from astropy.units import Quantity
+from astropy.units import Quantity, UnitConversionError
 
 from daxa import NUM_CORES
 from daxa.archive.base import Archive
+from daxa.exceptions import NoDependencyProcessError
 from daxa.process._cleanup import _last_process
 from daxa.process.erosita._common import _esass_process_setup, ALLOWED_EROSITA_MISSIONS, esass_call, _is_valid_flag
-from daxa.exceptions import NoDependencyProcessError
+
 
 @_last_process(ALLOWED_EROSITA_MISSIONS, 1)
 @esass_call
-def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV'), hi_en: Quantity = Quantity(10, 'keV'),
-                      flag: int = 0xc0000000, flag_invert: bool = True, pattern: int = 15, num_cores: int = NUM_CORES,
-                      disable_progress: bool = False, timeout: Quantity = None):
+def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV'),
+                      hi_en: Quantity = Quantity(10, 'keV'), flag: int = 0xc0000000, flag_invert: bool = True,
+                      pattern: int = 15, num_cores: int = NUM_CORES, disable_progress: bool = False,
+                      timeout: Quantity = None):
 
     """
     The function wraps the eROSITA eSASS task evtool, which is used for selecting events.
@@ -26,20 +29,21 @@ def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV
     which are ready to be analysed.
 
     :param Archive obs_archive: An Archive instance containing eROSITA mission instances with observations for
-        which cleaned event lists should be created. This function will fail if no eROSITA missions are present in the archive.
+        which cleaned event lists should be created. This function will fail if no eROSITA missions are present in
+        the archive.
     :param Quantity lo_en: The lower bound of an energy filter to be applied to the cleaned, filtered, event lists. If
         'lo_en' is set to an Astropy Quantity, then 'hi_en' must be as well. Default is None, in which case no
         energy filter is applied.
     :param Quantity hi_en: The upper bound of an energy filter to be applied to the cleaned, filtered, event lists. If
         'hi_en' is set to an Astropy Quantity, then 'lo_en' must be as well. Default is None, in which case no
         energy filter is applied.
-    :param int flag: FLAG parameter to select events based on owner, information, rejection, quality, and corrupted data. The eROSITA
-        website contains the full description of event flags in section 1.1.2 of the following link:
-        https://erosita.mpe.mpg.de/edr/DataAnalysis/prod_descript/EventFiles_edr.html. The default parameter will remove all events 
-        flagged as either singly corrupt or as part of a corrupt frame.
+    :param int flag: FLAG parameter to select events based on owner, information, rejection, quality, and corrupted
+        data. The eROSITA website contains the full description of event flags in section 1.1.2 of the following link:
+        https://erosita.mpe.mpg.de/edr/DataAnalysis/prod_descript/EventFiles_edr.html. The default parameter will
+        remove all events flagged as either singly corrupt or as part of a corrupt frame.
     :param bool flag_invert: If set to True, this function will discard all events selected by the flag parameter.
-    :param int pattern: Selects events of a certain pattern chosen by the integer key. The default of 15 selects all four of the
-        recognized legal patterns.
+    :param int pattern: Selects events of a certain pattern chosen by the integer key. The default of 15 selects
+        all four of the recognized legal patterns.
     :param int num_cores: The number of cores to use, default is set to 90% of available.
     :param bool disable_progress: Setting this to true will turn off the eSASS generation progress bar.
     :param Quantity timeout: The amount of time each individual process is allowed to run for, the default is None.
@@ -84,16 +88,16 @@ def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV
             raise ValueError("{} is not a valid eSASS flag, see the eROSITA website"
                              " for valid flags.".format(flag))
 
-    # Checking user has input flag_invert as a boolean
+    # Checking user has input flag_invert as a boolean
     if not isinstance(flag_invert, bool):
         raise TypeError("The flag_invert parameter must be a boolean.")
 
-    # Checking user has input pattern as an integer
+    # Checking user has input pattern as an integer
     if not isinstance(pattern, int):
         raise TypeError("The pattern parameter must be an integer between 1 and 15 inclusive.")
 
-    #  Checking user has input a valid pattern
-    if (pattern <= 0 or pattern >= 16):
+    # Checking user has input a valid pattern
+    if pattern <= 0 or pattern >= 16:
         raise ValueError("Valid eROSITA patterns are between 1 and 15 inclusive")
 
     # Converting the parameters to the correct format for the esass command
@@ -114,8 +118,8 @@ def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV
     miss_final_paths = {}
     miss_extras = {}
 
-    # Just grabs the eROSITA missions, we already know there will be at least one because otherwise _esass_process_setup
-    #  would have thrown an error
+    # Just grabs the eROSITA missions, we already know there will be at least one because otherwise
+    #  _esass_process_setup would have thrown an error
     erosita_miss = [mission for mission in obs_archive if mission.name in ALLOWED_EROSITA_MISSIONS]
     # We are iterating through erosita missions (options could include erosita_cal_pv for instance).
     for miss in erosita_miss:
@@ -129,12 +133,13 @@ def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV
 
         # Checking that any valid observations are left after the get_obs_to_process function is run
         if len(all_obs_info) == 0:
-            raise FileNotFoundError("No valid observations have been found, so cleanded_evt_lists may not be run.")
+            raise FileNotFoundError("No valid observations have been found, so cleaned_evt_lists may not be run.")
 
         # all_obs_info is a list of lists, where each list is of the format: [ObsID, Inst, 'usable'].
-        #   There is a new list for each instrument, but I just want to loop over the ObsID in the following bit of code, 
-        #   I also want to know all the instruments that the ObsID contains events for
-        #   So here I am just making a dictionary of the format: {ObsID: insts}
+        # There is a new list for each instrument, but I just want to loop over the ObsID in the following
+        #  bit of code,
+        # I also want to know all the instruments that the ObsID contains events for
+        # So here I am just making a dictionary of the format: {ObsID: insts}
         # Getting unique obs_ids in all_obs_info
         obs_ids = list(set([all_obs_info_list[0] for all_obs_info_list in all_obs_info]))
         obs_info_dict = {}
@@ -145,13 +150,13 @@ def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV
             #   and append it to the dict
             obs_info_dict[obs] = ''.join(ch for ch in insts if ch.isdigit())
 
-        # Counter for number of ObsIDs that flaregti has not been run succesfully on
+        # Counter for number of ObsIDs that flaregti has not been run successfully on
         bad_obs_counter = 0
         # We iterate through the valid identifying information
         for obs_id in obs_info_dict:
             try:
                 # Checking that flaregti has been run successfully on this observation so that it can be cleaned
-                # then only writing a command for ObsIDs that have had flaregti sucessfully run on them
+                # Then only writing a command for ObsIDs that have had flaregti successfully run on them
                 obs_archive.check_dependence_success(miss.name, obs_id, 'flaregti')
 
                 # Getting the insts associated with this obs for file naming purposes 
@@ -182,8 +187,8 @@ def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV
                 if not os.path.exists(temp_dir):
                     os.makedirs(temp_dir)
 
-                cmd = evtool_cmd.format(d=temp_dir, ef=evt_list_file, of=filt_evt_name, f=flag, fi=flag_invert, p=pattern,
-                    emin=lo_en, emax=hi_en, fep=filt_evt_path)
+                cmd = evtool_cmd.format(d=temp_dir, ef=evt_list_file, of=filt_evt_name, f=flag, fi=flag_invert,
+                                        p=pattern, emin=lo_en, emax=hi_en, fep=filt_evt_path)
 
                 # Now store the bash command, the path, and extra info in the dictionaries
                 miss_cmds[miss.name][obs_id] = cmd
@@ -193,26 +198,17 @@ def cleaned_evt_lists(obs_archive: Archive, lo_en: Quantity = Quantity(0.2, 'keV
 
             except NoDependencyProcessError:
                 # If archive.check_dependence_success raises this error, it means flaregti was not run
-                # sucessfully, and so a warning will be raised saying this observation has not been cleaned
+                # successfully, and so a warning will be raised saying this observation has not been cleaned
                 bad_obs_counter += 1
                 pass
 
         # If no observations have had flaregti run successfully, then no events can be cleaned
         if bad_obs_counter == len(obs_info_dict):
             raise NoDependencyProcessError("The required process flaregti has not been run successfully"
-                                            "for any data in {mn}".format(mn=miss.name))
+                                           "for any data in {mn}".format(mn=miss.name))
 
     # This is just used for populating a progress bar during the process run
     process_message = 'Generating final event lists'
 
-    return miss_cmds, miss_final_paths, miss_extras, process_message, num_cores, disable_progress, timeout, esass_in_docker
-
-
-
-
-
-
-
-
-
-
+    return (miss_cmds, miss_final_paths, miss_extras, process_message, num_cores, disable_progress, timeout,
+            esass_in_docker)

daxa/process/erosita/clean.py

@@ -1,5 +1,5 @@
-# This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
-# Last modified by David J Turner ([email protected]) Thu Apr 20 2023, 10:52. Copyright (c) The Contributors
+#  This code is a part of the Democratising Archival X-ray Astronomy (DAXA) module.
+#  Last modified by David J Turner ([email protected]) 01/02/2024, 09:04. Copyright (c) The Contributors
 import os
 from random import randint
 from typing import Union
@@ -8,13 +8,13 @@
 
 from daxa import NUM_CORES
 from daxa.archive.base import Archive
-from daxa.exceptions import NoDependencyProcessError
-from daxa.process.erosita.setup import sb_rate
 from daxa.process.erosita._common import _esass_process_setup, ALLOWED_EROSITA_MISSIONS, esass_call
+from daxa.process.erosita.setup import sb_rate
 
 # Adding this to the enabled astropy units so that it can be used in flaregti to define thresholds
 add_enabled_units([sb_rate])
 
+
 @esass_call
 def flaregti(obs_archive: Archive, pimin: Quantity = Quantity(200, 'eV'), pimax: Quantity = Quantity(10000, 'eV'), mask_pimin: Quantity = Quantity(200, 'eV'), 
             mask_pimax: Quantity = Quantity(10000, 'eV'), binsize: int = 1200, detml: Union[float, int] = 10, timebin: Quantity = Quantity(20, 's'),