diff --git a/CHANGELOG.md b/CHANGELOG.md index fe10a139..b5776df1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,9 +5,12 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [0.X.X] - 2023-XX-XX * Bug Fix * New window needs to be integer for calculate_imf_steadiness +* Documentation + * Added example of how to export data for archival * Maintenance * Implemented unit tests for cleaning warnings * Use pip install for readthedocs + * Moved references and acknowledgements to methods files * Added tests for OMNI HRO routines ## [0.0.5] - 2023-06-27 diff --git a/docs/archival.rst b/docs/archival.rst new file mode 100644 index 00000000..ad1b1d5e --- /dev/null +++ b/docs/archival.rst @@ -0,0 +1,69 @@ +Building data files for archival at NASA SPDF +============================================= + +The codes and routines at :py:mod:`pysatNASA` are designed for end-users of NASA data +products. However, pysat in general has also been used to build operational +instruments for generating archival data to be uploaded to the Space Physics +Data Facility (SPDF) at NASA. + +In general, such instruments should include separate naming conventions. An +example of this is the REACH data, where netCDF4 files are generated for +archival purposes as part of the :py:mod:`ops_reach` package, but can be accessed by +the end user through :py:mod:`pysatNASA`. + +In general, a :py:class:`pysat.Instrument` object can be constructed for any +dataset. Full instructions and conventions can be found +`here `_. In the +case of the REACH data, the operational code reads in a series of csv files and +updates the metadata according to user specifications. Once the file is loaded, +it can be exported to a netCDF4 file via pysat. In the simplest case, this is + +:: + + reach = pysat.Instrument(inst_module=aero_reach, tag='l1b', inst_id=inst_id) + pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch') + + +However, there are additional options when translating pysat metadata to SPDF +preferred formats. An example of this is + +:: + + # Use meta translation table to include SPDF preferred format. + # Note that multiple names are output for compliance with pysat. + # Using the most generalized form for labels for future compatibility. + meta_dict = {reach.meta.labels.min_val: ['VALIDMIN'], + reach.meta.labels.max_val: ['VALIDMAX'], + reach.meta.labels.units: ['UNITS'], + reach.meta.labels.name: ['CATDESC', 'LABLAXIS', 'FIELDNAM'], + reach.meta.labels.notes: ['VAR_NOTES'], + reach.meta.labels.fill_val: ['_FillValue'], + 'Depend_0': ['DEPEND_0'], + 'Format': ['FORMAT'], + 'Monoton': ['MONOTON'], + 'Var_Type': ['VAR_TYPE']} + + pysat.utils.io.inst_to_netcdf(reach, 'output_file.nc', epoch_name='Epoch', + meta_translation=meta_dict, + export_pysat_info=False) + + +In this case, note that the pysat 'name' label is output to three different +metadata values required by the ITSP standards. Additionally, the +:py:attr:`export_pysat_info` option is set to false here. This drops several +internal :py:mod:`pysat` metadata values before writing to file. + +A full guide to SPDF metadata standards can be found +`here `_. + +Other best practices for archival include adding the operational software version +to the metadata header before writing. The pysat version will be automatically +written to the metadata. + +:: + + reach.meta.header.Software_version = ops_reach.__version__ + + +A full example script to generate output files can be found at +https://github.com/jklenzing/ops_reach/blob/main/scripts/netcdf_gen.py diff --git a/docs/index.rst b/docs/index.rst index cda7cf4f..926c4f76 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,7 @@ CDAWeb interface. supported_constellations.rst examples.rst develop_guide.rst + archival.rst migration_guide.rst history.rst diff --git a/pysatNASA/instruments/formosat1_ivm.py b/pysatNASA/instruments/formosat1_ivm.py index 3ac177d9..ceccc95c 100644 --- a/pysatNASA/instruments/formosat1_ivm.py +++ b/pysatNASA/instruments/formosat1_ivm.py @@ -26,9 +26,9 @@ import functools from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import formosat as mm_formosat from pysatNASA.instruments.methods import general as mm_nasa # ---------------------------------------------------------------------------- @@ -47,31 +47,7 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - self.acknowledgements = ' '.join(('Data provided through NASA CDAWeb', - 'Key Parameters - Shin-Yi Su', - '(Institute of Space Science,', - 'National Central University,', - 'Taiwan, R.O.C.)')) - self.references = ' '.join(('Yeh, H.C., S.‐Y. Su, Y.C. Yeh, J.M. Wu,', - 'R. A. Heelis, and B. J. Holt, Scientific', - 'mission of the IPEI payload on board', - 'ROCSAT‐1, Terr. Atmos. Ocean. Sci., 9,', - 'suppl., 1999a.\n', - 'Yeh, H.C., S.‐Y. Su, R.A. Heelis, and', - 'J.M. Wu, The ROCSAT‐1 IPEI preliminary', - 'results, Vertical ion drift statistics,', - 'Terr. Atmos. Ocean. Sci., 10, 805,', - '1999b.')) - logger.info(self.acknowledgements) - - return +init = functools.partial(mm_nasa.init, module=mm_formosat, name=name) # Use default clean diff --git a/pysatNASA/instruments/iss_fpmu.py b/pysatNASA/instruments/iss_fpmu.py index c98f907c..d65272e3 100644 --- a/pysatNASA/instruments/iss_fpmu.py +++ b/pysatNASA/instruments/iss_fpmu.py @@ -28,10 +28,10 @@ import functools from pysat.instruments.methods import general as mm_gen -from pysat import logger from pysatNASA.instruments.methods import cdaweb as cdw from pysatNASA.instruments.methods import general as mm_nasa +from pysatNASA.instruments.methods import iss as mm_iss # ---------------------------------------------------------------------------- # Instrument attributes @@ -49,34 +49,7 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - ackn_str = ' '.join(('Data provided through NASA CDAWeb. Contact', - 'Rob.Suggs@nasa.gov for support and use.')) - logger.info(ackn_str) - self.acknowledgements = ackn_str - self.references = ' '.join(('V. N. Coffey et al., "Validation of the', - 'Plasma Densities and Temperatures From', - 'the ISS Floating Potential Measurement', - 'Unit," in IEEE Transactions on Plasma', - 'Science, vol. 36, no. 5, pp. 2301-2308,', - 'Oct. 2008,', - 'doi: 10.1109/TPS.2008.2004271.\n', - 'A. Barjatya, C.M. Swenson, D.C.', - 'Thompson, and K.H. Wright Jr., Data', - 'analysis of the Floating Potential', - 'Measurement Unit aboard the', - 'International Space Station, Rev. Sci.', - 'Instrum. 80, 041301 (2009),', - 'https://doi.org/10.1063/1.3116085')) - - return +init = functools.partial(mm_nasa.init, module=mm_iss, name=name) # Use default clean diff --git a/pysatNASA/instruments/methods/__init__.py b/pysatNASA/instruments/methods/__init__.py index 8542fcbc..4459d353 100644 --- a/pysatNASA/instruments/methods/__init__.py +++ b/pysatNASA/instruments/methods/__init__.py @@ -5,9 +5,11 @@ from pysatNASA.instruments.methods import cnofs # noqa F401 from pysatNASA.instruments.methods import de2 # noqa F401 from pysatNASA.instruments.methods import dmsp # noqa F401 +from pysatNASA.instruments.methods import formosat # noqa F401 from pysatNASA.instruments.methods import general # noqa F401 from pysatNASA.instruments.methods import gps # noqa F401 from pysatNASA.instruments.methods import icon # noqa F401 +from pysatNASA.instruments.methods import iss # noqa F401 from pysatNASA.instruments.methods import jhuapl # noqa F401 from pysatNASA.instruments.methods import omni # noqa F401 from pysatNASA.instruments.methods import ses14 # noqa F401 diff --git a/pysatNASA/instruments/methods/formosat.py b/pysatNASA/instruments/methods/formosat.py new file mode 100644 index 00000000..41ee0f22 --- /dev/null +++ b/pysatNASA/instruments/methods/formosat.py @@ -0,0 +1,14 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for C/NOFS data.""" + +ackn_str = ' '.join(('Data provided through NASA CDAWeb Key Parameters -', + 'Shin-Yi Su (Institute of Space Science, National Central', + 'University, Taiwan, R.O.C.)')) +refs = {'ivm': ' '.join(('Yeh, H.C., S.‐Y. Su, Y.C. Yeh, J.M. Wu, R. A.', + 'Heelis, and B. J. Holt, Scientific mission of the', + 'IPEI payload on board ROCSAT‐1, Terr. Atmos. Ocean.', + 'Sci., 9, suppl., 1999a.\n', + 'Yeh, H.C., S.‐Y. Su, R.A. Heelis, and J.M. Wu, The', + 'ROCSAT‐1 IPEI preliminary results, Vertical ion', + 'drift statistics, Terr. Atmos. Ocean. Sci., 10, 805,', + '1999b.'))} diff --git a/pysatNASA/instruments/methods/iss.py b/pysatNASA/instruments/methods/iss.py new file mode 100644 index 00000000..24ed41ce --- /dev/null +++ b/pysatNASA/instruments/methods/iss.py @@ -0,0 +1,29 @@ +# -*- coding: utf-8 -*- +"""Provides non-instrument specific routines for C/NOFS data.""" + +ackn_str = ' '.join(("R.M. Suggs, S.L. Koontz, NASA Johnson Space Center", + "Contact Rob Suggs for support and use.", + "Rob.Suggs@nasa.gov. Please acknowledge the data", + "providers and CDAWeb when using these data.")) + +refs = {'fpmu': ' '.join(('V. N. Coffey et al., "Validation of the Plasma', + 'Densities and Temperatures From the ISS Floating', + 'Potential Measurement Unit," in IEEE Transactions', + 'on Plasma Science, vol. 36, no. 5, pp. 2301-2308,', + 'Oct. 2008, doi: 10.1109/TPS.2008.2004271.', + '\n', + 'A. Barjatya, C.M. Swenson, D.C. Thompson, and K.H.', + 'Wright Jr., Data analysis of the Floating Potential', + 'Measurement Unit aboard the International Space', + 'Station, Rev. Sci. Instrum. 80, 041301 (2009),', + 'https://doi.org/10.1063/1.3116085', + '\n', + 'Debchoudhury, S., Barjatya, A., Minow, J. I.,', + 'Coffey, V. N., & Chandler, M. O. (2021).', + 'Observations and validation of plasma density,', + 'temperature, and O+ abundance from a Langmuir', + 'probe onboard the International Space Station.', + 'Journal of Geophysical Research: Space', + 'Physics, 126, e2021JA029393.', + 'https://doi.org/10.1029/2021JA029393')) + } diff --git a/pysatNASA/instruments/methods/omni.py b/pysatNASA/instruments/methods/omni.py index ba4fc584..178e91cb 100644 --- a/pysatNASA/instruments/methods/omni.py +++ b/pysatNASA/instruments/methods/omni.py @@ -9,6 +9,17 @@ import pysat +ackn_str = ' '.join(('For full acknowledgement info, please see:', + 'https://omniweb.gsfc.nasa.gov/html/citing.html')) + +refs = {'hro': ' '.join(('J.H. King and N.E. Papitashvili, Solar', + 'wind spatial scales in and comparisons', + 'of hourly Wind and ACE plasma and', + 'magnetic field data, J. Geophys. Res.,', + 'Vol. 110, No. A2, A02209,', + '10.1029/2004JA010649.'))} + + def time_shift_to_magnetic_poles(inst): """Shift OMNI times to intersection with the magnetic pole. diff --git a/pysatNASA/instruments/omni_hro.py b/pysatNASA/instruments/omni_hro.py index d7a205cf..b37f2954 100644 --- a/pysatNASA/instruments/omni_hro.py +++ b/pysatNASA/instruments/omni_hro.py @@ -44,10 +44,10 @@ import pandas as pds import warnings -import pysat from pysat.instruments.methods import general as mm_gen from pysatNASA.instruments.methods import cdaweb as cdw +from pysatNASA.instruments.methods import general as mm_nasa from pysatNASA.instruments.methods import omni as mm_omni # ---------------------------------------------------------------------------- @@ -68,25 +68,7 @@ # ---------------------------------------------------------------------------- # Instrument methods - -def init(self): - """Initialize the Instrument object with instrument specific values. - - Runs once upon instantiation. - - """ - - ackn_str = ''.join(('For full acknowledgement info, please see: ', - 'https://omniweb.gsfc.nasa.gov/html/citing.html')) - self.acknowledgements = ackn_str - self.references = ' '.join(('J.H. King and N.E. Papitashvili, Solar', - 'wind spatial scales in and comparisons', - 'of hourly Wind and ACE plasma and', - 'magnetic field data, J. Geophys. Res.,', - 'Vol. 110, No. A2, A02209,', - '10.1029/2004JA010649.')) - pysat.logger.info(ackn_str) - return +init = functools.partial(mm_nasa.init, module=mm_omni, name=name) def clean(self):