From 1fa719fa11264ac4fe014bffeb761ecb5e15572e Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:24:41 -0500 Subject: [PATCH 01/15] ENH: added general LISIRD functions Added general functions to support LISIRD downloads and data sets. --- .../instruments/methods/__init__.py | 1 + .../instruments/methods/lisird.py | 183 ++++++++++++++++++ 2 files changed, 184 insertions(+) create mode 100644 pysatSpaceWeather/instruments/methods/lisird.py diff --git a/pysatSpaceWeather/instruments/methods/__init__.py b/pysatSpaceWeather/instruments/methods/__init__.py index dfb2c612..7a886b23 100644 --- a/pysatSpaceWeather/instruments/methods/__init__.py +++ b/pysatSpaceWeather/instruments/methods/__init__.py @@ -1,3 +1,4 @@ from pysatSpaceWeather.instruments.methods import general # noqa F401 from pysatSpaceWeather.instruments.methods import ace, dst # noqa F401 from pysatSpaceWeather.instruments.methods import f107, kp_ap # noqa F401 +from pysatSpaceWeather.instruments.methods import lisird # noqa F401 diff --git a/pysatSpaceWeather/instruments/methods/lisird.py b/pysatSpaceWeather/instruments/methods/lisird.py new file mode 100644 index 00000000..f3f7eeca --- /dev/null +++ b/pysatSpaceWeather/instruments/methods/lisird.py @@ -0,0 +1,183 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*-. +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Provides support functions for the LASP LISIRD data base.""" + +import datetime as dt +import json +import numpy as np +import os +import pandas as pds +import requests + +import pysat + +ackn = "".join(["LASP Interactive Solar Irradiance Data Center provides ", + "access to many solar datasets generated by researchers at ", + "LASP and other institutions."]) + + +def references(platform, name, tag, inst_id): + """Provide references for different Instrument data products. + + Parameters + ---------- + platform : str + Instrument platform + name : str + Instrument name + tag : str + Instrument tag + inst_id : str + Instrument ID + + Returns + ------- + refs : str + String of references + + """ + + refs = {'sw': {'mgii': { + 'composite': { + '': ''.join(["Viereck, R. A., Floyd, L. E., Crane, P. C., Woods, ", + "T. N., Knapp, B. G., Rottman, G., Weber, M., Puga,", + " L. C., and DeLand, M. T. (2004), A composite Mg ", + "II index spanning from 1978 to 2003, Space Weather", + ", 2, S10005, doi:10.1029/2004SW000084."])}, + 'sorce': { + '': "\n".join([ + "".join(["Snow, M, William E. McClintock, Thomas N. Woods, ", + "Oran R. White, Jerald W. Harder, and Gary Rottman ", + "(2005). The Mg II Index from SORCE, Solar Phys., ", + "230, 1, 325-344."]), + "".join(["Heath, D. and Schlesinger, B. (1986). The Mg 280-nm ", + "doublet as a monitor of changes in solar ", + "ultraviolet irradiance, JGR, 91, 8672-8682."])])}}}} + + return refs[platform][name][tag][inst_id] + + +def build_lisird_url(lisird_data_name, start, stop): + """Build a LASP LISIRD direct download URL. + + Parameters + ---------- + lisird_data_name : str + Name of the data set on the LISARD server + start : dt.datetime + Start time + stop : dt.datetime + Stop time + + Returns + ------- + url : str + URL that will download the desired data + + """ + + # Define the formatting for the start and stop times + tfmt = "%Y-%m-%dT%H:%M:%S.000Z" + + url = "".join(["https://lasp.colorado.edu/lisird/latis/dap/", + lisird_data_name, ".json?&time>=", start.strftime(tfmt), + '&time<=', stop.strftime(tfmt), + "&format_time(yyyy-MM-dd'T'HH:mm:ss.SSS)"]) + + return url + + +def download(date_array, data_path, local_file_prefix, local_date_fmt, + lisird_data_name, freq, update_files=True, fill_vals=None): + """Download routine for LISIRD data. + + Parameters + ---------- + date_array : array-like + Sequence of dates for which files will be downloaded. + data_path : str + Path to data directory. + local_file_prefix : str + Prefix for local files, e.g., 'tag_' or 'tag_monthly_' + local_date_fmt : str + String format for the local filename, e.g., '%Y-%m-%d' or '%Y-%m' + lisird_data_name : str + Name of the data set on the LISARD server + freq : pds.DateOffset or dt.timedelta + Offset to add to the start date to ensure all data is downloaded + (inclusive) + update_files : bool + Re-download data for files that already exist if True (default=False) + fill_vals : dict or NoneType + Dict of fill values to replace with NaN by variable name or None to + leave alone + + Raises + ------ + IOError + If there is a gateway timeout when downloading data + KeyError + If the `fill_vals` input does not match the downloaded data + + """ + # Initialize the fill_vals dict, if necessary + if fill_vals is None: + fill_vals = {} + + # Cycle through all the dates + for dl_date in date_array: + # Build the local filename + local_file = os.path.join( + data_path, ''.join([local_file_prefix, + dl_date.strftime(local_date_fmt), '.txt'])) + + # Determine if the download should occur + if update_files or not os.path.isfile(local_file): + # Get the URL for the desired data + url = build_lisird_url(lisird_data_name, dl_date, dl_date + freq) + + # The data is returned as a JSON file + req = requests.get(url) + + # Process the JSON file + if req.text.find('Gateway Timeout') >= 0: + raise IOError(''.join(['Gateway timeout when requesting ', + 'file using command: ', url])) + + if req.ok: + raw_dict = json.loads(req.text)[lisird_data_name] + data = pds.DataFrame.from_dict(raw_dict['samples']) + if data.empty: + pysat.logger.warn("no data for {:}".format(dl_date)) + else: + # The URL specifies the time format, so break it down + frac_sec = [int(tval.split('.')[-1]) + for tval in data['time']] + times = [dt.datetime.strptime(tval.split('.')[0], + '%Y-%m-%dT%H:%M:%S') + + dt.timedelta(microseconds=frac_sec[i] * 6) + for i, tval in enumerate(data.pop('time'))] + data.index = times + + # Replace fill value with NaNs + for var in fill_vals.keys(): + if var in data.columns: + idx, = np.where(data[var] == fill_vals[var]) + data.iloc[idx, :] = np.nan + else: + raise KeyError(''.join(['unknown fill value ', + 'variable name supplied: ', + var])) + + # Create a local CSV file + data.to_csv(local_file, header=True) + else: + pysat.logger.info("".join(["Data not downloaded for ", + dl_date.strftime("%d %b %Y"), + ", date may be out of range ", + "for the database."])) + return From f125e8e16aa044fe606388dd4be2f45ab0126d8d Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:25:07 -0500 Subject: [PATCH 02/15] ENH: use LISIRD in F10.7 Use the new LISIRD functions in the F10.7 instrument. --- pysatSpaceWeather/instruments/sw_f107.py | 61 ++---------------------- 1 file changed, 5 insertions(+), 56 deletions(-) diff --git a/pysatSpaceWeather/instruments/sw_f107.py b/pysatSpaceWeather/instruments/sw_f107.py index 2cf9cb4b..f0895afd 100644 --- a/pysatSpaceWeather/instruments/sw_f107.py +++ b/pysatSpaceWeather/instruments/sw_f107.py @@ -57,16 +57,15 @@ import datetime as dt import ftplib -import json import numpy as np import os import pandas as pds import pysat import requests import sys -import warnings from pysatSpaceWeather.instruments.methods import f107 as mm_f107 +from pysatSpaceWeather.instruments.methods import lisird from pysatSpaceWeather.instruments.methods import general logger = pysat.logger @@ -464,60 +463,10 @@ def download(date_array, tag, inst_id, data_path, update_files=False): date_array[-1], freq='MS') # Download from LASP, by month - for dl_date in date_array: - # Create the name to which the local file will be saved - str_date = dl_date.strftime('%Y-%m') - data_file = os.path.join(data_path, - 'f107_monthly_{:s}.txt'.format(str_date)) - - if update_files or not os.path.isfile(data_file): - # Set the download webpage - dstr = ''.join(['http://lasp.colorado.edu/lisird/latis/dap/', - 'noaa_radio_flux.json?time%3E=', - dl_date.strftime('%Y-%m-%d'), - 'T00:00:00.000Z&time%3C=', - (dl_date + pds.DateOffset(months=1) - - pds.DateOffset(days=1)).strftime('%Y-%m-%d'), - 'T00:00:00.000Z']) - - # The data is returned as a JSON file - req = requests.get(dstr) - - # Process the JSON file - if req.text.find('Gateway Timeout') >= 0: - raise IOError(''.join(['Gateway timeout when requesting ', - 'file using command: ', dstr])) - - if req.ok: - raw_dict = json.loads(req.text)['noaa_radio_flux'] - data = pds.DataFrame.from_dict(raw_dict['samples']) - if data.empty: - warnings.warn("no data for {:}".format(dl_date), - UserWarning) - else: - # The file format changed over time - try: - # This is the new data format - times = [dt.datetime.strptime(time, '%Y%m%d') - for time in data.pop('time')] - except ValueError: - # Accepts old file formats - times = [dt.datetime.strptime(time, '%Y %m %d') - for time in data.pop('time')] - data.index = times - - # Replace fill value with NaNs - for var in data.columns: - idx, = np.where(data[var] == -99999.0) - data.iloc[idx, :] = np.nan - - # Create a local CSV file - data.to_csv(data_file, header=True) - else: - pysat.logger.info("".join(["Data not downloaded for ", - dl_date.strftime("%d %b %Y"), - ", date may be out of range ", - "for the database."])) + freq = pds.DateOffset(months=1) - pds.DateOffset(days=1) + lisird.download(date_array, data_path, 'f107_monthly_', '%Y-%m', + 'noaa_radio_flux', freq, update_files, + {'f107_adjusted': -99999.0, 'f107_observed': -99999.0}) elif tag == 'prelim': ftp = ftplib.FTP('ftp.swpc.noaa.gov') # Connect to host, default port From d41a9c455c876c5c3e790695759694ad466a0876 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:25:40 -0500 Subject: [PATCH 03/15] ENH: added MgII Instrument Added an instrument for the LASP MgII core-to-wing ratio index. --- pysatSpaceWeather/instruments/__init__.py | 4 +- pysatSpaceWeather/instruments/sw_mgii.py | 311 ++++++++++++++++++++++ 2 files changed, 313 insertions(+), 2 deletions(-) create mode 100644 pysatSpaceWeather/instruments/sw_mgii.py diff --git a/pysatSpaceWeather/instruments/__init__.py b/pysatSpaceWeather/instruments/__init__.py index 76cb3645..ac91f07d 100644 --- a/pysatSpaceWeather/instruments/__init__.py +++ b/pysatSpaceWeather/instruments/__init__.py @@ -1,7 +1,7 @@ from pysatSpaceWeather.instruments import methods # noqa F401 __all__ = ['ace_epam', 'ace_mag', 'ace_sis', 'ace_swepam', - 'sw_dst', 'sw_f107', 'sw_kp'] + 'sw_dst', 'sw_f107', 'sw_kp', 'sw_mgii'] for inst in __all__: - exec("from pysatSpaceWeather.instruments import {x}".format(x=inst)) + exec("from pysatSpaceWeather.instruments import {inst}".format(inst=inst)) diff --git a/pysatSpaceWeather/instruments/sw_mgii.py b/pysatSpaceWeather/instruments/sw_mgii.py new file mode 100644 index 00000000..0d3da147 --- /dev/null +++ b/pysatSpaceWeather/instruments/sw_mgii.py @@ -0,0 +1,311 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*-. +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Supports the MgII core-to-wing ratio index. + +Properties +---------- +platform + 'sw' +name + 'mgii' +tag + - 'composite' Composite data set of MgII core-to-wing index + - 'sorce' SORCE SOLSTICE MgII core-to-wing index + +Examples +-------- +Download a month of the composite MgII data and load the second day of the +month. +:: + + stime = dt.datetime(1981, 2, 2) + mgii = pysat.Instrument('sw', 'mgii', tag='composite') + mgii.download(start=stime) + mgii.load(date=stime) + +""" + +import datetime as dt +import numpy as np +import pandas as pds +import warnings + +import pysat + +from pysatSpaceWeather.instruments.methods import general +from pysatSpaceWeather.instruments.methods import lisird + + +# ---------------------------------------------------------------------------- +# Instrument attributes + +platform = 'sw' +name = 'mgii' +tags = {'composite': 'Composite data set of MgII core-to-wing index', + 'sorce': 'SORCE SOLSTICE MgII core-to-wing index'} + +# Dict keyed by inst_id that lists supported tags for each inst_id +inst_ids = {'': [tag for tag in tags.keys()]} + +# Dict keyed by inst_id that lists supported tags and a good day of test data +# generate todays date to support loading forecast data +now = dt.datetime.utcnow() +today = dt.datetime(now.year, now.month, now.day) +tomorrow = today + pds.DateOffset(days=1) + +# ---------------------------------------------------------------------------- +# Instrument test attributes + +_test_dates = {'': {'composite': dt.datetime(1981, 11, 6), + 'sorce': dt.datetime(2005, 3, 6)}} + +# ---------------------------------------------------------------------------- +# Instrument methods + +preprocess = general.preprocess + + +def init(self): + """Initialize the Instrument object with instrument specific values.""" + + # Set the required Instrument attributes + self.acknowledgements = lisird.ackn + self.references = lisird.references(self.platform, self.name, self.tag, + self.inst_id) + pysat.logger.info(self.acknowledgements) + + return + + +def clean(self): + """Clean the MGII data, empty function as this is not necessary. + + Warnings + -------- + Warning raised for SORCE data, which has an uncertainty parameter that + should be used to select quality data. + + """ + + if self.tag == 'sorce': + warnings.warn(''.join(["The SORCE MGII core-to-wing ratio has an ", + "associated uncertaintly ('unc') that should ", + 'be considered when using the data']), + UserWarning) + return + + +# ---------------------------------------------------------------------------- +# Instrument functions + + +def load(fnames, tag='', inst_id=''): + """Load MGII core-to-wing ratio index files. + + Parameters + ---------- + fnames : pandas.Series + Series of filenames. + tag : str + Instrument tag. (default='') + inst_id : str + Instrument ID, not used. (default='') + + Returns + ------- + data : pandas.DataFrame + Object containing satellite data. + meta : pysat.Meta + Object containing metadata such as column names and units. + + See Also + -------- + pysat.instruments.methods.general.load_csv_data + + """ + # Get the desired file dates and file names from the daily indexed list + file_dates = list() + if tag == 'composite': + unique_files = list() + for fname in fnames: + file_dates.append(dt.datetime.strptime(fname[-10:], '%Y-%m-%d')) + if fname[0:-11] not in unique_files: + unique_files.append(fname[0:-11]) + fnames = unique_files + + # Load the CSV data files + data = pysat.instruments.methods.general.load_csv_data( + fnames, read_csv_kwargs={"index_col": 0, "parse_dates": True}) + + # If there is a date range, downselect here + if len(file_dates) > 0: + idx, = np.where((data.index >= min(file_dates)) + & (data.index < max(file_dates) + dt.timedelta(days=1))) + data = data.iloc[idx, :] + + # Initalize the metadata + meta = pysat.Meta() + notes = { + 'composite': ''.join([ + 'The Mg II Index is a proxy for solar chromospheric variability. ', + 'This composite data record is based on the work of Viereck et ', + 'al. (2004) Space Weather, vol 2, CiteID S10005 for measurements', + ' from 1978 through 2003. For this time range, the Upper ', + 'Atmosphere Research Satellite (UARS) Solar Ultraviolet Spectral ', + 'Irradiance Monitor (SUSIM), UARS Solar Stellar Irradiance ', + 'Comparison Experiment (SOLSTICE), ERS-2/Global Ozone Monitoring ', + 'Experiment (GOME) and five NOAA solar backscatter ultraviolet ', + 'data sets were used. Initially, the best data sets were ', + 'selected. Then the gaps in the record were filled with data ', + 'from various other Mg II data sets. Where no alternate data were', + ' available, a cubic spline function was used to bridge the ', + 'missing data. In some cases the data gaps were too long for ', + 'reasonable spline fits (more than 5 days), and for these gaps ', + 'the F10.7 cm flux data were scaled to fill the gaps.\nStarting ', + 'in 2003, the data from SORCE SOLSTICE is used exclusively. The ', + 'SOLSTICE spectra have been convolved with a 1.1 nm triangular ', + 'response function to improve the long-term agreement with other ', + 'measurements. All of the data sets have been normalized to a ', + 'commonscale to create a single long-term record.\nFor more ', + 'details please see https://lasp.colorado.edu/home/sorce/', + 'instruments/solstice/solstice-data-product-release-notes/.']), + 'sorce': ''.join([ + 'The SORCE Magnesium II core-to-wing index is produced from SORCE', + ' SOLSTICE instrument measurements using the definition in Snow ', + 'et al. [2005]. The spectral resolution of SORCE SOLSTICE (0.1 ', + 'nm) allows the emission cores of the Mg II doublet to be fully ', + 'resolved and modeled with Gaussian functions. To determine the ', + 'wing irradiances, the spectrum is convolved with a 1.1 nm ', + 'triangular bandpass and then measured at the four wavelengths ', + 'used by NOAA as described in Heath and Schlesinger [1986]. This', + ' method has several advantages; most importantly it extracts a ', + 'more precise measurement of solar activity [Snow et al., 2005]. ', + 'A simple linear correlation with the standard NOAA data product ', + 'can be used to scale this SORCE measurement to be compatible ', + 'with the long-term composite Mg II index maintained by NOAA.'])} + meta['mg_index'] = {meta.labels.units: '', + meta.labels.name: 'MG II index', + meta.labels.notes: notes[tag], + meta.labels.desc: 'MG II core-to-wing ratio index', + meta.labels.fill_val: np.nan, + meta.labels.min_val: 0, + meta.labels.max_val: np.inf} + + if 'unc' in data.columns: + meta['unc'] = {meta.labels.units: '', + meta.labels.name: 'MG II Index Uncertainty', + meta.labels.notes: "".join([ + "As described in Snow et al. (2005), the formal ", + "uncertainty in each SOLSTICE MgII index ", + "measurement is about 0.65%. This is derived from ", + "the statistical uncertainty of the underlying ", + "spectral measurements and the way they are ", + "combined to produce the index."]), + meta.labels.desc: + 'MG II core-to-wing ratio index uncertainty', + meta.labels.fill_val: np.nan, + meta.labels.min_val: 0, + meta.labels.max_val: np.inf} + + return data, meta + + +def list_files(tag='', inst_id='', data_path='', format_str=None): + """List local MGII core-to-wing ratio index files. + + Parameters + ---------- + tag : str + Instrument tag, accepts any value from `tags`. (default='') + inst_id : str + Instrument ID, not used. (default='') + data_path : str + Path to data directory. (default='') + format_str : str or NoneType + User specified file format. If None is specified, the default + formats associated with the supplied tags are used. (default=None) + + Returns + ------- + out_files : pysat._files.Files + A class containing the verified available files + + Note + ---- + Called by pysat. Not intended for direct use by user. + + """ + + # If the format string is not supplied, build the default string + if format_str is None: + format_str = "".join([ + "mgii_", tag, "_{year:04d}-{month:02d}", + '.txt' if tag == 'composite' else '-{day:02d}.txt']) + + # Get the files using the default function + out_files = pysat.Files.from_os(data_path=data_path, format_str=format_str) + + if tag == 'composite': + # Files are by month, going to add date to monthly filename for + # each day of the month. The load routine will load a month of + # data and use the appended date to select out appropriate data. + if not out_files.empty: + out_files.loc[out_files.index[-1] + pds.DateOffset(months=1) + - pds.DateOffset(days=1)] = out_files.iloc[-1] + out_files = out_files.asfreq('D', 'pad') + out_files = out_files + '_' + out_files.index.strftime( + '%Y-%m-%d') + + return out_files + + +def download(date_array, tag, inst_id, data_path): + """Download MGII core-to-wing index data from the appropriate repository. + + Parameters + ---------- + date_array : array-like + Sequence of dates for which files will be downloaded. + tag : str + Denotes type of file to load. + inst_id : str + Specifies the ID for the Instrument (not used). + data_path : str + Path to data directory. + + Raises + ------ + IOError + If a problem is encountered connecting to the gateway or retrieving + data from the repository. + + """ + + # Set the download input options + local_file_prefix = "mgii_{:s}_".format(tag) + lisird_data_name = "{:s}_mg_index".format(tag) + + if tag == "composite": + local_file_prefix = "mgii_composite_" + local_date_fmt = "%Y-%m" + freq = pds.DateOffset(months=1, seconds=-1) + + # Adjust the date array if monthly downloads are desired + if date_array.freq != 'MS': + date_array = pysat.utils.time.create_date_range( + dt.datetime(date_array[0].year, date_array[0].month, 1), + date_array[-1], freq='MS') + + elif tag == "sorce": + local_date_fmt = "%Y-%m-%d" + freq = dt.timedelta(seconds=86399) + + # Download the desired data + lisird.download(date_array, data_path, local_file_prefix, local_date_fmt, + lisird_data_name, freq) + + return From dbb2c30fe7371e401a844a255b56bf902cbccd91 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:27:32 -0500 Subject: [PATCH 04/15] DOC: updated docs API Updated the documentation API. --- docs/methods.rst | 9 +++++++++ docs/supported_instruments.rst | 15 +++++++++++++++ 2 files changed, 24 insertions(+) diff --git a/docs/methods.rst b/docs/methods.rst index 4092bc73..71e91fed 100644 --- a/docs/methods.rst +++ b/docs/methods.rst @@ -45,3 +45,12 @@ filter for other data sets. .. automodule:: pysatSpaceWeather.instruments.methods.kp_ap :members: + +LISIRD +------ +Contains functions to support LASP Interactive Solar IRradiance Datacenter +(LISIRD) data sets and downloads. + + +.. automodule:: pysatSpaceWeather.instruments.methods.lisird + :members: diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 33aa8aee..fc3f7422 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -116,3 +116,18 @@ are available from the German Research Centre for Geosciences at Potsdam, .. automodule:: pysatSpaceWeather.instruments.sw_kp :members: + + +.. _mgii-inst: + +MgII Core-to-Wing Ratio +----------------------- + +The core-to-wing ratio of the solar MgII line is a proxy for solar chromospheric +variability. It has been used to extract a precise measurement of solar +activity at Earth. The two data sets provided by LASP together provide index +values from 1978 through 2020. + + +.. automodule:: pysatSpaceWeather.instruments.sw_mgii + :members: From f39704dc8e693d4b3321f14d756d988eeaac3c0a Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:28:22 -0500 Subject: [PATCH 05/15] DOC: updated CHANGELOG Updated the changelog with a description of this pull request. --- CHANGELOG.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 2cebf5b7..dfa7e165 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,6 +3,12 @@ Change Log All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). +[0.0.9] - 2023-01-18 +-------------------- +* Enhancements + * Added an instrument for the LASP MgII core-to-wing index + * Added functions for general LISIRD downloads + [0.0.8] - 2022-11-29 -------------------- * Bugs From 612419fda0f105a2369a75f0eba37d953f42ecad Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:33:04 -0500 Subject: [PATCH 06/15] STY: updated warning to log warning Changed a UserWarning to a logger warning. --- pysatSpaceWeather/instruments/sw_mgii.py | 18 +++++------------- 1 file changed, 5 insertions(+), 13 deletions(-) diff --git a/pysatSpaceWeather/instruments/sw_mgii.py b/pysatSpaceWeather/instruments/sw_mgii.py index 0d3da147..c0550404 100644 --- a/pysatSpaceWeather/instruments/sw_mgii.py +++ b/pysatSpaceWeather/instruments/sw_mgii.py @@ -32,7 +32,6 @@ import datetime as dt import numpy as np import pandas as pds -import warnings import pysat @@ -82,20 +81,13 @@ def init(self): def clean(self): - """Clean the MGII data, empty function as this is not necessary. - - Warnings - -------- - Warning raised for SORCE data, which has an uncertainty parameter that - should be used to select quality data. - - """ + """Clean the MGII data, empty function as this is not necessary.""" if self.tag == 'sorce': - warnings.warn(''.join(["The SORCE MGII core-to-wing ratio has an ", - "associated uncertaintly ('unc') that should ", - 'be considered when using the data']), - UserWarning) + pysat.logger.warn(''.join(["The SORCE MGII core-to-wing ratio has an ", + "associated uncertaintly ('unc') that ", + "should be considered when using the ", + 'data']), UserWarning) return From 38c5b6acd282e220fb03de31942a6a8299649d55 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:37:52 -0500 Subject: [PATCH 07/15] STY: made imports alphabetical Made the imports alphabetical and imported only one module per line. --- pysatSpaceWeather/instruments/methods/__init__.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pysatSpaceWeather/instruments/methods/__init__.py b/pysatSpaceWeather/instruments/methods/__init__.py index 7a886b23..12743d76 100644 --- a/pysatSpaceWeather/instruments/methods/__init__.py +++ b/pysatSpaceWeather/instruments/methods/__init__.py @@ -1,4 +1,6 @@ +from pysatSpaceWeather.instruments.methods import ace # noqa F401 +from pysatSpaceWeather.instruments.methods import dst # noqa F401 +from pysatSpaceWeather.instruments.methods import f107 # noqa F401 from pysatSpaceWeather.instruments.methods import general # noqa F401 -from pysatSpaceWeather.instruments.methods import ace, dst # noqa F401 -from pysatSpaceWeather.instruments.methods import f107, kp_ap # noqa F401 +from pysatSpaceWeather.instruments.methods import kp_ap # noqa F401 from pysatSpaceWeather.instruments.methods import lisird # noqa F401 From d81590d7d7ce236cb48c3391ade577462215bdb9 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:40:19 -0500 Subject: [PATCH 08/15] STY: updated import order Updated the import order in the F10.7 Instrument module. --- pysatSpaceWeather/instruments/sw_f107.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/sw_f107.py b/pysatSpaceWeather/instruments/sw_f107.py index f0895afd..1d672709 100644 --- a/pysatSpaceWeather/instruments/sw_f107.py +++ b/pysatSpaceWeather/instruments/sw_f107.py @@ -65,8 +65,8 @@ import sys from pysatSpaceWeather.instruments.methods import f107 as mm_f107 -from pysatSpaceWeather.instruments.methods import lisird from pysatSpaceWeather.instruments.methods import general +from pysatSpaceWeather.instruments.methods import lisird logger = pysat.logger From 1d1415dcca427bdfbc67b2e68e928c756f691110 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 13:54:14 -0500 Subject: [PATCH 09/15] BUG: fixed logger warning format Fixed the logger warning format by using the non-deprecated call and not including bad input args. --- pysatSpaceWeather/instruments/sw_mgii.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysatSpaceWeather/instruments/sw_mgii.py b/pysatSpaceWeather/instruments/sw_mgii.py index c0550404..58ff2949 100644 --- a/pysatSpaceWeather/instruments/sw_mgii.py +++ b/pysatSpaceWeather/instruments/sw_mgii.py @@ -84,10 +84,10 @@ def clean(self): """Clean the MGII data, empty function as this is not necessary.""" if self.tag == 'sorce': - pysat.logger.warn(''.join(["The SORCE MGII core-to-wing ratio has an ", - "associated uncertaintly ('unc') that ", - "should be considered when using the ", - 'data']), UserWarning) + pysat.logger.warning(''.join(["The SORCE MGII core-to-wing ratio has ", + "an associated uncertaintly ('unc') ", + "that should be considered when using ", + 'the data'])) return From 09ca7488637762ef6ad66212742a44dc9c291ca4 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 17:31:45 -0500 Subject: [PATCH 10/15] DEP: updated deprecated logger function Update the deprecated logger call 'warn' to 'warning'. --- pysatSpaceWeather/instruments/methods/lisird.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/methods/lisird.py b/pysatSpaceWeather/instruments/methods/lisird.py index f3f7eeca..97dcefc6 100644 --- a/pysatSpaceWeather/instruments/methods/lisird.py +++ b/pysatSpaceWeather/instruments/methods/lisird.py @@ -152,7 +152,7 @@ def download(date_array, data_path, local_file_prefix, local_date_fmt, raw_dict = json.loads(req.text)[lisird_data_name] data = pds.DataFrame.from_dict(raw_dict['samples']) if data.empty: - pysat.logger.warn("no data for {:}".format(dl_date)) + pysat.logger.warning("no data for {:}".format(dl_date)) else: # The URL specifies the time format, so break it down frac_sec = [int(tval.split('.')[-1]) From 1087147d6a6bb40648cf55c3984432853f796c88 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 17:32:15 -0500 Subject: [PATCH 11/15] TST: added unit tests for LISIRD functions Added unit tests for accessible, uncovered lines in the LISIRD functions. --- .../tests/test_methods_lisird.py | 55 +++++++++++++++++++ 1 file changed, 55 insertions(+) create mode 100644 pysatSpaceWeather/tests/test_methods_lisird.py diff --git a/pysatSpaceWeather/tests/test_methods_lisird.py b/pysatSpaceWeather/tests/test_methods_lisird.py new file mode 100644 index 00000000..1dfa06ae --- /dev/null +++ b/pysatSpaceWeather/tests/test_methods_lisird.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*-. +# Full license can be found in License.md +# Full author list can be found in .zenodo.json file +# DOI:10.5281/zenodo.3986138 +# ---------------------------------------------------------------------------- +"""Tests for the LISIRD functions.""" + +import datetime as dt +import logging +import pytest + +from pysatSpaceWeather import test_data_path +from pysatSpaceWeather.instruments import sw_mgii +from pysatSpaceWeather.instruments.methods import lisird + + +class TestLISIRDFunctions(object): + """Unit tests for LISIRD functions.""" + + def setup_method(self): + """Create the test environment.""" + self.in_args = [[dt.datetime(2001, 1, 1)], test_data_path, 'test_', + '%Y-%m-%d', 'sorce_mg_index', + dt.timedelta(seconds=86399)] + + return + + def teardown_method(self): + """Clean up the test environment.""" + + del self.in_args + return + + def test_download_empty_file(self, caplog): + """Test a logger warning is raised when no data is found.""" + + with caplog.at_level(logging.WARNING, logger='pysat'): + lisird.download(*self.in_args) + + # Test the warning + captured = caplog.text + assert captured.find('no data for') >= 0 + return + + def test_download_bad_fill_columns(self): + """Test raises KeyError with unexpected column names from fill_vals.""" + + self.in_args[0] += dt.timedelta(years=5) + + with pytest.raises(KeyError) as kerr: + lisird.download(*self.in_args, fill_vals={"NOPE": -999.0}) + + assert str(kerr).find('unknown fill value variable name supplied') >= 0 + return From c04f510f7750b58e622c640fe7f23bf1b3061079 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Fri, 2 Dec 2022 17:34:27 -0500 Subject: [PATCH 12/15] BUG: update F10.7 freq Update the F10.7 frequency definition to use DateOffset correctly. --- pysatSpaceWeather/instruments/sw_f107.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/instruments/sw_f107.py b/pysatSpaceWeather/instruments/sw_f107.py index 1d672709..11e7bf54 100644 --- a/pysatSpaceWeather/instruments/sw_f107.py +++ b/pysatSpaceWeather/instruments/sw_f107.py @@ -463,7 +463,7 @@ def download(date_array, tag, inst_id, data_path, update_files=False): date_array[-1], freq='MS') # Download from LASP, by month - freq = pds.DateOffset(months=1) - pds.DateOffset(days=1) + freq = pds.DateOffset(months=1, seconds=-1) lisird.download(date_array, data_path, 'f107_monthly_', '%Y-%m', 'noaa_radio_flux', freq, update_files, {'f107_adjusted': -99999.0, 'f107_observed': -99999.0}) From 7278a3724cbc74e252c217d79f2eab111e2fa6c3 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 17:36:54 -0500 Subject: [PATCH 13/15] STY: made flake8 changes Removed whitespace and unused imports. --- pysatSpaceWeather/tests/test_methods_lisird.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/pysatSpaceWeather/tests/test_methods_lisird.py b/pysatSpaceWeather/tests/test_methods_lisird.py index 1dfa06ae..6de87521 100644 --- a/pysatSpaceWeather/tests/test_methods_lisird.py +++ b/pysatSpaceWeather/tests/test_methods_lisird.py @@ -11,7 +11,6 @@ import pytest from pysatSpaceWeather import test_data_path -from pysatSpaceWeather.instruments import sw_mgii from pysatSpaceWeather.instruments.methods import lisird @@ -23,7 +22,6 @@ def setup_method(self): self.in_args = [[dt.datetime(2001, 1, 1)], test_data_path, 'test_', '%Y-%m-%d', 'sorce_mg_index', dt.timedelta(seconds=86399)] - return def teardown_method(self): @@ -47,7 +45,7 @@ def test_download_bad_fill_columns(self): """Test raises KeyError with unexpected column names from fill_vals.""" self.in_args[0] += dt.timedelta(years=5) - + with pytest.raises(KeyError) as kerr: lisird.download(*self.in_args, fill_vals={"NOPE": -999.0}) From c934ce6c60389445f46620c20b789c02fb2e794c Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 17:39:26 -0500 Subject: [PATCH 14/15] STY: remote flake8 complaint This is the order the flake8 CI wants. --- pysatSpaceWeather/tests/test_methods_lisird.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/tests/test_methods_lisird.py b/pysatSpaceWeather/tests/test_methods_lisird.py index 6de87521..2e80ec69 100644 --- a/pysatSpaceWeather/tests/test_methods_lisird.py +++ b/pysatSpaceWeather/tests/test_methods_lisird.py @@ -10,8 +10,8 @@ import logging import pytest -from pysatSpaceWeather import test_data_path from pysatSpaceWeather.instruments.methods import lisird +from pysatSpaceWeather import test_data_path class TestLISIRDFunctions(object): From eba48cd253cea300e634a067194deb73e59e3563 Mon Sep 17 00:00:00 2001 From: "Angeline G. Burrell" Date: Fri, 2 Dec 2022 17:56:04 -0500 Subject: [PATCH 15/15] BUG: fixed time adjustment Fixed the time adjustment to use the correct kwarg. --- pysatSpaceWeather/tests/test_methods_lisird.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatSpaceWeather/tests/test_methods_lisird.py b/pysatSpaceWeather/tests/test_methods_lisird.py index 2e80ec69..7ea36189 100644 --- a/pysatSpaceWeather/tests/test_methods_lisird.py +++ b/pysatSpaceWeather/tests/test_methods_lisird.py @@ -44,7 +44,7 @@ def test_download_empty_file(self, caplog): def test_download_bad_fill_columns(self): """Test raises KeyError with unexpected column names from fill_vals.""" - self.in_args[0] += dt.timedelta(years=5) + self.in_args[0][0] += dt.timedelta(weeks=160) with pytest.raises(KeyError) as kerr: lisird.download(*self.in_args, fill_vals={"NOPE": -999.0})