From 22654275c67008ced5d9e1475cf7936127f9a531 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 12:08:26 -0400 Subject: [PATCH 01/60] MAINT: update version --- pyproject.toml | 2 +- setup.cfg | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index 686c8ac5..63b726aa 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "setuptools.build_meta" [project] name = "pysatNASA" -version = "0.0.5" +version = "0.0.6" description = "pysat support for NASA Instruments" readme = "README.md" requires-python = ">=3.6" diff --git a/setup.cfg b/setup.cfg index 08661e10..32dc6c9c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -2,7 +2,7 @@ [metadata] name = pysatNASA -version = 0.0.5 +version = 0.0.6 [flake8] max-line-length = 80 From 888eec6288ff3a4711b935234891f046a86dc4ca Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 12:09:01 -0400 Subject: [PATCH 02/60] MAINT: add tentative release date --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a0e1e4..372e9427 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.0.6] - 2024-XX-XX +## [0.0.6] - 2024-07-31 * New Instruments * DE2 VEFIMAGB - electric and magnetic field on the same cadence * MAVEN mag From b30b0b56e9d087949610b25474e6ef3f93862ebb Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:09:30 -0400 Subject: [PATCH 03/60] MAINT: add new pytest marker --- pyproject.toml | 1 + 1 file changed, 1 insertion(+) diff --git a/pyproject.toml b/pyproject.toml index 63b726aa..8213f9ee 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -84,6 +84,7 @@ markers = [ "download", "no_download", "load_options", + "new_tests", "first", "second" ] From 800975069662a24173bc62732b1efe176fa61445 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:10:29 -0400 Subject: [PATCH 04/60] MAINT: logger.warning --- pysatNASA/instruments/de2_vefi.py | 2 +- pysatNASA/instruments/methods/cdaweb.py | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 28f13a15..516e39cb 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -151,7 +151,7 @@ def load(fnames, tag='', inst_id='', **kwargs): if tag == '': # Warn user that e-field data is dropped. estr = 'E-field data dropped' - pysat.logger.warn(estr) + pysat.logger.warning(estr) # Drop E-field data if 'use_cdflib' in kwargs.keys(): diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 1a3b1382..e277c844 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -164,7 +164,7 @@ def load(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), else: if not use_cdflib: estr = 'The `use_cdflib` option is not currently enabled for xarray' - pysat.logger.warn(estr) + pysat.logger.warning(estr) data, meta = load_xarray(fnames, tag=tag, inst_id=inst_id, epoch_name=epoch_name, @@ -264,8 +264,8 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), tdata = tdata.loc[date:date2, :] ldata.append(tdata) except ValueError as verr: - logger.warn("unable to load {:}: {:}".format(fname, - str(verr))) + logger.warning( + "unable to load {:}: {:}".format(fname, str(verr))) else: # Basic data return with CDF(lfname) as cdf: @@ -274,8 +274,8 @@ def load_pandas(fnames, tag='', inst_id='', file_cadence=dt.timedelta(days=1), flatten_twod=flatten_twod) ldata.append(temp_data) except ValueError as verr: - logger.warn("unable to load {:}: {:}".format(lfname, - str(verr))) + logger.warning( + "unable to load {:}: {:}".format(lfname, str(verr))) # Combine individual files together if len(ldata) > 0: From af3fba57156bf00516e26a86b929ef44537ce350 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:11:02 -0400 Subject: [PATCH 05/60] MAINT: pandas freq --- pysatNASA/instruments/methods/cdaweb.py | 4 ++-- pysatNASA/instruments/methods/omni.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index e277c844..5bd2261f 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -847,7 +847,7 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, if 'month' in search_dir['keys']: search_times = pds.date_range(start, stop + pds.DateOffset(months=1), - freq='M') + freq='ME') for time in search_times: subdir = format_dir.format(year=time.year, month=time.month) url_list.append('/'.join((remote_url, subdir))) @@ -859,7 +859,7 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, else: search_times = pds.date_range(start, stop + pds.DateOffset(years=1), - freq='Y') + freq='YE') for time in search_times: doy = int(time.strftime('%j')) subdir = format_dir.format(year=time.year, day=doy) diff --git a/pysatNASA/instruments/methods/omni.py b/pysatNASA/instruments/methods/omni.py index a6c3df46..40f12df0 100644 --- a/pysatNASA/instruments/methods/omni.py +++ b/pysatNASA/instruments/methods/omni.py @@ -59,7 +59,7 @@ def time_shift_to_magnetic_poles(inst): inst['BSN_x'] = inst['BSN_x'].fillna(method='pad') # Make sure there are no gaps larger than a minute. - inst.data = inst.data.resample('1T').interpolate('time') + inst.data = inst.data.resample('1min').interpolate('time') time_x = inst['BSN_x'] * 6371.2 / -inst['Vx'] idx, = np.where(np.isnan(time_x)) From 4008745ffcab9e95e2f5826f56376edf17e4fc32 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:11:32 -0400 Subject: [PATCH 06/60] MAINT: ffill and bfill --- pysatNASA/instruments/methods/omni.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/methods/omni.py b/pysatNASA/instruments/methods/omni.py index 40f12df0..a4c1f9c5 100644 --- a/pysatNASA/instruments/methods/omni.py +++ b/pysatNASA/instruments/methods/omni.py @@ -51,12 +51,12 @@ def time_shift_to_magnetic_poles(inst): # Need to fill in Vx to get an estimate of what is going on. inst['Vx'] = inst['Vx'].interpolate('nearest') - inst['Vx'] = inst['Vx'].fillna(method='backfill') - inst['Vx'] = inst['Vx'].fillna(method='pad') + inst['Vx'] = inst['Vx'].bfill() + inst['Vx'] = inst['Vx'].ffill() inst['BSN_x'] = inst['BSN_x'].interpolate('nearest') - inst['BSN_x'] = inst['BSN_x'].fillna(method='backfill') - inst['BSN_x'] = inst['BSN_x'].fillna(method='pad') + inst['BSN_x'] = inst['BSN_x'].bfill() + inst['BSN_x'] = inst['BSN_x'].ffill() # Make sure there are no gaps larger than a minute. inst.data = inst.data.resample('1min').interpolate('time') From 53178d402cc1a2aa8478577fe2e74f72c7b88fe5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:11:51 -0400 Subject: [PATCH 07/60] MAINT: iloc --- pysatNASA/instruments/methods/omni.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/omni.py b/pysatNASA/instruments/methods/omni.py index a4c1f9c5..b456044d 100644 --- a/pysatNASA/instruments/methods/omni.py +++ b/pysatNASA/instruments/methods/omni.py @@ -164,12 +164,14 @@ def calculate_imf_steadiness(inst, steady_window=15, min_window_frac=0.75, if steady: del_min = int((inst.data.index[i] - inst.data.index[i - 1]).total_seconds() / 60.0) - if np.isnan(cv) or np.isnan(ca_std[i]) or del_min > sample_rate: + if np.any([np.isnan(cv), + np.isnan(ca_std.iloc[i]), + del_min > sample_rate]): # Reset the steadiness flag if fill values are encountered, or # if an entry is missing steady = False - if cv <= max_bmag_cv and ca_std[i] <= max_clock_angle_std: + if cv <= max_bmag_cv and ca_std.iloc[i] <= max_clock_angle_std: # Steadiness conditions have been met if steady: imf_steady[i] = imf_steady[i - 1] From 50f1953ea25960c93f0dec77f488173572e599da Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:12:06 -0400 Subject: [PATCH 08/60] MAINT: pysat headers --- pysatNASA/tests/test_instruments.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 821dfe58..7a9eb1b9 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -89,13 +89,13 @@ def test_load_cdflib(self, inst_dict): target = 'Fake Data to be cleared' test_inst.data = [target] try: - test_inst.load(date=date, use_header=True, use_cdflib=True) + test_inst.load(date=date, use_cdflib=True) except ValueError as verr: # Check if instrument is failing due to strict time flag if str(verr).find('Loaded data') > 0: test_inst.strict_time_flag = False with warnings.catch_warnings(record=True) as war: - test_inst.load(date=date, use_header=True) + test_inst.load(date=date) assert len(war) >= 1 categories = [war[j].category for j in range(0, len(war))] assert UserWarning in categories From 6d379d4d3eb36b48efcefabfffb25d428897a63f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:12:52 -0400 Subject: [PATCH 09/60] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 372e9427..cdb43e5e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -38,6 +38,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added version cap for sphinx_rtd_theme * Include standard tests for ICON IVM-B * Update NEP29 standards for Jun 2024 + * Updated standards for pandas, numpy, and pysat ## [0.0.5] - 2023-06-27 * New Instruments From 50a717f91b016a6ac422d65eb73a734c8fed0814 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:41:59 -0400 Subject: [PATCH 10/60] MAINT: update GA versions --- .github/workflows/docs.yml | 4 ++-- .github/workflows/main.yml | 4 ++-- .github/workflows/pip_rc_install.yml | 4 ++-- .github/workflows/pysat_rc.yml | 4 ++-- 4 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/docs.yml b/.github/workflows/docs.yml index 2b6d7e88..9e9736bf 100644 --- a/.github/workflows/docs.yml +++ b/.github/workflows/docs.yml @@ -16,9 +16,9 @@ jobs: name: Documentation tests steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 45e17fae..d6ab5ffa 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -27,9 +27,9 @@ jobs: name: Python ${{ matrix.python-version }} on ${{ matrix.os }} with numpy ${{ matrix.numpy_ver }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pip_rc_install.yml b/.github/workflows/pip_rc_install.yml index cc60a7dc..e0502661 100644 --- a/.github/workflows/pip_rc_install.yml +++ b/.github/workflows/pip_rc_install.yml @@ -17,9 +17,9 @@ jobs: name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 85afb492..84bf06b4 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -18,9 +18,9 @@ jobs: name: Python ${{ matrix.python-version }} on ${{ matrix.os }} runs-on: ${{ matrix.os }} steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v4 + uses: actions/setup-python@v5 with: python-version: ${{ matrix.python-version }} From 8cbc2a94a60239c9b0ec7971fd7b885e01bf36b6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 14:42:17 -0400 Subject: [PATCH 11/60] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cdb43e5e..06613778 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -39,6 +39,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Include standard tests for ICON IVM-B * Update NEP29 standards for Jun 2024 * Updated standards for pandas, numpy, and pysat + * Updated versions in GitHub Actions ## [0.0.5] - 2023-06-27 * New Instruments From 00b0d5209354efc7fc6ed6f1d1645b517a1c566b Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 16 Jul 2024 16:04:56 -0400 Subject: [PATCH 12/60] MAINT: usage of dims --- pysatNASA/instruments/methods/jhuapl.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/methods/jhuapl.py b/pysatNASA/instruments/methods/jhuapl.py index 4a3e3bb0..a855c1e2 100644 --- a/pysatNASA/instruments/methods/jhuapl.py +++ b/pysatNASA/instruments/methods/jhuapl.py @@ -282,15 +282,15 @@ def load_sdr_aurora(fnames, name='', tag='', inst_id='', pandas_format=False, # Ensure identical day and night dimensions for GUVI if name == 'guvi': - if sdata.dims['nAlongDay'] != sdata.dims['nAlongNight']: + if sdata.sizes['nAlongDay'] != sdata.sizes['nAlongNight']: raise ValueError('Along-track day and night dimensions differ') if 'nCrossDay' in rename_dims.keys(): - if sdata.dims['nCrossDay'] != sdata.dims['nCrossNight']: + if sdata.sizes['nCrossDay'] != sdata.sizes['nCrossNight']: raise ValueError(''.join([ 'Cross-track day and night dimensions differ ', - '{:} != {:}'.format(sdata.dims['nCrossDay'], - sdata.dims['nCrossNight'])])) + '{:} != {:}'.format(sdata.sizes['nCrossDay'], + sdata.sizes['nCrossNight'])])) # Combine identical dimensions and rename some time dimensions sdata = sdata.rename_dims(rename_dims) From f2d3f0fbff848ddf49fa6b2f95c9a9fefe59c574 Mon Sep 17 00:00:00 2001 From: Aadarsh Govada Date: Mon, 22 Jul 2024 16:28:11 -0400 Subject: [PATCH 13/60] Try coveralls pip installation --- .github/workflows/main.yml | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index d6ab5ffa..e646c8fb 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,16 +96,17 @@ jobs: curl -L https://github.com/coverallsapp/coverage-reporter/releases/latest/download/coveralls-windows.exe -o coveralls.exe ./coveralls.exe report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} + - name: Install Coveralls on MacOS + if: startswith(matrix.os, 'macos') + run: | + pip install coveralls + - name: Coveralls Parallel (MacOS) if: startsWith(matrix.os, 'macos') env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} COVERALLS_PARALLEL: true run: | - brew tap coverallsapp/coveralls --quiet - brew install coveralls --quiet - ls -lh .coverage - ls -lh coverage.xml coveralls report coverage.xml --format=cobertura --verbose --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} --debug --dry-run finish: From 93df6df5a1e69f41097a24b6c94c795f91998d19 Mon Sep 17 00:00:00 2001 From: Aadarsh-Govada <159968006+Aadarsh-Govada@users.noreply.github.com> Date: Mon, 22 Jul 2024 17:37:58 -0400 Subject: [PATCH 14/60] Use verbose and showlocals pytest versions --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index e646c8fb..cb917344 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -76,7 +76,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest + run: pytest -v --showlocals - name: Coveralls Parallel (Ubuntu) if: startsWith(matrix.os, 'ubuntu') From ca77acf8a6c64a5c2d8d617f65b69e3ed7512344 Mon Sep 17 00:00:00 2001 From: Aadarsh-Govada <159968006+Aadarsh-Govada@users.noreply.github.com> Date: Mon, 29 Jul 2024 14:51:05 -0400 Subject: [PATCH 15/60] Cap cdasws version at 1.8.2 --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index 3684998a..f035fe4d 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ beautifulsoup4 -cdasws +cdasws<=1.8.2 cdflib>=0.4.4 lxml netCDF4 From 9c77cabcb309309652caa1b763e01142f350d5cc Mon Sep 17 00:00:00 2001 From: Aadarsh Govada Date: Tue, 30 Jul 2024 10:05:03 -0400 Subject: [PATCH 16/60] Restore MacOS brew coveralls installation --- .github/workflows/main.yml | 20 ++++++-------------- 1 file changed, 6 insertions(+), 14 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index cb917344..53b81e49 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,23 +52,19 @@ jobs: python setup.py develop cd ../pysatNASA pip install . - - name: Install NEP29 dependencies if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} pip install --upgrade-strategy only-if-needed .[test] - - name: Install standard dependencies if: ${{ matrix.test_config == 'latest'}} run: | pip install .[test] - - name: Set up pysat run: | mkdir pysatData python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" - - name: Test PEP8 compliance run: flake8 . --count --select=D,E,F,H,W --show-source --statistics @@ -76,7 +72,7 @@ jobs: run: flake8 . --count --exit-zero --max-complexity=10 --statistics - name: Test with pytest - run: pytest -v --showlocals + run: pytest - name: Coveralls Parallel (Ubuntu) if: startsWith(matrix.os, 'ubuntu') @@ -86,7 +82,6 @@ jobs: run: | curl -sL https://coveralls.io/coveralls-linux.tar.gz | tar -xz ./coveralls report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} - - name: Coveralls Parallel (Windows) if: startsWith(matrix.os, 'windows') env: @@ -95,20 +90,17 @@ jobs: run: | curl -L https://github.com/coverallsapp/coverage-reporter/releases/latest/download/coveralls-windows.exe -o coveralls.exe ./coveralls.exe report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} - - - name: Install Coveralls on MacOS - if: startswith(matrix.os, 'macos') - run: | - pip install coveralls - - name: Coveralls Parallel (MacOS) if: startsWith(matrix.os, 'macos') env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} COVERALLS_PARALLEL: true run: | + brew tap coverallsapp/coveralls --quiet + brew install coveralls --quiet + ls -lh .coverage + ls -lh coverage.xml coveralls report coverage.xml --format=cobertura --verbose --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} --debug --dry-run - finish: name: Finish Coverage Analysis needs: build @@ -121,4 +113,4 @@ jobs: COVERALLS_PARALLEL: true run: | curl -sL https://coveralls.io/coveralls-linux.tar.gz | tar -xz - ./coveralls done --build-number ${{ github.run_number }} + ./coveralls done --build-number ${{ github.run_number }} \ No newline at end of file From d4af585dc1f582bb415f4a88d1f2c1461abc9388 Mon Sep 17 00:00:00 2001 From: Aadarsh Govada Date: Tue, 30 Jul 2024 10:41:33 -0400 Subject: [PATCH 17/60] Remove --quiet from coveralls installation on MacOS --- .github/workflows/main.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53b81e49..6f86b746 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -96,8 +96,8 @@ jobs: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} COVERALLS_PARALLEL: true run: | - brew tap coverallsapp/coveralls --quiet - brew install coveralls --quiet + brew tap coverallsapp/coveralls + brew install coveralls ls -lh .coverage ls -lh coverage.xml coveralls report coverage.xml --format=cobertura --verbose --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} --debug --dry-run From ec99d151ea3432b40668df63aca6c07397267993 Mon Sep 17 00:00:00 2001 From: Aadarsh Govada Date: Tue, 30 Jul 2024 11:55:03 -0400 Subject: [PATCH 18/60] Revert to rcfile coveralls publishing for MacOS --- .github/workflows/main.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 6f86b746..58897909 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -90,17 +90,13 @@ jobs: run: | curl -L https://github.com/coverallsapp/coverage-reporter/releases/latest/download/coveralls-windows.exe -o coveralls.exe ./coveralls.exe report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} - - name: Coveralls Parallel (MacOS) + - name: Publish results to coveralls (MacOS) if: startsWith(matrix.os, 'macos') env: COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} COVERALLS_PARALLEL: true run: | - brew tap coverallsapp/coveralls - brew install coveralls - ls -lh .coverage - ls -lh coverage.xml - coveralls report coverage.xml --format=cobertura --verbose --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} --debug --dry-run + coveralls --rcfile=pyproject.toml --service=github finish: name: Finish Coverage Analysis needs: build From 47b06949327eb5af5a62a79a703ac8e43bdc644a Mon Sep 17 00:00:00 2001 From: Aadarsh Govada Date: Tue, 30 Jul 2024 14:19:50 -0400 Subject: [PATCH 19/60] DOC: Update zenodo.json --- .zenodo.json | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.zenodo.json b/.zenodo.json index 433d6dbf..b708c1df 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -50,6 +50,11 @@ { "name": "Spence, Carey", "orcid": "0000-0001-8340-5625" + }, + { + "affiliation": "Universities Space Research Association, Goddard Space Flight Center", + "name": "Govada, Aadarsh", + "orcid": "0009-0004-7873-5899" } ] } From 573830ad088b15c1f2f1b7df1c108739e10f2a53 Mon Sep 17 00:00:00 2001 From: Aadarsh Govada Date: Tue, 30 Jul 2024 14:21:34 -0400 Subject: [PATCH 20/60] Update CHANGELOG.md --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 06613778..7a1ed72e 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Allow graceful failure with no files in jhuapl load functions * New window needs to be integer for calculate_imf_steadiness * Fixed a bug where cdas_download may drop the requested end date file + * Reverted the coveralls integration to the GitHub service for MacOS runs * Documentation * Added example of how to export data for archival * Updated documentation refs From 763b105d900dad0ec17223b46f22927db1a83e11 Mon Sep 17 00:00:00 2001 From: Aadarsh-Govada <159968006+Aadarsh-Govada@users.noreply.github.com> Date: Tue, 30 Jul 2024 22:20:32 -0400 Subject: [PATCH 21/60] MAINT: Remove cdasws version cap --- requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements.txt b/requirements.txt index f035fe4d..3684998a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,5 +1,5 @@ beautifulsoup4 -cdasws<=1.8.2 +cdasws cdflib>=0.4.4 lxml netCDF4 From a0cbeee486a5ec933db28fc0ef7a3cfcbd2553a1 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 31 Jul 2024 14:14:14 -0400 Subject: [PATCH 22/60] BUG: test returned output Test returned output to provide an empty list if no remote files were found. --- pysatNASA/instruments/methods/cdaweb.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 1a3b1382..e60d3d74 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -981,7 +981,9 @@ def cdas_list_remote_files(tag='', inst_id='', start=None, stop=None, og_files = cdas.get_original_files(dataset=dataset, start=start, end=stop) - if series_out: + if og_files[1] is None: + file_list = pds.Series(dtype=str) if series_out else [] + elif series_out: name_list = [os.path.basename(f['Name']) for f in og_files[1]] t_stamp = [pds.Timestamp(f['StartTime'][:10]) for f in og_files[1]] file_list = pds.Series(data=name_list, index=t_stamp) From 39d4d631ec2a9d7b8ea8fa73f6163a9c3375cd29 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Wed, 31 Jul 2024 14:15:29 -0400 Subject: [PATCH 23/60] DOC: update changelog Added a line to the changelog. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 88a0e1e4..ebcca592 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Allow graceful failure with no files in jhuapl load functions * New window needs to be integer for calculate_imf_steadiness * Fixed a bug where cdas_download may drop the requested end date file + * Fixed a bug where cdas_list_remote_files errored without remote data * Documentation * Added example of how to export data for archival * Updated documentation refs From 8dc0ced62796a4904b11dd848e5bd7ce3724891d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 1 Aug 2024 10:21:49 -0400 Subject: [PATCH 24/60] MAINT: fix broken link --- pysatNASA/instruments/cnofs_vefi.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index 80bfc905..5f8e9b39 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -23,9 +23,10 @@ spacecraft is to enable an accurate V x B measurement along the spacecraft trajectory. In order to provide an in-flight calibration of the magnetic field data, we compare the most recent POMME model (the POtsdam Magnetic Model of the -Earth, https://geomag.us/models/pomme5.html) with the actual magnetometer -measurements to help determine a set of calibration parameters for the gains, -offsets, and non-orthogonality matrix of the sensor axes. The calibrated +Earth, https://geomag.colorado.edu/pomme-5-magnetic-model-of-the-earth.html) +with the actual magnetometer measurements to help determine a set of calibration +parameters for the gains, offsets, and non-orthogonality matrix of the sensor +axes. The calibrated magnetic field measurements are provided in the data file here. The VEFI magnetic field data file currently contains the following variables: B_north Magnetic field in the north direction From 251831a3ecaf3d3e06134e365c85af600c671fcb Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Thu, 1 Aug 2024 10:22:19 -0400 Subject: [PATCH 25/60] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7a1ed72e..d5fbede0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -28,6 +28,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Added example of how to export data for archival * Updated documentation refs * Add keywords to zenodo + * Fixed broken links * Deprecations * Deprecated '' tag for de2_vefi module, support moved to de2_vefimagb * Maintenance From 93924ddb46c639a60195e48d13c11299b57dd230 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 1 Aug 2024 16:54:35 -0400 Subject: [PATCH 26/60] Apply suggestions from code review Co-authored-by: Angeline Burrell Co-authored-by: Russell Stoneback --- CHANGELOG.md | 2 +- docs/supported_instruments.rst | 4 ++-- pyproject.toml | 2 +- pysatNASA/instruments/de2_vefi.py | 10 ++++++++-- pysatNASA/instruments/de2_vefimagb.py | 2 +- pysatNASA/instruments/dmsp_ssusi.py | 6 +++--- pysatNASA/instruments/maven_insitu_kp.py | 10 +++++----- pysatNASA/instruments/maven_mag.py | 2 +- pysatNASA/instruments/maven_sep.py | 8 ++++---- pysatNASA/instruments/methods/maven.py | 7 +------ pysatNASA/instruments/reach_dosimeter.py | 10 +--------- pysatNASA/instruments/timed_guvi.py | 6 +++--- 12 files changed, 31 insertions(+), 38 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index c4475214..730199a1 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,7 +5,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). ## [0.0.6] - 2024-07-31 * New Instruments * DE2 VEFIMAGB - electric and magnetic field on the same cadence - * MAVEN mag + * MAVEN MAG * MAVEN SEP * MAVEN in situ key parameters * REACH Dosimeter diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index d29d4c3f..70c23ab3 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -188,8 +188,8 @@ JPL GPS .. _maven_insitu_kp: -MAVEN INSITU KP ---------------- +MAVEN IN SITU KP +---------------- .. automodule:: pysatNASA.instruments.maven_insitu_kp :members: diff --git a/pyproject.toml b/pyproject.toml index 8213f9ee..5d8f1d4e 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -44,7 +44,7 @@ dependencies = [ "netCDF4", "numpy", "pandas", - "pysat >= 3.1", + "pysat >= 3.2", "requests", "scipy >= 1.4", "xarray" diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index 516e39cb..b4c342df 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -43,7 +43,7 @@ tag '', 'dca', 'ac' inst_id - none supported + None supported Warnings @@ -133,7 +133,9 @@ def load(fnames, tag='', inst_id='', **kwargs): inst_id : str Instrument ID used to identify particular data set to be loaded. This input is nominally provided by pysat itself. (default='') - + **kwargs : unpacked dict + Optional kwargs that will be passed to the + `pysatNASA.instruments.methods.cdaweb.load_xarray` function Returns ------- data : pds.DataFrame @@ -146,6 +148,10 @@ def load(fnames, tag='', inst_id='', **kwargs): Several variables relating to time stored in different formats are dropped. These are redundant and complicate the load procedure. + See Also + -------- + pysatNASA.instruments.methods.cdaweb.load_xarray + """ if tag == '': diff --git a/pysatNASA/instruments/de2_vefimagb.py b/pysatNASA/instruments/de2_vefimagb.py index 83ec6432..65186628 100644 --- a/pysatNASA/instruments/de2_vefimagb.py +++ b/pysatNASA/instruments/de2_vefimagb.py @@ -111,7 +111,7 @@ # Set the load routine def load(fnames, tag='', inst_id='', **kwargs): - """Load DE2 VEFI data. + """Load DE2 VEFI MAG B-field data. This routine is called as needed by pysat. It is not intended for direct user interaction. diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index 96cee90d..f23f6a34 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -130,12 +130,12 @@ def clean(self): def concat_data(self, new_data, combine_times=False, **kwargs): - """Concatonate data to self.data for DMSP SSUSI data. + """Concatenate data to self.data for DMSP SSUSI data. Parameters ---------- new_data : xarray.Dataset or list of such objects - New data objects to be concatonated + New data objects to be concatenated combine_times : bool For SDR data, optionally combine the different datetime coordinates into a single time coordinate (default=False) @@ -154,7 +154,7 @@ def concat_data(self, new_data, combine_times=False, **kwargs): if self.tag in ['sdr-disk', 'sdr2-dist']: time_dims.append('time_auroral') - # Concatonate using the appropriate method for the number of time + # Concatenate using the appropriate method for the number of time # dimensions jhuapl.concat_data(self, time_dims, new_data, combine_times=combine_times, **kwargs) diff --git a/pysatNASA/instruments/maven_insitu_kp.py b/pysatNASA/instruments/maven_insitu_kp.py index d8239e73..ae75c944 100644 --- a/pysatNASA/instruments/maven_insitu_kp.py +++ b/pysatNASA/instruments/maven_insitu_kp.py @@ -7,9 +7,9 @@ # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- -"""Module for the MAVEN insitu instruments. +"""Module for the MAVEN in situ instruments. -Supports the in situ Key Parameter (kp) data from multiple instruments +Supports the in situ Key Parameter (KP) data from multiple instruments onboard the Mars Atmosphere and Volatile Evolution (MAVEN) satellite. Accesses local data in CDF format. @@ -32,9 +32,9 @@ :: import pysat - insitu = pysat.Instrument(platform='maven', name='insitu_kp') - insitu.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) - insitu.load(2020, 1, use_header=True) + insitukp = pysat.Instrument(platform='maven', name='insitu_kp') + insitukp.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) + insitukp.load(2020, 1, use_header=True) """ diff --git a/pysatNASA/instruments/maven_mag.py b/pysatNASA/instruments/maven_mag.py index 738d1fed..6f46b77d 100644 --- a/pysatNASA/instruments/maven_mag.py +++ b/pysatNASA/instruments/maven_mag.py @@ -7,7 +7,7 @@ # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- -"""Module for the MAVEN mag instrument. +"""Module for the MAVEN magnetometer. Supports the Magnetometer (MAG) onboard the Mars Atmosphere and Volatile Evolution (MAVEN) satellite. diff --git a/pysatNASA/instruments/maven_sep.py b/pysatNASA/instruments/maven_sep.py index 9a16945b..123ed2b2 100644 --- a/pysatNASA/instruments/maven_sep.py +++ b/pysatNASA/instruments/maven_sep.py @@ -7,7 +7,7 @@ # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- -"""Module for the MAVEN sep instrument. +"""Module for the MAVEN Solar Energetic Particle instrument. Supports the Solar Energetic Particle (SEP) data from onboard the Mars Atmosphere and Volatile Evolution (MAVEN) satellite. @@ -31,9 +31,9 @@ :: import pysat - insitu = pysat.Instrument(platform='maven', name='sep', inst_id='s1') - insitu.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) - insitu.load(2020, 1, use_header=True) + sep = pysat.Instrument(platform='maven', name='sep', inst_id='s1') + sep.download(dt.datetime(2020, 1, 1), dt.datetime(2020, 1, 31)) + sep.load(2020, 1, use_header=True) """ diff --git a/pysatNASA/instruments/methods/maven.py b/pysatNASA/instruments/methods/maven.py index 1a00fb74..e935ebd7 100644 --- a/pysatNASA/instruments/methods/maven.py +++ b/pysatNASA/instruments/methods/maven.py @@ -7,12 +7,7 @@ # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- -"""Provides non-instrument specific routines for MAVEN data. - -Created on Thu Jul 13 11:21:01 2023 - -@author: tesman -""" +"""Provides non-instrument specific routines for MAVEN data.""" ackn_str = ''.join(('Jakosky, B.M., Lin, R.P., Grebowsky, J.M. et al.', ' The Mars Atmosphere and Volatile Evolution', diff --git a/pysatNASA/instruments/reach_dosimeter.py b/pysatNASA/instruments/reach_dosimeter.py index d56cc0c5..914a3783 100644 --- a/pysatNASA/instruments/reach_dosimeter.py +++ b/pysatNASA/instruments/reach_dosimeter.py @@ -14,7 +14,7 @@ The Responsive Environmental Assessment Commercially Hosted (REACH) constellation is collection of 32 small sensors hosted on six orbital planes of -the Iridium-Next space vehicles in low earth orbit. Each sensor contains two +the Iridium-Next space vehicles in Low Earth Orbit. Each sensor contains two micro-dosimeters sensitive to the passage of charged particles from the Earth's radiation belts. There are six distinct dosimeter types spread among the 64 individual sensors, which are unique in shielding and electronic threshold. @@ -33,8 +33,6 @@ '101', '102', '105', '108', '113', '114', '115', '116', '133', '134', '135', '136', '137', '138', '139', '140', '148', '149', '162', '163', '164', '165', '166', '169', '170', '171', '172', '173', '175', '176', '180', '181' - - """ import datetime as dt @@ -69,8 +67,6 @@ # ---------------------------------------------------------------------------- # Instrument methods - - # Use standard init routine init = functools.partial(mm_nasa.init, module=mm_reach, name=name) @@ -84,8 +80,6 @@ def preprocess(self): self.acknowledgements = self.meta.header.Acknowledgement return - - # ---------------------------------------------------------------------------- # Instrument functions # @@ -160,8 +154,6 @@ def load(fnames, tag=None, inst_id=None): meta.header = MetaHeader(new_header) return data, meta - - # Support download routine download_tags = {iid: {'': 'REACH-VID-{iid}_DOSIMETER-L1C'.format(iid=iid)} for iid in inst_ids.keys()} diff --git a/pysatNASA/instruments/timed_guvi.py b/pysatNASA/instruments/timed_guvi.py index 5e1ef6ac..b97c6e5b 100644 --- a/pysatNASA/instruments/timed_guvi.py +++ b/pysatNASA/instruments/timed_guvi.py @@ -138,12 +138,12 @@ def clean(self): def concat_data(self, new_data, combine_times=False, **kwargs): - """Concatonate data to self.data for TIMED GUVI data. + """Concatenate data to self.data for TIMED GUVI data. Parameters ---------- new_data : xarray.Dataset or list of such objects - New data objects to be concatonated + New data objects to be concatenated combine_times : bool For SDR data, optionally combine the different datetime coordinates into a single time coordinate (default=False) @@ -164,7 +164,7 @@ def concat_data(self, new_data, combine_times=False, **kwargs): elif self.tag == 'sdr-spectrograph': time_dims.extend(['time_gaim_day', 'time_gaim_night']) - # Concatonate using the appropriate method for the number of time + # Concatenate using the appropriate method for the number of time # dimensions jhuapl.concat_data(self, time_dims, new_data, combine_times=combine_times, **kwargs) From 52e4baee6790de9ee76abc5ad55111b5e87e1ba7 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 5 Aug 2024 15:05:23 -0400 Subject: [PATCH 27/60] DOC: updated GOLD docstring Updated the format in a GOLD docstring. --- pysatNASA/instruments/ses14_gold.py | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/pysatNASA/instruments/ses14_gold.py b/pysatNASA/instruments/ses14_gold.py index 61bb8caa..3d657f8c 100644 --- a/pysatNASA/instruments/ses14_gold.py +++ b/pysatNASA/instruments/ses14_gold.py @@ -158,21 +158,19 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, Instrument ID (default='') start : dt.datetime or NoneType Starting time for file list. A None value will start with the first - file found. - (default=None) + file found. (default=None) stop : dt.datetime or NoneType Ending time for the file list. A None value will stop with the last - file found. - (default=None) + file found. (default=None) series_out : bool - boolean to determine output type. True for pandas series of file names, - and False for a list of the full web address. - (default=True) + Boolean to determine output type. True for pandas series of file names, + and False for a list of the full web address. (default=True) Returns ------- - file_list : list - A list containing the verified available files + file_list : pds.Series or list + A Series or list (if tag is not 'tlimb' and `series_out` is False) + containing the verified available files """ From 2ffeeab7b49452d699b8bbf15723b08b6bb0b051 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 5 Aug 2024 15:10:19 -0400 Subject: [PATCH 28/60] ENH: added SSUSI file format function Added a function to provide the correct filename format for DMSP SSUSI data based on the date. --- pysatNASA/instruments/methods/dmsp.py | 53 +++++++++++++++++++++++++++ 1 file changed, 53 insertions(+) diff --git a/pysatNASA/instruments/methods/dmsp.py b/pysatNASA/instruments/methods/dmsp.py index 16b55753..aa406beb 100644 --- a/pysatNASA/instruments/methods/dmsp.py +++ b/pysatNASA/instruments/methods/dmsp.py @@ -9,6 +9,8 @@ # ---------------------------------------------------------------------------- """Provides non-instrument specific routines for the DMSP data.""" +import datetime as dt + ackn_str = "".join(["This Defense Meteorological Satellite Program (DMSP) ", "satellite data is provided through CDAWeb"]) @@ -22,3 +24,54 @@ 'Sensing, and Instrumentation for Atmospheric and ', 'Space Research IV, (30 January 2002); ', 'doi:10.1117/12.454268'))} + +# The DMSP SSUSI filename format in SPDF fully changes on 81-2023 to have a +# 6-padded revision number (previously 5-digit revision number +fmt_swap_time = dt.datetime(2023, 3, 22) + + +def ssusi_fname(ftimes, tag=None, inst_id=None): + """Provide a DMSP SSUSI filename format for the desired time. + + Parameters + ---------- + ftimes : list of dt.datetime + List of dates and times to retrieve the filename format. + tag : str or NoneType + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default=None) + inst_id : str or NoneType + Satellite ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default=None) + + Returns + ------- + file_fmts : list + List of filename formats for the desired times, if either `tag` or + `inst_id` are not supplied (NoneType provided), these will be included + as first-level format options, while the date, version, and cycle are + second-level format options. Otherwise the date, version, and cycle are + first-level format options. + + """ + file_fmts = list() + for ftime in ftimes: + if ftime < fmt_swap_time: + file_fmt = ''.join(['dmsp{inst_id:s}_ssusi_{tag:s}_{{year:04d}}', + '{{day:03d}}T{{hour:02d}}{{minute:02d}}', + '{{second:02d}}-???????T??????-REV?????_vA', + '{{version:1d}}.?.?r{{cycle:03d}}.nc']) + else: + file_fmt = ''.join(['dmsp{inst_id:s}_ssusi_{tag:s}_{{year:04d}}', + '{{day:03d}}T{{hour:02d}}{{minute:02d}}', + '{{second:02d}}-???????T??????-REV??????_vA', + '{{version:1d}}.?.?r{{cycle:03d}}.nc']) + + # If desired, format the tag and inst_id + if tag is not None and inst_id is not None: + file_fmt = file_fmt.format(tag=tag, inst_id=inst_id) + + # Save to the list + file_fmts.append(file_fmt) + + return file_fmts From 1f48f00b743046d0f0d18090b659db7562953dd5 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 5 Aug 2024 15:11:19 -0400 Subject: [PATCH 29/60] BUG: fix file format Added custom `download`, `list_files`, and `list_remote_files` methods that account for the changes in the file format. --- pysatNASA/instruments/dmsp_ssusi.py | 166 ++++++++++++++++++++++++---- 1 file changed, 146 insertions(+), 20 deletions(-) diff --git a/pysatNASA/instruments/dmsp_ssusi.py b/pysatNASA/instruments/dmsp_ssusi.py index 96cee90d..d06d1cc9 100644 --- a/pysatNASA/instruments/dmsp_ssusi.py +++ b/pysatNASA/instruments/dmsp_ssusi.py @@ -61,6 +61,7 @@ import datetime as dt import functools +import pandas as pds import pysat from pysat.instruments.methods import general as mm_gen @@ -163,32 +164,157 @@ def concat_data(self, new_data, combine_times=False, **kwargs): # ---------------------------------------------------------------------------- # Instrument functions -# -# Use the default CDAWeb and pysat methods +remote_dir = ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', + '/data/{tag:s}/{{year:4d}}/{{day:03d}}/')) + # Set the list_files routine -fname = ''.join(['dmsp{inst_id:s}_ssusi_{tag:s}_{{year:04d}}{{day:03d}}T', - '{{hour:02d}}{{minute:02d}}{{second:02d}}-???????T??????-REV', - '?????_vA{{version:1d}}.?.?r{{cycle:03d}}.nc']) -supported_tags = {sat_id: {tag: fname.format(tag=tag, inst_id=sat_id) - for tag in tags.keys()} - for sat_id in inst_ids.keys()} -list_files = functools.partial(mm_gen.list_files, - supported_tags=supported_tags) +def list_files(tag='', inst_id='', data_path='', **kwargs): + """Return a Pandas Series of every file for DMSP SSUSI data. + + Parameters + ---------- + tag : str + Tag name used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + inst_id : str + Instrument ID used to identify particular data set to be loaded. + This input is nominally provided by pysat itself. (default='') + data_path : str + Path to data directory. This input is nominally provided by pysat + itself. (default='') + **kwargs : dict + Dict of kwargs allowed by `pysat.instruments.general.list_files` + + Returns + ------- + out : pds.Series + A Series containing the verified available files + + See Also + -------- + pysat.Files.from_os, pysat.instruments.general.list_files + + """ + # There are two potential file formats for DMSP SSUSI data, check both + file_fmts = mm_dmsp.ssusi_fname( + [mm_dmsp.fmt_swap_time - dt.timedelta(days=1), mm_dmsp.fmt_swap_time], + tag=tag, inst_id=inst_id) + + out_list = list() + for file_fmt in file_fmts: + supported_tags = {inst_id: {tag: file_fmt}} + out_list.append(mm_gen.list_files(tag=tag, inst_id=inst_id, + data_path=data_path, + supported_tags=supported_tags, + **kwargs)) + + # Combine the outputs + out = pds.concat(out_list) + return out + # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/dmsp/dmsp{inst_id:s}/ssusi/', - '/data/{tag:s}/{{year:4d}}/{{day:03d}}/')), - 'fname': fname} -download_tags = { - sat_id: {tag: {btag: basic_tag[btag].format(tag=tag, inst_id=sat_id) - for btag in basic_tag.keys()} for tag in tags.keys()} - for sat_id in inst_ids.keys()} -download = functools.partial(cdw.download, supported_tags=download_tags) +def download(date_array, tag='', inst_id='', data_path=None): + """Download DMSP SSUSI data. + + Parameters + ---------- + date_array : array-like + Array of datetimes to download data for. Provided by pysat. + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + data_path : str or NoneType + Path to data directory. If None is specified, the value previously + set in Instrument.files.data_path is used. (default=None) + + """ + # Initalize the supported tags kwarg + supported_tags = {inst_id: {tag: {'remote_dir': remote_dir.format( + tag=tag, inst_id=inst_id)}}} + + # Determine the filename format for the desired period of time + file_fmts = mm_dmsp.ssusi_fname([date_array[0], date_array[-1]], tag, + inst_id) + + # Proceed differently if there are one or two potential file formats + supported_tags[inst_id][tag]['fname'] = file_fmts[0] + if file_fmts[0] == file_fmts[1]: + cdw.download(date_array, data_path, tag=tag, inst_id=inst_id, + supported_tags=supported_tags) + else: + # Get a mask for the time array + swap_mask = date_array < mm_dmsp.fmt_swap_time + + # Download the first set of data + cdw.download(date_array[swap_mask], data_path, tag=tag, inst_id=inst_id, + supported_tags=supported_tags) + + # Download the second set of data + supported_tags[inst_id][tag]['fname'] = file_fmts[1] + cdw.download(date_array[~swap_mask], data_path, tag=tag, + inst_id=inst_id, supported_tags=supported_tags) + return + # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, - supported_tags=download_tags) +def list_remote_files(tag='', inst_id='', start=None, stop=None): + """List every file for remote DMSP SSUSI data. + + Parameters + ---------- + tag : str + Data product tag (default='') + inst_id : str + Instrument ID (default='') + start : dt.datetime or NoneType + Starting time for file list. A None value will start with the first + file found. (default=None) + stop : dt.datetime or NoneType + Ending time for the file list. A None value will stop with the last + file found. (default=None) + + Returns + ------- + file_list : pds.Series + A Series containing the verified available files + + """ + # Initalize the supported tags kwarg + supported_tags = {inst_id: {tag: {'remote_dir': remote_dir.format( + tag=tag, inst_id=inst_id)}}} + + # Determine the filename format for the desired period of time + start_time = dt.datetime(1900, 1, 1) if start is None else start + stop_time = dt.datetime.now(tz=dt.timezone.utc) if stop is None else stop + file_fmts = mm_dmsp.ssusi_fname([start_time, stop_time], tag, inst_id) + + # Proceed differently if there are one or two potential file formats + supported_tags[inst_id][tag]['fname'] = file_fmts[0] + if file_fmts[0] == file_fmts[1]: + file_list = cdw.list_remote_files(tag=tag, inst_id=inst_id, start=start, + stop=stop, + supported_tags=supported_tags) + else: + # Get the first set of files + file_list_start = cdw.list_remote_files(tag=tag, inst_id=inst_id, + start=start, + stop=mm_dmsp.fmt_swap_time, + supported_tags=supported_tags) + + # Get the second set of files + supported_tags[inst_id][tag]['fname'] = file_fmts[1] + file_list_stop = cdw.list_remote_files(tag=tag, inst_id=inst_id, + start=mm_dmsp.fmt_swap_time, + stop=stop, + supported_tags=supported_tags) + + # Join the two file lists + file_list = pds.concat([file_list_start, file_list_stop]) + + return file_list # Set the load routine From 31e4327e9c92c3506323191b68fbcc87c6d0a892 Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 5 Aug 2024 15:11:47 -0400 Subject: [PATCH 30/60] DOC: update changelog Update the changelog to include a line describing the addition to the DMSP instrument methods. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index ebcca592..950c6cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -14,6 +14,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Allow files to be unzipped after download * Added custom `concat_data` method to JHUAPL methods, for TIMED-GUVI and DMSP-SSUSI data + * Added time-dependent, file format function for DMSP SSUSI to DMSP methods * Added cleaning to TIMED-GUVI SDR imaging data * Bug Fixes * Fix general clean routine to skip transformation matrices From be1bbe390f9f098eea6c38b71fbc15b9b4ecc2ef Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 5 Aug 2024 15:22:03 -0400 Subject: [PATCH 31/60] TST: update instrument tests Update the instrument tests. If `supported_tags` is not present, do not require it. It is only required if the Instrument has 'cdf' files. --- pysatNASA/tests/test_instruments.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index 821dfe58..d9e9a2ef 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -50,11 +50,12 @@ skip_cdf_list = ['de2_vefimagb'] for inst in instruments['download']: - fname = inst['inst_module'].supported_tags[inst['inst_id']][inst['tag']] - if '.cdf' in fname: - temp_inst, _ = clslib.initialize_test_inst_and_date(inst) - if temp_inst.pandas_format and temp_inst.name not in skip_cdf_list: - instruments['cdf'].append(inst) + if hasattr(inst['inst_module'], 'supported_tags'): + fname = inst['inst_module'].supported_tags[inst['inst_id']][inst['tag']] + if '.cdf' in fname: + temp_inst, _ = clslib.initialize_test_inst_and_date(inst) + if temp_inst.pandas_format and temp_inst.name not in skip_cdf_list: + instruments['cdf'].append(inst) class TestInstruments(clslib.InstLibTests): From 4d627af74614187864e1ba0ed9ddf26b32fcc35d Mon Sep 17 00:00:00 2001 From: Angeline Burrell Date: Mon, 5 Aug 2024 15:28:24 -0400 Subject: [PATCH 32/60] STY: fix blank lines Fix PEP8 compliance for blank lines after functions. --- pysatNASA/instruments/reach_dosimeter.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/pysatNASA/instruments/reach_dosimeter.py b/pysatNASA/instruments/reach_dosimeter.py index d56cc0c5..44e59ddf 100644 --- a/pysatNASA/instruments/reach_dosimeter.py +++ b/pysatNASA/instruments/reach_dosimeter.py @@ -91,6 +91,7 @@ def preprocess(self): # # Use the default CDAWeb and pysat methods + # Set the list_files routine datestr = '{year:04d}{month:02d}{day:02d}' fname = 'reach-vid-{inst_id}_dosimeter-l1c_{datestr}_v{{version:01d}}.nc' @@ -100,6 +101,16 @@ def preprocess(self): supported_tags=supported_tags) +# Support download routine +download_tags = {iid: {'': 'REACH-VID-{iid}_DOSIMETER-L1C'.format(iid=iid)} + for iid in inst_ids.keys()} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) + +# Support listing files currently on CDAWeb +list_remote_files = functools.partial(cdw.cdas_list_remote_files, + supported_tags=download_tags) + + def load(fnames, tag=None, inst_id=None): """Load REACH data into `pandas.DataFrame` and `pysat.Meta` objects. @@ -160,13 +171,3 @@ def load(fnames, tag=None, inst_id=None): meta.header = MetaHeader(new_header) return data, meta - - -# Support download routine -download_tags = {iid: {'': 'REACH-VID-{iid}_DOSIMETER-L1C'.format(iid=iid)} - for iid in inst_ids.keys()} -download = functools.partial(cdw.cdas_download, supported_tags=download_tags) - -# Support listing files currently on CDAWeb -list_remote_files = functools.partial(cdw.cdas_list_remote_files, - supported_tags=download_tags) From 0c552cdfd857cce4fce34b178bd45e9d07ba8657 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 9 Aug 2024 16:27:33 -0400 Subject: [PATCH 33/60] BUG: fix broken link --- pysatNASA/instruments/cnofs_vefi.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index 5f8e9b39..81071eaf 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -23,10 +23,9 @@ spacecraft is to enable an accurate V x B measurement along the spacecraft trajectory. In order to provide an in-flight calibration of the magnetic field data, we compare the most recent POMME model (the POtsdam Magnetic Model of the -Earth, https://geomag.colorado.edu/pomme-5-magnetic-model-of-the-earth.html) -with the actual magnetometer measurements to help determine a set of calibration -parameters for the gains, offsets, and non-orthogonality matrix of the sensor -axes. The calibrated +Earth, https://geomag.orgmodels/pomme5.html) with the actual magnetometer +measurements to help determine a set of calibration parameters for the gains, +offsets, and non-orthogonality matrix of the sensor axes. The calibrated magnetic field measurements are provided in the data file here. The VEFI magnetic field data file currently contains the following variables: B_north Magnetic field in the north direction From c4dd40a8448843ebdef0c826aee85de0bdb5ab58 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 9 Aug 2024 17:12:27 -0400 Subject: [PATCH 34/60] BUG: link --- pysatNASA/instruments/cnofs_vefi.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/cnofs_vefi.py b/pysatNASA/instruments/cnofs_vefi.py index 81071eaf..a297f719 100644 --- a/pysatNASA/instruments/cnofs_vefi.py +++ b/pysatNASA/instruments/cnofs_vefi.py @@ -23,7 +23,7 @@ spacecraft is to enable an accurate V x B measurement along the spacecraft trajectory. In order to provide an in-flight calibration of the magnetic field data, we compare the most recent POMME model (the POtsdam Magnetic Model of the -Earth, https://geomag.orgmodels/pomme5.html) with the actual magnetometer +Earth, https://geomag.org/models/pomme5.html) with the actual magnetometer measurements to help determine a set of calibration parameters for the gains, offsets, and non-orthogonality matrix of the sensor axes. The calibrated magnetic field measurements are provided in the data file here. The VEFI From 39f21258d5ba62fd9ed6a043ee1412ea21334c45 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 12 Aug 2024 10:22:50 -0400 Subject: [PATCH 35/60] STY: whitespace --- .github/workflows/main.yml | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 58897909..a38ada92 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -52,19 +52,23 @@ jobs: python setup.py develop cd ../pysatNASA pip install . + - name: Install NEP29 dependencies if: ${{ matrix.test_config == 'NEP29'}} run: | pip install numpy==${{ matrix.numpy_ver }} pip install --upgrade-strategy only-if-needed .[test] + - name: Install standard dependencies if: ${{ matrix.test_config == 'latest'}} run: | pip install .[test] + - name: Set up pysat run: | mkdir pysatData python -c "import pysat; pysat.params['data_dirs'] = 'pysatData'" + - name: Test PEP8 compliance run: flake8 . --count --select=D,E,F,H,W --show-source --statistics @@ -82,6 +86,7 @@ jobs: run: | curl -sL https://coveralls.io/coveralls-linux.tar.gz | tar -xz ./coveralls report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} + - name: Coveralls Parallel (Windows) if: startsWith(matrix.os, 'windows') env: @@ -90,6 +95,7 @@ jobs: run: | curl -L https://github.com/coverallsapp/coverage-reporter/releases/latest/download/coveralls-windows.exe -o coveralls.exe ./coveralls.exe report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} + - name: Publish results to coveralls (MacOS) if: startsWith(matrix.os, 'macos') env: @@ -97,6 +103,7 @@ jobs: COVERALLS_PARALLEL: true run: | coveralls --rcfile=pyproject.toml --service=github + finish: name: Finish Coverage Analysis needs: build From c32c10be555f37b320d92aea2fc1e5a6ea5a4e81 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 13 Aug 2024 10:14:06 -0400 Subject: [PATCH 36/60] MAINT: use coveralls app --- .github/workflows/main.yml | 46 ++++++++++---------------------------- pyproject.toml | 1 - 2 files changed, 12 insertions(+), 35 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a38ada92..a8562cf5 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,42 +78,20 @@ jobs: - name: Test with pytest run: pytest - - name: Coveralls Parallel (Ubuntu) - if: startsWith(matrix.os, 'ubuntu') - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - COVERALLS_PARALLEL: true - run: | - curl -sL https://coveralls.io/coveralls-linux.tar.gz | tar -xz - ./coveralls report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} - - - name: Coveralls Parallel (Windows) - if: startsWith(matrix.os, 'windows') - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - COVERALLS_PARALLEL: true - run: | - curl -L https://github.com/coverallsapp/coverage-reporter/releases/latest/download/coveralls-windows.exe -o coveralls.exe - ./coveralls.exe report --parallel --repo-token=${{ secrets.COVERALLS_REPO_TOKEN }} --build-number ${{ github.run_number }} + - name: Coveralls Parallel + uses: coverallsapp/github-action@v2 + with: + flag-name: run-${{ join(matrix.*, '-') }} + parallel: true - - name: Publish results to coveralls (MacOS) - if: startsWith(matrix.os, 'macos') - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - COVERALLS_PARALLEL: true - run: | - coveralls --rcfile=pyproject.toml --service=github - finish: name: Finish Coverage Analysis needs: build - if: always() - runs-on: "ubuntu-latest" + if: ${{ always() }} + runs-on: ubuntu-latest steps: - - name: Coveralls Finish - env: - COVERALLS_REPO_TOKEN: ${{ secrets.COVERALLS_REPO_TOKEN }} - COVERALLS_PARALLEL: true - run: | - curl -sL https://coveralls.io/coveralls-linux.tar.gz | tar -xz - ./coveralls done --build-number ${{ github.run_number }} \ No newline at end of file + - name: Coveralls Finished + uses: coverallsapp/github-action@v2 + with: + parallel-finished: true + diff --git a/pyproject.toml b/pyproject.toml index 5d8f1d4e..49306051 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -53,7 +53,6 @@ dependencies = [ [project.optional-dependencies] pysatcdf = ["pysatCDF"] test = [ - "coveralls", "flake8", "flake8-docstrings", "hacking >= 1.0", From 5d779d0eb1e1b9603f9afda36c43ecd0a7677293 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 13 Aug 2024 10:14:34 -0400 Subject: [PATCH 37/60] BUG: whitespace --- .github/workflows/main.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index a8562cf5..4c42bd23 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -78,7 +78,7 @@ jobs: - name: Test with pytest run: pytest - - name: Coveralls Parallel + - name: Coveralls Parallel uses: coverallsapp/github-action@v2 with: flag-name: run-${{ join(matrix.*, '-') }} From 0a4db1c401b2b1c0cf1ddabac0eee2b0ae20ede6 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 13 Aug 2024 13:36:14 -0400 Subject: [PATCH 38/60] MAINT: use coveralls app in rc test --- .github/workflows/pysat_rc.yml | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/.github/workflows/pysat_rc.yml b/.github/workflows/pysat_rc.yml index 84bf06b4..db17f76c 100644 --- a/.github/workflows/pysat_rc.yml +++ b/.github/workflows/pysat_rc.yml @@ -42,20 +42,20 @@ jobs: - name: Test with pytest run: pytest - - name: Publish results to coveralls - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - COVERALLS_PARALLEL: true - run: coveralls --rcfile=pyproject.toml --service=github + - name: Coveralls Parallel + uses: coverallsapp/github-action@v2 + with: + flag-name: run-${{ join(matrix.*, '-') }} + parallel: true finish: name: Finish Coverage Analysis needs: build + if: ${{ always() }} runs-on: ubuntu-latest steps: - name: Coveralls Finished - env: - GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} - run: | - pip install --upgrade coveralls - coveralls --service=github --finish + uses: coverallsapp/github-action@v2 + with: + parallel-finished: true + From e827fe621b2337ff9bc10cd39371f376438721e3 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Tue, 13 Aug 2024 13:36:22 -0400 Subject: [PATCH 39/60] DOC: update changelog --- CHANGELOG.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index db7498b4..03f0557d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.0.6] - 2024-07-31 +## [0.0.6] - 2024-08-26 * New Instruments * DE2 VEFIMAGB - electric and magnetic field on the same cadence * MAVEN MAG @@ -44,6 +44,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Update NEP29 standards for Jun 2024 * Updated standards for pandas, numpy, and pysat * Updated versions in GitHub Actions + * Implement coveralls app in GitHUb Actions ## [0.0.5] - 2023-06-27 * New Instruments From 47f7c3ca4fc55959bb03f0b7e888f1b1799a9410 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 14 Aug 2024 10:03:47 -0400 Subject: [PATCH 40/60] Update CHANGELOG.md Co-authored-by: Angeline Burrell --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 03f0557d..8755300c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -44,7 +44,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Update NEP29 standards for Jun 2024 * Updated standards for pandas, numpy, and pysat * Updated versions in GitHub Actions - * Implement coveralls app in GitHUb Actions + * Implement coveralls app in GitHub Actions ## [0.0.5] - 2023-06-27 * New Instruments From cd1204823753aa4fd0dc1185b26de72547484f63 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 20 Aug 2024 14:23:00 -0400 Subject: [PATCH 41/60] MAINT: ensure backward compat with pandas --- pysatNASA/instruments/methods/cdaweb.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 04d37c5e..f85a7bde 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -18,6 +18,7 @@ import datetime as dt import numpy as np import os +from packaging.version import Version as pack_ver import pandas as pds import requests import tempfile @@ -845,9 +846,15 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, if 'year' in search_dir['keys']: url_list = [] if 'month' in search_dir['keys']: + # TODO(#242): remove if/else once support for older pandas is + # dropped. + if pack_ver(pds.__version__) >= pack_ver('2.2.0'): + freq_key = 'ME' + else: + freq_key = 'M' search_times = pds.date_range(start, stop + pds.DateOffset(months=1), - freq='ME') + freq=freq_key) for time in search_times: subdir = format_dir.format(year=time.year, month=time.month) url_list.append('/'.join((remote_url, subdir))) @@ -857,9 +864,16 @@ def list_remote_files(tag='', inst_id='', start=None, stop=None, + pds.DateOffset(days=1), freq='D') else: + + # TODO(#242): remove if/else once support for older pandas + # is dropped. + if pack_ver(pds.__version__) >= pack_ver('2.2.0'): + freq_key = 'YE' + else: + freq_key = 'Y' search_times = pds.date_range(start, stop + pds.DateOffset(years=1), - freq='YE') + freq=freq_key) for time in search_times: doy = int(time.strftime('%j')) subdir = format_dir.format(year=time.year, day=doy) From 1c8fae8a22a2d3b91aea09c1191df4300b74d4cf Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 22 Aug 2024 17:23:42 -0400 Subject: [PATCH 42/60] STY: cleanup testlines in GA ops --- .github/workflows/main.yml | 7 ------- 1 file changed, 7 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 4c42bd23..5a3ee62f 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -45,13 +45,6 @@ jobs: pip install "cdflib<1.0" pip install -r requirements.txt pip install -r test_requirements.txt - cd .. - git clone https://github.com/pysat/pysat.git - cd pysat - git checkout pip_rc_install - python setup.py develop - cd ../pysatNASA - pip install . - name: Install NEP29 dependencies if: ${{ matrix.test_config == 'NEP29'}} From 27385a55de65c533a95059bcfa65b46d32660bb8 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 23 Aug 2024 09:57:28 -0400 Subject: [PATCH 43/60] Apply suggestions from code review Co-authored-by: Angeline Burrell Co-authored-by: Jonathon Smith <36175570+JonathonMSmith@users.noreply.github.com> --- .zenodo.json | 1 + pysatNASA/instruments/maven_insitu_kp.py | 9 +++------ pysatNASA/instruments/maven_mag.py | 9 +++------ pysatNASA/instruments/maven_sep.py | 16 ++++------------ pysatNASA/instruments/methods/cdaweb.py | 3 ++- 5 files changed, 13 insertions(+), 25 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index b708c1df..1f4cc42a 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -5,6 +5,7 @@ "atmosphere", "thermosphere", "magnetosphere", + "mars", "heliosphere", "observations", "models", diff --git a/pysatNASA/instruments/maven_insitu_kp.py b/pysatNASA/instruments/maven_insitu_kp.py index ae75c944..b11bb6f8 100644 --- a/pysatNASA/instruments/maven_insitu_kp.py +++ b/pysatNASA/instruments/maven_insitu_kp.py @@ -88,14 +88,11 @@ list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/maven/insitu/kp-4sec/', - 'cdfs/{year:04d}/{month:02d}')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'MVN_INSITU_KP-4SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) diff --git a/pysatNASA/instruments/maven_mag.py b/pysatNASA/instruments/maven_mag.py index 6f46b77d..2a5741a1 100644 --- a/pysatNASA/instruments/maven_mag.py +++ b/pysatNASA/instruments/maven_mag.py @@ -86,14 +86,11 @@ list_files = functools.partial(mm_gen.list_files, supported_tags=supported_tags) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/maven/mag/l2/sunstate-1sec', - '/cdfs/{year:04d}/{month:02d}')), - 'fname': fname} -download_tags = {'': {'': basic_tag}} -download = functools.partial(cdw.download, supported_tags=download_tags) +download_tags = {'': {'': 'MVN_MAG_L2-SUNSTATE-1SEC'}} +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) # Set the load routine diff --git a/pysatNASA/instruments/maven_sep.py b/pysatNASA/instruments/maven_sep.py index 123ed2b2..ef577c5f 100644 --- a/pysatNASA/instruments/maven_sep.py +++ b/pysatNASA/instruments/maven_sep.py @@ -93,22 +93,14 @@ supported_tags=supported_tags) # Set the download routine -basic_tag = {'remote_dir': ''.join(('/pub/data/maven/sep/l2/s1-cal-svy-full', - '/{year:04d}/{month:02d}')), - 'fname': fname} - -basic_tag2 = {'remote_dir': ''.join(('/pub/data/maven/sep/l2/s2-cal-svy-full', - '/{year:04d}/{month:02d}')), - 'fname': fname2} - -download_tags = {'s1': {'': basic_tag}, - 's2': {'': basic_tag2}} +download_tags = {'s1': {'': 'MVN_SEP_L2_S1-CAL-SVY-FULL'}, + 's2': {'': 'MVN_SEP_L2_S2-CAL-SVY-FULL'}} # Set the download routine -download = functools.partial(cdw.download, supported_tags=download_tags) +download = functools.partial(cdw.cdas_download, supported_tags=download_tags) # Set the list_remote_files routine -list_remote_files = functools.partial(cdw.list_remote_files, +list_remote_files = functools.partial(cdw.cdas_list_remote_files, supported_tags=download_tags) # Set the load routine diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 04d37c5e..d0e435f5 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -610,7 +610,8 @@ def get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): fname : str Name of file on the remote server. temp_path : str - Path to temporary directory. (Default=None) + Path to temporary directory. Must be specified if zip_method is True. + (Default=None) zip_method : str The method used to zip the file. Supports 'zip' and None. If None, downloads files directly. (default=None) From 563b5f1b264ac83d0ba986774e1e996413a14b37 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 26 Aug 2024 17:15:01 -0400 Subject: [PATCH 44/60] Apply suggestions from code review --- docs/archival.rst | 6 +++++- pysatNASA/tests/test_instruments.py | 2 ++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/docs/archival.rst b/docs/archival.rst index 833905d5..9191e737 100644 --- a/docs/archival.rst +++ b/docs/archival.rst @@ -52,7 +52,11 @@ preferred formats. An example of this is: In this case, note that the pysat 'name' label is output to three different metadata values required by the ITSP standards. Additionally, the :py:attr:`export_pysat_info` option is set to false here. This drops several -internal :py:mod:`pysat` metadata values before writing to file. +internal :py:mod:`pysat` metadata values before writing to file. Note that +this includes the default acknowledgements and references objects. These +are set manually to avoid conflicts between the original dataset and the +new dataset, as well as keeping in line with requirements with potentially +different data servers. An example can be found in the sample script below. A full guide to SPDF metadata standards can be found `at SPDF `_. diff --git a/pysatNASA/tests/test_instruments.py b/pysatNASA/tests/test_instruments.py index ae87dc84..138dec02 100644 --- a/pysatNASA/tests/test_instruments.py +++ b/pysatNASA/tests/test_instruments.py @@ -47,6 +47,8 @@ instruments['cdf'] = [] # Create list of pandas instruments where this is not needed +# In general, this is for xarray instruments that are not supported +# by pysatCDF skip_cdf_list = ['de2_vefimagb'] for inst in instruments['download']: From 3a4cc1d5d4fe05aa70e2a54af6d70c3230977242 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 26 Aug 2024 17:17:39 -0400 Subject: [PATCH 45/60] DOC: formosat note --- docs/supported_instruments.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/supported_instruments.rst b/docs/supported_instruments.rst index 70c23ab3..4d368542 100644 --- a/docs/supported_instruments.rst +++ b/docs/supported_instruments.rst @@ -125,6 +125,8 @@ DMSP SSUSI FORMOSAT-1 IVM -------------- +Note that the Ion Velocity Meter data product is managed here since the repository is +at CDAWeb. For other FORMOSAT-X data products, please see pysatCDAAC. .. automodule:: pysatNASA.instruments.formosat1_ivm :members: From c35cb48f742a0dad2f3cd4c8a207c51da89cefd4 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Mon, 26 Aug 2024 17:17:51 -0400 Subject: [PATCH 46/60] DOC: vefimag warning info --- pysatNASA/instruments/de2_vefi.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pysatNASA/instruments/de2_vefi.py b/pysatNASA/instruments/de2_vefi.py index b4c342df..bcc6e9a0 100644 --- a/pysatNASA/instruments/de2_vefi.py +++ b/pysatNASA/instruments/de2_vefi.py @@ -49,6 +49,8 @@ Warnings -------- - Currently no cleaning routine. +- The deprecated '' tag will drop the E-field data. To use this data + product, please use the new de2_vefimagb instrument. """ From d56ec0bc1f57149ec278d4261241d8f7378df46f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Tue, 27 Aug 2024 12:56:05 -0400 Subject: [PATCH 47/60] Update docs/archival.rst --- docs/archival.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/archival.rst b/docs/archival.rst index 9191e737..e8c5d0e2 100644 --- a/docs/archival.rst +++ b/docs/archival.rst @@ -56,7 +56,7 @@ internal :py:mod:`pysat` metadata values before writing to file. Note that this includes the default acknowledgements and references objects. These are set manually to avoid conflicts between the original dataset and the new dataset, as well as keeping in line with requirements with potentially -different data servers. An example can be found in the sample script below. +different data servers. An example can be found in the [REACH Operational Software](https://github.com/jklenzing/ops_reach/blob/main/ops_reach/instruments/methods/reach.py). A full guide to SPDF metadata standards can be found `at SPDF `_. From 53a3b7a32e3583db993ae5452df2d30b6ff5d478 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 29 Aug 2024 10:56:18 -0400 Subject: [PATCH 48/60] Apply suggestions from code review --- ACKNOWLEDGEMENTS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/ACKNOWLEDGEMENTS.md b/ACKNOWLEDGEMENTS.md index 22c05861..2035a263 100644 --- a/ACKNOWLEDGEMENTS.md +++ b/ACKNOWLEDGEMENTS.md @@ -19,6 +19,7 @@ Missions - NASA Scintillation Observations and Response of the Ionosphere to Electrodynamics (SORTIE) - NASA Scintillation Prediction Observations Research Task (SPORT) +- NOAA Constellation Observing System for Meteorology Ionosphere and Climate (COSMIC-2) Disclaimers =========== From 2f278739a4e921fe67d5077c96c14dfa3911579d Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:22:34 -0400 Subject: [PATCH 49/60] BUG: error if temp_dir unset --- pysatNASA/instruments/methods/cdaweb.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 5a56cd07..62ef1e3f 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -542,6 +542,7 @@ def download(date_array, data_path, tag='', inst_id='', supported_tags=None, stop=date_array[-1]) # Create temproary directory if files need to be unzipped. + # Use one temp dir for all files if needed. if 'zip_method' in inst_dict.keys(): zip_method = inst_dict['zip_method'] temp_dir = tempfile.TemporaryDirectory() @@ -577,10 +578,11 @@ def download(date_array, data_path, tag='', inst_id='', supported_tags=None, with requests.get(remote_path) as req: if req.status_code != 404: if zip_method: - get_file(req.content, data_path, fname, - temp_path=temp_dir.name, zip_method=zip_method) + _get_file(req.content, data_path, fname, + temp_path=temp_dir.name, + zip_method=zip_method) else: - get_file(req.content, data_path, fname) + _get_file(req.content, data_path, fname) logger.info(''.join(('Successfully downloaded ', fname, '.'))) else: @@ -599,7 +601,7 @@ def download(date_array, data_path, tag='', inst_id='', supported_tags=None, return -def get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): +def _get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): """Retrieve a file, unzipping if necessary. Parameters @@ -617,11 +619,18 @@ def get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): The method used to zip the file. Supports 'zip' and None. If None, downloads files directly. (default=None) + Warnings + -------- + - Warns if temp_path not set when unzipping. + """ if zip_method: # Use a temporary location. - dl_fname = os.path.join(temp_path, fname) + if temp_path: + dl_fname = os.path.join(temp_path, fname) + else: + raise ValueError('Temp path needs to be set if unzipping') else: # Use the pysat data directory. dl_fname = os.path.join(data_path, fname) From c29d81d7c98dbf8cbb5dd31d9e8064f88557d0f5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 29 Aug 2024 14:22:47 -0400 Subject: [PATCH 50/60] TST: test _get_file --- pysatNASA/tests/test_methods_cdaweb.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pysatNASA/tests/test_methods_cdaweb.py b/pysatNASA/tests/test_methods_cdaweb.py index c5184a06..666e0f2c 100644 --- a/pysatNASA/tests/test_methods_cdaweb.py +++ b/pysatNASA/tests/test_methods_cdaweb.py @@ -109,8 +109,8 @@ def test_bad_zip_warning_get_files(self, caplog): temp_dir = tempfile.TemporaryDirectory() with caplog.at_level(logging.WARNING, logger='pysat'): - cdw.get_file(req.content, '.', 'test.txt', temp_path=temp_dir.name, - zip_method='badzip') + cdw._get_file(req.content, '.', 'test.txt', temp_path=temp_dir.name, + zip_method='badzip') captured = caplog.text # Check for appropriate warning @@ -119,6 +119,14 @@ def test_bad_zip_warning_get_files(self, caplog): return + def test_get_file_unzip_without_temp_path(self): + """Test that warning when cdf file does not have expected params.""" + + with pytest.raises(ValueError) as excinfo: + cdw._get_file('remote_file', 'fake_path', 'fname', zip_method='zip') + assert str(excinfo.value).find('Temp path needs') >= 0 + return + @pytest.mark.parametrize("bad_key,bad_val,err_msg", [("tag", "badval", "inst_id / tag combo unknown."), ("inst_id", "badval", From 97d4e960aa779878d222e6d13f175d64988eb41f Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 29 Aug 2024 16:38:59 -0400 Subject: [PATCH 51/60] Update pysatNASA/instruments/methods/cdaweb.py --- pysatNASA/instruments/methods/cdaweb.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/methods/cdaweb.py b/pysatNASA/instruments/methods/cdaweb.py index 62ef1e3f..b0e71803 100644 --- a/pysatNASA/instruments/methods/cdaweb.py +++ b/pysatNASA/instruments/methods/cdaweb.py @@ -619,9 +619,9 @@ def _get_file(remote_file, data_path, fname, temp_path=None, zip_method=None): The method used to zip the file. Supports 'zip' and None. If None, downloads files directly. (default=None) - Warnings - -------- - - Warns if temp_path not set when unzipping. + Raises + ------ + ValueError if temp_path not specified for zip_method """ From 8dba3ee39eb74f67dab3b761c5e71ee44e008ff1 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 29 Aug 2024 19:44:59 -0400 Subject: [PATCH 52/60] Update pysatNASA/instruments/templates/template_cdaweb_instrument.py --- pysatNASA/instruments/templates/template_cdaweb_instrument.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/pysatNASA/instruments/templates/template_cdaweb_instrument.py b/pysatNASA/instruments/templates/template_cdaweb_instrument.py index 8c145998..ff69fe90 100644 --- a/pysatNASA/instruments/templates/template_cdaweb_instrument.py +++ b/pysatNASA/instruments/templates/template_cdaweb_instrument.py @@ -11,7 +11,8 @@ Copy and modify this file as needed when adding a new Instrument to pysat. -DO NOT include the NRL distribution statement in your new file. +DO NOT include the NRL distribution statement in your new file. Statements for new +files will have to be approved by NRL separately before the versioned release. This is a good area to introduce the instrument, provide background on the mission, operations, instrumenation, and measurements. From b1f002d8cd3ff064163306598a42da2d615543c5 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Fri, 30 Aug 2024 23:56:55 -0400 Subject: [PATCH 53/60] STY: fix 80 char line length --- .../instruments/templates/template_cdaweb_instrument.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/templates/template_cdaweb_instrument.py b/pysatNASA/instruments/templates/template_cdaweb_instrument.py index ff69fe90..30822080 100644 --- a/pysatNASA/instruments/templates/template_cdaweb_instrument.py +++ b/pysatNASA/instruments/templates/template_cdaweb_instrument.py @@ -11,8 +11,9 @@ Copy and modify this file as needed when adding a new Instrument to pysat. -DO NOT include the NRL distribution statement in your new file. Statements for new -files will have to be approved by NRL separately before the versioned release. +DO NOT include the NRL distribution statement in your new file. Statements for +new files will have to be approved by NRL separately before the versioned +release. This is a good area to introduce the instrument, provide background on the mission, operations, instrumenation, and measurements. @@ -64,7 +65,8 @@ # The platform and name strings associated with this instrument # need to be defined at the top level. # These attributes will be copied over to the Instrument object by pysat. -# The strings used here should also be used to name this file `platform_name.py` +# The strings used here should also be used to name this file +# `platform_name.py` platform = '' name = '' From 9de4cd880d04b3d42c20dfd046261a4154767dea Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 18 Sep 2024 11:21:52 -0400 Subject: [PATCH 54/60] MAINT: cycle ops env test to 3.9 --- .github/workflows/main.yml | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 5a3ee62f..54692314 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -19,8 +19,8 @@ jobs: numpy_ver: "1.24" os: ubuntu-latest test_config: "NEP29" - - python-version: "3.6.8" - numpy_ver: "1.19.5" + - python-version: "3.9" + numpy_ver: "1.23.5" os: "ubuntu-20.04" test_config: "Ops" @@ -42,9 +42,7 @@ jobs: run: | sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} - pip install "cdflib<1.0" - pip install -r requirements.txt - pip install -r test_requirements.txt + pip install --upgrade-strategy only-if-needed .[test] - name: Install NEP29 dependencies if: ${{ matrix.test_config == 'NEP29'}} From 9f9a27ab5ed7f8fa4618f3ec3109966a05234265 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 18 Sep 2024 11:22:02 -0400 Subject: [PATCH 55/60] DOC: update changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8755300c..319524f0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -45,6 +45,7 @@ This project adheres to [Semantic Versioning](https://semver.org/). * Updated standards for pandas, numpy, and pysat * Updated versions in GitHub Actions * Implement coveralls app in GitHub Actions + * Cycled Operational Environment testing ## [0.0.5] - 2023-06-27 * New Instruments From 6e1ef04df4f0023566bedcf1437c98970005413a Mon Sep 17 00:00:00 2001 From: Jeff Klenzing Date: Wed, 18 Sep 2024 11:32:15 -0400 Subject: [PATCH 56/60] BUG: apt get --- .github/workflows/main.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 54692314..2c599693 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -40,7 +40,6 @@ jobs: - name: Install Operational dependencies if: ${{ matrix.test_config == 'Ops'}} run: | - sudo apt-get install libhdf5-serial-dev netcdf-bin libnetcdf-dev pip install --no-cache-dir numpy==${{ matrix.numpy_ver }} pip install --upgrade-strategy only-if-needed .[test] From 4847f53bedf55bfb66385323092dffdbdaac6fb7 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 2 Oct 2024 14:35:45 -0400 Subject: [PATCH 57/60] Update pysatNASA/instruments/templates/template_cdaweb_instrument.py Co-authored-by: Russell Stoneback --- .../instruments/templates/template_cdaweb_instrument.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pysatNASA/instruments/templates/template_cdaweb_instrument.py b/pysatNASA/instruments/templates/template_cdaweb_instrument.py index 30822080..d6670c50 100644 --- a/pysatNASA/instruments/templates/template_cdaweb_instrument.py +++ b/pysatNASA/instruments/templates/template_cdaweb_instrument.py @@ -11,9 +11,9 @@ Copy and modify this file as needed when adding a new Instrument to pysat. -DO NOT include the NRL distribution statement in your new file. Statements for -new files will have to be approved by NRL separately before the versioned -release. +DO NOT include the NRL distribution statement in your new file. Contributions +by NRL developers will need to be cleared for classified or controlled information +separately from the pysat pull request process. This is a good area to introduce the instrument, provide background on the mission, operations, instrumenation, and measurements. From 97067ceaa74851ad6862f2822211d7120736dfd0 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 2 Oct 2024 15:29:26 -0400 Subject: [PATCH 58/60] Apply suggestions from code review Co-authored-by: Russell Stoneback --- .../instruments/templates/template_cdaweb_instrument.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/pysatNASA/instruments/templates/template_cdaweb_instrument.py b/pysatNASA/instruments/templates/template_cdaweb_instrument.py index d6670c50..8f971f5e 100644 --- a/pysatNASA/instruments/templates/template_cdaweb_instrument.py +++ b/pysatNASA/instruments/templates/template_cdaweb_instrument.py @@ -4,6 +4,8 @@ # Full author list can be found in .zenodo.json file # DOI:10.5281/zenodo.3986131 # +# Review Status for Classified or Controlled Information by NRL +# ------------------------------------------------------------- # DISTRIBUTION STATEMENT A: Approved for public release. Distribution is # unlimited. # ---------------------------------------------------------------------------- @@ -12,8 +14,8 @@ Copy and modify this file as needed when adding a new Instrument to pysat. DO NOT include the NRL distribution statement in your new file. Contributions -by NRL developers will need to be cleared for classified or controlled information -separately from the pysat pull request process. +by NRL developers will need to be cleared for classified or controlled +information separately from the pysat pull request process. This is a good area to introduce the instrument, provide background on the mission, operations, instrumenation, and measurements. From cb7d953e7039abc07dace650bca6182231b2a113 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Wed, 2 Oct 2024 16:52:56 -0400 Subject: [PATCH 59/60] Update pysatNASA/instruments/templates/template_cdaweb_instrument.py --- pysatNASA/instruments/templates/template_cdaweb_instrument.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pysatNASA/instruments/templates/template_cdaweb_instrument.py b/pysatNASA/instruments/templates/template_cdaweb_instrument.py index 8f971f5e..c95f8e25 100644 --- a/pysatNASA/instruments/templates/template_cdaweb_instrument.py +++ b/pysatNASA/instruments/templates/template_cdaweb_instrument.py @@ -14,7 +14,7 @@ Copy and modify this file as needed when adding a new Instrument to pysat. DO NOT include the NRL distribution statement in your new file. Contributions -by NRL developers will need to be cleared for classified or controlled +by NRL developers will need to be cleared for classified or controlled information separately from the pysat pull request process. This is a good area to introduce the instrument, provide background From f306c96294c1c724245dc788e8a73cca9a83aa25 Mon Sep 17 00:00:00 2001 From: Jeff Klenzing <19592220+jklenzing@users.noreply.github.com> Date: Thu, 3 Oct 2024 16:59:18 -0400 Subject: [PATCH 60/60] MAINT: update release, authors --- .zenodo.json | 20 ++++++++++---------- CHANGELOG.md | 2 +- 2 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.zenodo.json b/.zenodo.json index 1f4cc42a..648769a0 100644 --- a/.zenodo.json +++ b/.zenodo.json @@ -19,16 +19,16 @@ "name": "Klenzing, Jeff", "orcid": "0000-0001-8321-6074" }, - { - "affiliation": "Cosmic Studio", - "name": "Stoneback, Russell", - "orcid": "0000-0001-7216-4336" - }, { "affiliation": "U.S. Naval Research Laboratory", "name": "Burrell, Angeline G.", "orcid": "0000-0001-8875-9326" }, + { + "affiliation": "Cosmic Studio", + "name": "Stoneback, Russell", + "orcid": "0000-0001-7216-4336" + }, { "affiliation": "Catholic University of America, Goddard Space Flight Center", "name": "Smith, Jonathon", @@ -44,6 +44,11 @@ "name": "Esman, Teresa", "orcid": "0000-0003-0382-6281" }, + { + "affiliation": "Universities Space Research Association, Goddard Space Flight Center", + "name": "Govada, Aadarsh", + "orcid": "0009-0004-7873-5899" + }, { "affiliation": "Predictive Science", "name": "Pembroke, Asher" @@ -51,11 +56,6 @@ { "name": "Spence, Carey", "orcid": "0000-0001-8340-5625" - }, - { - "affiliation": "Universities Space Research Association, Goddard Space Flight Center", - "name": "Govada, Aadarsh", - "orcid": "0009-0004-7873-5899" } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index 319524f0..85b3381c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,7 +2,7 @@ All notable changes to this project will be documented in this file. This project adheres to [Semantic Versioning](https://semver.org/). -## [0.0.6] - 2024-08-26 +## [0.0.6] - 2024-10-03 * New Instruments * DE2 VEFIMAGB - electric and magnetic field on the same cadence * MAVEN MAG