diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index a5a44c47..d9ba9162 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -9,11 +9,11 @@ jobs: container: image: continuumio/miniconda3:latest steps: - - name: Checkout Files - uses: actions/checkout@v4 - - name: Run Docker to Build - run: | - conda config --append channels conda-forge - conda config --append channels noaa-gfdl - conda install conda-build conda-verify - conda build . + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build conda-verify + conda build . diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index e2df48e7..7acb2271 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -7,42 +7,58 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v4 - - name: Set up Python - uses: actions/setup-python@v5 - with: - python-version: '>=3.9' + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '>=3.9' + + - name: Add conda to system path + run: | + # $CONDA is an env var pointing to root of miniconda dir + echo $CONDA/bin >> $GITHUB_PATH + + - name: Create fre-cli environment + run: | + # create environment containing all dependencies + # the env cannot be explicitly activated in github CI/CD + conda env create -f environment.yml --name fre-cli - - name: Add conda to system path - run: | - # $CONDA is an environment variable pointing to the root of the miniconda directory - echo $CONDA/bin >> $GITHUB_PATH + # add conda env's executables to github's PATH equiv. + echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH + + # use *conda environment's pip* to install fre-cli + # called w/ full path to conda's python for explicitness + # called as a module (-m pip) for explicitness + $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli . + + - name: Run pytest in fre-cli environment + run: | + # add conda env's executables to github's PATH equiv. + # does this need to be done twice? + echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - - name: Create fre-cli environment - run: | - # create environment fre-cli will be installed into - conda env create -f environment.yml --name fre-cli - # try to make sure the right things are in GITHUB_PATH - echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - # install fre-cli w pip - $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli . + # are we talking to the right python? + which python + python --version + $CONDA/envs/fre-cli/bin/python --version + + # run pytest, not explicitly from the conda env? + which pytest + pytest --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov=fre fre/ + + - name: Run pylint in fre-cli environment + run: | + # add conda env's executables to github's PATH equiv. + # does this need to be done a third time? + echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - - name: Run pytest in fre-cli environment - run: | - # try to make sure the right things are in GITHUB_PATH - echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - which python - python --version - $CONDA/envs/fre-cli/bin/python --version - # run pytest - pytest --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov=fre fre/ - - - name: Run pylint in fre-cli environment - run: | - # try to make sure the right things are in GITHUB_PATH - echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH - which python - python --version - $CONDA/envs/fre-cli/bin/python --version - # run pytest - pylint fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." + # are we talking to the right python? + which python + python --version + $CONDA/envs/fre-cli/bin/python --version + + # run pylint, not explicitly from the conda env? + which pylint + # ignore toothless complaint pylint has about netCDF4's stuff + pylint --ignored-modules netCDF4 fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo." diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index b2cfce1d..ee93e977 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -17,6 +17,13 @@ jobs: pip install --upgrade sphinx-rtd-theme - name: Sphinx build run: | + # list content of docs dir before spinx-apidoc + ls docs/ + # generate .rst from doc strings in fre/ modules + sphinx-apidoc --output-dir docs fre/ --separate + # list content of docs dir after spinx-apidoc + ls docs/ + # build content in docs/ dir, target is build/ dir sphinx-build docs build - name: Deploy to GitHub Pages uses: peaceiris/actions-gh-pages@v3 diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index b4d00b43..e36a72ea 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -2,20 +2,20 @@ name: publish_conda on: push: branches: - - main + - main jobs: publish: runs-on: ubuntu-latest container: image: continuumio/miniconda3:latest steps: - - name: Checkout Files - uses: actions/checkout@v4 - - name: Run Docker to Build and Upload - run: | - conda config --append channels conda-forge - conda config --append channels noaa-gfdl - conda install conda-build anaconda-client conda-verify - export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} - conda config --set anaconda_upload yes - conda build . + - name: Checkout Files + uses: actions/checkout@v4 + - name: Run Conda to Build and Publish + run: | + conda config --append channels conda-forge + conda config --append channels noaa-gfdl + conda install conda-build anaconda-client conda-verify + export ANACONDA_API_TOKEN=${{ secrets.ANACONDA_TOKEN }} + conda config --set anaconda_upload yes + conda build . diff --git a/docs/conf.py b/docs/conf.py index fa898651..32399782 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -14,7 +14,7 @@ # -- General configuration --------------------------------------------------- # https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration -extensions = [] +extensions = ['sphinx.ext.autodoc'] templates_path = ['_templates'] exclude_patterns = [] diff --git a/fre/README-tool-template.md b/fre/README.md similarity index 94% rename from fre/README-tool-template.md rename to fre/README.md index 870d3772..4570b317 100644 --- a/fre/README-tool-template.md +++ b/fre/README.md @@ -1,3 +1,4 @@ + +* Refer to fre-cli [README.md](https://github.com/NOAA-GFDL/fre-cli/blob/main/README.md) for usage and tips. + + diff --git a/fre/app/__init__.py b/fre/app/__init__.py index c71f9893..c8eeb56b 100644 --- a/fre/app/__init__.py +++ b/fre/app/__init__.py @@ -1,4 +1,4 @@ -from .maskAtmosPlevel import maskAtmosPlevel_subtool -from .freapp import appCli +''' for fre.app imports ''' +from .freapp import app_cli -__all__ = ["maskAtmosPlevel_subtool", "appCli"] +__all__ = ["app_cli"] diff --git a/fre/app/freapp.py b/fre/app/freapp.py index da3dfb84..cdb0ff9a 100644 --- a/fre/app/freapp.py +++ b/fre/app/freapp.py @@ -1,12 +1,19 @@ +#!/usr/bin/env python3 +''' fre app calls ''' + +import time + import click -from .maskAtmosPlevel import maskAtmosPlevel_subtool + +from .mask_atmos_plevel import mask_atmos_plevel_subtool from .generate_time_averages.generate_time_averages import generate @click.group(help=click.style(" - access fre app subcommands", fg=(250,154,90))) -def appCli(): - pass +def app_cli(): + ''' entry point to fre app click commands ''' + -@appCli.command() +@app_cli.command() @click.option("-i", "--infile", type=str, help="Input NetCDF file containing pressure-level output to be masked", @@ -20,11 +27,12 @@ def appCli(): required=True) @click.pass_context def mask_atmos_plevel(context, infile, outfile, psfile): + # pylint: disable=unused-argument """Mask out pressure level diagnostic output below land surface""" - context.forward(maskAtmosPlevel_subtool) + context.forward(mask_atmos_plevel_subtool) -@appCli.command() +@app_cli.command() @click.option("-i", "--inf", type=str, required=True, @@ -57,13 +65,15 @@ def mask_atmos_plevel(context, infile, outfile, psfile): help="Compute standard deviations for time-averages as well") @click.pass_context def gen_time_averages(context, inf, outf, pkg, var, unwgt, avg_type, stddev_type): + # pylint: disable=unused-argument """ - generate time averages for specified set of netCDF files. Example: generate-time-averages.py /path/to/your/files/ + generate time averages for specified set of netCDF files. + Example: generate-time-averages.py /path/to/your/files/ """ - import time start_time=time.perf_counter() context.forward(generate) - print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)') # need to change to a click.echo, not sure if echo supports f strings + # need to change to a click.echo, not sure if echo supports f strings + print(f'Finished in total time {round(time.perf_counter() - start_time , 2)} second(s)') if __name__ == "__main__": - appCli() + app_cli() diff --git a/fre/app/generate_time_averages/frepytoolsTimeAverager.py b/fre/app/generate_time_averages/frepytoolsTimeAverager.py index 7f21e7c9..9a9dd693 100644 --- a/fre/app/generate_time_averages/frepytoolsTimeAverager.py +++ b/fre/app/generate_time_averages/frepytoolsTimeAverager.py @@ -1,4 +1,9 @@ ''' class for python-native routine usuing netCDF4 and numpy to crunch time-averages ''' + +import math +import numpy +from netCDF4 import Dataset + from .timeAverager import timeAverager class frepytoolsTimeAverager(timeAverager): @@ -9,8 +14,9 @@ class inheriting from abstract base class timeAverager ''' def generate_timavg(self, infile=None, outfile=None): - ''' frepytools approach in a python-native manner. deliberately avoids pre-packaged routines. ''' - assert(self.pkg=="fre-python-tools") + ''' frepytools approach in a python-native manner. + deliberately avoids pre-packaged routines. ''' + assert self.pkg=="fre-python-tools" if __debug__: print(locals()) #input argument details @@ -21,10 +27,6 @@ def generate_timavg(self, infile=None, outfile=None): print(f'ERROR: avg_type={self.avg_type} not supported at this time.') return 1 - import math - import numpy - from netCDF4 import Dataset - # (TODO) file I/O should be a sep function, no? make tests, extend nc_fin = Dataset(infile, 'r') if nc_fin.file_format != 'NETCDF4': @@ -36,8 +38,8 @@ def generate_timavg(self, infile=None, outfile=None): # attempt to determine target var w/ bronx convention if self.var is not None: targ_var = self.var - else: - targ_var = infile.split('/').pop().split('.')[-2] # this can be replaced w/ a regex search maybe + else: # this can be replaced w/ a regex search maybe + targ_var = infile.split('/').pop().split('.')[-2] if __debug__: print(f'targ_var={targ_var}') @@ -67,7 +69,9 @@ def generate_timavg(self, infile=None, outfile=None): fin_dims = nc_fin.dimensions num_time_bnds = fin_dims['time'].size if not self.unwgt: #compute sum of weights - wgts = numpy.moveaxis( time_bnds,0,-1)[1][:].copy() - numpy.moveaxis( time_bnds,0,-1)[0][:].copy() +# wgts = numpy.moveaxis( time_bnds,0,-1 )[1][:].copy() - numpy.moveaxis( time_bnds,0,-1 )[0][:].copy() + wgts = numpy.moveaxis( time_bnds,0,-1 )[1][:].copy() \ + - numpy.moveaxis( time_bnds,0,-1 )[0][:].copy() wgts_sum=sum(wgts) if __debug__: print(f'wgts_sum={wgts_sum}') @@ -134,9 +138,11 @@ def generate_timavg(self, infile=None, outfile=None): - # write output file, (TODO) make this a sep function, make tests, extend, consider compression particular;y for NETCDF file writing - # with Dataset( outfile, 'w', format='NETCDF4', persist=True ) as nc_fout: # consider this approach instead - #nc_fout= Dataset( outfile, 'w', format='NETCDF4', persist=True ) # how should we handle file formats (HDF5, Zarr, older NETCDF)? + # write output file + # (TODO) make this a sep function, make tests, extend, + # (TODO) consider compression particular;y for NETCDF file writing + # consider this approach instead: + # with Dataset( outfile, 'w', format='NETCDF4', persist=True ) as nc_fout: nc_fout= Dataset( outfile, 'w', format=nc_fin.file_format, persist=True ) # (TODO) make this a sep function, make tests, extend @@ -156,7 +162,7 @@ def generate_timavg(self, infile=None, outfile=None): print(f'could not get nc file attribute: {ncattr}. moving on.') unwritten_ncattr_list.append(ncattr) if len(unwritten_ncattr_list)>0: - print(f'WARNING: Some global attributes ({unwritten_ncattr_list}) were not successfully written.') + print(f'WARNING: Some global attributes ({unwritten_ncattr_list}) were not written.') print('------- DONE writing output attributes. --------') ## @@ -166,11 +172,13 @@ def generate_timavg(self, infile=None, outfile=None): unwritten_dims_list=[] for key in fin_dims: try: - if key=='time': # this strongly influences the final data structure shape of the averages. - #nc_fout.createDimension( dimname=key, size=None ) # if set to None, and lets say you try to write - # # e.g. the original 'time_bnds' (which has 60 time steps) - # # the array holding the avg. value will suddently have 60 time steps - # # even though only 1 is needed, 59 time steps will have no data + if key=='time': + # this strongly influences the final data structure shape of the averages. + # if set to None, and lets say you try to write + # e.g. the original 'time_bnds' (which has 60 time steps) + # the array holding the avg. value will suddently have 60 time steps + # even though only 1 is needed, 59 time steps will have no data + #nc_fout.createDimension( dimname=key, size=None ) nc_fout.createDimension( dimname=key, size=1) else: nc_fout.createDimension( dimname=key, size=fin_dims[key].size ) @@ -178,22 +186,26 @@ def generate_timavg(self, infile=None, outfile=None): print(f'problem. cannot read/write dimension {key}') unwritten_dims_list.append(key) if len(unwritten_dims_list)>0: - print(f'WARNING: Some dimensions ({unwritten_dims_list}) were not successfully written.') + print(f'WARNING: Some dimensions ({unwritten_dims_list}) were not written.') print('------ DONE writing output dimensions. ------- \n') ## # (TODO) make this a sep function, make tests, extend # first write the data we care most about- those we computed + # copying metadata, not fully correct + # but not far from wrong according to CF + # cell_methods must be changed TO DO print(f'\n------- writing data for data {targ_var} -------- ') nc_fout.createVariable(targ_var, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions) - nc_fout.variables[targ_var].setncatts(nc_fin[targ_var].__dict__) # copying metadata, not fully correct - # but not far from wrong according to CF - # cell_methods must be changed TO DO + nc_fout.variables[targ_var].setncatts(nc_fin[targ_var].__dict__) + + nc_fout.variables[targ_var][:]=avgvals if self.stddev_type is not None: stddev_varname=targ_var+'_'+self.stddev_type+'_stddev' - nc_fout.createVariable(stddev_varname, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions) + nc_fout.createVariable( + stddev_varname, nc_fin[targ_var].dtype, nc_fin[targ_var].dimensions ) nc_fout.variables[stddev_varname].setncatts(nc_fin[targ_var].__dict__) nc_fout.variables[stddev_varname][:]=stddevs print('---------- DONE writing output variables. ---------') @@ -217,15 +229,15 @@ def generate_timavg(self, infile=None, outfile=None): print(f'nc_fin[var].shape={nc_fin[var].shape}') #print(f'len(nc_fout.variables[{var}])={len(nc_fout.variables[var])}') nc_fout.variables[var][:] = [ nc_fin[var][0] ] - print(f'is it a time variable? {self.var_has_time_units(nc_fin.variables[var])}') + print(f'time variable? {self.var_has_time_units(nc_fin.variables[var])}') else: continue if len(unwritten_var_list)>0: - print(f'WARNING: some variables\' data ({unwritten_var_list}) was not successfully written.') + print(f'WARNING: some variables\' data ({unwritten_var_list}) was not written.') if len(unwritten_var_ncattr_dict)>0: - print(f'WARNING: some variables\' metadata was not successfully written.') - print(f'WARNING: relevant variable (key) and attribute (value) pairs: \n{unwritten_var_ncattr_dict}') + print('WARNING: some variables\' metadata was not successfully written.') + print(f'WARNING: relevant variable/attr pairs: \n{unwritten_var_ncattr_dict}') print('---------- DONE writing output variables. ---------') ## diff --git a/fre/app/generate_time_averages/tests/test_generate_time_averages.py b/fre/app/generate_time_averages/tests/test_generate_time_averages.py index 82400990..b8db14b0 100644 --- a/fre/app/generate_time_averages/tests/test_generate_time_averages.py +++ b/fre/app/generate_time_averages/tests/test_generate_time_averages.py @@ -1,18 +1,20 @@ +''' for testing fre app generate-time-averages ''' import pathlib as pl import pytest # not a test function, to be called by test functions below. -def run_avgtype_pkg_calculations(infile=None,outfile=None, pkg=None, avg_type=None, unwgt=None, stddev_type=None): - assert all( [infile is not None, outfile is not None, +def run_avgtype_pkg_calculations(infile=None,outfile=None, pkg=None, avg_type=None, unwgt=None, + stddev_type=None): + assert all( [infile is not None, outfile is not None, pkg is not None, avg_type is not None, unwgt is not None] ) if pl.Path(outfile).exists(): - print(f'output test file exists. deleting before remaking.') + print('output test file exists. deleting before remaking.') pl.Path(outfile).unlink() #delete file so we check that it can be recreated from fre.app.generate_time_averages import generate_time_averages as gtas - gtas.generate_time_average(infile = infile, outfile = outfile, - pkg = pkg, unwgt = unwgt, - avg_type = avg_type, stddev_type = stddev_type) + gtas.generate_time_average(infile = infile, outfile = outfile, + pkg = pkg, unwgt = unwgt, + avg_type = avg_type, stddev_type = stddev_type) return pl.Path(outfile).exists() ### preamble tests. if these fail, none of the others will succeed. ----------------- @@ -30,23 +32,26 @@ def test_time_avg_input_file_exists(): ### cdo avgs, unweighted, all/seasonal/monthly ------------------------ def test_monthly_cdo_time_unwgt_avgs(): ''' generates an unweighted monthly time averaged file using cdo ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'ymonmean_unwgt_'+test_file_name), - pkg='cdo',avg_type='month',unwgt=True) - + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'ymonmean_unwgt_'+test_file_name), + pkg='cdo',avg_type='month',unwgt=True ) + def test_seasonal_cdo_time_unwgt_avgs(): ''' generates an unweighted seasonal time averaged file using cdo ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'yseasmean_unwgt_'+test_file_name), - pkg='cdo',avg_type='seas',unwgt=True) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'yseasmean_unwgt_'+test_file_name), + pkg='cdo',avg_type='seas',unwgt=True ) def test_cdo_time_unwgt_avgs(): ''' generates an unweighted time averaged file using cdo ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'timmean_unwgt_'+test_file_name), - pkg='cdo',avg_type='all',unwgt=True) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'timmean_unwgt_'+test_file_name), + pkg='cdo',avg_type='all',unwgt=True ) + - #### cdo avgs, weighted, all/seasonal/monthly ------------------------ ## (TODO) WRITE THESE VERSIONS FOR CDOTIMEAVERAGER CLASS THEN MAKE THESE TESTS ##def test_monthly_cdo_time_avgs(): @@ -54,29 +59,33 @@ def test_cdo_time_unwgt_avgs(): def test_cdo_time_avgs(): ''' generates a weighted time averaged file using cdo ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'timmean_'+test_file_name), - pkg='cdo',avg_type='all',unwgt=False) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'timmean_'+test_file_name), + pkg='cdo',avg_type='all',unwgt=False ) ### cdo stddevs, unweighted, all/seasonal/monthly ------------------------ def test_monthly_cdo_time_unwgt_stddevs(): ''' generates a monthly time averaged file using cdo ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'ymonstddev1_unwgt_'+test_file_name), - pkg='cdo',avg_type='month',stddev_type='samp', unwgt=True) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'ymonstddev1_unwgt_'+test_file_name), + pkg='cdo',avg_type='month',stddev_type='samp', unwgt=True ) def test_seasonal_cdo_time_unwgt_stddevs(): ''' generates a seasonal time averaged file using cdo ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'yseasstddev1_unwgt_'+test_file_name), - pkg='cdo',avg_type='seas',stddev_type='samp',unwgt=True) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'yseasstddev1_unwgt_'+test_file_name), + pkg='cdo',avg_type='seas',stddev_type='samp',unwgt=True ) def test_cdo_time_unwgt_stddevs(): ''' generates a time averaged file using cdo ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'yseasmean_unwgt_'+test_file_name), - pkg='cdo',avg_type='all',stddev_type='samp', unwgt=True) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'yseasmean_unwgt_'+test_file_name), + pkg='cdo',avg_type='all',stddev_type='samp', unwgt=True ) #### cdo stddevs, weighted, all/seasonal/monthly ----------------------- @@ -84,35 +93,39 @@ def test_cdo_time_unwgt_stddevs(): #def test_monthly_cdo_time_stddevs(): #def test_seasonal_cdo_time_stddevs(): #def test_cdo_time_stddevs(): - + ## frepythontools avgs+stddevs, weighted+unweighted, all ------------------------ def test_fre_python_tools_time_avgs(): ''' generates a time averaged file using fre_python_tools's version ''' ''' weighted average, no std deviation ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'frepytools_timavg_'+test_file_name), - pkg='fre-python-tools',avg_type='all', unwgt=False) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'frepytools_timavg_'+test_file_name), + pkg='fre-python-tools',avg_type='all', unwgt=False ) def test_fre_python_tools_time_unwgt_avgs(): ''' generates a time averaged file using fre_python_tools's version ''' ''' weighted average, no std deviation ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'frepytools_unwgt_timavg_'+test_file_name), - pkg='fre-python-tools',avg_type='all', unwgt=True) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'frepytools_unwgt_timavg_'+test_file_name), + pkg='fre-python-tools',avg_type='all', unwgt=True ) def test_fre_python_tools_time_avgs_stddevs(): ''' generates a time averaged file using fre_python_tools's version ''' ''' weighted average, no std deviation ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'frepytools_stddev_'+test_file_name), - pkg='fre-python-tools',avg_type='all', stddev_type='samp', unwgt=False) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'frepytools_stddev_'+test_file_name), + pkg='fre-python-tools',avg_type='all', stddev_type='samp', unwgt=False ) def test_fre_python_tools_time_unwgt_avgs_stddevs(): ''' generates a time averaged file using fre_python_tools's version ''' ''' weighted average, no std deviation ''' - assert run_avgtype_pkg_calculations(infile = (time_avg_file_dir+test_file_name), - outfile = (time_avg_file_dir+'frepytools_unwgt_stddev_'+test_file_name), - pkg='fre-python-tools',avg_type='all', stddev_type='samp', unwgt=True) + assert run_avgtype_pkg_calculations( + infile = (time_avg_file_dir+test_file_name), + outfile = (time_avg_file_dir+'frepytools_unwgt_stddev_'+test_file_name), + pkg='fre-python-tools',avg_type='all', stddev_type='samp', unwgt=True ) ## (TODO) WRITE THESE VERSIONS FOR FREPYTOOLSTIMEAVERAGER CLASS THEN MAKE THESE TESTS #def test_monthly_fre_python_tools_time_avgs(): @@ -128,7 +141,8 @@ def test_fre_python_tools_time_unwgt_avgs_stddevs(): ## this will only work at GFDL. dev convenience only. -#alt_str_fre_nctools_inf='tests/time_avg_test_files/fre_nctools_timavg_CLI_test_r8_b_atmos_LWP_1979_5y.nc' +#alt_str_fre_nctools_inf= \ +# 'tests/time_avg_test_files/fre_nctools_timavg_CLI_test_r8_b_atmos_LWP_1979_5y.nc' #def test_fre_nctools_time_avgs(): # ''' generates a time averaged file using fre_python_tools's version ''' # ''' weighted average, no std deviation ''' @@ -136,11 +150,12 @@ def test_fre_python_tools_time_unwgt_avgs_stddevs(): # all_outfile=time_avg_file_dir+'frenctools_timavg_'+test_file_name # # if pl.Path(all_outfile).exists(): -# print(f'output test file exists. deleting before remaking.') +# print('output test file exists. deleting before remaking.') # pl.Path(all_outfile).unlink() #delete file so we check that it can be recreated # # from fre_python_tools.generate_time_averages import generate_time_averages as gtas -# gtas.generate_time_average(infile = infile, outfile = all_outfile, pkg='fre-nctools', unwgt=False, avg_type='all') +# gtas.generate_time_average(infile = infile, outfile = all_outfile, +# pkg='fre-nctools', unwgt=False, avg_type='all') # assert pl.Path(all_outfile).exists() @@ -182,13 +197,14 @@ def test_compare_fre_python_tools_to_fre_nctools(): for lat in range(0,len(diff_pytools_nctools_timavg[0])): for lon in range(0,len(diff_pytools_nctools_timavg[0][0])): print(f'lat={lat},lon={lon}') - print(f'diff_pytools_nctools_timavg[0][lat][lon]={diff_pytools_nctools_timavg[0][lat][lon]}') + #diff_at_latlon=diff_pytools_nctools_timavg[0][lat][lon] + #print(f'diff_pytools_nctools_timavg[0][lat][lon]={diff_at_latlon}') if lon>10: break break - + non_zero_count=np.count_nonzero(diff_pytools_nctools_timavg[:]) - #assert (non_zero_count == 0.) # bad way to check for zero. - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + #assert (non_zero_count == 0.) # bad way to check for zero. + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) @pytest.mark.skip(reason='test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer') def test_compare_fre_python_tools_to_cdo(): @@ -218,7 +234,7 @@ def test_compare_fre_python_tools_to_cdo(): break non_zero_count=np.count_nonzero(diff_pytools_cdo_timavg[:]) - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) def test_compare_unwgt_fre_python_tools_to_unwgt_cdo(): @@ -247,7 +263,7 @@ def test_compare_unwgt_fre_python_tools_to_unwgt_cdo(): break non_zero_count=np.count_nonzero(diff_pytools_cdo_timavg[:]) - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) @pytest.mark.skip(reason='test fails b.c. cdo cannot bitwise-reproduce fre-nctools answer') def test_compare_cdo_to_fre_nctools(): @@ -262,7 +278,8 @@ def test_compare_cdo_to_fre_nctools(): probably because you are not at GFDL! run the shell script \ example if you would like to see this pass. otherwise, \ i will error right after this message.') - alt_str_fre_nctools_inf='tests/time_avg_test_files/fre_nctools_timavg_CLI_test_r8_b_atmos_LWP_1979_5y.nc' + alt_str_fre_nctools_inf = \ + 'tests/time_avg_test_files/fre_nctools_timavg_CLI_test_r8_b_atmos_LWP_1979_5y.nc' try: fre_nctools_inf=nc.Dataset(alt_str_fre_nctools_inf,'r') except: @@ -284,4 +301,4 @@ def test_compare_cdo_to_fre_nctools(): break non_zero_count=np.count_nonzero(diff_cdo_nctools_timavg[:]) - assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) + assert not( (non_zero_count > 0.) or (non_zero_count < 0.) ) diff --git a/fre/app/maskAtmosPlevel.py b/fre/app/mask_atmos_plevel.py similarity index 74% rename from fre/app/maskAtmosPlevel.py rename to fre/app/mask_atmos_plevel.py index fd6d5896..0e7b2c28 100755 --- a/fre/app/maskAtmosPlevel.py +++ b/fre/app/mask_atmos_plevel.py @@ -1,10 +1,11 @@ #!/usr/bin/env python - -# This script contains the refineDiags that produce data at the same -# frequency as the input data (no reduction) such as surface albedo, -# masking fields,... -# It can accept any file and will only compute refineDiags in fields -# are present. +''' +This script contains the refineDiags that produce data at the same +frequency as the input data (no reduction) such as surface albedo, +masking fields,... +It can accept any file and will only compute refineDiags in fields +are present. +''' import os import netCDF4 as nc @@ -12,24 +13,25 @@ import click @click.command() -def maskAtmosPlevel_subtool(infile, outfile, psfile): +def mask_atmos_plevel_subtool(infile, outfile, psfile): + ''' click entry point to fre cmor mask-atmos-plevel''' + # Error if outfile exists if os.path.exists(outfile): - raise Exception(f"ERROR: Output file '{outfile}' already exists") + raise FileExistsError(f"ERROR: Output file {outfile} already exists") # Open ps dataset if not os.path.exists(psfile): - raise Exception(f"ERROR: Input surface pressure file '{psfile}' does not exist") + raise FileNotFoundError(f"ERROR: Input surface pressure file {psfile} does not exist") ds_ps = xr.open_dataset(psfile) # Exit with message if "ps" not available if "ps" not in list(ds_ps.variables): - print(f"WARNING: File '{infile}' does not contain surface pressure, so exiting") - return None + raise ValueError(f"ERROR: File {infile} does not contain surface pressure. exit.") # Open input dataset if not os.path.exists(infile): - raise Exception(f"ERROR: Input file '{infile}' does not exist") + raise FileNotFoundError(f"ERROR: Input file {infile} does not exist") ds_in = xr.open_dataset(infile) # The trigger for atmos masking is a variable attribute "needs_atmos_masking = True". @@ -49,61 +51,59 @@ def maskAtmosPlevel_subtool(infile, outfile, psfile): # Write the output file if anything was done if ds_out.variables: - print(f"Modifying variables '{list(ds_out.variables)}', appending into new file '{outfile}'") + print(f"Modifying variables: {list(ds_out.variables)}\n appending into new file {outfile}") write_dataset(ds_out, ds_in, outfile) else: - print(f"No variables modified, so not writing output file '{outfile}'") - return None + print(f"No variables modified, so not writing output file {outfile}") + def preprocess(ds): """add needs_atmos_masking attribute if var ends with _unmsk""" - for var in list(ds.variables): if var.endswith('_unmsk'): ds[var].attrs['needs_atmos_masking'] = True - return ds def mask_field_above_surface_pressure(ds, var, ds_ps): """mask data with pressure larger than surface pressure""" - plev = pressure_coordinate(ds, var) # broadcast pressure coordinate and surface pressure to # the dimensions of the variable to mask plev_extended, _ = xr.broadcast(plev, ds[var]) ps_extended, _ = xr.broadcast(ds_ps["ps"], ds[var]) + # masking do not need looping masked = xr.where(plev_extended > ps_extended, 1.0e20, ds[var]) + # copy attributes, but it doesn't include the missing values attrs = ds[var].attrs.copy() + # add the missing values back attrs['missing_value'] = 1.0e20 attrs['_FillValue'] = 1.0e20 masked.attrs = attrs + # transpose dims like the original array masked = masked.transpose(*ds[var].dims) - print(f"Processed {var}") return masked -def pressure_coordinate(ds, varname, verbose=False): +def pressure_coordinate(ds, varname):#, verbose=False): """check if dataArray has pressure coordinate fitting requirements and return it""" pressure_coord = None - for dim in list(ds[varname].dims): if dim in list(ds.variables): # dim needs to have values in file if ds[dim].attrs["long_name"] == "pressure": pressure_coord = ds[dim] elif ("coordinates" in ds.attrs) and (ds[dim].attrs["units"] == "Pa"): pressure_coord = ds[dim] - return pressure_coord @@ -118,37 +118,29 @@ def write_dataset(ds, template, outfile): for var in list(template.variables): if var in list(ds.variables): continue - else: - ds[var] = template[var] - ds[var].attrs = template[var].attrs.copy() - + ds[var] = template[var] + ds[var].attrs = template[var].attrs.copy() + # write to file ds.to_netcdf(outfile, unlimited_dims="time") - return None def set_netcdf_encoding(ds, pressure_vars): """set preferred options for netcdf encoding""" - all_vars = list(ds.variables) encoding = {} - - for var in do_not_encode_vars + pressure_vars: + #for var in do_not_encode_vars + pressure_vars: #what was here in first place + for var in pressure_vars: #remove unused variable if var in all_vars: - encoding.update({var: dict(_FillValue=None)}) - + encoding.update( { var : + { '_FillValue':None} } ) return encoding -def post_write(filename, ds, var_with_bounds, bounds_variables): +def post_write(filename, var_with_bounds, bounds_variables): """fix a posteriori attributes that xarray.to_netcdf did not do properly using low level netcdf lib""" - f = nc.Dataset(filename, "a") - for var, bndvar in zip(var_with_bounds, bounds_variables): f.variables[var].setncattr("bounds", bndvar) - f.close() - - return None diff --git a/fre/catalog/__init__.py b/fre/catalog/__init__.py index 2cb1eeeb..13e87b07 100644 --- a/fre/catalog/__init__.py +++ b/fre/catalog/__init__.py @@ -1,4 +1,4 @@ -#from .gen_intake_gfdl import build_script -from .frecatalog import catalogCli +''' for fre.catalog imports ''' +from .frecatalog import catalog_cli -__all__ = ["catalogCli"] +__all__ = ["catalog_cli"] diff --git a/fre/catalog/frecatalog.py b/fre/catalog/frecatalog.py index c395a4bb..3223002e 100644 --- a/fre/catalog/frecatalog.py +++ b/fre/catalog/frecatalog.py @@ -1,15 +1,20 @@ +''' +entry point for fre catalog subcommands +''' + import click -import catalogbuilder +#import catalogbuilder from catalogbuilder.scripts import gen_intake_gfdl from catalogbuilder.scripts import test_catalog + @click.group(help=click.style(" - access fre catalog subcommands", fg=(64,94,213))) -def catalogCli(): +def catalog_cli(): ''' entry point for click into fre catalog cli calls ''' - pass -@catalogCli.command() + +@catalog_cli.command() #TODO arguments dont have help message. So consider changing arguments to options? @click.argument('input_path', required = False, nargs = 1) #, help = 'The directory path with the datasets to be cataloged. E.g a GFDL PP path till /pp') @@ -25,17 +30,20 @@ def catalogCli(): @click.pass_context def builder(context, input_path = None, output_path = None, config = None, filter_realm = None, filter_freq = None, filter_chunk = None, overwrite = False, append = False): + # pylint: disable=unused-argument """ - Generate .csv and .json files for catalog """ - context.forward(gen_intake_gfdl.main) + context.forward(gen_intake_gfdl.create_catalog_cli) -@catalogCli.command() +@catalog_cli.command() @click.argument('json_path', nargs = 1 , required = True) @click.argument('json_template_path', nargs = 1 , required = False) -@click.option('-tf', '--test-failure', is_flag=True, default = False, help="Errors are only printed. Program will not exit.") +@click.option('-tf', '--test-failure', is_flag=True, default = False, + help="Errors are only printed. Program will not exit.") @click.pass_context def validate(context, json_path, json_template_path, test_failure): + # pylint: disable=unused-argument """ - Validate a catalog against catalog schema """ context.forward(test_catalog.main) if __name__ == "__main__": - catalogCli() + catalog_cli() diff --git a/fre/check/__init__.py b/fre/check/__init__.py index 6216e5dd..ef351297 100644 --- a/fre/check/__init__.py +++ b/fre/check/__init__.py @@ -1,4 +1,6 @@ +''' for fre.check imports ''' + from .frecheckexample import check_test_function -from .frecheck import checkCli +from .frecheck import check_cli -__all__ = ["check_test_function", "checkCli"] +__all__ = ["check_test_function", "check_cli"] diff --git a/fre/check/frecheck.py b/fre/check/frecheck.py index 6c259dbe..9640d138 100644 --- a/fre/check/frecheck.py +++ b/fre/check/frecheck.py @@ -1,16 +1,20 @@ +''' fre check ''' + import click + from .frecheckexample import check_test_function @click.group(help=click.style(" - access fre check subcommands", fg=(162,91,232))) -def checkCli(): - pass +def check_cli(): + ''' entry point to fre check click commands ''' -@checkCli.command() +@check_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') @click.pass_context def function(context, uppercase): + # pylint: disable=unused-argument """ - Execute fre check test """ context.forward(check_test_function) if __name__ == "__main__": - checkCli() + check_cli() diff --git a/fre/check/frecheckexample.py b/fre/check/frecheckexample.py index aa9e84b6..daac462a 100644 --- a/fre/check/frecheckexample.py +++ b/fre/check/frecheckexample.py @@ -7,7 +7,7 @@ import click @click.command() -def check_test_function(uppercase): +def check_test_function(uppercase=None): """Execute fre list testfunction2.""" statement = "testingtestingtestingtesting" if uppercase: diff --git a/fre/cmor/CMORmixer.py b/fre/cmor/CMORmixer.py index 824aeda7..501261cf 100644 --- a/fre/cmor/CMORmixer.py +++ b/fre/cmor/CMORmixer.py @@ -2,15 +2,13 @@ ''' see README.md for CMORmixer.py usage ''' -import os, sys +import os import time as tm -import numpy, json -import cmor -import netCDF4 as nc -import string +import json from shutil import copyfile -#import shutil +import netCDF4 as nc import click +import cmor global nameOfset, GFDL_vars_file, CMIP_output, GFDL_real_vars_file diff --git a/fre/cmor/__init__.py b/fre/cmor/__init__.py index 0e02ef4a..0f8bd3ff 100644 --- a/fre/cmor/__init__.py +++ b/fre/cmor/__init__.py @@ -1,4 +1,5 @@ +''' for fre.cmor imports ''' from .CMORmixer import cmor_run_subtool -from .frecmor import cmorCli +from .frecmor import cmor_cli -__all__ = ["cmor_run_subtool", "cmorCli"] +__all__ = ["cmor_run_subtool", "cmor_cli"] diff --git a/fre/cmor/frecmor.py b/fre/cmor/frecmor.py index 06f37d92..c693b1e6 100644 --- a/fre/cmor/frecmor.py +++ b/fre/cmor/frecmor.py @@ -1,11 +1,14 @@ +''' fre cmor ''' + import click + from .CMORmixer import cmor_run_subtool @click.group(help=click.style(" - access fre cmor subcommands", fg=(232,91,204))) -def cmorCli(): - pass +def cmor_cli(): + ''' entry point to fre cmor click commands ''' -@cmorCli.command() +@cmor_cli.command() @click.option("-d", "--indir", type=str, help="Input directory", @@ -28,8 +31,9 @@ def cmorCli(): required=True) @click.pass_context def run(context, indir, outdir, varlist, table_config, exp_config): + # pylint: disable=unused-argument """Rewrite climate model output""" context.forward(cmor_run_subtool) if __name__ == "__main__": - cmorCli() + cmor_cli() diff --git a/fre/fre.py b/fre/fre.py index ea2afa6d..9dd65eb0 100644 --- a/fre/fre.py +++ b/fre/fre.py @@ -12,16 +12,16 @@ @click.group( cls = LazyGroup, - lazy_subcommands = {"pp": ".pp.frepp.ppCli", - "catalog": ".catalog.frecatalog.catalogCli", - "list": ".list.frelist.listCli", - "check": ".check.frecheck.checkCli", - "run": ".run.frerun.runCli", - "test": ".test.fretest.testCli", - "yamltools": ".yamltools.freyamltools.yamltoolsCli", - "make": ".make.fremake.makeCli", - "app": ".app.freapp.appCli", - "cmor": ".cmor.frecmor.cmorCli" }, + lazy_subcommands = {"pp": ".pp.frepp.pp_cli", + "catalog": ".catalog.frecatalog.catalog_cli", + "list": ".list.frelist.list_cli", + "check": ".check.frecheck.check_cli", + "run": ".run.frerun.run_cli", + "test": ".test.fretest.test_cli", + "yamltools": ".yamltools.freyamltools.yamltools_cli", + "make": ".make.fremake.make_cli", + "app": ".app.freapp.app_cli", + "cmor": ".cmor.frecmor.cmor_cli" }, help = click.style( "'fre' is the main CLI click group that houses the other tool groups as lazy subcommands.", fg='cyan') @@ -35,7 +35,7 @@ def fre(): ''' entry point function to subgroup functions ''' - pass + if __name__ == '__main__': fre() diff --git a/fre/list/__init__.py b/fre/list/__init__.py index 9275abc0..555dae55 100644 --- a/fre/list/__init__.py +++ b/fre/list/__init__.py @@ -1,4 +1,5 @@ +''' for fre.list imports ''' from .frelistexample import list_test_function -from .frelist import listCli +from .frelist import list_cli -__all__ = ["list_test_function", "listCli"] +__all__ = ["list_test_function", "list_cli"] diff --git a/fre/list/frelist.py b/fre/list/frelist.py index a944385b..fea3b6ea 100644 --- a/fre/list/frelist.py +++ b/fre/list/frelist.py @@ -1,16 +1,20 @@ +''' fre list ''' + import click + from .frelistexample import list_test_function @click.group(help=click.style(" - access fre list subcommands", fg=(232,204,91))) -def listCli(): - pass +def list_cli(): + ''' entry point to fre list click commands ''' -@listCli.command() +@list_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') @click.pass_context def function(context, uppercase): + # pylint: disable=unused-argument """ - Execute fre list test """ context.forward(list_test_function) if __name__ == "__main__": - listCli() + list_cli() diff --git a/fre/list/frelistexample.py b/fre/list/frelistexample.py index 1effeb9e..7e40e195 100644 --- a/fre/list/frelistexample.py +++ b/fre/list/frelistexample.py @@ -7,7 +7,7 @@ import click @click.command() -def list_test_function(uppercase): +def list_test_function(uppercase=None): """Execute fre list testfunction2.""" statement = "testingtestingtestingtesting" if uppercase: diff --git a/fre/make/__init__.py b/fre/make/__init__.py index ff03e974..a35633b9 100644 --- a/fre/make/__init__.py +++ b/fre/make/__init__.py @@ -1,17 +1,15 @@ -''' -for proper import of fre make subcommands -''' +''' for fre.make imports ''' from .createCheckout import checkout_create from .createCompile import compile_create from .createDocker import dockerfile_create from .createMakefile import makefile_create from .runFremake import fremake_run -from .fremake import makeCli +from .fremake import make_cli __all__ = ["checkout_create", "compile_create", "dockerfile_create", "makefile_create", "fremake_run", - "makeCli"] + "make_cli"] diff --git a/fre/make/createCheckout.py b/fre/make/createCheckout.py index e86ac006..97c34cc6 100644 --- a/fre/make/createCheckout.py +++ b/fre/make/createCheckout.py @@ -1,10 +1,10 @@ #!/usr/bin/python3 -from .gfdlfremake import varsfre, platformfre, yamlfre, checkout, targetfre -import click import os import logging import sys +import click +from .gfdlfremake import varsfre, platformfre, yamlfre, checkout, targetfre @click.command() def checkout_create(yamlfile,platform,target,no_parallel_checkout,jobs,execute,verbose): diff --git a/fre/make/createCompile.py b/fre/make/createCompile.py index 94f39b82..24f23255 100644 --- a/fre/make/createCompile.py +++ b/fre/make/createCompile.py @@ -1,11 +1,12 @@ #!/usr/bin/python3 -from .gfdlfremake import varsfre, platformfre, yamlfre, targetfre, buildBaremetal -from multiprocessing.dummy import Pool -import logging import os -import click import sys +import logging +from multiprocessing.dummy import Pool + +import click +from .gfdlfremake import varsfre, platformfre, yamlfre, targetfre, buildBaremetal @click.command() def compile_create(yamlfile,platform,target,jobs,parallel,execute,verbose): diff --git a/fre/make/createDocker.py b/fre/make/createDocker.py index 577ab42f..fee5775f 100644 --- a/fre/make/createDocker.py +++ b/fre/make/createDocker.py @@ -1,9 +1,9 @@ #!/usr/bin/python3 -from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker -import click import os import sys +import click +from .gfdlfremake import varsfre, targetfre, makefilefre, platformfre, yamlfre, buildDocker @click.command() def dockerfile_create(yamlfile, platform, target, execute): diff --git a/fre/make/createMakefile.py b/fre/make/createMakefile.py index 03eaaa01..e520f7c7 100644 --- a/fre/make/createMakefile.py +++ b/fre/make/createMakefile.py @@ -1,9 +1,8 @@ #!/usr/bin/python3 -from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre -import click import os -import logging +import click +from .gfdlfremake import makefilefre, varsfre, targetfre, yamlfre @click.command() def makefile_create(yamlfile,platform,target): diff --git a/fre/make/fremake.py b/fre/make/fremake.py index 10e437b7..27cec2db 100644 --- a/fre/make/fremake.py +++ b/fre/make/fremake.py @@ -30,10 +30,10 @@ @click.group(help=click.style(" - access fre make subcommands", fg=(210,73,57))) -def makeCli(): +def make_cli(): pass -@makeCli.command() +@make_cli.command() @click.option("-y", "--yamlfile", type = str, @@ -75,7 +75,7 @@ def run_fremake(context, yamlfile, platform, target, parallel, jobs, no_parallel context.forward(fremake_run) #### -@makeCli.command() +@make_cli.command() @click.option("-y", "--yamlfile", type = str, @@ -117,7 +117,7 @@ def create_checkout(context,yamlfile,platform,target,no_parallel_checkout,jobs,e context.forward(checkout_create) ##### -@makeCli.command +@make_cli.command @click.option("-y", "--yamlfile", type = str, @@ -140,7 +140,7 @@ def create_makefile(context,yamlfile,platform,target): ##### -@makeCli.command +@make_cli.command @click.option("-y", "--yamlfile", type = str, @@ -183,7 +183,7 @@ def create_compile(context,yamlfile,platform,target,jobs,parallel,execute,verbos -@makeCli.command +@make_cli.command @click.option("-y", "--yamlfile", type = str, @@ -210,4 +210,4 @@ def create_dockerfile(context,yamlfile,platform,target,execute): if __name__ == "__main__": - makeCli() + make_cli() diff --git a/fre/make/runFremake.py b/fre/make/runFremake.py index 49072b2f..e45d6837 100644 --- a/fre/make/runFremake.py +++ b/fre/make/runFremake.py @@ -5,14 +5,14 @@ ## \author Bennett Chang ## \description Script for fremake is used to create and run a code checkout script and compile a model. -import click -import subprocess import os -import yaml import logging -from .gfdlfremake import targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal from multiprocessing.dummy import Pool +import click + +from .gfdlfremake import targetfre, varsfre, yamlfre, checkout, makefilefre, buildDocker, buildBaremetal + @click.command() def fremake_run(yamlfile, platform, target, parallel, jobs, no_parallel_checkout, verbose): diff --git a/fre/pp/__init__.py b/fre/pp/__init__.py index bfe00341..4ac71ea1 100644 --- a/fre/pp/__init__.py +++ b/fre/pp/__init__.py @@ -1,3 +1,5 @@ +''' for fre.pp imports ''' + from .checkoutScript import checkoutTemplate from .configure_script_yaml import yamlInfo from .configure_script_xml import convert @@ -5,7 +7,7 @@ from .install import install_subtool from .run import pp_run_subtool from .status import status_subtool -from .frepp import ppCli +from .frepp import pp_cli from .wrapper import runFre2pp __all__ = ["checkoutTemplate", @@ -15,6 +17,5 @@ "install_subtool", "pp_run_subtool", "status_subtool", - "ppCli", + "pp_cli", "runFre2pp"] - diff --git a/fre/pp/checkoutScript.py b/fre/pp/checkoutScript.py index ac594888..5095a52c 100644 --- a/fre/pp/checkoutScript.py +++ b/fre/pp/checkoutScript.py @@ -5,12 +5,11 @@ import os import sys -from pathlib import Path import subprocess from subprocess import PIPE from subprocess import STDOUT -import click import re +import click ############################################# diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 11b6aeb9..78f8eb75 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -36,8 +36,8 @@ def validate_yaml(file): #################### def join_constructor(loader, node): """ - Allows FRE properties defined - in main yaml to be concatenated. + Allows FRE properties defined + in main yaml to be concatenated. """ seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) @@ -109,7 +109,7 @@ def consolidate_yamls(mainyaml,experiment, platform,target): f1.write(f"\n### {i.upper()} settings ###\n") #copy expyaml into combined shutil.copyfileobj(f2,f1) - + return combined #################### @@ -160,7 +160,7 @@ def set_rose_suite(yamlfile,rose_suite): # set rose-suite items if pp is not None: for i in pp.values(): - if not isinstance(i,list): + if not isinstance(i,list): for key,value in i.items(): # rose-suite.conf is somewhat finicky with quoting # cylc validate will reveal any complaints @@ -228,14 +228,14 @@ def _yamlInfo(yamlfile,experiment,platform,target): # Load the combined yaml final_yaml = yaml_load(comb_yaml) - + # Clean combined yaml to validate # If keys exists, delete: keys_clean=["fre_properties","shared","experiments"] for kc in keys_clean: if kc in final_yaml.keys(): del final_yaml[kc] - + with open("combined.yaml",'w') as f: yaml.safe_dump(final_yaml,f,sort_keys=False) diff --git a/fre/pp/frepp.py b/fre/pp/frepp.py index ac48ddc2..b3456e31 100644 --- a/fre/pp/frepp.py +++ b/fre/pp/frepp.py @@ -1,3 +1,5 @@ +''' fre pp ''' + import click from .checkoutScript import checkoutTemplate from .configure_script_yaml import yamlInfo @@ -9,168 +11,130 @@ from .wrapper import runFre2pp @click.group(help=click.style(" - access fre pp subcommands", fg=(57,139,210))) -def ppCli(): - pass +def pp_cli(): + ''' entry point to fre pp click commands ''' + # fre pp status -@ppCli.command() -@click.option("-e", - "--experiment", - type=str, +@pp_cli.command() +@click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-p", - "--platform", - type=str, +@click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", - "--target", - type=str, - help="Target name", - required=True) +@click.option("-t", "--target", type=str, + help="Target name", + required=True) @click.pass_context def status(context, experiment, platform, target): + # pylint: disable=unused-argument """ - Report status of PP configuration""" context.forward(status_subtool) # fre pp run -@ppCli.command() -@click.option("-e", - "--experiment", - type=str, +@pp_cli.command() +@click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-p", - "--platform", - type=str, +@click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", - "--target", - type=str, - help="Target name", - required=True) +@click.option("-t", "--target", type=str, + help="Target name", + required=True) @click.pass_context def run(context, experiment, platform, target): + # pylint: disable=unused-argument """ - Run PP configuration""" context.forward(pp_run_subtool) # fre pp validate -@ppCli.command() -@click.option("-e", - "--experiment", - type=str, +@pp_cli.command() +@click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-p", - "--platform", - type=str, +@click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", - "--target", - type=str, - help="Target name", - required=True) +@click.option("-t", "--target", type=str, + help="Target name", + required=True) @click.pass_context def validate(context, experiment, platform, target): + # pylint: disable=unused-argument """ - Validate PP configuration""" context.forward(validate_subtool) # fre pp install -@ppCli.command() -@click.option("-e", - "--experiment", - type=str, +@pp_cli.command() +@click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-p", - "--platform", - type=str, +@click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", - "--target", - type=str, - help="Target name", - required=True) +@click.option("-t", "--target", type=str, + help="Target name", + required=True) @click.pass_context def install(context, experiment, platform, target): + # pylint: disable=unused-argument """ - Install PP configuration""" context.forward(install_subtool) -@ppCli.command() -@click.option("-y", - "--yamlfile", - type=str, +@pp_cli.command() +@click.option("-y", "--yamlfile", type=str, help="YAML file to be used for parsing", required=True) -@click.option("-e", - "--experiment", - type=str, +@click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-p", - "--platform", - type=str, +@click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", - "--target", - type=str, - help="Target name", - required=True) +@click.option("-t", "--target", type=str, + help="Target name", + required=True) @click.pass_context def configure_yaml(context,yamlfile,experiment,platform,target): + # pylint: disable=unused-argument """ - Execute fre pp configure """ context.forward(yamlInfo) -@ppCli.command() -@click.option("-e", - "--experiment", - type=str, +@pp_cli.command() +@click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-p", - "--platform", - type=str, +@click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", - "--target", - type=str, - help="Target name", - required=True) -@click.option("-b", - "--branch", +@click.option("-t", "--target", type=str, + help="Target name", + required=True) +@click.option("-b", "--branch", show_default=True, - default="main", - type=str, - help=" ".join(["Name of fre2/workflows/postproc branch to clone;" - "defaults to 'main'. Not intended for production use," - "but needed for branch testing."]) - ) + default="main", type=str, + help="Name of fre2/workflows/postproc branch to clone; " \ + "defaults to 'main'. Not intended for production use, " \ + "but needed for branch testing." ) @click.pass_context def checkout(context, experiment, platform, target, branch='main'): + # pylint: disable=unused-argument """ - Execute fre pp checkout """ context.forward(checkoutTemplate) -@ppCli.command() -@click.option('-x', - '--xml', +@pp_cli.command() +@click.option('-x', '--xml', required=True, help="Required. The Bronx XML") -@click.option('-p', - '--platform', +@click.option('-p', '--platform', required=True, help="Required. The Bronx XML Platform") -@click.option('-t', - '--target', +@click.option('-t', '--target', required=True, help="Required. The Bronx XML Target") -@click.option('-e', - '--experiment', +@click.option('-e', '--experiment', required=True, help="Required. The Bronx XML Experiment") @click.option('--do_analysis', @@ -178,36 +142,34 @@ def checkout(context, experiment, platform, target, branch='main'): default=False, help="Optional. Runs the analysis scripts.") @click.option('--historydir', - help="Optional. History directory to reference. " \ - "If not specified, the XML's default will be used.") + help="Optional. History directory to reference. " \ + "If not specified, the XML's default will be used.") @click.option('--refinedir', - help="Optional. History refineDiag directory to reference. " \ - "If not specified, the XML's default will be used.") + help="Optional. History refineDiag directory to reference. " \ + "If not specified, the XML's default will be used.") @click.option('--ppdir', - help="Optional. Postprocessing directory to reference. " \ - "If not specified, the XML's default will be used.") + help="Optional. Postprocessing directory to reference. " \ + "If not specified, the XML's default will be used.") @click.option('--do_refinediag', is_flag=True, default=False, help="Optional. Process refineDiag scripts") @click.option('--pp_start', - help="Optional. Starting year of postprocessing. " \ - "If not specified, a default value of '0000' " \ - "will be set and must be changed in rose-suite.conf") + help="Optional. Starting year of postprocessing. " \ + "If not specified, a default value of '0000' " \ + "will be set and must be changed in rose-suite.conf") @click.option('--pp_stop', - help="Optional. Ending year of postprocessing. " \ - "If not specified, a default value of '0000' " \ + help="Optional. Ending year of postprocessing. " \ + "If not specified, a default value of '0000' " \ "will be set and must be changed in rose-suite.conf") @click.option('--validate', is_flag=True, - help="Optional. Run the Cylc validator " \ + help="Optional. Run the Cylc validator " \ "immediately after conversion") -@click.option('-v', - '--verbose', +@click.option('-v', '--verbose', is_flag=True, help="Optional. Display detailed output") -@click.option('-q', - '--quiet', +@click.option('-q', '--quiet', is_flag=True, help="Optional. Display only serious messages and/or errors") @click.option('--dual', @@ -216,44 +178,36 @@ def checkout(context, experiment, platform, target, branch='main'): @click.pass_context def configure_xml(context, xml, platform, target, experiment, do_analysis, historydir, refinedir, ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual): + # pylint: disable=unused-argument """ - Converts a Bronx XML to a Canopy rose-suite.conf """ context.forward(convert) #fre pp wrapper -@ppCli.command() -@click.option("-e", - "--experiment", - type=str, +@pp_cli.command() +@click.option("-e", "--experiment", type=str, help="Experiment name", required=True) -@click.option("-p", - "--platform", - type=str, +@click.option("-p", "--platform", type=str, help="Platform name", required=True) -@click.option("-t", - "--target", - type=str, - help="Target name", - required=True) -@click.option("-c", - "--config-file", - type=str, - help="Path to a configuration file in either XML or YAML", - required=True) -@click.option("-b", - "--branch", +@click.option("-t", "--target", type=str, + help="Target name", + required=True) +@click.option("-c", "--config-file", type=str, + help="Path to a configuration file in either XML or YAML", + required=True) +@click.option("-b", "--branch", show_default=True, - default="main", - type=str, - help=" ".join(["Name of fre2/workflows/postproc branch to clone;" - "defaults to 'main'. Not intended for production use," - "but needed for branch testing."]) - ) + default="main", type=str, + help="Name of fre2/workflows/postproc branch to clone; " \ + "defaults to 'main'. Not intended for production use, " \ + "but needed for branch testing." ) @click.pass_context def wrapper(context, experiment, platform, target, config_file, branch='main'): + # pylint: disable=unused-argument """ - Execute fre pp steps in order """ context.forward(runFre2pp) if __name__ == "__main__": - ppCli() + ''' entry point for click to fre pp commands ''' + pp_cli() diff --git a/fre/pp/install.py b/fre/pp/install.py index d344c5fa..9ffc00ee 100644 --- a/fre/pp/install.py +++ b/fre/pp/install.py @@ -1,6 +1,6 @@ #!/usr/bin/env python +''' fre pp install ''' -import os import subprocess import click @@ -18,4 +18,5 @@ def _install_subtool(experiment, platform, target): @click.command() def install_subtool(experiment, platform, target): + ''' entry point to install for click ''' return _install_subtool(experiment, platform, target) diff --git a/fre/pp/run.py b/fre/pp/run.py index 444d6d5b..57f2c427 100644 --- a/fre/pp/run.py +++ b/fre/pp/run.py @@ -1,10 +1,9 @@ #!/usr/bin/env python +''' fre pp run ''' -import os import subprocess import click - def _pp_run_subtool(experiment, platform, target): """ Start or restart the Cylc workflow identified by: @@ -17,4 +16,5 @@ def _pp_run_subtool(experiment, platform, target): @click.command() def pp_run_subtool(experiment, platform, target): + ''' entry point to run for click ''' return _pp_run_subtool(experiment, platform, target) diff --git a/fre/pp/status.py b/fre/pp/status.py index 0eaa9079..6e2c07e1 100644 --- a/fre/pp/status.py +++ b/fre/pp/status.py @@ -1,6 +1,6 @@ #!/usr/bin/env python +''' fre pp status ''' -import os import subprocess import click @@ -17,4 +17,5 @@ def _status_subtool(experiment, platform, target): @click.command() def status_subtool(experiment, platform, target): + ''' entry point to status for click ''' return _status_subtool(experiment, platform, target) diff --git a/fre/pp/validate.py b/fre/pp/validate.py index 350c3cb3..9c07340f 100644 --- a/fre/pp/validate.py +++ b/fre/pp/validate.py @@ -1,4 +1,5 @@ #!/usr/bin/env python +''' fre pp validate ''' import os import subprocess @@ -16,13 +17,14 @@ def _validate_subtool(experiment, platform, target): os.chdir(directory) # Run the Rose validation macros - cmd = f"rose macro --validate" + cmd = "rose macro --validate" subprocess.run(cmd, shell=True, check=True) # Validate the Cylc configuration - cmd = f"cylc validate ." + cmd = "cylc validate ." subprocess.run(cmd, shell=True, check=True) @click.command() def validate_subtool(experiment, platform, target): + ''' entry point to validate for click ''' return _validate_subtool(experiment, platform, target) diff --git a/fre/pp/wrapper.py b/fre/pp/wrapper.py index d3ca3802..965b9b2b 100644 --- a/fre/pp/wrapper.py +++ b/fre/pp/wrapper.py @@ -14,25 +14,24 @@ import sys import os -import subprocess -from subprocess import PIPE, STDOUT -from subprocess import STDOUT -import click -import re import time - -#Add path to this file to the pythonpath for local imports -import_dir = os.path.dirname(os.path.abspath(__file__)) -sys.path.append(import_dir) +#import subprocess +#from subprocess import PIPE, STDOUT +#from subprocess import STDOUT +import click # Import from the local packages -from checkoutScript import _checkoutTemplate -from configure_script_xml import _convert -from configure_script_yaml import _yamlInfo -from validate import _validate_subtool -from install import _install_subtool -from run import _pp_run_subtool -from status import _status_subtool +from .checkoutScript import _checkoutTemplate +from .configure_script_xml import _convert +from .configure_script_yaml import _yamlInfo +from .validate import _validate_subtool +from .install import _install_subtool +from .run import _pp_run_subtool +from .status import _status_subtool + +##Add path to this file to the pythonpath for local imports +#import_dir = os.path.dirname(os.path.abspath(__file__)) +#sys.path.append(import_dir) @click.command() def runFre2pp(experiment, platform, target, config_file, branch): diff --git a/fre/pytest.ini b/fre/pytest.ini index 20575029..0bcbff6a 100644 --- a/fre/pytest.ini +++ b/fre/pytest.ini @@ -1,11 +1,11 @@ [pytest] testpaths = fre/tests -# fre/catalog/tests + fre/catalog/tests # fre/check/tests # fre/cmor/tests # fre/list/tests -# fre/make/tests + fre/make/tests fre/pp/tests # fre/run/tests # fre/test/tests diff --git a/fre/run/__init__.py b/fre/run/__init__.py index e707693d..cdd61f12 100644 --- a/fre/run/__init__.py +++ b/fre/run/__init__.py @@ -1,8 +1,6 @@ -''' -for proper import of fre run subcommands -''' +''' for fre.run imports ''' from .frerunexample import run_test_function -from .frerun import runCli +from .frerun import run_cli -__all__ = ["run_test_function", "runCli"] +__all__ = ["run_test_function", "run_cli"] diff --git a/fre/run/frerun.py b/fre/run/frerun.py index 3c0a5459..c2c611c3 100644 --- a/fre/run/frerun.py +++ b/fre/run/frerun.py @@ -6,15 +6,16 @@ from .frerunexample import run_test_function @click.group(help=click.style(" - access fre run subcommands", fg=(164,29,132))) -def runCli(): - pass +def run_cli(): + ''' entry point to fre run click commands ''' -@runCli.command() +@run_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') @click.pass_context def function(context, uppercase): + # pylint: disable=unused-argument """ - Execute fre run test """ context.forward(run_test_function) if __name__ == "__main__": - runCli() + run_cli() diff --git a/fre/run/frerunexample.py b/fre/run/frerunexample.py index 55cb1feb..a4335144 100644 --- a/fre/run/frerunexample.py +++ b/fre/run/frerunexample.py @@ -7,8 +7,8 @@ import click @click.command() -def run_test_function(uppercase): - """Execute fre list testfunction2.""" +def run_test_function(uppercase=None): + """Execute fre run run_test_function""" statement = "testingtestingtestingtesting" if uppercase: statement = statement.upper() diff --git a/fre/test/__init__.py b/fre/test/__init__.py index a29104d6..0dedd8fe 100644 --- a/fre/test/__init__.py +++ b/fre/test/__init__.py @@ -1,4 +1,5 @@ +''' for fre.test imports ''' from .fretestexample import test_test_function -from .fretest import testCli +from .fretest import test_cli -__all__ = ["test_test_function", "testCli"] +__all__ = ["test_test_function", "test_cli"] diff --git a/fre/test/fretest.py b/fre/test/fretest.py index 69236d4c..172ac855 100644 --- a/fre/test/fretest.py +++ b/fre/test/fretest.py @@ -1,16 +1,21 @@ +''' +entry point for fre test subcommands +''' + import click from .fretestexample import test_test_function @click.group(help=click.style(" - access fre test subcommands", fg=(92,164,29))) -def testCli(): - pass +def test_cli(): + ''' entry point to fre test click commands ''' -@testCli.command() +@test_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') @click.pass_context def function(context, uppercase): + # pylint: disable=unused-argument """ - Execute fre test test """ context.forward(test_test_function) if __name__ == "__main__": - testCli() + test_cli() diff --git a/fre/test/fretestexample.py b/fre/test/fretestexample.py index ad465774..86d0f6ce 100644 --- a/fre/test/fretestexample.py +++ b/fre/test/fretestexample.py @@ -7,8 +7,8 @@ import click @click.command() -def test_test_function(uppercase): - """Execute fre list testfunction2.""" +def test_test_function(uppercase=None): + """Execute fre list test_test_function""" statement = "testingtestingtestingtesting" if uppercase: statement = statement.upper() diff --git a/fre/tests/test_fre_app_cli.py b/fre/tests/test_fre_app_cli.py new file mode 100644 index 00000000..87bc496a --- /dev/null +++ b/fre/tests/test_fre_app_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre app" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_app(): + ''' fre app ''' + result = runner.invoke(fre.fre, args=["app"]) + assert result.exit_code == 0 + +def test_cli_fre_app_help(): + ''' fre app --help ''' + result = runner.invoke(fre.fre, args=["app", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_app_opt_dne(): + ''' fre app optionDNE ''' + result = runner.invoke(fre.fre, args=["app", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_catalog_cli.py b/fre/tests/test_fre_catalog_cli.py new file mode 100644 index 00000000..fee3e0a3 --- /dev/null +++ b/fre/tests/test_fre_catalog_cli.py @@ -0,0 +1,38 @@ +#!/usr/bin/env python3 +''' test "fre catalog" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_catalog(): + ''' fre catalog ''' + result = runner.invoke(fre.fre, args=["catalog"]) + assert result.exit_code == 0 + +def test_cli_fre_catalog_help(): + ''' fre catalog --help ''' + result = runner.invoke(fre.fre, args=["catalog", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_catalog_opt_dne(): + ''' fre catalog optionDNE ''' + result = runner.invoke(fre.fre, args=["catalog", "optionDNE"]) + assert result.exit_code == 2 + +def test_cli_fre_catalog_builder(): + ''' fre catalog builder ''' + result = runner.invoke(fre.fre, args=["catalog", "builder"]) + assert all( [ + result.exit_code == 1, + 'No paths given, using yaml configuration' + in result.stdout.split('\n') + ] + ) + +def test_cli_fre_catalog_builder_help(): + ''' fre catalog builder --help ''' + result = runner.invoke(fre.fre, args=["catalog", "builder", "--help"]) + assert result.exit_code == 0 diff --git a/fre/tests/test_fre_check_cli.py b/fre/tests/test_fre_check_cli.py new file mode 100644 index 00000000..b00dc834 --- /dev/null +++ b/fre/tests/test_fre_check_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre check" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_check(): + ''' fre check ''' + result = runner.invoke(fre.fre, args=["check"]) + assert result.exit_code == 0 + +def test_cli_fre_check_help(): + ''' fre check --help ''' + result = runner.invoke(fre.fre, args=["check", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_check_opt_dne(): + ''' fre check optionDNE ''' + result = runner.invoke(fre.fre, args=["check", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_cli.py b/fre/tests/test_fre_cli.py index 9a3e3141..daff4fbd 100644 --- a/fre/tests/test_fre_cli.py +++ b/fre/tests/test_fre_cli.py @@ -1,151 +1,23 @@ #!/usr/bin/env python3 - -from fre import fre +''' test "fre" calls ''' from click.testing import CliRunner -runner = CliRunner() -#tests are structured in the manner of: -#https://click.palletsprojects.com/en/8.1.x/testing/ -#general intent for these tests is that each fre tool has 3 commandline tests: -#command, help, command does not exist +from fre import fre -#Test list: -#fre -#-- fre app -#-- fre catalog -#-- fre check -#-- fre cmor -#-- fre list -#-- fre make -#-- fre pp -#-- fre run +runner = CliRunner() def test_cli_fre(): + ''' fre ''' result = runner.invoke(fre.fre) - #print(f'exit code of runner result is {result.exit_code}') - #print(f'output of runner result is {result.output}') assert result.exit_code == 0 def test_cli_fre_help(): - result = runner.invoke(fre.fre,args='--help') - #print(f'exit code of runner result is {result.exit_code}') - #print(f'output of runner result is {result.output}') + ''' fre --help ''' + result = runner.invoke(fre.fre, args='--help') assert result.exit_code == 0 def test_cli_fre_option_dne(): - result = runner.invoke(fre.fre,args='optionDNE') - #print(f'exit code of runner result is {result.exit_code}') - #print(f'output of runner result is {result.output}') - assert result.exit_code == 2 - -#-- fre app - -def test_cli_fre_app(): - result = runner.invoke(fre.fre, args=["app"]) - assert result.exit_code == 0 - -def test_cli_fre_app_help(): - result = runner.invoke(fre.fre, args=['--help', "app"]) - assert result.exit_code == 0 - -def test_cli_fre_app_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "app"]) - assert result.exit_code == 2 - -#-- fre catalog -def test_cli_fre_catalog(): - result = runner.invoke(fre.fre, args=["catalog"]) - assert result.exit_code == 0 - -def test_cli_fre_catalog_builder_help(): - result = runner.invoke(fre.fre, args=["catalog", "builder", "--help"]) - assert result.exit_code == 0 - -def test_cli_fre_catalog_help(): - result = runner.invoke(fre.fre, args=['--help', "catalog"]) - assert result.exit_code == 0 - -def test_cli_fre_catalog_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "catalog"]) - assert result.exit_code == 2 - -#-- fre check - -def test_cli_fre_check(): - result = runner.invoke(fre.fre, args=["check"]) - assert result.exit_code == 0 - -def test_cli_fre_check_help(): - result = runner.invoke(fre.fre, args=['--help', "check"]) - assert result.exit_code == 0 - -def test_cli_fre_check_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "check"]) - assert result.exit_code == 2 - -#-- fre cmor - -def test_cli_fre_cmor(): - result = runner.invoke(fre.fre, args=["cmor"]) - assert result.exit_code == 0 - -def test_cli_fre_cmor_help(): - result = runner.invoke(fre.fre, args=['--help', "cmor"]) - assert result.exit_code == 0 - -def test_cli_fre_cmor_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "cmor"]) - assert result.exit_code == 2 - -#-- fre list -def test_cli_fre_list(): - result = runner.invoke(fre.fre, args=["list"]) - assert result.exit_code == 0 - -def test_cli_fre_list_help(): - result = runner.invoke(fre.fre, args=['--help', "list"]) - assert result.exit_code == 0 - -def test_cli_fre_list_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "list"]) - assert result.exit_code == 2 - -#-- fre make -def test_cli_fre_make(): - result = runner.invoke(fre.fre, args=["make"]) - assert result.exit_code == 0 - -def test_cli_fre_make_help(): - result = runner.invoke(fre.fre, args=['--help', "make"]) - assert result.exit_code == 0 - -def test_cli_fre_make_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "make"]) - assert result.exit_code == 2 - -#-- fre pp -def test_cli_fre_pp(): - result = runner.invoke(fre.fre, args=["pp"]) - assert result.exit_code == 0 - -def test_cli_fre_pp_help(): - result = runner.invoke(fre.fre, args=['--help', "pp"]) - assert result.exit_code == 0 - -def test_cli_fre_pp_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "pp"]) - assert result.exit_code == 2 - -#-- fre run -def test_cli_fre_run(): - result = runner.invoke(fre.fre, args=["run"]) - assert result.exit_code == 0 - -def test_cli_fre_run_help(): - result = runner.invoke(fre.fre, args=['--help', "run"]) - assert result.exit_code == 0 - -def test_cli_fre_run_opt_dne(): - result = runner.invoke(fre.fre, args=['optionDNE', "run"]) + ''' fre optionDNE ''' + result = runner.invoke(fre.fre, args='optionDNE') assert result.exit_code == 2 diff --git a/fre/tests/test_fre_cmor_cli.py b/fre/tests/test_fre_cmor_cli.py new file mode 100644 index 00000000..51ce52b2 --- /dev/null +++ b/fre/tests/test_fre_cmor_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre cmor" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_cmor(): + ''' fre cmor ''' + result = runner.invoke(fre.fre, args=["cmor"]) + assert result.exit_code == 0 + +def test_cli_fre_cmor_help(): + ''' fre cmor --help ''' + result = runner.invoke(fre.fre, args=["cmor", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_cmor_opt_dne(): + ''' fre cmor optionDNE ''' + result = runner.invoke(fre.fre, args=["cmor", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_list_cli.py b/fre/tests/test_fre_list_cli.py new file mode 100644 index 00000000..effbb261 --- /dev/null +++ b/fre/tests/test_fre_list_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre list" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_list(): + ''' fre list ''' + result = runner.invoke(fre.fre, args=["list"]) + assert result.exit_code == 0 + +def test_cli_fre_list_help(): + ''' fre list --help ''' + result = runner.invoke(fre.fre, args=["list", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_list_opt_dne(): + ''' fre list optionDNE ''' + result = runner.invoke(fre.fre, args=["list", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_make_cli.py b/fre/tests/test_fre_make_cli.py new file mode 100644 index 00000000..6d6ba41a --- /dev/null +++ b/fre/tests/test_fre_make_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre make" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_make(): + ''' fre make ''' + result = runner.invoke(fre.fre, args=["make"]) + assert result.exit_code == 0 + +def test_cli_fre_make_help(): + ''' fre make --help ''' + result = runner.invoke(fre.fre, args=["make", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_make_opt_dne(): + ''' fre make optionDNE ''' + result = runner.invoke(fre.fre, args=["make", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_pp_cli.py b/fre/tests/test_fre_pp_cli.py new file mode 100644 index 00000000..4fef6afa --- /dev/null +++ b/fre/tests/test_fre_pp_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre pp" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_pp(): + ''' fre pp ''' + result = runner.invoke(fre.fre, args=["pp"]) + assert result.exit_code == 0 + +def test_cli_fre_pp_help(): + ''' fre pp --help ''' + result = runner.invoke(fre.fre, args=["pp", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_pp_opt_dne(): + ''' fre pp optionDNE ''' + result = runner.invoke(fre.fre, args=["pp", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_run_cli.py b/fre/tests/test_fre_run_cli.py new file mode 100644 index 00000000..42215490 --- /dev/null +++ b/fre/tests/test_fre_run_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre run" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_run(): + ''' fre run ''' + result = runner.invoke(fre.fre, args=["run"]) + assert result.exit_code == 0 + +def test_cli_fre_run_help(): + ''' fre run --help ''' + result = runner.invoke(fre.fre, args=["run", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_run_opt_dne(): + ''' fre run optionDNE ''' + result = runner.invoke(fre.fre, args=["run", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_test_cli.py b/fre/tests/test_fre_test_cli.py new file mode 100644 index 00000000..a82da835 --- /dev/null +++ b/fre/tests/test_fre_test_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre test" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_test(): + ''' fre test ''' + result = runner.invoke(fre.fre, args=["test"]) + assert result.exit_code == 0 + +def test_cli_fre_test_help(): + ''' fre test --help ''' + result = runner.invoke(fre.fre, args=["test", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_test_opt_dne(): + ''' fre test optionDNE ''' + result = runner.invoke(fre.fre, args=["test", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/tests/test_fre_yamltools_cli.py b/fre/tests/test_fre_yamltools_cli.py new file mode 100644 index 00000000..fd6bd1c2 --- /dev/null +++ b/fre/tests/test_fre_yamltools_cli.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +''' test "fre yamltools" calls ''' + +from click.testing import CliRunner + +from fre import fre + +runner = CliRunner() + +def test_cli_fre_yamltools(): + ''' fre yamltools ''' + result = runner.invoke(fre.fre, args=["yamltools"]) + assert result.exit_code == 0 + +def test_cli_fre_yamltools_help(): + ''' fre yamltools --help ''' + result = runner.invoke(fre.fre, args=["yamltools", "--help"]) + assert result.exit_code == 0 + +def test_cli_fre_yamltools_opt_dne(): + ''' fre yamltools optionDNE ''' + result = runner.invoke(fre.fre, args=["yamltools", "optionDNE"]) + assert result.exit_code == 2 diff --git a/fre/yamltools/__init__.py b/fre/yamltools/__init__.py index 9c94eeb6..369b1a42 100644 --- a/fre/yamltools/__init__.py +++ b/fre/yamltools/__init__.py @@ -1,7 +1,5 @@ -''' -for proper import of fre yamltools subcommands -''' +''' for fre.yamltools imports ''' from .freyamltoolsexample import yamltools_test_function -from .freyamltools import yamltoolsCli +from .freyamltools import yamltools_cli -__all__ = ["yamltools_test_function", "yamltoolsCli"] +__all__ = ["yamltools_test_function", "yamltools_cli"] diff --git a/fre/yamltools/data_table/is_valid_data_table_yaml.py b/fre/yamltools/data_table/is_valid_data_table_yaml.py index e3169435..17d08fbd 100644 --- a/fre/yamltools/data_table/is_valid_data_table_yaml.py +++ b/fre/yamltools/data_table/is_valid_data_table_yaml.py @@ -18,15 +18,12 @@ * You should have received a copy of the GNU Lesser General Public * License along with FMS. If not, see . *********************************************************************** -""" - -""" Determine if a yaml data_table is valid. + Determine if a yaml data_table is valid. Run `python3 is_valid_data_table_yaml.py -h` for more details Author: Uriel Ramirez 05/27/2022 """ import yaml -import sys import click def check_gridname(grid_name): diff --git a/fre/yamltools/diag_table/diag_table_to_yaml.py b/fre/yamltools/diag_table/diag_table_to_yaml.py index 4eb52333..5fe7f9b3 100755 --- a/fre/yamltools/diag_table/diag_table_to_yaml.py +++ b/fre/yamltools/diag_table/diag_table_to_yaml.py @@ -24,10 +24,10 @@ """ import copy as cp -import click from os import path +import click import yaml -from .. import __version__, TableParseError +from .. import __version__#, TableParseError def main(): #: parse user input diff --git a/fre/yamltools/diag_table/is_valid_diag_table_yaml.py b/fre/yamltools/diag_table/is_valid_diag_table_yaml.py index 8b5fe22f..e9ce12a5 100644 --- a/fre/yamltools/diag_table/is_valid_diag_table_yaml.py +++ b/fre/yamltools/diag_table/is_valid_diag_table_yaml.py @@ -19,16 +19,14 @@ * You should have received a copy of the GNU Lesser General Public * License along with FMS. If not, see . *********************************************************************** -""" - -""" Determine if a yaml diag_table is valid. + Determine if a yaml diag_table is valid. Run `python3 is_valid_diag_table_yaml.py -h` for more details Author: Uriel Ramirez 05/27/2022 """ -import yaml import sys import argparse +import yaml parser = argparse.ArgumentParser(prog='is_valid_diag_table_yaml', \ description="Determine if a yaml diag_table is valid. \ diff --git a/fre/yamltools/field_table/field_table_to_yaml.py b/fre/yamltools/field_table/field_table_to_yaml.py index 2c0cff90..1067da74 100755 --- a/fre/yamltools/field_table/field_table_to_yaml.py +++ b/fre/yamltools/field_table/field_table_to_yaml.py @@ -18,14 +18,13 @@ * You should have received a copy of the GNU Lesser General Public * License along with FMS. If not, see . *********************************************************************** -""" -""" Converts a legacy ascii field_table to a yaml field_table. +Converts a legacy ascii field_table to a yaml field_table. Author: Eric Stofferahn 07/14/2022 + """ import re -import sys from collections import OrderedDict import click import yaml diff --git a/fre/yamltools/freyamltools.py b/fre/yamltools/freyamltools.py index 125d0005..f603f803 100644 --- a/fre/yamltools/freyamltools.py +++ b/fre/yamltools/freyamltools.py @@ -1,16 +1,19 @@ +''' fre yamltools ''' + import click from .freyamltoolsexample import yamltools_test_function @click.group(help=click.style(" - access fre yamltools subcommands", fg=(202,177,95))) -def yamltoolsCli(): - pass +def yamltools_cli(): + ''' entry point to fre yamltools click commands ''' -@yamltoolsCli.command() +@yamltools_cli.command() @click.option('--uppercase', '-u', is_flag=True, help = 'Print statement in uppercase.') @click.pass_context def function(context, uppercase): + # pylint: disable=unused-argument """ - Execute fre yamltools test """ context.forward(yamltools_test_function) if __name__ == "__main__": - yamltoolsCli() + yamltools_cli() diff --git a/fre/yamltools/freyamltoolsexample.py b/fre/yamltools/freyamltoolsexample.py index 22fa6bef..e86fb206 100644 --- a/fre/yamltools/freyamltoolsexample.py +++ b/fre/yamltools/freyamltoolsexample.py @@ -7,7 +7,7 @@ import click @click.command() -def yamltools_test_function(uppercase): +def yamltools_test_function(uppercase=None): """Execute fre list testfunction2.""" statement = "testingtestingtestingtesting" if uppercase: