Merge branch 'main' into cmor-snack-prep

.github/workflows/build_conda.yml

@@ -7,10 +7,15 @@ jobs:
   build:
     runs-on: ubuntu-latest
     container:
-      image: continuumio/miniconda3:latest
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
     steps:
       - name: Checkout Files
         uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - name: Add mkmf to PATH
+        run: |
+          echo $PWD/mkmf/bin >> $GITHUB_PATH
       - name: Run Conda to Build
         run: |
           conda config --append channels conda-forge

.github/workflows/create_test_conda_env.yml

@@ -5,66 +5,54 @@ on: [push]
 jobs:
   build-linux:
     runs-on: ubuntu-latest
+    container:
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
     steps:
       - uses: actions/checkout@v4
-      - name: Set up Python
-        uses: actions/setup-python@v5
         with:
-          python-version: '>=3.9'
-
-      - name: Add conda to system path
-        run: |
-          # $CONDA is an env var pointing to root of miniconda dir
-          echo $CONDA/bin >> $GITHUB_PATH
-      
+          submodules: 'recursive'
       - name: Create fre-cli environment
         run: |
           # create environment containing all dependencies
           # the env cannot be explicitly activated in github CI/CD
           conda env create -f environment.yml --name fre-cli
 
-          # add conda env's executables to github's PATH equiv.
+          # sets CONDA to wherever it may be on the image
+          source /root/.bashrc
+
+          # add conda env's executables and mkmf to github's PATH equiv.
           echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
-          
+          echo $PWD/mkmf/bin >> $GITHUB_PATH
+
           # use *conda environment's pip* to install fre-cli
           # called w/ full path to conda's python for explicitness
           # called as a module (-m pip) for explicitness
-          $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli .      
+          $CONDA/envs/fre-cli/bin/python -m pip install --prefix $CONDA/envs/fre-cli .
 
       - name: Run pytest in fre-cli environment
         run: |
-          # try to make sure the right things are in GITHUB_PATH
-          echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
-          
-          # are we talking to the right python?
-          which python
-          python --version
-          $CONDA/envs/fre-cli/bin/python --version
-          
+          # add spack installed binaries to front of path so that
+          # conda's netcdf/hdf5 installs don't break compilation tests
+          export path_save=$PATH
+          export PATH="/opt/views/view/bin:$PATH"
+
           # run pytest
           pytest --junit-xml=pytest_results.xml --config-file=fre/pytest.ini --cov-config=fre/coveragerc --cov-report=xml --cov=fre fre/
-          
-          # install genbadge to generate coverage badge based on xml
+
+          # restore original path and install genbadge to generate coverage badge based on xml
+          export PATH="$path_save"
           pip install genbadge
           genbadge coverage -v -i coverage.xml -o docs/cov_badge.svg
           genbadge tests -v -i pytest_results.xml -o docs/pytest_badge.svg
-        
+
       - name: Run pylint in fre-cli environment
         run: |
-          # try to make sure the right things are in GITHUB_PATH
-          echo $CONDA/envs/fre-cli/bin >> $GITHUB_PATH
-
-          # are we talking to the right python?
-          which python
-          python --version
-          $CONDA/envs/fre-cli/bin/python --version
-          
-          # run pylint, ignored modules avoid warnings arising from code internal to those modules 
+          # run pylint, ignored modules avoid warnings arising from code internal to those modules
           pylint --max-args 6 -ry --ignored-modules netCDF4,cmor fre/ || echo "pylint returned non-zero exit code. preventing workflow from dying with this echo."
-      
+
       - name: Install Sphinx and Build Documentation
         run: |
-          pip install sphinx renku-sphinx-theme sphinx-rtd-theme          
+          pip install sphinx renku-sphinx-theme sphinx-rtd-theme
           pip install --upgrade sphinx-rtd-theme
           sphinx-apidoc --output-dir docs fre/ --separate
           sphinx-build docs build

.github/workflows/publish_conda.yml

@@ -7,10 +7,15 @@ jobs:
   publish:
     runs-on: ubuntu-latest
     container:
-      image: continuumio/miniconda3:latest
+      image: ghcr.io/noaa-gfdl/fre-cli:miniconda24.7.1_gcc14.2.0
     steps:
       - name: Checkout Files
         uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+      - name: Add mkmf to PATH
+        run: |
+          echo $PWD/mkmf/bin >> $GITHUB_PATH
       - name: Run Conda to Build and Publish
         run: |
           conda config --append channels conda-forge

.gitmodules

@@ -0,0 +1,6 @@
+[submodule "mkmf"]
+	path = mkmf
+	url = https://github.com/NOAA-GFDL/mkmf
+[submodule "fre/gfdl_msd_schemas"]
+	path = fre/gfdl_msd_schemas
+	url = https://github.com/NOAA-GFDL/gfdl_msd_schemas

CONTRIBUTING.md

@@ -1,7 +1,7 @@
 ## **For Developers**
 
 * Developers are free to use this repository's `README.md` to familiarize with the CLI and save time from having to install any dependencies, but development within a Conda environment is heavily recommended regardless
-* Gain access to the repository with `git clone [email protected]:NOAA-GFDL/fre-cli.git` or your fork's link (recommended) and an SSH RSA key
+* Gain access to the repository with `git clone --recursive [email protected]:NOAA-GFDL/fre-cli.git` or your fork's link (recommended) and an SSH RSA key
     - Once inside the repository, developers can test local changes by running a `pip install .` inside of the root directory to install the fre-cli package locally with the newest local changes on top of the installed Conda fre-cli dependencies
     - Test as a normal user would use the CLI
 * Create a GitHub issue to reflect your contribution's background and reference it with Git commits

docs/api.rst

@@ -0,0 +1,4 @@
+=============
+API
+=============
+Auto-harvested goodness, coming soon.

docs/badges.rst

@@ -1,3 +1,5 @@
+.. this file is explicitly for the hyperlinkage in the base README.md to the badge image files
+======
 Badges
 ======
 

docs/contributing_to_doc.rst

@@ -0,0 +1,62 @@
+.. last updated early Nov 2024.
+   could use some refinement
+===========================
+Documentation-Documentation
+===========================
+
+Welcome to ``fre-cli``'s Documentation-documentation- where we document how the documentation is
+documented
+
+How to Contribute to ``fre-cli``'s documentation
+================================================
+
+
+
+fork and poke at the settings
+-----------------------------
+
+* Fork ``fre-cli`` on github
+
+* On github, navigate to your ``fre-cli`` fork, and click “settings”
+
+* In “settings”, click “pages”
+
+* In “pages”, under “build and deployment”, make sure “source” is set to “Deploy from a branch”
+
+* Under that, find “Branch”, make sure the branch selected is ``gh-pages``
+
+* The branch ``gh-pages`` is "automagic”- i.e. do not change anything about it nor create a new one,
+  nor interact with anything in that branch directly
+
+
+enable workflows for your fork
+------------------------------
+
+note: this step may depend on user-specific settings!
+
+* Back on top where you found “settings”, find and click “actions” to the left
+
+* Enable running the workflow actions assoc with the ``fre-cli`` repo under ``.github/workflows``
+
+
+run your fork's first workflow
+------------------------------
+
+* The documentation builds as the last steps to ``create_test_conda_env.yml`` when theres a push to ``main``
+
+* To get your first workflow run on your fork, comment out the ``github.ref == ‘refs/heads/main’`` bit
+  so that it runs when you push to any branch, and make a small, trivial, commit somewhere to your
+  remote fork
+
+* You should be able to find the deployed webpage from a successful workflow at
+  https://your_username.github.io/fre-cli (if you did not change the fork’s name from ``fre-cli``, that is)
+
+* If you’re only editing docs, you can make the turn-around time on your workflow ~3 min faster by
+  commenting-out the ``pylint`` and ``pytest`` steps in ``create_test_conda_env.yml``, and disable running the
+  ``build_conda.yml`` workflow
+
+
+
+Other Helpful Things
+====================
+`restructured text cheat-sheet <https://gist.github.com/SMotaal/24006b13b354e6edad0c486749171a70#sections>`_

docs/for-developers.rst

@@ -0,0 +1,118 @@
+===============
+For developers
+===============
+
+Developers are free to use the user guide above to familiarize with the CLI and save time from
+having to install any dependencies, but development within a Conda environment is heavily
+recommended regardless.
+
+Gain access to the repository with ``git clone --recursive [email protected]:NOAA-GFDL/fre-cli.git`` or your fork's
+link (recommended) and an SSH RSA key. Once inside the repository, developers can test local changes
+by running a ``pip install .`` inside of the root directory to install the ``fre-cli`` package locally
+with the newest local changes. Test as a normal user would use the CLI.
+
+
+Adding New Tools
+================
+
+
+From Other Repositories
+-----------------------
+
+Currently, the solution to this task is to approach it using Conda packages. The tool that is being
+added must reside within a repository that contains a ``meta.yaml`` that includes Conda dependencies
+like the one in this repository and ideally a ``setup.py`` (may be subject to change due to deprecation)
+that may include any potentially needed pip dependencies
+
+* Once published as a Conda package, ideally on the `NOAA-GFDL conda channel <https://anaconda.org/NOAA-GFDL>`_,
+  an addition can be made to the ``run`` section under ``requirements`` in ``meta.yaml`` of the ``fre-cli``
+  following the syntax ``channel::package``
+
+* On pushes to the main branch, the package located at https://anaconda.org/NOAA-GFDL/fre-cli will automatically
+  be updated using by the workflow defined in ``.github/workflows/publish_conda.yml``
+
+
+Checklist
+---------
+
+For the new tool you are trying to develop, there are a few criteria to satisfy
+
+1. Create a subdirectory for the tool group inside the ``fre/`` directory; i.e. ``fre/<tool>``
+
+2. Add an ``__init__.py`` inside of ``fre/<tool>`` 
+
+* typically this file should be empty, but it depends on the ``<tool>``'s needs
+* even if empty, the file facillitates module importability and must be present
+
+3. Add a file named ``fre/<tool>/fre<tool>.py``. This will serve as the main entry point for ``fre``
+   into the ``<tool>``'s functionality
+
+4. Add a ``click`` group named after ``<tool>`` within ``fre/<tool>/fre<tool>.py``
+
+* This ``click`` group will contain all the functionality under the ``<tool>``
+
+5. Create separate files as needed for different commands; do not code out the full
+   implemetation of ``<tool>`` inside of a ``click`` command within ``fre/<tool>/fre<tool>.py``.
+
+* better yet, consider what structure your tool may need in the future for maintainability's sake
+* if you need, specify a ``<subtool>`` like ``fre/<tool>/<subtool>``. ``fre/app`` currently has
+  this structure
+
+6. Be sure to import the contents of the needed subcommand scripts inside of ``fre<tool>.py``
+
+* i.e. from ``fre.<tool>.toolCommandScript import *``
+
+7. At this point, you can copy and paste the parts of your main ``click`` command from its script
+   into ``fre<tool>.py`` when implementing the function reflective of the command function
+
+* Everything will remain the same; i.e. arguments, options, etc.
+
+* However, this new function within ``fre<tool>.py`` must a new line after the arguments, options,
+  and other command components; ``@click.pass_context``
+
+* Along with this, a new argument ``context`` must now be added to the parameters of the command
+  (preferably at the beginning, but it won't break it if it's not)
+
+8. From here, all that needs to be added after defining the command with a name is
+   ``context.forward(mainFunctionOfToolCommand)``, and done!
+
+9. The last step is to replicate the command in the same way as done in ``fre<tool>.py``
+	inside of ``fre.py``, but make sure to add ``from fre import <tool>`` and
+	``from fre.<tool> import *``
+
+Please refer to this issue when encountering naming issues:
+`NOAA-GFDL#31 <https://github.com/NOAA-GFDL/fre-cli/issues/31>`_
+
+
+Example ``fre/`` Directory Structure
+------------------------------------
+
+``fre/``
+├── ``__init__.py``
+├── ``fre.py``
+├── ``fre<tool>``
+│   ├── ``__init__.py``
+│   ├── ``toolCommandScript.py``
+│   └── ``fre<tool>.py``
+
+
+``MANIFEST.in``
+---------------
+
+In the case where non-python files like templates, examples, and outputs are to be included in the ``fre-cli`` package,
+``MANIFEST.in`` can provide the solution. Ensure that the file exists within the correct folder, and add a line to the
+``MANIFEST.in`` file saying something like ``include fre/fre<tool>/fileName.fileExtension``
+
+* For more efficiency, if there are multiple files of the same type needed, the ``MANIFEST.in`` addition can be something
+  like ``recursive-include fre/fre<tool> *.fileExtension`` which would recursively include every file matching that
+  ``fileExtension`` within the specified directory and its respective subdirectories.
+
+
+Adding Documentation
+--------------------
+
+see section "Documentation-Documentation"
+
+
+
+

docs/index.rst

@@ -1,23 +1,28 @@
-.. Fre-Cli documentation master file, created by
-   sphinx-quickstart on Wed Mar  6 22:28:21 2024.
-   You can adapt this file completely to your liking, but it should at least
-   contain the root `toctree` directive.
+.. Fre-Cli documentation master file, created by sphinx-quickstart on Wed Mar  6 22:28:21 2024.
+   You can adapt this file completely to your liking, but it should at least contain the root
+   \`toctree\` directive (no backslashes)
+   Some sphinx markdown examples:
+   https://gist.github.com/SMotaal/24006b13b354e6edad0c486749171a70
 
-Welcome to Fre-Cli's documentation!
-===================================
+=======================================
+Welcome to ``fre-cli``'s documentation!
+=======================================
 
+.. the entry in the toc must be the .rst filename.
+   what shows in the webpage is the first header or title
 .. toctree::
-   :maxdepth: 2
+   :maxdepth: 1
    :caption: Contents:
 
+   what-is-fre
    setup
    usage
-   subtools
-   FAQ
-   badges
+   tools
+   api
+   for-developers
 
-Indices and tables
-==================
+Indices
+=======
 
 * :ref:`genindex`
 * :ref:`modindex`