Merge branch 'main' into add-gettid-null-model

docs/index.rst

@@ -11,7 +11,7 @@ Welcome to ``fre-cli``'s documentation!
 .. the entry in the toc must be the .rst filename.
    what shows in the webpage is the first header or title
 .. toctree::
-   :maxdepth: 1
+   :maxdepth: 2
    :caption: Contents:
 
    what-is-fre

fre/__init__.py

@@ -0,0 +1,10 @@
+# turn xxxx.y into xxxx.0y
+import importlib.metadata
+version_unexpanded = importlib.metadata.version('fre-cli')
+version_unexpanded_split = version_unexpanded.split('.')
+if len(version_unexpanded_split[1]) == 1:
+    version_minor = "0" + version_unexpanded_split[1]
+else:
+    version_minor = version_unexpanded_split[1]
+version = version_unexpanded_split[0] + '.' + version_minor
+__version__ = version

fre/fre.py

@@ -27,10 +27,9 @@
         fg='cyan')
 )
 
+
 @click.version_option(
-    package_name = "fre-cli",
-    message = click.style("%(package)s | %(version)s",
-                          fg = (155,255,172) )
+    package_name = "fre-cli"
 )
 
 def fre():

fre/make/tests/compilation/test_fre_make_run_fremake.py

@@ -1,9 +1,13 @@
 ''' test "fre make run-fremake" calls '''
 
 import os
-from fre.make import runFremake
 from pathlib import Path
 
+import pytest
+
+from fre.make import runFremake
+
+
 # command options
 YAMLFILE = "fre/make/tests/null_example/null_model.yaml"
 PLATFORM = [ "ci.gnu" ]
@@ -14,6 +18,7 @@
 # get HOME dir to check output
 HOME_DIR = os.environ["HOME"]
 
+@pytest.mark.skip(reason='failing: fix in development, see PR 275')
 def test_fre_make_run_fremake_null_model_serial_compile():
     ''' run fre make with run-fremake subcommand and build the null model experiment with gnu'''
     runFremake.fremake_run(YAMLFILE, PLATFORM, TARGET, False, 1, False, False)

fre/pp/checkout_script.py

@@ -0,0 +1,98 @@
+'''
+Description: Checkout script which accounts for 4 different scenarios: 
+1. branch not given, folder does not exist,
+2. branch given, folder does not exist, 
+3. branch not given, folder exists, 
+4. branch given and folder exists
+'''
+import os
+import sys
+import subprocess
+
+import click
+
+import fre
+
+FRE_WORKFLOWS_URL = 'https://github.com/NOAA-GFDL/fre-workflows.git'
+
+def checkout_template(experiment = None, platform = None, target = None, branch = None):
+    """
+    Checkout the workflow template files from the repo
+    """
+    ## Chdir back to here before we exit this routine
+    go_back_here = os.getcwd()
+
+    # branch and version parameters
+    default_tag = fre.__version__
+    print(f'(checkout_script) default_tag is {default_tag}')
+
+
+    # check args + set the name of the directory
+    if None in [experiment, platform, target]:
+        raise ValueError( 'one of these are None: experiment / platform / target = \n'
+                         f'{experiment} / {platform} / {target}' )
+    name = f"{experiment}__{platform}__{target}"
+
+    # Create the directory if it doesn't exist
+    directory = os.path.expanduser("~/cylc-src")
+    try:
+        os.makedirs(directory, exist_ok = True)
+    except Exception as exc:
+        raise OSError(
+            '(checkoutScript) directory {directory} wasnt able to be created. exit!') from exc
+
+    print(f'(checkout_script) branch is {branch}')
+    checkout_exists = os.path.isdir(f'{directory}/{name}')
+    git_clone_branch_arg = branch if branch is not None else default_tag
+    if branch is not None:
+        print('(checkout_script) WARNING using default_tag as branch argument for git clone!')
+
+
+    if not checkout_exists: # scenarios 1+2, checkout doesn't exist, branch specified (or not)
+        clone_output = subprocess.run( ['git', 'clone','--recursive',
+                                        f'--branch={git_clone_branch_arg}',
+                                        FRE_WORKFLOWS_URL, f'{directory}/{name}'],
+                                       capture_output = True, text = True, check = True)
+        print(f'(checkout_script) output git clone command: {clone_output}')
+
+    else:     # the repo checkout does exist, scenarios 3 and 4.
+        os.chdir(f'{directory}/{name}')
+
+        name_path_tag_subproc_out = subprocess.run(["git","describe","--tags"],
+                                                   capture_output = True,
+                                                   text = True, check = True).stdout
+        if branch is not None:
+            name_path_tag = name_path_tag_subproc_out.split('*')
+            name_path_branch = subprocess.run(["git","branch"],
+                                              capture_output = True,
+                                              text = True, check = True).stdout.split()[0]
+            if all( [ default_tag not in name_path_tag,
+                      name_path_branch != branch ] ):
+                sys.exit(
+                    f"Tag and branch of prexisting directory {directory}/{name} does not match "
+                    "fre --version or branch requested"                                            )
+        else:
+            name_path_tag = name_path_tag_subproc_out.split()[0]
+            if not default_tag in name_path_tag:
+                sys.exit(
+                    f"Tag of prexisting directory {directory}/{name} does not match fre --version")
+
+    # make sure we are back where we should be
+    if os.getcwd() != go_back_here:
+        os.chdir(go_back_here)
+
+    return 0
+
+#############################################
+
+@click.command()
+def _checkout_template(experiment, platform, target, branch = None):
+    '''
+    Wrapper script for calling checkout_template - allows the decorated version
+    of the function to be separate from the undecorated version
+    '''
+    return checkout_template(experiment, platform, target, branch)
+
+
+if __name__ == '__main__':
+    checkout_template()

fre/pp/configure_script_xml.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python3
 '''
  Primary Usage: fre-bronx-to-canopy -x XML -e EXP -p PLATFORM -t TARGET
 

fre/pp/configure_script_yaml.py

@@ -1,4 +1,3 @@
-#!/usr/bin/env python
 """
     Script creates rose-apps and rose-suite
     files for the workflow from the pp yaml.

fre/pp/frepp.py

@@ -1,14 +1,16 @@
 ''' fre pp '''
 
 import click
-from .checkoutScript import _checkoutTemplate
-from .configure_script_yaml import _yamlInfo
-from .configure_script_xml import convert
-from .validate import _validate_subtool
-from .install import install_subtool
-from .run import pp_run_subtool
-from .status import status_subtool
-from .wrapper import runFre2pp
+
+from fre.pp import checkout_script 
+from fre.pp import configure_script_yaml 
+from fre.pp import configure_script_xml 
+#from fre.pp import validate 
+from fre.pp import install 
+from fre.pp import run 
+from fre.pp import trigger 
+from fre.pp import status 
+from fre.pp import wrapper 
 
 @click.group(help=click.style(" - access fre pp subcommands", fg=(57,139,210)))
 def pp_cli():
@@ -30,7 +32,7 @@ def pp_cli():
 def status(context, experiment, platform, target):
     # pylint: disable=unused-argument
     """ - Report status of PP configuration"""
-    context.forward(status_subtool)
+    context.forward(status.status_subtool)
 
 # fre pp run
 @pp_cli.command()
@@ -47,7 +49,7 @@ def status(context, experiment, platform, target):
 def run(context, experiment, platform, target):
     # pylint: disable=unused-argument
     """ - Run PP configuration"""
-    context.forward(pp_run_subtool)
+    context.forward(run.pp_run_subtool)
 
 # fre pp validate
 @pp_cli.command()
@@ -81,7 +83,7 @@ def validate(context, experiment, platform, target):
 def install(context, experiment, platform, target):
     # pylint: disable=unused-argument
     """ - Install PP configuration"""
-    context.forward(install_subtool)
+    context.forward(install.install_subtool)
 
 @pp_cli.command()
 @click.option("-y", "--yamlfile", type=str,
@@ -100,7 +102,7 @@ def install(context, experiment, platform, target):
 def configure_yaml(context,yamlfile,experiment,platform,target):
     # pylint: disable=unused-argument
     """ - Execute fre pp configure """
-    context.forward(_yamlInfo)
+    context.forward(configure_script_yaml._yamlInfo)
 
 @pp_cli.command()
 @click.option("-e", "--experiment", type=str,
@@ -113,16 +115,13 @@ def configure_yaml(context,yamlfile,experiment,platform,target):
               help="Target name",
               required=True)
 @click.option("-b", "--branch",
-              show_default=True,
-              default="main", type=str,
-              help="Name of fre2/workflows/postproc branch to clone; " \
-                   "defaults to 'main'. Not intended for production use, " \
-                   "but needed for branch testing."  )
+              required=False,
+              help="fre-workflows branch/tag to clone; default is $(fre --version)")
 @click.pass_context
-def checkout(context, experiment, platform, target, branch='main'):
+def checkout(context, experiment, platform, target, branch=None):
     # pylint: disable=unused-argument
     """ - Execute fre pp checkout """
-    context.forward(_checkoutTemplate)
+    context.forward(checkout_script._checkout_template)
 
 @pp_cli.command()
 @click.option('-x', '--xml',
@@ -180,7 +179,7 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo
                   ppdir, do_refinediag, pp_start, pp_stop, validate, verbose, quiet, dual):
     # pylint: disable=unused-argument
     """ - Converts a Bronx XML to a Canopy rose-suite.conf """
-    context.forward(convert)
+    context.forward(configure_script_xml.convert)
 
 #fre pp wrapper
 @pp_cli.command()
@@ -190,23 +189,42 @@ def configure_xml(context, xml, platform, target, experiment, do_analysis, histo
 @click.option("-p", "--platform", type=str,
               help="Platform name",
               required=True)
-@click.option("-t", "--target", type=str,
+@click.option("-T", "--target", type=str,
               help="Target name",
               required=True)
 @click.option("-c", "--config-file", type=str,
               help="Path to a configuration file in either XML or YAML",
               required=True)
+@click.option("-t", "--time",
+              required=False,
+              help="Time whose history files are ready")
 @click.option("-b", "--branch",
-              show_default=True,
-              default="main", type=str,
-              help="Name of fre2/workflows/postproc branch to clone; " \
-                   "defaults to 'main'. Not intended for production use, " \
-                   "but needed for branch testing."             )
+              required=False,
+              help="fre-workflows branch/tag to clone; default is $(fre --version)")
 @click.pass_context
-def wrapper(context, experiment, platform, target, config_file, branch='main'):
+def wrapper(context, experiment, platform, target, config_file, time=None, branch=None):
     # pylint: disable=unused-argument
     """ - Execute fre pp steps in order """
-    context.forward(runFre2pp)
+    context.forward(wrapper.runFre2pp)
+
+@pp_cli.command()
+@click.option("-e", "--experiment", type=str,
+              help="Experiment name",
+              required=True)
+@click.option("-p", "--platform", type=str,
+              help="Platform name",
+              required=True)
+@click.option("-T", "--target", type=str,
+              help="Target name",
+              required=True)
+@click.option("-t", "--time",
+              required=True,
+              help="Time whose history files are ready")
+@click.pass_context
+def trigger(context, experiment, platform, target, time):
+    # pylint: disable=unused-argument
+    """ - Start postprocessing for a particular time """
+    context.forward(trigger._trigger)
 
 if __name__ == "__main__":
     ''' entry point for click to fre pp commands '''