Do not create a combined yaml file anymore

fre/pp/configure_script_yaml.py

@@ -15,7 +15,8 @@
 import yaml
 import metomi.rose.config
 
-import fre.yamltools.combine_yamls as cy
+import fre.yamltools.combine_yamls_script as cy
+import fre.yamltools.pp_info_parser as ppip
 
 ####################
 def yaml_load(yamlfile):
@@ -33,9 +34,6 @@ def validate_yaml(yamlfile):
     """
      Using the schema.json file, the yaml format is validated.
     """
-    # Load the yaml
-    yml = yaml_load(yamlfile)
-
     schema_dir = Path(__file__).resolve().parents[1]
     schema_path = os.path.join(schema_dir, 'gfdl_msd_schemas', 'FRE', 'fre_pp.json')
     # Load the json schema: .load() (vs .loads()) reads and parses the json in one)
@@ -45,7 +43,7 @@ def validate_yaml(yamlfile):
     # Validate yaml
     # If the yaml is not valid, the schema validation will raise errors and exit
     try:
-        validate(instance=yml,schema=schema)
+        validate(instance=yamlfile,schema=schema)
         print("\nCombined yaml VALID \n")
     except:
         raise ValueError("\nCombined yaml NOT VALID.\n")
@@ -165,26 +163,28 @@ def yaml_info(yamlfile = None, experiment = None, platform = None, target = None
     rose_suite,rose_regrid,rose_remap = rose_init(e,p,t)
 
     # Combine model, experiment, and analysis yamls
-    comb = cy.init_pp_yaml(yml,e,p,t)
-    full_combined = cy.get_combined_ppyaml(comb)
+    cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}")
+    outfile = os.path.join(cylc_dir, f"{e}.yaml")
+
+    full_yamldict = cy.consolidate_yamls(yamlfile = yml,
+                                         experiment = e,
+                                         platform = p,
+                                         target = t,
+                                         use = "pp",
+                                         output = outfile)
 
     # Validate yaml
-    validate_yaml(full_combined)
-    # Load the combined yaml
-    comb_pp_yaml = yaml_load(full_combined)
+    validate_yaml(full_yamldict)
 
     ## PARSE COMBINED YAML TO CREATE CONFIGS
     # Set rose-suite items
-    set_rose_suite(comb_pp_yaml,rose_suite)
+    set_rose_suite(full_yamldict,rose_suite) ####comb_pp_yaml,rose_suite)
 
     # Set regrid and remap rose app items
-    set_rose_apps(comb_pp_yaml,rose_regrid,rose_remap)
+    set_rose_apps(full_yamldict,rose_regrid,rose_remap) ####comb_pp_yaml,rose_regrid,rose_remap)
 
-    # write output files
+    # Write output files
     print("Writing output files...")
-    cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}")
-    outfile = os.path.join(cylc_dir, f"{e}.yaml")
-    shutil.copyfile(full_combined, outfile)
     print("  " + outfile)
 
     dumper = metomi.rose.config.ConfigDumper()

fre/tests/test_fre_pp_cli.py

@@ -94,9 +94,7 @@ def test_cli_fre_pp_configure_yaml_fail1():
                                              "-p", "BAR",
                                              "-t", "BAZ",
                                              "-y", "BOO"              ] )
-    assert all( [ result.exit_code == 1,
-                  isinstance(result.exception, FileNotFoundError )
-               ] )
+    assert result.exit_code == 1 
 
 
 #-- fre pp install

fre/yamltools/combine_yamls.py

@@ -1,5 +1,5 @@
 """
-Script combines the model yaml with the compile, platform, and experiment yamls.
+OLDER script that combines the model yaml with the compile, platform, and experiment yamls.
 """
 
 ## TO-DO:

fre/yamltools/combine_yamls_script.py

@@ -0,0 +1,134 @@
+import os
+import shutil
+
+from pathlib import Path
+import click
+import yaml
+import fre.yamltools.compile_info_parser as cip
+import fre.yamltools.pp_info_parser as ppip
+import pprint
+
+def join_constructor(loader, node):
+    """
+    Allows FRE properties defined
+    in main yaml to be concatenated.
+    """
+    seq = loader.construct_sequence(node)
+    return ''.join([str(i) for i in seq])
+
+def output_yaml(cleaned_yaml,experiment,output):
+    """
+    Write out the combined yaml dictionary info
+    to a file if --output is specified
+    """
+    filename = output
+    with open(filename,'w') as out:
+        out.write(yaml.dump(cleaned_yaml,default_flow_style=False,sort_keys=False))
+
+## Functions to combine the yaml files ##
+def get_combined_compileyaml(comb,output=None):
+    """
+    Combine the model, compile, and platform yamls
+    Arguments:
+    comb : combined yaml object
+    """
+    try:
+        (yaml_content, loaded_yaml)=comb.combine_model()
+    except:
+        raise ValueError("ERR: Could not merge model information.")
+
+    # Merge compile into combined file to create updated yaml_content/yaml
+    try:
+        (yaml_content, loaded_yaml) = comb.combine_compile(yaml_content, loaded_yaml)
+    except: 
+        raise ValueError("ERR: Could not merge compile yaml information.")
+
+    # Merge platforms.yaml into combined file
+    try:
+        (yaml_content,loaded_yaml) = comb.combine_platforms(yaml_content, loaded_yaml)
+    except: 
+        raise ValueError("ERR: Could not merge platform yaml information.")
+
+    # Clean the yaml
+    cleaned_yaml = comb.clean_yaml(yaml_content)
+
+    # OUTPUT IF NEEDED
+    if output is not None:
+        output_yaml(cleaned_yaml,experiment=None,output=output)
+    else:
+        print("Combined yaml information saved as dictionary")
+
+    return cleaned_yaml
+
+def get_combined_ppyaml(comb,experiment,output=None):
+    """
+    Combine the model, experiment, and analysis yamls
+    Arguments:
+    comb : combined yaml object
+    """
+    try:
+        # Merge model into combined file
+        (yaml_content, loaded_yaml) = comb.combine_model()
+    except:
+        raise ValueError("ERR: Could not merge model information.")
+
+    try:
+        # Merge pp experiment yamls into combined file
+        comb_pp_updated_list = comb.combine_experiment(yaml_content, loaded_yaml)
+    except:
+        raise ValueError("ERR: Could not merge pp experiment yaml information")
+
+    try:
+        # Merge analysis yamls, if defined, into combined file
+        comb_analysis_updated_list = comb.combine_analysis(yaml_content, loaded_yaml)
+    except:
+        raise ValueError("ERR: Could not merge analysis yaml information")
+
+    try:
+        # Merge model/pp and model/analysis yamls if more than 1 is defined
+        # (without overwriting the yaml)
+        full_combined = comb.merge_multiple_yamls(comb_pp_updated_list, comb_analysis_updated_list,loaded_yaml)
+    except: 
+        raise ValueError("ERR: Could not merge multiple pp and analysis information together.")
+
+    # Clean the yaml
+    cleaned_yaml = comb.clean_yaml(full_combined)
+
+    # OUTPUT IF NEEDED
+    if output is not None:
+        output_yaml(cleaned_yaml,experiment,output)
+    else:
+        print("Combined yaml information saved as dictionary")
+
+    return cleaned_yaml
+
+def consolidate_yamls(yamlfile,experiment,platform,target,use,output):
+    """
+    Depending on `use` argument passed, either create the final
+    combined yaml for compilation or post-processing
+    """
+    if use == "compile":
+        combined = cip.InitCompileYaml(yamlfile, platform, target, join_constructor)
+
+        if output is None :
+            yml_dict = get_combined_compileyaml(combined)
+        else:
+            yml_dict = get_combined_compileyaml(combined,output)
+            print(f"Combined yaml file located here: {os.getcwd()}/{output}")
+
+    elif use =="pp":
+        combined = ppip.InitPPYaml(yamlfile, experiment, platform, target, join_constructor)
+
+        if output is None:
+            yml_dict = get_combined_ppyaml(combined,experiment)
+        else:
+            yml_dict = get_combined_ppyaml(combined,experiment,output)
+            print(f"Combined yaml file located here: {os.getcwd()}/{output}")
+
+    else:
+        raise ValueError("'use' value is not valid; must be 'compile' or 'pp'") 
+
+    return yml_dict
+# Use parseyaml function to parse created edits.yaml
+if __name__ == '__main__':
+    consolidate_yamls()

fre/yamltools/compile_info_parser.py

@@ -0,0 +1,126 @@
+import os
+import yaml
+
+def get_compile_paths(full_path,loaded_yml):
+    """
+    Find and return the paths for the compile
+    and platform yamls
+    """
+    for key,value in loaded_yml.items():
+        if key == "build":
+            py_path = os.path.join(full_path,value.get("platformYaml"))
+            cy_path = os.path.join(full_path,value.get("compileYaml"))
+
+    return (py_path, cy_path)
+
+## COMPILE CLASS ##
+class InitCompileYaml():
+    """ class holding routines for initalizing compilation yamls """
+    def __init__(self,yamlfile,platform,target,join_constructor):
+        """
+        Process to combine yamls applicable to compilation
+        """
+        self.yml = yamlfile
+        #self.name = yamlfile.split(".")[0]
+        self.namenopath = self.yml.split("/")[-1].split(".")[0]
+        self.platform = platform
+        self.target = target
+
+        # Register tag handler
+        yaml.add_constructor('!join', join_constructor)
+
+        # Path to the main model yaml
+        self.mainyaml_dir = os.path.dirname(self.yml)
+
+        # Create combined compile yaml
+        print("Combining yaml files into one dictionary: ")
+
+    def combine_model(self):
+        """
+        Create the combined.yaml and merge it with the model yaml
+        """
+        # Define click options in string
+        yaml_content = (f'name: &name "{self.namenopath}"\n'
+                        f'platform: &platform "{self.platform}"\n'
+                        f'target: &target "{self.target}"\n')
+
+        # Read model yaml as string
+        with open(self.yml,'r') as f:
+            model_content = f.read()
+
+        # Combine information as strings
+        yaml_content += model_content
+
+        # Load string as yaml
+        yml=yaml.load(yaml_content, Loader=yaml.Loader)
+
+        # Return the combined string and loaded yaml
+        print(f"   model yaml: {self.yml}")
+        return (yaml_content, yml)
+
+    def combine_compile(self,yaml_content,loaded_yaml):
+        """
+        Combine compile yaml with the defined combined.yaml
+        """
+        self.mainyaml_dir = os.path.dirname(self.yml)
+
+        # Get compile info
+        (py_path,cy_path) = get_compile_paths(self.mainyaml_dir,loaded_yaml)
+
+        # copy compile yaml info into combined yaml
+        if cy_path is not None:
+            with open(cy_path,'r') as cf:
+                compile_content = cf.read()
+
+        # Combine information as strings
+        yaml_content += compile_content
+
+        # Load string as yaml
+        yml=yaml.load(yaml_content, Loader=yaml.Loader)
+
+        # Return the combined string and loaded yaml
+        print(f"   compile yaml: {cy_path}")
+        return (yaml_content,yml)
+
+    def combine_platforms(self, yaml_content, loaded_yaml):
+        """
+        Combine platforms yaml with the defined combined.yaml
+        """
+        self.mainyaml_dir = os.path.dirname(self.yml)
+
+        # Get compile info
+        (py_path,cy_path) = get_compile_paths(self.mainyaml_dir, loaded_yaml)
+
+        # copy compile yaml info into combined yaml
+        if py_path is not None:
+            with open(py_path,'r') as pf:
+                platform_content = pf.read()
+
+        # Combine information as strings
+        yaml_content += platform_content
+
+        # Load string as yaml
+        yml=yaml.load(yaml_content, Loader=yaml.Loader)
+
+        # Return the combined string and loaded yaml
+        print(f"   platforms yaml: {py_path}")
+        return (yaml_content, yml)
+
+    def clean_yaml(self, yaml_content):
+        """
+        Clean the yaml; remove unnecessary sections in
+        final combined yaml.
+        """
+        # Load the yaml
+        yml_dict=yaml.load(yaml_content, Loader=yaml.Loader)
+
+        # Clean the yaml
+        # If keys exists, delete:
+        keys_clean=["fre_properties", "shared", "experiments"]
+        for kc in keys_clean:
+            if kc in yml_dict.keys():
+                del yml_dict[kc]
+
+        cleaned_yml = yaml.safe_dump(yml_dict,default_flow_style=False,sort_keys=False)
+
+        return yml_dict  #either return dictionary OR string (cleaned_yml) - string works for fremake but dictionary works for pp and list

fre/yamltools/freyamltools.py

@@ -1,7 +1,7 @@
 ''' fre yamltools '''
 
 import click
-from .combine_yamls import _consolidate_yamls
+from fre.yamltools import combine_yamls_script
 
 @click.group(help=click.style(" - access fre yamltools subcommands", fg=(202,177,95)))
 def yamltools_cli():
@@ -23,22 +23,24 @@ def yamltools_cli():
               help="Platform name",
               required=True)
 @click.option("-t",
-                "--target",
-                type=str,
-                help="Target name",
-                required=True)
+              "--target",
+              type=str,
+              help="Target name",
+              required=True)
 @click.option("--use",
               type=click.Choice(['compile','pp']),
               help="Process user is combining yamls for. Can pass 'compile' or 'pp'",
               required=True)
-@click.pass_context
-def combine_yamls(context,yamlfile,experiment,platform,target,use):
-    # pylint: disable=unused-argument
+@click.option("-o",
+              "--output",
+              type=str,
+              help="Output")
+def combine_yamls(yamlfile,experiment,platform,target,use,output):
     """ 
     - Combine the model yaml with the compile, platform,
     experiment, and analysis yamls
     """
-    context.forward(_consolidate_yamls)
+    combine_yamls_script.consolidate_yamls(yamlfile,experiment,platform,target,use,output)
 
 if __name__ == "__main__":
     yamltools_cli()