From 9629ea5a159428f31a17f1706b7eff51be6372d6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 10 Jun 2024 14:36:01 -0400 Subject: [PATCH 01/27] #99 Make pp yaml more generic - update configure script to set experiment, platform, and target from click options --- fre/pp/configure_script_yaml.py | 12 ++++++++---- fre/pp/ppyaml_test/pp.yaml | 9 ++++++--- 2 files changed, 14 insertions(+), 7 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 7f9e437a..e212fab6 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -67,11 +67,15 @@ def yamlInfo(yamlfile,experiment,platform,target): for configkey,configvalue in dict.items(): if configvalue != None: k=configkey.upper() - if configvalue == True or configvalue == False: - rose_suite.set(keys=['template variables', k], value=f'{configvalue}') + if k in ("HISTORY_DIR", "PP_DIR", "ANALYSIS"): + # replace generic variables in pp yaml + stem=e.split("_",1)[0] + exp=e.split("_",1)[1] + replace=configvalue.replace("$(stem)",stem).replace("$(expname)",exp).replace("$(platform)",p).replace("$(target)",t) + rose_suite.set(keys=['template variables', k], value=f'{replace}') else: rose_suite.set(keys=['template variables', k], value=f'"{configvalue}"') - + if key == "components": for i in value: comp = i.get('type') @@ -119,7 +123,7 @@ def yamlInfo(yamlfile,experiment,platform,target): outfile = os.path.join(cylc_dir, "app", "remap-pp-components", "rose-app.conf") dumper(rose_remap, outfile) print(" " + outfile) - + # Use parseyaml function to parse created edits.yaml if __name__ == '__main__': yamlInfo() diff --git a/fre/pp/ppyaml_test/pp.yaml b/fre/pp/ppyaml_test/pp.yaml index b8ab4006..45439f82 100644 --- a/fre/pp/ppyaml_test/pp.yaml +++ b/fre/pp/ppyaml_test/pp.yaml @@ -21,10 +21,13 @@ rose-suite: do_analysis: False directories: - history_dir: "/archive/c2b/am5/2022.01/c96L65_am5f1a0r0_amip/gfdl.ncrc5-intel22-classic-prod-openmp/history" - pp_dir: "/archive/$USER/canopy/am5/c96L65_amip/gfdl.ncrc5-deploy-prod-openmp/pp" + history_dir: "/archive/c2b/$(stem)/2022.01/$(expname)/$(platform)-$(target)/history" + #"/archive/c2b/am5/2022.01/c96L65_am5f1a0r0_amip/gfdl.ncrc5-intel22-classic-prod-openmp/history" + pp_dir: "/archive/$USER/canopy/$(stem)/$(expname)/$(platform)-$(target)/pp" + #"/archive/$USER/canopy/am5/c96L65_amip/gfdl.ncrc5-deploy-prod-openmp/pp" ptmp_dir: "/xtmp/$USER/ptmp" - analysis: "/nbhome/$USER/canopy/analysis/am5_c96L65_amip" + analysis: "/nbhome/$USER/canopy/analysis/$(expname)" + #"/nbhome/$USER/canopy/analysis/am5_c96L65_amip" components: - type: "atmos_cmip" From 2cdb751c8d607e452b500343188a3c88c09edaf9 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 12 Jun 2024 09:28:18 -0400 Subject: [PATCH 02/27] #99 Fix some pylint errors --- fre/pp/configure_script_yaml.py | 231 +++++++++++++++++--------------- 1 file changed, 122 insertions(+), 109 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index e212fab6..e1b53e55 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -1,129 +1,142 @@ #!/usr/bin/env python +""" + Script creates rose-apps and rose-suite + files for the workflow from the pp yaml. +""" import os -import yaml -import click -from jsonschema import validate, ValidationError, SchemaError import json import shutil +from jsonschema import validate, ValidationError, SchemaError +import yaml +import click import metomi.rose.config ######VALIDATE##### package_dir = os.path.dirname(os.path.abspath(__file__)) schema_path = os.path.join(package_dir, 'schema.json') -def validateYaml(file): - # Load the json schema: .load() (vs .loads()) reads and parses the json in one - with open(schema_path) as s: - schema = json.load(s) +def validate_yaml(file): + """ + Using the schema.json file, the yaml format is validated. + """ + # Load the json schema: .load() (vs .loads()) reads and parses the json in one + with open(schema_path) as s: + schema = json.load(s) - # Validate yaml - # If the yaml is not valid, the schema validation will raise errors and exit - if validate(instance=file,schema=schema) == None: - print("YAML VALID") + # Validate yaml + # If the yaml is not valid, the schema validation will raise errors and exit + if validate(instance=file,schema=schema) is None: + print("YAML VALID") ################### @click.command() def yamlInfo(yamlfile,experiment,platform,target): - e = experiment - p = platform - t = target - yml = yamlfile - - # initialize rose suite config - rose_suite = metomi.rose.config.ConfigNode() - rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"') - rose_suite.set(keys=['template variables', 'CLEAN_WORK'], value='True') - rose_suite.set(keys=['template variables', 'PTMP_DIR'], value='"/xtmp/$USER/ptmp"') - rose_suite.set(keys=['template variables', 'DO_STATICS'], value='True') - rose_suite.set(keys=['template variables', 'DO_TIMEAVGS'], value='True') - rose_suite.set(keys=['template variables', 'DO_ATMOS_PLEVEL_MASKING'], value='True') - # disagreeable; these should be optional - rose_suite.set(keys=['template variables', 'DO_ANALYSIS_ONLY'], value='False') - rose_suite.set(keys=['template variables', 'DO_MDTF'], value='False') - rose_suite.set(keys=['template variables', 'PP_DEFAULT_XYINTERP'], value='0,0') - - # initialize rose regrid config - rose_regrid = metomi.rose.config.ConfigNode() - rose_regrid.set(keys=['command', 'default'], value='regrid-xy') - - # initialize rose remap config - rose_remap = metomi.rose.config.ConfigNode() - rose_remap.set(keys=['command', 'default'], value='remap-pp-components') - - # set some rose suite vars - rose_suite.set(keys=['template variables', 'EXPERIMENT'], value=f'"{e}"') - rose_suite.set(keys=['template variables', 'PLATFORM'], value=f'"{p}"') - rose_suite.set(keys=['template variables', 'TARGET'], value=f'"{t}"') - - with open(yml,'r') as f: - y=yaml.safe_load(f) - valyml = validateYaml(y) - - for key,value in y.items(): - if key == "rose-suite": - for suiteconfiginfo,dict in value.items(): - for configkey,configvalue in dict.items(): - if configvalue != None: - k=configkey.upper() - if k in ("HISTORY_DIR", "PP_DIR", "ANALYSIS"): - # replace generic variables in pp yaml - stem=e.split("_",1)[0] - exp=e.split("_",1)[1] - replace=configvalue.replace("$(stem)",stem).replace("$(expname)",exp).replace("$(platform)",p).replace("$(target)",t) - rose_suite.set(keys=['template variables', k], value=f'{replace}') - else: - rose_suite.set(keys=['template variables', k], value=f'"{configvalue}"') - - if key == "components": - for i in value: - comp = i.get('type') - sources = i.get('sources') - interp_method = i.get('interpMethod') - - # set remap items - rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - # if xyInterp doesnt exist, grid is native - if i.get("xyInterp") == None: - rose_remap.set(keys=[f'{comp}', 'grid'], value='native') - # if xyInterp exists, component can be regridded - else: - interp_split = i.get('xyInterp').split(',') - rose_remap.set(keys=[f'{comp}', 'grid'], value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') - - # set regrid items - if i.get("xyInterp") != None: - rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') - rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') - rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') - interp_split = i.get('xyInterp').split(',') - rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridType'], value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') - - # write output files - print("Writing output files...") - cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") - - outfile = os.path.join(cylc_dir, 'pp.yaml') - shutil.copyfile(yml, outfile) - print(" " + outfile) - - dumper = metomi.rose.config.ConfigDumper() - outfile = os.path.join(cylc_dir, "rose-suite.conf") - dumper(rose_suite, outfile) - print(" " + outfile) - - outfile = os.path.join(cylc_dir, "app", "regrid-xy", "rose-app.conf") - dumper(rose_regrid, outfile) - print(" " + outfile) - - outfile = os.path.join(cylc_dir, "app", "remap-pp-components", "rose-app.conf") - dumper(rose_remap, outfile) - print(" " + outfile) - + """ + Using a valid pp.yaml, the rose-app and rose-suite + configuration files are created in the cylc-src + directory. The pp.yaml is also copied to the + cylc-src directory. + """ + e = experiment + p = platform + t = target + yml = yamlfile + + # initialize rose suite config + rose_suite = metomi.rose.config.ConfigNode() + rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"') + rose_suite.set(keys=['template variables', 'CLEAN_WORK'], value='True') + rose_suite.set(keys=['template variables', 'PTMP_DIR'], value='"/xtmp/$USER/ptmp"') + rose_suite.set(keys=['template variables', 'DO_STATICS'], value='True') + rose_suite.set(keys=['template variables', 'DO_TIMEAVGS'], value='True') + rose_suite.set(keys=['template variables', 'DO_ATMOS_PLEVEL_MASKING'], value='True') + # disagreeable; these should be optional + rose_suite.set(keys=['template variables', 'DO_ANALYSIS_ONLY'], value='False') + rose_suite.set(keys=['template variables', 'DO_MDTF'], value='False') + rose_suite.set(keys=['template variables', 'PP_DEFAULT_XYINTERP'], value='0,0') + + # initialize rose regrid config + rose_regrid = metomi.rose.config.ConfigNode() + rose_regrid.set(keys=['command', 'default'], value='regrid-xy') + + # initialize rose remap config + rose_remap = metomi.rose.config.ConfigNode() + rose_remap.set(keys=['command', 'default'], value='remap-pp-components') + + # set some rose suite vars + rose_suite.set(keys=['template variables', 'EXPERIMENT'], value=f'"{e}"') + rose_suite.set(keys=['template variables', 'PLATFORM'], value=f'"{p}"') + rose_suite.set(keys=['template variables', 'TARGET'], value=f'"{t}"') + + with open(yml,"r") as f: + y=yaml.safe_load(f) + validate_yaml(y) + + for key,value in y.items(): + if key == "rose-suite": + for suiteconfiginfo,dicts in value.items(): + for configkey,configvalue in dicts.items(): + if configvalue is not None: + k=configkey.upper() + if k in ("HISTORY_DIR", "PP_DIR", "ANALYSIS"): + # replace generic variables in pp yaml + stem=e.split("_",1)[0] + exp=e.split("_",1)[1] + replacevars=configvalue.replace("$(stem)",stem).replace("$(expname)",exp).replace("$(platform)",p).replace("$(target)",t) + rose_suite.set(keys=['template variables', k], value=f'{replacevars}') + else: + rose_suite.set(keys=['template variables', k], value=f'"{configvalue}"') + + if key == "components": + for i in value: + comp = i.get('type') + sources = i.get('sources') + interp_method = i.get('interpMethod') + + # set remap items + rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + # if xyInterp doesnt exist, grid is native + if i.get("xyInterp") is None: + rose_remap.set(keys=[f'{comp}', 'grid'], value='native') + # if xyInterp exists, component can be regridded + else: + interp_split = i.get('xyInterp').split(',') + rose_remap.set(keys=[f'{comp}', 'grid'], value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') + + # set regrid items + if i.get("xyInterp") is not None: + rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') + rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') + rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') + interp_split = i.get('xyInterp').split(',') + rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridType'], value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') + + # write output files + print("Writing output files...") + cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") + + outfile = os.path.join(cylc_dir, 'pp.yaml') + shutil.copyfile(yml, outfile) + print(" " + outfile) + + dumper = metomi.rose.config.ConfigDumper() + outfile = os.path.join(cylc_dir, "rose-suite.conf") + dumper(rose_suite, outfile) + print(" " + outfile) + + outfile = os.path.join(cylc_dir, "app", "regrid-xy", "rose-app.conf") + dumper(rose_regrid, outfile) + print(" " + outfile) + + outfile = os.path.join(cylc_dir, "app", "remap-pp-components", "rose-app.conf") + dumper(rose_remap, outfile) + print(" " + outfile) + # Use parseyaml function to parse created edits.yaml if __name__ == '__main__': yamlInfo() From 6332f857d5f9952e57ba53a4b9ab1b1c82ca43f0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 17 Jun 2024 09:52:09 -0400 Subject: [PATCH 03/27] #99 Remove comments --- fre/pp/ppyaml_test/pp.yaml | 3 --- 1 file changed, 3 deletions(-) diff --git a/fre/pp/ppyaml_test/pp.yaml b/fre/pp/ppyaml_test/pp.yaml index 45439f82..4d0964ed 100644 --- a/fre/pp/ppyaml_test/pp.yaml +++ b/fre/pp/ppyaml_test/pp.yaml @@ -22,12 +22,9 @@ rose-suite: directories: history_dir: "/archive/c2b/$(stem)/2022.01/$(expname)/$(platform)-$(target)/history" - #"/archive/c2b/am5/2022.01/c96L65_am5f1a0r0_amip/gfdl.ncrc5-intel22-classic-prod-openmp/history" pp_dir: "/archive/$USER/canopy/$(stem)/$(expname)/$(platform)-$(target)/pp" - #"/archive/$USER/canopy/am5/c96L65_amip/gfdl.ncrc5-deploy-prod-openmp/pp" ptmp_dir: "/xtmp/$USER/ptmp" analysis: "/nbhome/$USER/canopy/analysis/$(expname)" - #"/nbhome/$USER/canopy/analysis/am5_c96L65_amip" components: - type: "atmos_cmip" From 60c15a66292bb9472ea4723a249c6a2b47b8f12c Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Tue, 18 Jun 2024 14:39:40 -0400 Subject: [PATCH 04/27] #99 Change `expname` to `name` and hardcode `stem` for now --- fre/pp/ppyaml_test/pp.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/fre/pp/ppyaml_test/pp.yaml b/fre/pp/ppyaml_test/pp.yaml index 4d0964ed..c675237f 100644 --- a/fre/pp/ppyaml_test/pp.yaml +++ b/fre/pp/ppyaml_test/pp.yaml @@ -21,10 +21,10 @@ rose-suite: do_analysis: False directories: - history_dir: "/archive/c2b/$(stem)/2022.01/$(expname)/$(platform)-$(target)/history" - pp_dir: "/archive/$USER/canopy/$(stem)/$(expname)/$(platform)-$(target)/pp" + history_dir: "/archive/c2b/am5/2022.01/$(name)/$(platform)-$(target)/history" + pp_dir: "/archive/$USER/canopy/am5/2022.01/$(name)/$(platform)-$(target)/pp" ptmp_dir: "/xtmp/$USER/ptmp" - analysis: "/nbhome/$USER/canopy/analysis/$(expname)" + analysis: "/nbhome/$USER/canopy/analysis/$(name)" components: - type: "atmos_cmip" From c58cb1df381a42d35acf1508af9912063fec6ca0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 5 Jul 2024 16:18:14 -0400 Subject: [PATCH 05/27] #99 Rename `pp.yaml` to `pp_c96L65.yaml` --- fre/pp/ppyaml_test/{pp.yaml => pp_c96L65.yaml} | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) rename fre/pp/ppyaml_test/{pp.yaml => pp_c96L65.yaml} (90%) diff --git a/fre/pp/ppyaml_test/pp.yaml b/fre/pp/ppyaml_test/pp_c96L65.yaml similarity index 90% rename from fre/pp/ppyaml_test/pp.yaml rename to fre/pp/ppyaml_test/pp_c96L65.yaml index c675237f..68be2e9d 100644 --- a/fre/pp/ppyaml_test/pp.yaml +++ b/fre/pp/ppyaml_test/pp_c96L65.yaml @@ -21,10 +21,10 @@ rose-suite: do_analysis: False directories: - history_dir: "/archive/c2b/am5/2022.01/$(name)/$(platform)-$(target)/history" - pp_dir: "/archive/$USER/canopy/am5/2022.01/$(name)/$(platform)-$(target)/pp" + history_dir: "/archive/c2b/$(stem)/$(name)/$(platform)-$(target)/history" + pp_dir: "/archive/$USER/canopy/$(stem)/$(name)/$(platform)-$(target)/pp" ptmp_dir: "/xtmp/$USER/ptmp" - analysis: "/nbhome/$USER/canopy/analysis/$(name)" + analysis: "/nbhome/$USER/canopy/analysis/$(name)" components: - type: "atmos_cmip" From 35429ae892cb82c5722b3402e3edc783ccee9cc7 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 5 Jul 2024 16:19:33 -0400 Subject: [PATCH 06/27] #99 Add yaml to hold other experiment yamls - can hold experiment yamls - hold FRE properties defined --- fre/pp/ppyaml_test/am5-yamls.yaml | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 fre/pp/ppyaml_test/am5-yamls.yaml diff --git a/fre/pp/ppyaml_test/am5-yamls.yaml b/fre/pp/ppyaml_test/am5-yamls.yaml new file mode 100644 index 00000000..5b0b6d40 --- /dev/null +++ b/fre/pp/ppyaml_test/am5-yamls.yaml @@ -0,0 +1,12 @@ +## FRE properties +projectID: "am5" +FRE_VERSION: &fre_version "canopy" +AM5_VERSION: &am5_version "2022.01" #"am5f7b12r1" +stem: "am5/2022.01" # stem=projectID/version + +# Experiments +exp: + - expname: "c96L65_am5f1a0r0_amip" + platform: "gfdl.ncrc5-intel22-classic" + target: "prod-openmp" + ppyaml: "pp_c96L65.yaml" From a1e6eb3f2f66fe13fb75cb8f8b75b51b54af4593 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 5 Jul 2024 16:45:07 -0400 Subject: [PATCH 07/27] #99 Load and parse main yaml - parse main yaml to use certain experiment yaml - parse main yaml for FRE properties defined --- fre/pp/configure_script_yaml.py | 37 ++++++++++++++++++++++++++------- 1 file changed, 30 insertions(+), 7 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index cefd06b2..e474c842 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -12,6 +12,30 @@ import click import metomi.rose.config +def yamlloads(yaml, experiment): + """ + Load the yaml that holds experiment yamls. + EXtract FRE properties defined in the yaml. + """ + with open(yaml,"r") as f: + y=yaml.safe_load(f) + + # Define properties (FRE properties) + stem=y.get("stem") + + for propkey,propvalue in y.items(): + if propkey == "exp": + for i in propvalue: + # if the experiment listed in the click options matches + # an experiment name listed in the main yaml, return + # that experiment yaml + if i.get("expname") == experiment: + # Load this specific pp yaml for rose suite and whatnot + expyaml=i.get("ppyaml") + + # return experiment yaml and defined FRE properties + return(expyaml,stem) + ######VALIDATE##### package_dir = os.path.dirname(os.path.abspath(__file__)) schema_path = os.path.join(package_dir, 'schema.json') @@ -30,7 +54,6 @@ def validate_yaml(file): print("YAML VALID") ################### - def _yamlInfo(yamlfile,experiment,platform,target): """ Using a valid pp.yaml, the rose-app and rose-suite @@ -43,6 +66,8 @@ def _yamlInfo(yamlfile,experiment,platform,target): t = target yml = yamlfile + exp,stem=yamlloads(yaml=yml,experiment=e) + # initialize rose suite config rose_suite = metomi.rose.config.ConfigNode() rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"') @@ -69,7 +94,7 @@ def _yamlInfo(yamlfile,experiment,platform,target): rose_suite.set(keys=['template variables', 'PLATFORM'], value=f'"{p}"') rose_suite.set(keys=['template variables', 'TARGET'], value=f'"{t}"') - with open(yml,"r") as f: + with open(exp,"r") as f: y=yaml.safe_load(f) validate_yaml(y) @@ -79,11 +104,10 @@ def _yamlInfo(yamlfile,experiment,platform,target): for configkey,configvalue in dicts.items(): if configvalue is not None: k=configkey.upper() + #if configvalue == True or configvalue == False: if k in ("HISTORY_DIR", "PP_DIR", "ANALYSIS"): # replace generic variables in pp yaml - stem=e.split("_",1)[0] - exp=e.split("_",1)[1] - replacevars=configvalue.replace("$(stem)",stem).replace("$(expname)",exp).replace("$(platform)",p).replace("$(target)",t) + replacevars=configvalue.replace("$(stem)",stem).replace("$(name)",e).replace("$(platform)",p).replace("$(target)",t) rose_suite.set(keys=['template variables', k], value=f'{replacevars}') else: rose_suite.set(keys=['template variables', k], value=f'"{configvalue}"') @@ -118,8 +142,7 @@ def _yamlInfo(yamlfile,experiment,platform,target): # write output files print("Writing output files...") cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") - - outfile = os.path.join(cylc_dir, 'pp.yaml') + outfile = os.path.join(cylc_dir, f"{e}.yaml") shutil.copyfile(yml, outfile) print(" " + outfile) From 7f01868637def9fbee66a76ee4b508f6f86bb052 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 8 Jul 2024 16:16:16 -0400 Subject: [PATCH 08/27] #99 Move `directories` section from experiment yaml to main yaml - directories defined are not experiment specific (should not be hard coded) - they are the same paths for each experiment - to limit duplication, they can be put in the main yaml instead --- fre/pp/ppyaml_test/am5-yamls.yaml | 18 +++++++++++------- fre/pp/ppyaml_test/pp_c96L65.yaml | 6 ------ 2 files changed, 11 insertions(+), 13 deletions(-) diff --git a/fre/pp/ppyaml_test/am5-yamls.yaml b/fre/pp/ppyaml_test/am5-yamls.yaml index 5b0b6d40..cd1cc6c0 100644 --- a/fre/pp/ppyaml_test/am5-yamls.yaml +++ b/fre/pp/ppyaml_test/am5-yamls.yaml @@ -1,12 +1,16 @@ ## FRE properties -projectID: "am5" -FRE_VERSION: &fre_version "canopy" -AM5_VERSION: &am5_version "2022.01" #"am5f7b12r1" -stem: "am5/2022.01" # stem=projectID/version +properties: + projectID: "am5" + FRE_VERSION: "canopy" + AM5_VERSION: "2022.01" + stem: "am5/2022.01" + +directories: + history_dir: "/archive/$USER/$(stem)/$(expname)/$(platform)-$(target)/history" + pp_dir: "/archive/$USER/canopy/$(stem)/$(expname)/$(platform)-$(target)/pp" + ptmp_dir: "/xtmp/$USER/ptmp" + analysis: "/nbhome/$USER/canopy/analysis/$(expname)" -# Experiments exp: - expname: "c96L65_am5f1a0r0_amip" - platform: "gfdl.ncrc5-intel22-classic" - target: "prod-openmp" ppyaml: "pp_c96L65.yaml" diff --git a/fre/pp/ppyaml_test/pp_c96L65.yaml b/fre/pp/ppyaml_test/pp_c96L65.yaml index 68be2e9d..b4f1905d 100644 --- a/fre/pp/ppyaml_test/pp_c96L65.yaml +++ b/fre/pp/ppyaml_test/pp_c96L65.yaml @@ -20,12 +20,6 @@ rose-suite: do_preanalysis: False do_analysis: False - directories: - history_dir: "/archive/c2b/$(stem)/$(name)/$(platform)-$(target)/history" - pp_dir: "/archive/$USER/canopy/$(stem)/$(name)/$(platform)-$(target)/pp" - ptmp_dir: "/xtmp/$USER/ptmp" - analysis: "/nbhome/$USER/canopy/analysis/$(name)" - components: - type: "atmos_cmip" sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip" From 690f0a77bdd275a6366d123bf5f08ee6280b1335 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 11 Jul 2024 10:01:14 -0400 Subject: [PATCH 09/27] #99. Rename `am5-yamls.yaml` to `am5.yaml` --- fre/pp/ppyaml_test/{am5-yamls.yaml => am5.yaml} | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) rename fre/pp/ppyaml_test/{am5-yamls.yaml => am5.yaml} (51%) diff --git a/fre/pp/ppyaml_test/am5-yamls.yaml b/fre/pp/ppyaml_test/am5.yaml similarity index 51% rename from fre/pp/ppyaml_test/am5-yamls.yaml rename to fre/pp/ppyaml_test/am5.yaml index cd1cc6c0..a45fc760 100644 --- a/fre/pp/ppyaml_test/am5-yamls.yaml +++ b/fre/pp/ppyaml_test/am5.yaml @@ -4,12 +4,13 @@ properties: FRE_VERSION: "canopy" AM5_VERSION: "2022.01" stem: "am5/2022.01" + xyInterp: "180,280" directories: - history_dir: "/archive/$USER/$(stem)/$(expname)/$(platform)-$(target)/history" - pp_dir: "/archive/$USER/canopy/$(stem)/$(expname)/$(platform)-$(target)/pp" + history_dir: "/archive/$USER/$(stem)/$(name)/$(platform)-$(target)/history" + pp_dir: "/archive/$USER/canopy/$(stem)/$(name)/$(platform)-$(target)/pp" ptmp_dir: "/xtmp/$USER/ptmp" - analysis: "/nbhome/$USER/canopy/analysis/$(expname)" + analysis: "/nbhome/$USER/canopy/analysis/$(name)" exp: - expname: "c96L65_am5f1a0r0_amip" From f517565a31be6fd4d300c8bc99d2dbb867310ab0 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 11 Jul 2024 10:05:38 -0400 Subject: [PATCH 10/27] #99 Rename am5.yaml file --- fre/pp/ppyaml_test/{am5.yaml => am5-pp.yaml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename fre/pp/ppyaml_test/{am5.yaml => am5-pp.yaml} (100%) diff --git a/fre/pp/ppyaml_test/am5.yaml b/fre/pp/ppyaml_test/am5-pp.yaml similarity index 100% rename from fre/pp/ppyaml_test/am5.yaml rename to fre/pp/ppyaml_test/am5-pp.yaml From 4bf41e010a027cfb35773e9bb0319e0be3786745 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Jul 2024 10:25:08 -0400 Subject: [PATCH 11/27] #99 Update test/example yamls - now 2: main yaml and experiment yaml (in yaml_include) --- fre/pp/ppyaml_test/am5-pp.yaml | 17 ---- fre/pp/ppyaml_test/am5.yaml | 91 +++++++++++++++++++ fre/pp/ppyaml_test/pp_c96L65.yaml | 81 ----------------- .../ppyaml_test/yaml_include/pp.c96_amip.yaml | 81 +++++++++++++++++ 4 files changed, 172 insertions(+), 98 deletions(-) delete mode 100644 fre/pp/ppyaml_test/am5-pp.yaml create mode 100644 fre/pp/ppyaml_test/am5.yaml delete mode 100644 fre/pp/ppyaml_test/pp_c96L65.yaml create mode 100644 fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml diff --git a/fre/pp/ppyaml_test/am5-pp.yaml b/fre/pp/ppyaml_test/am5-pp.yaml deleted file mode 100644 index a45fc760..00000000 --- a/fre/pp/ppyaml_test/am5-pp.yaml +++ /dev/null @@ -1,17 +0,0 @@ -## FRE properties -properties: - projectID: "am5" - FRE_VERSION: "canopy" - AM5_VERSION: "2022.01" - stem: "am5/2022.01" - xyInterp: "180,280" - -directories: - history_dir: "/archive/$USER/$(stem)/$(name)/$(platform)-$(target)/history" - pp_dir: "/archive/$USER/canopy/$(stem)/$(name)/$(platform)-$(target)/pp" - ptmp_dir: "/xtmp/$USER/ptmp" - analysis: "/nbhome/$USER/canopy/analysis/$(name)" - -exp: - - expname: "c96L65_am5f1a0r0_amip" - ppyaml: "pp_c96L65.yaml" diff --git a/fre/pp/ppyaml_test/am5.yaml b/fre/pp/ppyaml_test/am5.yaml new file mode 100644 index 00000000..1f33c1ed --- /dev/null +++ b/fre/pp/ppyaml_test/am5.yaml @@ -0,0 +1,91 @@ +# reusable variables +define: &AM5_VERSION "am5f7b12r1" +define: &FRE_STEM !join [am5/, *AM5_VERSION] + +# amip +define: &EXP_AMIP_START "1979" +define: &EXP_AMIP_END "2020" +define: &ANA_AMIP_START "1980" +define: &ANA_AMIP_END "2020" + +define: &PP_AMIP_CHUNK96 "1yr" +define: &PP_AMIP_CHUNK384 "1yr" +define: &PP_XYINTERP96 "180,288" +define: &PP_XYINTERP384 "720,1152" + +# climo +define: &EXP_CLIMO_START96 "0001" +define: &EXP_CLIMO_END96 "0011" +define: &ANA_CLIMO_START96 "0002" +define: &ANA_CLIMO_END96 "0011" + +define: &EXP_CLIMO_START384 "0001" +define: &EXP_CLIMO_END384 "0006" +define: &ANA_CLIMO_START384 "0002" +define: &ANA_CLIMO_END384 "0006" + +# coupled +define: &PP_CPLD_CHUNK_A "5yr" +define: &PP_CPLD_CHUNK_B "20yr" + +# grids +define: &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + +# directories shared across tools +shared_directories: + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" + +# shared pp settings +shared_postprocess: + settings: + history_segment: "P1Y" + site: "ppan" + switches: + do_statics: True + do_timeavgs: True + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + +experiments: + - name: "c96L65_am5f7b12r1_amip" + pp: + - "yaml_include/pp.c96_amip.yaml" + - name: "c96L65_am5f7b12r1_pdclim1850F" + pp: + - "yaml_include/pp.c96_clim.yaml" + - name: "c96L65_am5f7b12r1_pdclim2010F" + pp: + - "yaml_include/pp.c96_clim.yaml" + - name: "c96L65_am5f7b12r1_pdclim2010AERF" + pp: + - "yaml_include/pp.c96_clim.yaml" + - name: "c384L65_am5f7b12r1_amip" + pp: + - "yaml_include/pp.c384_amip.yaml" + - name: "c384L65_am5f7b12r1_pdclim2010F" + pp: + - "yaml_include/pp.c384_clim.yaml" + - name: "c384L65_am5f7b12r1_pdclim1850F" + pp: + - "yaml_include/pp.c384_clim.yaml" + - name: "c384L65_am5f7b12r1_pdclim2010AERF" + pp: + - "yaml_include/pp.c384_clim.yaml" + - name: "c384L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg" + pp: + - "yaml_include/pp.c384_amip.yaml" + - "yaml_include/pp.om4.yaml" + - name: "c96L65_am5f7b12r1_OM4_p25_piControl_noBLING_DynVeg" + pp: + - "yaml_include/pp.c96_amip.yaml" + - "yaml_include/pp.om4.yaml" + - name: "c96L65_am5f7b12r1_amip_cosp" + pp: + - "yaml_include/pp.c96_amip.yaml" diff --git a/fre/pp/ppyaml_test/pp_c96L65.yaml b/fre/pp/ppyaml_test/pp_c96L65.yaml deleted file mode 100644 index b4f1905d..00000000 --- a/fre/pp/ppyaml_test/pp_c96L65.yaml +++ /dev/null @@ -1,81 +0,0 @@ -# "FRE properties" :) -define1: &xyInterp "180,288" - -rose-suite: - settings: - history_segment: "P1Y" - pp_start: "1980" - pp_stop: "1988" - pp_chunk_a: "P1Y" - pp_chunk_b: "P5Y" - pp_components: "atmos atmos_scalar land" - pp_grid_spec: "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" - - switches: - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: False - -components: - - type: "atmos_cmip" - sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos" - sources: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_level" - sources: "atmos_level_cmip" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_month_aer" - sources: "atmos_month_aer" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order1" - inputRealm: 'atmos' - - type: "atmos_diurnal" - sources: "atmos_diurnal" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_scalar" - sources: "atmos_scalar" - - type: "aerosol_cmip" - xyInterp: *xyInterp - sources: "aerosol_month_cmip" - sourceGrid: "cubedsphere" - interpMethod: "conserve_order1" - inputRealm: 'atmos' - - type: "land" - sources: "land_month" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order1" - inputRealm: 'land' - - type: "land_cmip" - sources: "land_month_cmip" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order1" - inputRealm: 'land' - -tmpdir: - tmpdirpath: - -install-exp-script: - - path: - - install-option: - install: diff --git a/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml b/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml new file mode 100644 index 00000000..4e5ad015 --- /dev/null +++ b/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml @@ -0,0 +1,81 @@ +# local reusable variable overrides +define: &custom_interp "180,360" + +# directory overrides +directories: + ptmp_dir: "/ptmp/$USER" + +postprocess: + # pp setting overrides + settings: + pp_start: *ANA_AMIP_START + pp_stop: *ANA_AMIP_END + pp_chunk_a: *PP_AMIP_CHUNK96 + pp_components: "atmos atmos_scalar" + switches: + do_statics: False + + # main pp instructions + components: + - type: "atmos_cmip" + sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *custom_interp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos" + sources: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos_level_cmip" + sources: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos_level" + sources: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos_month_aer" + sources: "atmos_month_aer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' + - type: "atmos_diurnal" + sources: "atmos_diurnal" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos_scalar" + sources: "atmos_scalar" + - type: "aerosol_cmip" + xyInterp: *PP_XYINTERP96 + sources: "aerosol_month_cmip" + sourceGrid: "cubedsphere" + interpMethod: "conserve_order1" + inputRealm: 'atmos' + - type: "land" + sources: "land_month" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + - type: "land_cmip" + sources: "land_month_cmip" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'land' + - type: "tracer_level" + sources: "atmos_tracer" + sourceGrid: "cubedsphere" + xyInterp: *PP_XYINTERP96 + interpMethod: "conserve_order1" + inputRealm: 'atmos' From 26572e63fa878fdbc21f296d3fb53675d7d82b0e Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Jul 2024 10:42:28 -0400 Subject: [PATCH 12/27] #99 Update script - add functions to combine main and experiment yamls - parse combined yaml correctly - to-do: - update validation --- fre/pp/configure_script_yaml.py | 228 ++++++++++++++++++++------------ 1 file changed, 147 insertions(+), 81 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index e474c842..8b99d25e 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -9,33 +9,9 @@ import shutil from jsonschema import validate, ValidationError, SchemaError import yaml -import click +from pathlib import Path import metomi.rose.config -def yamlloads(yaml, experiment): - """ - Load the yaml that holds experiment yamls. - EXtract FRE properties defined in the yaml. - """ - with open(yaml,"r") as f: - y=yaml.safe_load(f) - - # Define properties (FRE properties) - stem=y.get("stem") - - for propkey,propvalue in y.items(): - if propkey == "exp": - for i in propvalue: - # if the experiment listed in the click options matches - # an experiment name listed in the main yaml, return - # that experiment yaml - if i.get("expname") == experiment: - # Load this specific pp yaml for rose suite and whatnot - expyaml=i.get("ppyaml") - - # return experiment yaml and defined FRE properties - return(expyaml,stem) - ######VALIDATE##### package_dir = os.path.dirname(os.path.abspath(__file__)) schema_path = os.path.join(package_dir, 'schema.json') @@ -53,7 +29,87 @@ def validate_yaml(file): if validate(instance=file,schema=schema) is None: print("YAML VALID") -################### +#################### +def join_constructor(loader, node): + """ + Allows FRE properties defined + in main yaml to be concatenated. + """ + seq = loader.construct_sequence(node) + return ''.join([str(i) for i in seq]) + +#################### +def yamlload(yamlfile): + """ + Load the given yaml and validate. + """ + # Regsiter tag handler + yaml.add_constructor('!join', join_constructor) + + # Load the main yaml and validate + with open(yamlfile,'r') as f: + y=yaml.load(f,Loader=yaml.Loader) + #validate_yaml(y) + +## TO-DO: +# - figure out way to safe_load (yaml_loader=yaml.SafeLoader?) +# - update validation + return y + +#################### +def consolidate_yamls(yamlfile,experiment, platform,target): + """ + Combine main yaml and experiment yaml into combined yamls + """ + # Path to new combined + combined=Path("combined.yaml") + # Create and write to combined yaml + with open(combined,"w") as f1: + f1.write('## Combined yamls\n') + f1.write(f'define: &name "{experiment}"\n') + f1.write(f'define: &platform "{platform}"\n') + f1.write(f'define: &target "{target}"\n\n') + + # Combine main yaml with + with open(yamlfile,'r') as f2: + f1.write("\n### MAIN YAML SETTINGS ###\n") + shutil.copyfileobj(f2,f1) #copy contents of main yaml into combined yaml + + # Load the new combined yaml and validate + cy=yamlload(combined) + + # Extract the experiment yaml + exp_list=[] + for i in cy.get("experiments"): + exp_list.append(i.get("name")) + + # Check is exp name is valid + if experiment not in exp_list: + raise Exception(f"{experiment} is not in the list of experiments") + + for i in cy.get("experiments"): + # Extract the experiment yaml + if experiment == i.get("name"): + expyaml=i.get("pp") + #else: + # simplify combine yaml?: remove name,ppyamls if it doesnt match experiment given + #remove i.get("name") and i.get("pp") associated with it + + if expyaml is not None: + #combine experiment yaml with combined yaml + with open(combined,"a") as f1: + for i in expyaml: + with open(i,'r') as f2: #copy expyaml into combined + f1.write(f"\n### {i.upper()} settings ###\n") + shutil.copyfileobj(f2,f1) + + return combined + +#################### +#def clean(): +# """ +# """ +#################### def _yamlInfo(yamlfile,experiment,platform,target): """ Using a valid pp.yaml, the rose-app and rose-suite @@ -66,16 +122,14 @@ def _yamlInfo(yamlfile,experiment,platform,target): t = target yml = yamlfile - exp,stem=yamlloads(yaml=yml,experiment=e) - # initialize rose suite config rose_suite = metomi.rose.config.ConfigNode() - rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"') - rose_suite.set(keys=['template variables', 'CLEAN_WORK'], value='True') - rose_suite.set(keys=['template variables', 'PTMP_DIR'], value='"/xtmp/$USER/ptmp"') - rose_suite.set(keys=['template variables', 'DO_STATICS'], value='True') - rose_suite.set(keys=['template variables', 'DO_TIMEAVGS'], value='True') - rose_suite.set(keys=['template variables', 'DO_ATMOS_PLEVEL_MASKING'], value='True') + #rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"') + #rose_suite.set(keys=['template variables', 'CLEAN_WORK'], value='True') + #rose_suite.set(keys=['template variables', 'PTMP_DIR'], value='"/xtmp/$USER/ptmp"') + #rose_suite.set(keys=['template variables', 'DO_STATICS'], value='True') + #rose_suite.set(keys=['template variables', 'DO_TIMEAVGS'], value='True') + #rose_suite.set(keys=['template variables', 'DO_ATMOS_PLEVEL_MASKING'], value='True') # disagreeable; these should be optional rose_suite.set(keys=['template variables', 'DO_ANALYSIS_ONLY'], value='False') rose_suite.set(keys=['template variables', 'DO_MDTF'], value='False') @@ -94,56 +148,68 @@ def _yamlInfo(yamlfile,experiment,platform,target): rose_suite.set(keys=['template variables', 'PLATFORM'], value=f'"{p}"') rose_suite.set(keys=['template variables', 'TARGET'], value=f'"{t}"') - with open(exp,"r") as f: - y=yaml.safe_load(f) - validate_yaml(y) - - for key,value in y.items(): - if key == "rose-suite": - for suiteconfiginfo,dicts in value.items(): - for configkey,configvalue in dicts.items(): - if configvalue is not None: - k=configkey.upper() - #if configvalue == True or configvalue == False: - if k in ("HISTORY_DIR", "PP_DIR", "ANALYSIS"): - # replace generic variables in pp yaml - replacevars=configvalue.replace("$(stem)",stem).replace("$(name)",e).replace("$(platform)",p).replace("$(target)",t) - rose_suite.set(keys=['template variables', k], value=f'{replacevars}') - else: - rose_suite.set(keys=['template variables', k], value=f'"{configvalue}"') - - if key == "components": - for i in value: - comp = i.get('type') - sources = i.get('sources') - interp_method = i.get('interpMethod') - - # set remap items - rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - # if xyInterp doesnt exist, grid is native - if i.get("xyInterp") is None: - rose_remap.set(keys=[f'{comp}', 'grid'], value='native') - # if xyInterp exists, component can be regridded - else: - interp_split = i.get('xyInterp').split(',') - rose_remap.set(keys=[f'{comp}', 'grid'], value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') - - # set regrid items - if i.get("xyInterp") is not None: - rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') - rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') - rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') - interp_split = i.get('xyInterp').split(',') - rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridType'], value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') + # Combine main and exp yamls into one new combine.yaml + comb_yaml = consolidate_yamls(yamlfile=yml, + experiment=e, + platform=p, + target=t) + + # Load the combined yaml + YAML = yamlload(comb_yaml) + + ## PARSE COMBINED YAML TO CREATE CONFIGS + # Remove unwanted keys + del YAML["experiments"] + del YAML["define"] + + for i in YAML.keys(): + #print(i) + if YAML.get(i) is not None: + for configkey,configvalue in YAML.get(i).items(): + if isinstance(configvalue,dict): ##for settings and switches + for i,j in configvalue.items(): + rose_suite.set(keys=['template variables', i.upper()], value=f'{j}') + elif isinstance(configvalue,str): ##for directories + rose_suite.set(keys=['template variables', configkey.upper()], + value=f'{configvalue}') + + components = YAML.get("postprocess").get("components") + for i in components: + comp = i.get('type') + sources = i.get('sources') + interp_method = i.get('interpMethod') + + # set remap items + rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + # if xyInterp doesnt exist, grid is native + if i.get("xyInterp") is None: + rose_remap.set(keys=[f'{comp}', 'grid'], value='native') + + # if xyInterp exists, component can be regridded + else: + interp_split = i.get('xyInterp').split(',') + rose_remap.set(keys=[f'{comp}', 'grid'], + value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') + + # set regrid items + if i.get("xyInterp") is not None: + print(i.values()) + rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') + rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') + rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') + interp_split = i.get('xyInterp').split(',') + rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridType'], + value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') # write output files print("Writing output files...") - cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") + #cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") + cylc_dir=os.path.join(os.path.expanduser("./")) outfile = os.path.join(cylc_dir, f"{e}.yaml") - shutil.copyfile(yml, outfile) + shutil.copyfile(YAML, outfile) print(" " + outfile) dumper = metomi.rose.config.ConfigDumper() @@ -165,7 +231,7 @@ def yamlInfo(yamlfile,experiment,platform,target): of the function to be separate from the undecorated version ''' return _yamlInfo(yamlfile,experiment,platform,target) - + # Use parseyaml function to parse created edits.yaml if __name__ == '__main__': yamlInfo() From 56059ffbc1fab43ef5590a22aa611f4b09e9be30 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Jul 2024 11:56:28 -0400 Subject: [PATCH 13/27] #99 Fix `cylc_dir` path - defined for my own development/troubleshooting --- fre/pp/configure_script_yaml.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 8b99d25e..bd85b242 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -7,9 +7,9 @@ import os import json import shutil -from jsonschema import validate, ValidationError, SchemaError -import yaml from pathlib import Path +from jsonschema import validate +import yaml import metomi.rose.config ######VALIDATE##### @@ -206,8 +206,7 @@ def _yamlInfo(yamlfile,experiment,platform,target): # write output files print("Writing output files...") - #cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") - cylc_dir=os.path.join(os.path.expanduser("./")) + cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") outfile = os.path.join(cylc_dir, f"{e}.yaml") shutil.copyfile(YAML, outfile) print(" " + outfile) From 7d8608c856f88c8abc32ba72510d17cf6ac0b8c6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Jul 2024 12:05:15 -0400 Subject: [PATCH 14/27] #99 Update comments --- fre/pp/configure_script_yaml.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index bd85b242..8922ad47 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -53,7 +53,7 @@ def yamlload(yamlfile): ## TO-DO: # - figure out way to safe_load (yaml_loader=yaml.SafeLoader?) -# - update validation +# - update validation and schema return y #################### @@ -83,7 +83,7 @@ def consolidate_yamls(yamlfile,experiment, platform,target): for i in cy.get("experiments"): exp_list.append(i.get("name")) - # Check is exp name is valid + # Check is exp name is valid experiment listed in main yaml if experiment not in exp_list: raise Exception(f"{experiment} is not in the list of experiments") @@ -103,6 +103,9 @@ def consolidate_yamls(yamlfile,experiment, platform,target): f1.write(f"\n### {i.upper()} settings ###\n") shutil.copyfileobj(f2,f1) +## TO-DO: +# - condition where there are multiple pp yamls + return combined #################### From be1f6ac4d82872a81ebc920657fa33d9ff37db02 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Thu, 18 Jul 2024 17:09:20 -0400 Subject: [PATCH 15/27] #99 Add click back in and create `rose-init` function - in process of fixing pylint messages --- fre/pp/configure_script_yaml.py | 58 +++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 24 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index d53ebf0e..b359c8f0 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -8,6 +8,7 @@ import json import shutil from pathlib import Path +import click from jsonschema import validate import yaml import metomi.rose.config @@ -103,9 +104,8 @@ def consolidate_yamls(yamlfile,experiment, platform,target): f1.write(f"\n### {i.upper()} settings ###\n") shutil.copyfileobj(f2,f1) -## TO-DO: +## TO-DO: # - condition where there are multiple pp yamls - return combined #################### @@ -113,18 +113,10 @@ def consolidate_yamls(yamlfile,experiment, platform,target): # """ # """ #################### -def _yamlInfo(yamlfile,experiment,platform,target): +def rose_init(experiment,platform,target): """ - Using a valid pp.yaml, the rose-app and rose-suite - configuration files are created in the cylc-src - directory. The pp.yaml is also copied to the - cylc-src directory. + Initialize the rose suite and app configurations. """ - e = experiment - p = platform - t = target - yml = yamlfile - # initialize rose suite config rose_suite = metomi.rose.config.ConfigNode() #rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"') @@ -138,6 +130,11 @@ def _yamlInfo(yamlfile,experiment,platform,target): rose_suite.set(keys=['template variables', 'DO_MDTF'], value='False') rose_suite.set(keys=['template variables', 'PP_DEFAULT_XYINTERP'], value='0,0') + # set some rose suite vars + rose_suite.set(keys=['template variables', 'EXPERIMENT'], value=f'"{experiment}"') + rose_suite.set(keys=['template variables', 'PLATFORM'], value=f'"{platform}"') + rose_suite.set(keys=['template variables', 'TARGET'], value=f'"{target}"') + # initialize rose regrid config rose_regrid = metomi.rose.config.ConfigNode() rose_regrid.set(keys=['command', 'default'], value='regrid-xy') @@ -146,10 +143,23 @@ def _yamlInfo(yamlfile,experiment,platform,target): rose_remap = metomi.rose.config.ConfigNode() rose_remap.set(keys=['command', 'default'], value='remap-pp-components') - # set some rose suite vars - rose_suite.set(keys=['template variables', 'EXPERIMENT'], value=f'"{e}"') - rose_suite.set(keys=['template variables', 'PLATFORM'], value=f'"{p}"') - rose_suite.set(keys=['template variables', 'TARGET'], value=f'"{t}"') + return(rose_suite,rose_regrid,rose_remap) + +#################### +def _yamlInfo(yamlfile,experiment,platform,target): + """ + Using a valid pp.yaml, the rose-app and rose-suite + configuration files are created in the cylc-src + directory. The pp.yaml is also copied to the + cylc-src directory. + """ + e = experiment + p = platform + t = target + yml = yamlfile + + # Initialize the rose configurations + rose_suite,rose_regrid,rose_remap=rose_init(e,p,t) # Combine main and exp yamls into one new combine.yaml comb_yaml = consolidate_yamls(yamlfile=yml, @@ -158,17 +168,17 @@ def _yamlInfo(yamlfile,experiment,platform,target): target=t) # Load the combined yaml - YAML = yamlload(comb_yaml) + final_yaml = yamlload(comb_yaml) ## PARSE COMBINED YAML TO CREATE CONFIGS # Remove unwanted keys - del YAML["experiments"] - del YAML["define"] + del final_yaml["experiments"] + del final_yaml["define"] - for i in YAML.keys(): + for i in final_yaml.keys(): #print(i) - if YAML.get(i) is not None: - for configkey,configvalue in YAML.get(i).items(): + if final_yaml.get(i) is not None: + for configkey,configvalue in final_yaml.get(i).items(): if isinstance(configvalue,dict): ##for settings and switches for i,j in configvalue.items(): rose_suite.set(keys=['template variables', i.upper()], value=f'{j}') @@ -176,7 +186,7 @@ def _yamlInfo(yamlfile,experiment,platform,target): rose_suite.set(keys=['template variables', configkey.upper()], value=f'{configvalue}') - components = YAML.get("postprocess").get("components") + components = final_yaml.get("postprocess").get("components") for i in components: comp = i.get('type') sources = i.get('sources') @@ -211,7 +221,7 @@ def _yamlInfo(yamlfile,experiment,platform,target): print("Writing output files...") cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") outfile = os.path.join(cylc_dir, f"{e}.yaml") - shutil.copyfile(YAML, outfile) + shutil.copyfile(final_yaml, outfile) print(" " + outfile) dumper = metomi.rose.config.ConfigDumper() From 11e14fa46eab9016e00a9fb13cb193d7ddc519be Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 22 Jul 2024 12:43:19 -0400 Subject: [PATCH 16/27] #99 Change `postprocess` and `directories` name --- fre/pp/ppyaml_test/am5.yaml | 4 ++-- fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/fre/pp/ppyaml_test/am5.yaml b/fre/pp/ppyaml_test/am5.yaml index 1f33c1ed..baeec9f4 100644 --- a/fre/pp/ppyaml_test/am5.yaml +++ b/fre/pp/ppyaml_test/am5.yaml @@ -32,7 +32,7 @@ define: &PP_CPLD_CHUNK_B "20yr" define: &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" # directories shared across tools -shared_directories: +directories: history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] @@ -40,7 +40,7 @@ shared_directories: fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" # shared pp settings -shared_postprocess: +postprocess: settings: history_segment: "P1Y" site: "ppan" diff --git a/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml b/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml index 4e5ad015..01317f0f 100644 --- a/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml +++ b/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml @@ -2,10 +2,10 @@ define: &custom_interp "180,360" # directory overrides -directories: +c96_amip_directories: ptmp_dir: "/ptmp/$USER" -postprocess: +c96_amip_postprocess: # pp setting overrides settings: pp_start: *ANA_AMIP_START From fd6e3c912cfbc2c76e5976441be8b7ab5f3ab0e5 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 23 Jul 2024 09:58:12 -0400 Subject: [PATCH 17/27] #99 Split up `yamlInfo` function and add in combining multiple exp yamls into one - split up `yamlInfo` functionality into functions to create rose-suite and rose-apps separately from main function - start to add in ability to combine multiple experiment yamls in combined yaml (if more than one listed in main yaml) --- fre/pp/configure_script_yaml.py | 179 +++++++++++++++++++------------- 1 file changed, 104 insertions(+), 75 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index b359c8f0..4eba0804 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -4,6 +4,10 @@ files for the workflow from the pp yaml. """ +## TO-DO: +# - figure out way to safe_load (yaml_loader=yaml.SafeLoader?) +# - update validation and schema + import os import json import shutil @@ -16,7 +20,6 @@ ######VALIDATE##### package_dir = os.path.dirname(os.path.abspath(__file__)) schema_path = os.path.join(package_dir, 'schema.json') - def validate_yaml(file): """ Using the schema.json file, the yaml format is validated. @@ -40,7 +43,7 @@ def join_constructor(loader, node): return ''.join([str(i) for i in seq]) #################### -def yamlload(yamlfile): +def yaml_load(yamlfile): """ Load the given yaml and validate. """ @@ -52,13 +55,10 @@ def yamlload(yamlfile): y=yaml.load(f,Loader=yaml.Loader) #validate_yaml(y) -## TO-DO: -# - figure out way to safe_load (yaml_loader=yaml.SafeLoader?) -# - update validation and schema return y #################### -def consolidate_yamls(yamlfile,experiment, platform,target): +def consolidate_yamls(mainyaml,experiment, platform,target): """ Combine main yaml and experiment yaml into combined yamls """ @@ -71,47 +71,47 @@ def consolidate_yamls(yamlfile,experiment, platform,target): f1.write(f'define: &platform "{platform}"\n') f1.write(f'define: &target "{target}"\n\n') - # Combine main yaml with - with open(yamlfile,'r') as f2: + # Combine main yaml with created combined.yaml + with open(mainyaml,'r') as f2: f1.write("\n### MAIN YAML SETTINGS ###\n") - shutil.copyfileobj(f2,f1) #copy contents of main yaml into combined yaml + #copy contents of main yaml into combined yaml + shutil.copyfileobj(f2,f1) # Load the new combined yaml and validate - cy=yamlload(combined) + cy=yaml_load(combined) - # Extract the experiment yaml + # Extract experiment yaml file paths from combined yaml exp_list=[] for i in cy.get("experiments"): exp_list.append(i.get("name")) - # Check is exp name is valid experiment listed in main yaml + # Check if exp name given is actually valid experiment listed in combined yaml if experiment not in exp_list: raise Exception(f"{experiment} is not in the list of experiments") + # Extract specific experiment yaml path for exp. provided + # if experiment matches name in list of experiments in yaml, extract file path for i in cy.get("experiments"): - # Extract the experiment yaml if experiment == i.get("name"): expyaml=i.get("pp") - #else: - # simplify combine yaml?: remove name,ppyamls if it doesnt match experiment given - #remove i.get("name") and i.get("pp") associated with it + # Combine experiment yaml with combined yaml + # Could be multtple experiment yamls associated with one experiment + # expname_list will be used later to set rose-suite items for each ppyaml (needed?) + expname_list=[] if expyaml is not None: - #combine experiment yaml with combined yaml with open(combined,"a") as f1: for i in expyaml: - with open(i,'r') as f2: #copy expyaml into combined + expname_list.append(i.split(".")[1]) + with open(i,'r') as f2: f1.write(f"\n### {i.upper()} settings ###\n") + #copy expyaml into combined shutil.copyfileobj(f2,f1) ## TO-DO: # - condition where there are multiple pp yamls - return combined + return (combined,expname_list) -#################### -#def clean(): -# """ -# """ #################### def rose_init(experiment,platform,target): """ @@ -145,6 +145,75 @@ def rose_init(experiment,platform,target): return(rose_suite,rose_regrid,rose_remap) +#################### +def set_rose_suite(yamlfile,rose_suite,expname_list): + """ + Set items in the rose suite configuration. + """ + + pp=yamlfile.get("postprocess") + dirs=yamlfile.get("directories") + + # set rose-suite items + if pp is not None: + for i in pp.values(): + for key,value in i.items(): + rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') + if dirs is not None: + for key,value in dirs.items(): + rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') + + for n in expname_list: + exp_pp=yamlfile.get(f"{n}_postprocess") + exp_dirs=yamlfile.get(f"{n}_directories") + + # set rose-suite items + if exp_pp is not None: + for i in exp_pp.values(): + if not isinstance(i,list): + for key,value in i.items(): + rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') + if exp_dirs is not None: + for key,value in exp_dirs.items(): + rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') + +#################### +def set_rose_apps(yamlfile,rose_regrid,rose_remap,expname_list): + """ + Set items in the regrid and remap rose app configurations. + """ + + for n in expname_list: + components = yamlfile.get(f"{n}_postprocess").get("components") + for i in components: + comp = i.get('type') + sources = i.get('sources') + interp_method = i.get('interpMethod') + + # set remap items + rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + # if xyInterp doesnt exist, grid is native + if i.get("xyInterp") is None: + rose_remap.set(keys=[f'{comp}', 'grid'], value='native') + + # if xyInterp exists, component can be regridded + else: + interp_split = i.get('xyInterp').split(',') + rose_remap.set(keys=[f'{comp}', 'grid'], + value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') + + # set regrid items + if i.get("xyInterp") is not None: + rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') + rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') + rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') + interp_split = i.get('xyInterp').split(',') + rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridType'], + value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') + #################### def _yamlInfo(yamlfile,experiment,platform,target): """ @@ -159,69 +228,29 @@ def _yamlInfo(yamlfile,experiment,platform,target): yml = yamlfile # Initialize the rose configurations - rose_suite,rose_regrid,rose_remap=rose_init(e,p,t) + rose_suite,rose_regrid,rose_remap = rose_init(e,p,t) # Combine main and exp yamls into one new combine.yaml - comb_yaml = consolidate_yamls(yamlfile=yml, - experiment=e, - platform=p, - target=t) + comb_yaml,expname_list = consolidate_yamls(mainyaml=yml, + experiment=e, + platform=p, + target=t) # Load the combined yaml - final_yaml = yamlload(comb_yaml) + final_yaml = yaml_load(comb_yaml) ## PARSE COMBINED YAML TO CREATE CONFIGS - # Remove unwanted keys - del final_yaml["experiments"] - del final_yaml["define"] - - for i in final_yaml.keys(): - #print(i) - if final_yaml.get(i) is not None: - for configkey,configvalue in final_yaml.get(i).items(): - if isinstance(configvalue,dict): ##for settings and switches - for i,j in configvalue.items(): - rose_suite.set(keys=['template variables', i.upper()], value=f'{j}') - elif isinstance(configvalue,str): ##for directories - rose_suite.set(keys=['template variables', configkey.upper()], - value=f'{configvalue}') - - components = final_yaml.get("postprocess").get("components") - for i in components: - comp = i.get('type') - sources = i.get('sources') - interp_method = i.get('interpMethod') - - # set remap items - rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - # if xyInterp doesnt exist, grid is native - if i.get("xyInterp") is None: - rose_remap.set(keys=[f'{comp}', 'grid'], value='native') - - # if xyInterp exists, component can be regridded - else: - interp_split = i.get('xyInterp').split(',') - rose_remap.set(keys=[f'{comp}', 'grid'], - value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') - - # set regrid items - if i.get("xyInterp") is not None: - print(i.values()) - rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') - rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') - rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') - interp_split = i.get('xyInterp').split(',') - rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridType'], - value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') + # set rose-suite items + set_rose_suite(final_yaml,rose_suite,expname_list) + + # set regrid and remap rose app items + set_rose_apps(final_yaml,rose_regrid,rose_remap,expname_list) # write output files print("Writing output files...") cylc_dir = os.path.join(os.path.expanduser("~/cylc-src"), f"{e}__{p}__{t}") outfile = os.path.join(cylc_dir, f"{e}.yaml") - shutil.copyfile(final_yaml, outfile) + shutil.copyfile(comb_yaml, outfile) print(" " + outfile) dumper = metomi.rose.config.ConfigDumper() From 5f640ca12000289fecd882cbee885e3c5a397daf Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 23 Jul 2024 12:55:56 -0400 Subject: [PATCH 18/27] #99 Add pytest for `configure_script_yaml` --- .../example_ppyamls}/am5.yaml | 0 .../yaml_include/pp.c384_clim.yml | 45 ++++++++++++++++ .../yaml_include/pp.c96_amip.yaml | 0 fre/pp/tests/test_configure_script_yaml.py | 53 +++++++++++++++++++ fre/pytest.ini | 2 +- 5 files changed, 99 insertions(+), 1 deletion(-) rename fre/pp/{ppyaml_test => tests/example_ppyamls}/am5.yaml (100%) create mode 100644 fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml rename fre/pp/{ppyaml_test => tests/example_ppyamls}/yaml_include/pp.c96_amip.yaml (100%) create mode 100644 fre/pp/tests/test_configure_script_yaml.py diff --git a/fre/pp/ppyaml_test/am5.yaml b/fre/pp/tests/example_ppyamls/am5.yaml similarity index 100% rename from fre/pp/ppyaml_test/am5.yaml rename to fre/pp/tests/example_ppyamls/am5.yaml diff --git a/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml b/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml new file mode 100644 index 00000000..589ead49 --- /dev/null +++ b/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml @@ -0,0 +1,45 @@ +# "FRE properties" :) +define1: &xyInterp "720,1152" + +c384_clim_directories: + history_dir: "/archive/oar.gfdl.am5/am5/am5f7b12r0/c384L65_am5f7b12r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/history" + pp_dir: "/archive/$USER/am5/am5f7b12r0/c384L65_am5f7b12r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp_canopy" + analysis_dir: "/nbhome/$USER/analysis_canopy/c384L65_am5f7b12r0_pdclim1850F" + ptmp_dir: "/xtmp/$USER/ptmp" + fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" + +c384_clim_postprocess: + switches: + do_statics: True + do_timeavgs: True + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True + + components: + - type: "atmos_cmip" + sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip" + sourceGrid: "cubedsphere" + xyInterp: *xyInterp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos" + sources: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *xyInterp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos_level_cmip" + sources: "atmos_level_cmip" + sourceGrid: "cubedsphere" + xyInterp: *xyInterp + interpMethod: "conserve_order2" + inputRealm: 'atmos' + - type: "atmos_level" + sources: "atmos_month" + sourceGrid: "cubedsphere" + xyInterp: *xyInterp + interpMethod: "conserve_order2" + inputRealm: 'atmos' diff --git a/fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml similarity index 100% rename from fre/pp/ppyaml_test/yaml_include/pp.c96_amip.yaml rename to fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py new file mode 100644 index 00000000..dee1df41 --- /dev/null +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -0,0 +1,53 @@ +import os +from pathlib import Path +from fre.pp import configure_script_yaml as csy + +# Set ex. yaml paths, input directory, output directory +CWD = Path.cwd() +yamls_dir = Path("tests/example_ppyamls") +in_dir = Path(f"{CWD}/{yamls_dir}") + +# Set what would be click options +experiment = "c96L65_am5f7b12r1_amip" +platform = "gfdl.ncrc5-intel22-classic" +target = "prod-openmp" + +# Set home for ~/cylc-src location in script +os.environ["HOME"]=str(Path(f"{CWD}/{yamls_dir}/yaml_out")) + +def test_mainyaml_exists(): + """ + Make sure main yaml file exists + """ + assert Path(f"{CWD}/{yamls_dir}/am5.yaml").exists() + +def test_expyaml_exists(): + """ + Make sure experiment yaml file exists + """ + assert Path(f"{CWD}/{yamls_dir}/yaml_include/pp.c96_amip.yaml").exists() + +def test_configure_script(): + """ + Tests success of confgure yaml script + Creates rose-suite, regrid rose-app, remap rose-app + TO-DO: will break this up for better tests + """ + # Go into example yamls location + os.chdir(in_dir) + + # Set output directory + out_dir = Path(f"{os.getenv('HOME')}/cylc-src/{experiment}__{platform}__{target}") + Path(out_dir).mkdir(parents=True,exist_ok=True) + + # Define main yaml + mainyaml = str(Path(f"{CWD}/{yamls_dir}/am5.yaml")) + + # Invoke configure_yaml_script.py + csy._yamlInfo(mainyaml,experiment,platform,target) + + # Check for configuration creation and final combined yaml + assert all([Path(f"{Path.cwd()}/yaml_out/cylc-src/{experiment}__{platform}__{target}/{experiment}.yaml").exists(), + Path(f"{Path.cwd()}/yaml_out/cylc-src/{experiment}__{platform}__{target}/rose-suite.conf").exists(), + Path(f"{Path.cwd()}/yaml_out/cylc-src/{experiment}__{platform}__{target}/app/regrid-xy/rose-app.conf").exists(), + Path(f"{Path.cwd()}/yaml_out/cylc-src/{experiment}__{platform}__{target}/app/remap-pp-components/rose-app.conf").exists()]) diff --git a/fre/pytest.ini b/fre/pytest.ini index 72cf74e3..20575029 100644 --- a/fre/pytest.ini +++ b/fre/pytest.ini @@ -6,7 +6,7 @@ testpaths = # fre/cmor/tests # fre/list/tests # fre/make/tests -# fre/pp/tests + fre/pp/tests # fre/run/tests # fre/test/tests # fre/yamltools/tests From 67edd2319ed2c81bb65051a8aa44e0296e4e1359 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 24 Jul 2024 15:22:58 -0400 Subject: [PATCH 19/27] #99 Update script to combine main yaml and experiment yaml and parse correctly --- fre/pp/configure_script_yaml.py | 136 ++++++++++++++------------------ 1 file changed, 61 insertions(+), 75 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 4eba0804..028f0213 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -6,7 +6,7 @@ ## TO-DO: # - figure out way to safe_load (yaml_loader=yaml.SafeLoader?) -# - update validation and schema +# - condition where there are multiple pp yamls import os import json @@ -41,7 +41,6 @@ def join_constructor(loader, node): """ seq = loader.construct_sequence(node) return ''.join([str(i) for i in seq]) - #################### def yaml_load(yamlfile): """ @@ -53,7 +52,6 @@ def yaml_load(yamlfile): # Load the main yaml and validate with open(yamlfile,'r') as f: y=yaml.load(f,Loader=yaml.Loader) - #validate_yaml(y) return y @@ -67,9 +65,9 @@ def consolidate_yamls(mainyaml,experiment, platform,target): # Create and write to combined yaml with open(combined,"w") as f1: f1.write('## Combined yamls\n') - f1.write(f'define: &name "{experiment}"\n') - f1.write(f'define: &platform "{platform}"\n') - f1.write(f'define: &target "{target}"\n\n') + f1.write(f'name: &name "{experiment}"\n') + f1.write(f'platform: &platform "{platform}"\n') + f1.write(f'target: &target "{target}"\n\n') # Combine main yaml with created combined.yaml with open(mainyaml,'r') as f2: @@ -96,7 +94,7 @@ def consolidate_yamls(mainyaml,experiment, platform,target): expyaml=i.get("pp") # Combine experiment yaml with combined yaml - # Could be multtple experiment yamls associated with one experiment + # Could be multiple experiment yamls associated with one experiment # expname_list will be used later to set rose-suite items for each ppyaml (needed?) expname_list=[] if expyaml is not None: @@ -107,10 +105,8 @@ def consolidate_yamls(mainyaml,experiment, platform,target): f1.write(f"\n### {i.upper()} settings ###\n") #copy expyaml into combined shutil.copyfileobj(f2,f1) - -## TO-DO: -# - condition where there are multiple pp yamls - return (combined,expname_list) + + return combined #################### def rose_init(experiment,platform,target): @@ -119,12 +115,6 @@ def rose_init(experiment,platform,target): """ # initialize rose suite config rose_suite = metomi.rose.config.ConfigNode() - #rose_suite.set(keys=['template variables', 'SITE'], value='"ppan"') - #rose_suite.set(keys=['template variables', 'CLEAN_WORK'], value='True') - #rose_suite.set(keys=['template variables', 'PTMP_DIR'], value='"/xtmp/$USER/ptmp"') - #rose_suite.set(keys=['template variables', 'DO_STATICS'], value='True') - #rose_suite.set(keys=['template variables', 'DO_TIMEAVGS'], value='True') - #rose_suite.set(keys=['template variables', 'DO_ATMOS_PLEVEL_MASKING'], value='True') # disagreeable; these should be optional rose_suite.set(keys=['template variables', 'DO_ANALYSIS_ONLY'], value='False') rose_suite.set(keys=['template variables', 'DO_MDTF'], value='False') @@ -146,73 +136,57 @@ def rose_init(experiment,platform,target): return(rose_suite,rose_regrid,rose_remap) #################### -def set_rose_suite(yamlfile,rose_suite,expname_list): +def set_rose_suite(yamlfile,rose_suite): """ Set items in the rose suite configuration. """ - pp=yamlfile.get("postprocess") dirs=yamlfile.get("directories") # set rose-suite items if pp is not None: for i in pp.values(): - for key,value in i.items(): - rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') + if not isinstance(i,list): + for key,value in i.items(): + rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') if dirs is not None: for key,value in dirs.items(): rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') - for n in expname_list: - exp_pp=yamlfile.get(f"{n}_postprocess") - exp_dirs=yamlfile.get(f"{n}_directories") - - # set rose-suite items - if exp_pp is not None: - for i in exp_pp.values(): - if not isinstance(i,list): - for key,value in i.items(): - rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') - if exp_dirs is not None: - for key,value in exp_dirs.items(): - rose_suite.set(keys=['template variables', key.upper()], value=f'{value}') - #################### -def set_rose_apps(yamlfile,rose_regrid,rose_remap,expname_list): +def set_rose_apps(yamlfile,rose_regrid,rose_remap): """ Set items in the regrid and remap rose app configurations. """ - - for n in expname_list: - components = yamlfile.get(f"{n}_postprocess").get("components") - for i in components: - comp = i.get('type') - sources = i.get('sources') - interp_method = i.get('interpMethod') - - # set remap items - rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - # if xyInterp doesnt exist, grid is native - if i.get("xyInterp") is None: - rose_remap.set(keys=[f'{comp}', 'grid'], value='native') - - # if xyInterp exists, component can be regridded - else: - interp_split = i.get('xyInterp').split(',') - rose_remap.set(keys=[f'{comp}', 'grid'], - value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') - - # set regrid items - if i.get("xyInterp") is not None: - rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') - rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') - rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') - rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') - interp_split = i.get('xyInterp').split(',') - rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') - rose_regrid.set(keys=[f'{comp}', 'outputGridType'], - value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') + components = yamlfile.get(f"postprocess").get("components") + for i in components: + comp = i.get('type') + sources = i.get('sources') + interp_method = i.get('interpMethod') + + # set remap items + rose_remap.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + # if xyInterp doesnt exist, grid is native + if i.get("xyInterp") is None: + rose_remap.set(keys=[f'{comp}', 'grid'], value='native') + + # if xyInterp exists, component can be regridded + else: + interp_split = i.get('xyInterp').split(',') + rose_remap.set(keys=[f'{comp}', 'grid'], + value=f'regrid-xy/{interp_split[0]}_{interp_split[1]}.{interp_method}') + + # set regrid items + if i.get("xyInterp") is not None: + rose_regrid.set(keys=[f'{comp}', 'sources'], value=f'{sources}') + rose_regrid.set(keys=[f'{comp}', 'inputRealm'], value=f'{i.get("inputRealm")}') + rose_regrid.set(keys=[f'{comp}', 'inputGrid'], value=f'{i.get("sourceGrid")}') + rose_regrid.set(keys=[f'{comp}', 'interpMethod'], value=f'{interp_method}') + interp_split = i.get('xyInterp').split(',') + rose_regrid.set(keys=[f'{comp}', 'outputGridLon'], value=f'{interp_split[1]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridLat'], value=f'{interp_split[0]}') + rose_regrid.set(keys=[f'{comp}', 'outputGridType'], + value=f'{interp_split[0]}_{interp_split[1]}.{interp_method}') #################### def _yamlInfo(yamlfile,experiment,platform,target): @@ -231,20 +205,32 @@ def _yamlInfo(yamlfile,experiment,platform,target): rose_suite,rose_regrid,rose_remap = rose_init(e,p,t) # Combine main and exp yamls into one new combine.yaml - comb_yaml,expname_list = consolidate_yamls(mainyaml=yml, - experiment=e, - platform=p, - target=t) + comb_yaml = consolidate_yamls(mainyaml=yml, + experiment=e, + platform=p, + target=t) # Load the combined yaml final_yaml = yaml_load(comb_yaml) + # Clean combined yaml to validate + del final_yaml["fre_properties"] + del final_yaml["shared"] + del final_yaml["experiments"] + + with open("combined.yaml",'w') as f: + yaml.safe_dump(final_yaml,f,sort_keys=False) + + # validate yaml + validate_yaml(final_yaml) + + ## PARSE COMBINED YAML TO CREATE CONFIGS - # set rose-suite items - set_rose_suite(final_yaml,rose_suite,expname_list) + # Set rose-suite items + set_rose_suite(final_yaml,rose_suite) - # set regrid and remap rose app items - set_rose_apps(final_yaml,rose_regrid,rose_remap,expname_list) + # Set regrid and remap rose app items + set_rose_apps(final_yaml,rose_regrid,rose_remap) # write output files print("Writing output files...") From c0bbb3d3dc0aa0da899d553f22bb7e0e6ebc9572 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 24 Jul 2024 15:27:32 -0400 Subject: [PATCH 20/27] #99 Update main yaml, experiment yaml, and schema for validation - update main yaml to have a `fre_properties` block - update experiment yaml to inherit like sections - also overwrites fields if defined again in experiment yaml - update schema.json to validate final `combined.yaml` - schema shows issues for validating separate yamls (main and experiment individaully) because validation does not seem to recognize `!join` (custom tags) --- fre/pp/schema.json | 137 +++++++----------- fre/pp/tests/example_ppyamls/am5.yaml | 90 ++++++------ .../yaml_include/pp.c384_clim.yml | 45 ------ .../yaml_include/pp.c96_amip.yaml | 15 +- 4 files changed, 113 insertions(+), 174 deletions(-) delete mode 100644 fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml diff --git a/fre/pp/schema.json b/fre/pp/schema.json index 7e19522a..e69b4b3d 100644 --- a/fre/pp/schema.json +++ b/fre/pp/schema.json @@ -3,86 +3,61 @@ "title": "Schema for PP Yaml", "type": "object", "properties": { - "rose-suite": { - "type": "object", - "properties": { - "settings": { - "type":"object", - "properties": { - "history_segment": {"type":"string"}, - "pp_start": {"type":"string"}, - "pp_stop": {"type":"string"}, - "site": {"type":"string"}, - "fre_analysis_home": {"type":["string","null"]}, - "pp_chunk_a": {"type":"string"}, - "pp_chunk_b": {"type":["string","null"]}, - "pp_grid_spec": {"type":"string"} - } - }, - "switches": { - "type":"object", - "properties": { - "clean_work": {"type":"boolean"}, - "do_mdtf": {"type":"boolean"}, - "do_statics": {"type":"boolean"}, - "do_timeavgs": {"type":"boolean"}, - "do_refinediag": {"type":"boolean"}, - "do_atmos_plevel_masking": {"type":"boolean"}, - "do_preanalysis": {"type":"boolean"}, - "do_analysis": {"type":"boolean"}, - "do_analysis_only": {"type":"boolean"} - } - }, - "directories": { - "type":"object", - "properties": { - "history_dir": {"type":"string"}, - "pp_dir": {"type":"string"}, - "ptmp_dir": {"type":"string"}, - "refinediag_scripts":{"type":["string","null"]}, - "preanalysis_script":{"type":["string","null"]}, - "history_refined":{"type":["string","null"]}, - "analysis":{"type":["string","null"]} - } - }, - "metadata": { - "type":"object", - "properties": { - "experiment":{"type":"string"}, - "platform":{"type":"string"}, - "target":{"type":"string"} - } - } - } - }, - "component": { - "type":"object", - "properties": { - "type": {"type":"string"}, - "sources": {"type":"string"}, - "sourceGrid": {"type":"string"}, - "xyInterp": {"type":"string"}, - "interpMethod": {"type":"string"}, - "inputRealm": {"type":"string"} - } - }, - "tmpdir": { - "type": "object", - "properties": { - "tmpdirpath": {"type":["string","null"]} - } - }, - "install-exp-script": { - "type": "array", - "properties": { - "path": {"type": "string"}, - "install-option": { - "type": "object", - "properties": { - "install": {"type": "string"} - } - } - } - } + "name": {"type": "string"}, + "platform": {"type": "string"}, + "target": {"type": "string"}, + "directories": { + "history_dir": {"type":"string"}, + "pp_dir": {"type":"string"}, + "ptmp_dir": {"type":"string"}, + "refinediag_scripts":{"type":["string","null"]}, + "preanalysis_script":{"type":["string","null"]}, + "history_refined":{"type":["string","null"]}, + "analysis":{"type":["string","null"]}, + "pp_grid_spec": {"type":"string"}, + "fre_analysis_home": {"type":["string","null"]} + }, + "postprocess": { + "type": "object", + "properties": { + "settings": { + "type:": "object", + "properties": { + "history_segment": {"type":"string"}, + "site": {"type":"string"}, + "pp_chunk_a": {"type":"string"}, + "pp_chunk_b": {"type":"string"}, + "pp_start": {"type":"string"}, + "pp_stop": {"type":"string"}, + "pp_components": {"type":"string"} + } + }, + "switches": { + "type": "object", + "properties": { + "clean_work": {"type":"boolean"}, + "do_mdtf": {"type":"boolean"}, + "do_statics": {"type":"boolean"}, + "do_timeavgs": {"type":"boolean"}, + "do_refinediag": {"type":"boolean"}, + "do_atmos_plevel_masking": {"type":"boolean"}, + "do_preanalysis": {"type":"boolean"}, + "do_analysis": {"type":"boolean"}, + "do_analysis_only": {"type":"boolean"} + } + }, + "components": { + "type": "array", + "properties": { + "type": {"type":"string"}, + "sources": {"type":"string"}, + "sourceGrid": {"type":"string"}, + "xyInterp": {"type":"string"}, + "interpMethod": {"type":"string"}, + "inputRealm": {"type":"string"} + } + } + } + } } } diff --git a/fre/pp/tests/example_ppyamls/am5.yaml b/fre/pp/tests/example_ppyamls/am5.yaml index baeec9f4..a2d6e5f6 100644 --- a/fre/pp/tests/example_ppyamls/am5.yaml +++ b/fre/pp/tests/example_ppyamls/am5.yaml @@ -1,57 +1,59 @@ # reusable variables -define: &AM5_VERSION "am5f7b12r1" -define: &FRE_STEM !join [am5/, *AM5_VERSION] +fre_properties: + - &AM5_VERSION "am5f7b12r1" + - &FRE_STEM !join [am5/, *AM5_VERSION] -# amip -define: &EXP_AMIP_START "1979" -define: &EXP_AMIP_END "2020" -define: &ANA_AMIP_START "1980" -define: &ANA_AMIP_END "2020" + # amip + - &EXP_AMIP_START "1979" + - &EXP_AMIP_END "2020" + - &ANA_AMIP_START "1980" + - &ANA_AMIP_END "2020" -define: &PP_AMIP_CHUNK96 "1yr" -define: &PP_AMIP_CHUNK384 "1yr" -define: &PP_XYINTERP96 "180,288" -define: &PP_XYINTERP384 "720,1152" + - &PP_AMIP_CHUNK96 "1yr" + - &PP_AMIP_CHUNK384 "1yr" + - &PP_XYINTERP96 "180,288" + - &PP_XYINTERP384 "720,1152" -# climo -define: &EXP_CLIMO_START96 "0001" -define: &EXP_CLIMO_END96 "0011" -define: &ANA_CLIMO_START96 "0002" -define: &ANA_CLIMO_END96 "0011" + # climo + - &EXP_CLIMO_START96 "0001" + - &EXP_CLIMO_END96 "0011" + - &ANA_CLIMO_START96 "0002" + - &ANA_CLIMO_END96 "0011" -define: &EXP_CLIMO_START384 "0001" -define: &EXP_CLIMO_END384 "0006" -define: &ANA_CLIMO_START384 "0002" -define: &ANA_CLIMO_END384 "0006" + - &EXP_CLIMO_START384 "0001" + - &EXP_CLIMO_END384 "0006" + - &ANA_CLIMO_START384 "0002" + - &ANA_CLIMO_END384 "0006" -# coupled -define: &PP_CPLD_CHUNK_A "5yr" -define: &PP_CPLD_CHUNK_B "20yr" + # coupled + - &PP_CPLD_CHUNK_A "5yr" + - &PP_CPLD_CHUNK_B "20yr" -# grids -define: &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" + # grids + - &GRID_SPEC96 "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar" +shared: # directories shared across tools -directories: - history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] - pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] - analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] - ptmp_dir: "/xtmp/$USER/ptmp" - fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" + directories: &shared_directories + history_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history] + pp_dir: !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp] + analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name] + ptmp_dir: "/xtmp/$USER/ptmp" + fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" -# shared pp settings -postprocess: - settings: - history_segment: "P1Y" - site: "ppan" - switches: - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: True + # shared pp settings + postprocess: + settings: &shared_settings + history_segment: "P1Y" + site: "ppan" + switches: &shared_switches + do_statics: True + do_timeavgs: True + clean_work: True + do_refinediag: False + do_atmos_plevel_masking: True + do_preanalysis: False + do_analysis: True experiments: - name: "c96L65_am5f7b12r1_amip" diff --git a/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml b/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml deleted file mode 100644 index 589ead49..00000000 --- a/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml +++ /dev/null @@ -1,45 +0,0 @@ -# "FRE properties" :) -define1: &xyInterp "720,1152" - -c384_clim_directories: - history_dir: "/archive/oar.gfdl.am5/am5/am5f7b12r0/c384L65_am5f7b12r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/history" - pp_dir: "/archive/$USER/am5/am5f7b12r0/c384L65_am5f7b12r0_pdclim1850F/gfdl.ncrc5-deploy-prod-openmp/pp_canopy" - analysis_dir: "/nbhome/$USER/analysis_canopy/c384L65_am5f7b12r0_pdclim1850F" - ptmp_dir: "/xtmp/$USER/ptmp" - fre_analysis_home: "/home/fms/local/opt/fre-analysis/test" - -c384_clim_postprocess: - switches: - do_statics: True - do_timeavgs: True - clean_work: True - do_refinediag: False - do_atmos_plevel_masking: True - do_preanalysis: False - do_analysis: True - - components: - - type: "atmos_cmip" - sources: "atmos_month_cmip atmos_8xdaily_cmip atmos_daily_cmip" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos" - sources: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_level_cmip" - sources: "atmos_level_cmip" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' - - type: "atmos_level" - sources: "atmos_month" - sourceGrid: "cubedsphere" - xyInterp: *xyInterp - interpMethod: "conserve_order2" - inputRealm: 'atmos' diff --git a/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml index 01317f0f..117c66c6 100644 --- a/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml +++ b/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml @@ -1,18 +1,25 @@ # local reusable variable overrides -define: &custom_interp "180,360" +fre_properties: + - &custom_interp "180,360" # directory overrides -c96_amip_directories: +#c96_amip_directories: +directories: + <<: *shared_directories ptmp_dir: "/ptmp/$USER" + pp_grid_spec: *GRID_SPEC96 -c96_amip_postprocess: +#c96_amip_postprocess: +postprocess: # pp setting overrides settings: + <<: *shared_settings pp_start: *ANA_AMIP_START pp_stop: *ANA_AMIP_END pp_chunk_a: *PP_AMIP_CHUNK96 pp_components: "atmos atmos_scalar" - switches: + switches: + <<: *shared_switches do_statics: False # main pp instructions From 25b088883e208a213bfcbd2e0683fd02cb31a55d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 24 Jul 2024 16:39:01 -0400 Subject: [PATCH 21/27] #99 Move `configure_script_yaml` test into `fre/test` --- fre/{pp => }/tests/example_ppyamls/am5.yaml | 0 .../tests/example_ppyamls/yaml_include/pp.c96_amip.yaml | 0 fre/{pp => }/tests/test_configure_script_yaml.py | 7 ++++--- 3 files changed, 4 insertions(+), 3 deletions(-) rename fre/{pp => }/tests/example_ppyamls/am5.yaml (100%) rename fre/{pp => }/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml (100%) rename fre/{pp => }/tests/test_configure_script_yaml.py (91%) diff --git a/fre/pp/tests/example_ppyamls/am5.yaml b/fre/tests/example_ppyamls/am5.yaml similarity index 100% rename from fre/pp/tests/example_ppyamls/am5.yaml rename to fre/tests/example_ppyamls/am5.yaml diff --git a/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml b/fre/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml similarity index 100% rename from fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml rename to fre/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/tests/test_configure_script_yaml.py similarity index 91% rename from fre/pp/tests/test_configure_script_yaml.py rename to fre/tests/test_configure_script_yaml.py index dee1df41..5c50cbd5 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/tests/test_configure_script_yaml.py @@ -1,10 +1,11 @@ import os from pathlib import Path -from fre.pp import configure_script_yaml as csy +#import configure_script_yaml as csy +import fre.pp.configure_script_yaml as csy -# Set ex. yaml paths, input directory, output directory +# Set example yaml paths, input directory CWD = Path.cwd() -yamls_dir = Path("tests/example_ppyamls") +yamls_dir = Path("fre/tests/example_ppyamls") in_dir = Path(f"{CWD}/{yamls_dir}") # Set what would be click options From fd309ab2343a17dfa5bb37713e4af31e3914da62 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 26 Jul 2024 10:28:58 -0400 Subject: [PATCH 22/27] #99 Put configure_script_yaml test back in `fre/pp/tests` --- fre/{ => pp}/tests/example_ppyamls/am5.yaml | 0 .../tests/example_ppyamls/yaml_include/pp.c96_amip.yaml | 0 fre/{ => pp}/tests/test_configure_script_yaml.py | 5 ++--- 3 files changed, 2 insertions(+), 3 deletions(-) rename fre/{ => pp}/tests/example_ppyamls/am5.yaml (100%) rename fre/{ => pp}/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml (100%) rename fre/{ => pp}/tests/test_configure_script_yaml.py (93%) diff --git a/fre/tests/example_ppyamls/am5.yaml b/fre/pp/tests/example_ppyamls/am5.yaml similarity index 100% rename from fre/tests/example_ppyamls/am5.yaml rename to fre/pp/tests/example_ppyamls/am5.yaml diff --git a/fre/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml similarity index 100% rename from fre/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml rename to fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml diff --git a/fre/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py similarity index 93% rename from fre/tests/test_configure_script_yaml.py rename to fre/pp/tests/test_configure_script_yaml.py index 5c50cbd5..2c3c6885 100644 --- a/fre/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -1,11 +1,10 @@ import os from pathlib import Path -#import configure_script_yaml as csy -import fre.pp.configure_script_yaml as csy +import configure_script_yaml as csy # Set example yaml paths, input directory CWD = Path.cwd() -yamls_dir = Path("fre/tests/example_ppyamls") +yamls_dir = Path("tests/example_ppyamls") in_dir = Path(f"{CWD}/{yamls_dir}") # Set what would be click options From bf0737dba6c25475b20a1389ebb18aa5061fc19d Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 26 Jul 2024 11:27:00 -0400 Subject: [PATCH 23/27] #99 Update path for github build action to work --- fre/pp/tests/test_configure_script_yaml.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/pp/tests/test_configure_script_yaml.py b/fre/pp/tests/test_configure_script_yaml.py index 2c3c6885..17ac8851 100644 --- a/fre/pp/tests/test_configure_script_yaml.py +++ b/fre/pp/tests/test_configure_script_yaml.py @@ -1,10 +1,10 @@ import os from pathlib import Path -import configure_script_yaml as csy +from fre.pp import configure_script_yaml as csy # Set example yaml paths, input directory CWD = Path.cwd() -yamls_dir = Path("tests/example_ppyamls") +yamls_dir = Path("fre/pp/tests/example_ppyamls") in_dir = Path(f"{CWD}/{yamls_dir}") # Set what would be click options From eadc5e3a4ffd2e9d581d34f48009f8eb3b810ced Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 26 Jul 2024 11:27:33 -0400 Subject: [PATCH 24/27] #99 Add instructions for `pp` tests --- fre/pp/README.md | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/fre/pp/README.md b/fre/pp/README.md index 5ee90688..80a558db 100644 --- a/fre/pp/README.md +++ b/fre/pp/README.md @@ -37,3 +37,13 @@ etc. ``` ### **PP Wrapper Decision Tree** ![pp_wrapper_decsiontree](https://github.com/NOAA-GFDL/fre-cli/assets/98476720/d3eaa237-1e29-4922-9d83-8d9d11925c54) + +### **Tests** + +To `fre pp` tests, return to root directory of the fre-cli repo and call just those tests with + + python -m pytest fre/pp/tests/[test script.py] + +Or run all tests with + + python -m pytests fre/pp/tests From ae8dfad65a7a816f1b8d9a96fa6e6062e3093c9f Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 26 Jul 2024 11:37:26 -0400 Subject: [PATCH 25/27] Update README.md --- fre/pp/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fre/pp/README.md b/fre/pp/README.md index 80a558db..669ba634 100644 --- a/fre/pp/README.md +++ b/fre/pp/README.md @@ -40,7 +40,7 @@ etc. ### **Tests** -To `fre pp` tests, return to root directory of the fre-cli repo and call just those tests with +To run `fre pp` tests, return to root directory of the fre-cli repo and call just those tests with python -m pytest fre/pp/tests/[test script.py] From 4a197f0f43205a59d155feb0cd05868c1c5b526e Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:06:02 -0400 Subject: [PATCH 26/27] #99 Add check for is keys exist in yaml --- fre/pp/configure_script_yaml.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 028f0213..834abe67 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -214,9 +214,10 @@ def _yamlInfo(yamlfile,experiment,platform,target): final_yaml = yaml_load(comb_yaml) # Clean combined yaml to validate - del final_yaml["fre_properties"] - del final_yaml["shared"] - del final_yaml["experiments"] + # If keys exists, delete: + for keys_clean in ["fre_properties","shared","experiments"]: + if final_yaml[keys_clean] is not None: + del final_yaml[keys_clean] with open("combined.yaml",'w') as f: yaml.safe_dump(final_yaml,f,sort_keys=False) From 14a6e0c924403035fc35dd282ade7ac685a6c540 Mon Sep 17 00:00:00 2001 From: Dana Singh <115384427+singhd789@users.noreply.github.com> Date: Fri, 26 Jul 2024 14:26:21 -0400 Subject: [PATCH 27/27] #99 Fix key check for cleaning combined.yaml --- fre/pp/configure_script_yaml.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/fre/pp/configure_script_yaml.py b/fre/pp/configure_script_yaml.py index 834abe67..de400f12 100644 --- a/fre/pp/configure_script_yaml.py +++ b/fre/pp/configure_script_yaml.py @@ -212,13 +212,14 @@ def _yamlInfo(yamlfile,experiment,platform,target): # Load the combined yaml final_yaml = yaml_load(comb_yaml) - + # Clean combined yaml to validate # If keys exists, delete: - for keys_clean in ["fre_properties","shared","experiments"]: - if final_yaml[keys_clean] is not None: - del final_yaml[keys_clean] - + keys_clean=["fre_properties","shared","experiments"] + for kc in keys_clean: + if kc in final_yaml.keys(): + del final_yaml[kc] + with open("combined.yaml",'w') as f: yaml.safe_dump(final_yaml,f,sort_keys=False)