#99 Update main yaml, experiment yaml, and schema for validation

- update main yaml to have a `fre_properties` block - update experiment yaml to inherit like sections - also overwrites fields if defined again in experiment yaml - update schema.json to validate final `combined.yaml` - schema shows issues for validating separate yamls (main and experiment individaully) because validation does not seem to recognize `!join` (custom tags)
NOAA-GFDL · Jul 24, 2024 · c0bbb3d · c0bbb3d
1 parent 67edd23
commit c0bbb3d
Show file tree

Hide file tree

Showing 4 changed files with 113 additions and 174 deletions.
diff --git a/fre/pp/schema.json b/fre/pp/schema.json
@@ -3,86 +3,61 @@
   "title": "Schema for PP Yaml",
   "type": "object",
   "properties": {
-    "rose-suite": {
-      "type": "object",
-      "properties": {
-        "settings": {
-          "type":"object",
-          "properties": {
-            "history_segment": {"type":"string"},
-            "pp_start": {"type":"string"},
-            "pp_stop": {"type":"string"},
-            "site": {"type":"string"},
-            "fre_analysis_home": {"type":["string","null"]},
-            "pp_chunk_a": {"type":"string"},
-            "pp_chunk_b": {"type":["string","null"]},
-            "pp_grid_spec": {"type":"string"}
-          }
-        },
-        "switches": {
-          "type":"object",
-          "properties": {
-            "clean_work": {"type":"boolean"},
-            "do_mdtf": {"type":"boolean"},
-            "do_statics": {"type":"boolean"},
-            "do_timeavgs": {"type":"boolean"},
-            "do_refinediag": {"type":"boolean"},
-            "do_atmos_plevel_masking": {"type":"boolean"},
-            "do_preanalysis": {"type":"boolean"},
-            "do_analysis": {"type":"boolean"},
-            "do_analysis_only": {"type":"boolean"}
-          }
-        },
-        "directories": {
-          "type":"object",
-          "properties": {
-            "history_dir": {"type":"string"},
-            "pp_dir": {"type":"string"},
-            "ptmp_dir": {"type":"string"},
-            "refinediag_scripts":{"type":["string","null"]},
-            "preanalysis_script":{"type":["string","null"]},
-            "history_refined":{"type":["string","null"]},
-            "analysis":{"type":["string","null"]}
-          }
-        },
-        "metadata": {
-          "type":"object",
-          "properties": {
-            "experiment":{"type":"string"},
-            "platform":{"type":"string"},
-            "target":{"type":"string"}
-          }
-        }
-      }
-    },
-    "component": {
-      "type":"object",
-      "properties": {
-        "type": {"type":"string"},
-        "sources": {"type":"string"},
-        "sourceGrid": {"type":"string"},
-        "xyInterp": {"type":"string"},
-        "interpMethod": {"type":"string"},
-        "inputRealm": {"type":"string"}
-      }
-    },
-    "tmpdir": {
-      "type": "object",
-      "properties": {
-        "tmpdirpath": {"type":["string","null"]}
-      }
-    },
-    "install-exp-script": {
-      "type": "array",
-      "properties": {
-        "path": {"type": "string"},
-        "install-option": {
-          "type": "object",
-          "properties": {
-            "install": {"type": "string"}
-          }
-        }
-      }
-    }
+       "name": {"type": "string"},
+       "platform": {"type": "string"},
+       "target": {"type": "string"},
+       "directories": {
+           "history_dir": {"type":"string"},
+           "pp_dir": {"type":"string"},
+           "ptmp_dir": {"type":"string"},
+           "refinediag_scripts":{"type":["string","null"]},
+           "preanalysis_script":{"type":["string","null"]},
+           "history_refined":{"type":["string","null"]},
+           "analysis":{"type":["string","null"]},
+           "pp_grid_spec": {"type":"string"},
+           "fre_analysis_home": {"type":["string","null"]}
+       },
+       "postprocess": {
+           "type": "object",
+           "properties": {
+               "settings": {
+                   "type:": "object", 
+                   "properties": {
+                       "history_segment": {"type":"string"},
+                       "site": {"type":"string"},
+                       "pp_chunk_a": {"type":"string"},
+                       "pp_chunk_b": {"type":"string"},
+                       "pp_start": {"type":"string"},
+                       "pp_stop": {"type":"string"},
+                       "pp_components": {"type":"string"}
+                   }
+               },
+               "switches": {
+                   "type": "object",
+                   "properties": {
+                       "clean_work": {"type":"boolean"},
+                       "do_mdtf": {"type":"boolean"},
+                       "do_statics": {"type":"boolean"},
+                       "do_timeavgs": {"type":"boolean"},
+                       "do_refinediag": {"type":"boolean"},
+                       "do_atmos_plevel_masking": {"type":"boolean"},
+                       "do_preanalysis": {"type":"boolean"},
+                       "do_analysis": {"type":"boolean"},
+                       "do_analysis_only": {"type":"boolean"}
+                   }
+               },
+               "components": {
+                   "type": "array",
+                   "properties": {
+                       "type": {"type":"string"},
+                       "sources": {"type":"string"},
+                       "sourceGrid": {"type":"string"},
+                       "xyInterp": {"type":"string"},
+                       "interpMethod": {"type":"string"},
+                       "inputRealm": {"type":"string"}
+                   }
+               }
+           }
+       }
   }
 }
diff --git a/fre/pp/tests/example_ppyamls/am5.yaml b/fre/pp/tests/example_ppyamls/am5.yaml
@@ -1,57 +1,59 @@
 # reusable variables
-define: &AM5_VERSION        "am5f7b12r1"
-define: &FRE_STEM           !join [am5/, *AM5_VERSION]
+fre_properties:
+  - &AM5_VERSION        "am5f7b12r1"
+  - &FRE_STEM           !join [am5/, *AM5_VERSION]
 
-# amip
-define: &EXP_AMIP_START     "1979"
-define: &EXP_AMIP_END       "2020"
-define: &ANA_AMIP_START     "1980"
-define: &ANA_AMIP_END       "2020"
+  # amip
+  - &EXP_AMIP_START     "1979"
+  - &EXP_AMIP_END       "2020"
+  - &ANA_AMIP_START     "1980"
+  - &ANA_AMIP_END       "2020"
 
-define: &PP_AMIP_CHUNK96    "1yr"
-define: &PP_AMIP_CHUNK384   "1yr"
-define: &PP_XYINTERP96      "180,288"
-define: &PP_XYINTERP384     "720,1152"
+  - &PP_AMIP_CHUNK96    "1yr"
+  - &PP_AMIP_CHUNK384   "1yr"
+  - &PP_XYINTERP96      "180,288"
+  - &PP_XYINTERP384     "720,1152"
 
-# climo
-define: &EXP_CLIMO_START96  "0001"
-define: &EXP_CLIMO_END96    "0011"
-define: &ANA_CLIMO_START96  "0002"
-define: &ANA_CLIMO_END96    "0011"
+  # climo
+  - &EXP_CLIMO_START96  "0001"
+  - &EXP_CLIMO_END96    "0011"
+  - &ANA_CLIMO_START96  "0002"
+  - &ANA_CLIMO_END96    "0011"
 
-define: &EXP_CLIMO_START384 "0001"
-define: &EXP_CLIMO_END384   "0006"
-define: &ANA_CLIMO_START384 "0002"
-define: &ANA_CLIMO_END384   "0006"
+  - &EXP_CLIMO_START384 "0001"
+  - &EXP_CLIMO_END384   "0006"
+  - &ANA_CLIMO_START384 "0002"
+  - &ANA_CLIMO_END384   "0006"
 
-# coupled
-define: &PP_CPLD_CHUNK_A    "5yr"
-define: &PP_CPLD_CHUNK_B    "20yr"
+  # coupled
+  - &PP_CPLD_CHUNK_A    "5yr"
+  - &PP_CPLD_CHUNK_B    "20yr"
 
-# grids
-define: &GRID_SPEC96         "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar"
+  # grids
+  - &GRID_SPEC96         "/archive/oar.gfdl.am5/model_gen5/inputs/c96_grid/c96_OM4_025_grid_No_mg_drag_v20160808.tar"
 
+shared:
 # directories shared across tools
-directories:
-  history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
-  pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
-  analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
-  ptmp_dir:     "/xtmp/$USER/ptmp"
-  fre_analysis_home: "/home/fms/local/opt/fre-analysis/test"
+  directories: &shared_directories
+    history_dir:  !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, history]
+    pp_dir:       !join [/archive/$USER/, *FRE_STEM, /, *name, /, *platform, -, *target, /, pp]
+    analysis_dir: !join [/nbhome/$USER/, *FRE_STEM, /, *name]
+    ptmp_dir:     "/xtmp/$USER/ptmp"
+    fre_analysis_home: "/home/fms/local/opt/fre-analysis/test"
 
-# shared pp settings
-postprocess:
-  settings:
-    history_segment:    "P1Y"
-    site:               "ppan"
-  switches:
-    do_statics:         True
-    do_timeavgs:        True
-    clean_work:         True
-    do_refinediag:      False
-    do_atmos_plevel_masking: True
-    do_preanalysis:     False
-    do_analysis:        True
+  # shared pp settings
+  postprocess: 
+    settings: &shared_settings
+      history_segment:    "P1Y"
+      site:               "ppan"
+    switches: &shared_switches
+      do_statics:         True
+      do_timeavgs:        True
+      clean_work:         True
+      do_refinediag:      False
+      do_atmos_plevel_masking: True
+      do_preanalysis:     False
+      do_analysis:        True
 
 experiments:
   - name: "c96L65_am5f7b12r1_amip"

diff --git a/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml b/fre/pp/tests/example_ppyamls/yaml_include/pp.c384_clim.yml
diff --git a/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml b/fre/pp/tests/example_ppyamls/yaml_include/pp.c96_amip.yaml
@@ -1,18 +1,25 @@
 # local reusable variable overrides
-define: &custom_interp "180,360"
+fre_properties: 
+  - &custom_interp "180,360"
 
 # directory overrides
-c96_amip_directories:
+#c96_amip_directories:
+directories:
+  <<: *shared_directories
   ptmp_dir: "/ptmp/$USER"
+  pp_grid_spec: *GRID_SPEC96
 
-c96_amip_postprocess:
+#c96_amip_postprocess:
+postprocess:
   # pp setting overrides
   settings:
+    <<: *shared_settings
     pp_start:   *ANA_AMIP_START
     pp_stop:    *ANA_AMIP_END
     pp_chunk_a: *PP_AMIP_CHUNK96
     pp_components: "atmos atmos_scalar"
-  switches:
+  switches: 
+    <<: *shared_switches
     do_statics: False
 
   # main pp instructions