From dcb0789558d64f59298a78211858bca6ec5cccd5 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 1 Nov 2024 16:45:01 -0400 Subject: [PATCH 01/13] #187 Add gfdl_msd_schemas as a submodule and reference schem to validate correctly --- .gitmodules | 3 + fre/gfdl_msd_schemas | 1 + fre/make/gfdlfremake/schema.json | 201 ------------------------------- fre/make/gfdlfremake/yamlfre.py | 5 +- 4 files changed, 7 insertions(+), 203 deletions(-) create mode 100644 .gitmodules create mode 160000 fre/gfdl_msd_schemas delete mode 100644 fre/make/gfdlfremake/schema.json diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..e5414cde --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "fre/gfdl_msd_schemas"] + path = fre/gfdl_msd_schemas + url = https://github.com/NOAA-GFDL/gfdl_msd_schemas diff --git a/fre/gfdl_msd_schemas b/fre/gfdl_msd_schemas new file mode 160000 index 00000000..04c8150b --- /dev/null +++ b/fre/gfdl_msd_schemas @@ -0,0 +1 @@ +Subproject commit 04c8150bc362304d82e60e765405135460b69f06 diff --git a/fre/make/gfdlfremake/schema.json b/fre/make/gfdlfremake/schema.json deleted file mode 100644 index 751bb9db..00000000 --- a/fre/make/gfdlfremake/schema.json +++ /dev/null @@ -1,201 +0,0 @@ -{ - "$schema": "http://json-schema.org/draft-06/schema#", - "type": "object", - "additionalProperties": false, - "properties": { - "name": { - "description": "The name of the experiment", - "type": "string" - }, - "platform": { - "description": "The platforms listed in the command", - "type": "string" - }, - "target": { - "description": "The targets listed in the command", - "type": "string" - }, - "build": { - "type": "object", - "additionalProperties": false, - "properties": { - "compileYaml": { - "description": "Path to the compile yaml.", - "type": "string" - }, - "platformYaml": { - "description": "Path to the platform yaml.", - "type": "string" - } - } - }, - "compile": { - "description": "The source code descriptions", - "$ref": "#/definitions/Compile" - }, - "platforms": { - "description": "FRE platforms", - "type": "array", - "items": {"$ref": "#/definitions/Platform"} - } - }, - "definitions": { - "Compile": { - "type": "object", - "properties": { - "experiment": { - "description": "The name of the model", - "type": "string" - }, - "container_addlibs": { - "description": "Libraries and packages needed for linking in the container", - "type": ["array","string","null"] - }, - "baremetal_linkerflags": { - "description": "Linker flags of libraries and packages needed for linking in the bare-metal build", - "type": ["array","string","null"] - }, - "src": { - "type": "array", - "items": {"$ref": "#/definitions/Src"} - } - } - }, - "Src": { - "type": "object", - "properties": { - "component": { - "description": "The name of the model component", - "type": "string" - }, - "repo": { - "anyOf": [ - { - "description": "The URL of the code repository", - "type": "array", - "items": { - "type": "string", - "format": "uri", - "qt-uri-protocols": [ - "https" - ], - "qt-uri-extensions": [ - ".git" - ] - } - }, - { - "description": "The URL of the code repository", - "type": "string", - "format": "uri", - "qt-uri-protocols": [ - "https" - ], - "qt-uri-extensions": [ - ".git" - ] - } - ] - }, - "cppdefs": { - "description": "String of CPPDEFs to include in compiling the component", - "type": "string" - }, - "branch": { - "anyOf": [ - { - "description": "The version of code to clone", - "type": "array", - "items": { - "type": "string" - } - }, - { - "description": "The version of code to clone", - "type": "string" - } - ] - }, - "otherFlags": { - "description": "String of Include flags necessary to retrieve other code needed", - "type": "string" - }, - "requires": { - "description": "list of componets that this component depends on", - "type": "array", - "items": {"type": "string"} - }, - "paths": { - "description": "A list of the paths in the component to compile", - "type": "array", - "items": {"type": "string"} - }, - "doF90Cpp": { - "description": "True if the preprocessor needs to be run", - "type": "boolean" - }, - "makeOverrides": { - "description": "Overrides openmp target for MOM6", - "type": "string" - } - } - }, - "Platform": { - "type": "object", - "properties": { - "name": { - "description": "The name of the platform", - "type": "string" - }, - "compiler": { - "description": "The compiler used to build the model", - "type": "string" - }, - "modulesInit": { - "description": "Array of commands to run before loading modules", - "type": "array", - "items": {"type": "string"} - }, - "modules": { - "description": "List (array) of modules to load", - "type": "array", - "items": { - "type": "string" - } - }, - "fc": { - "description": "The Fortran compiler", - "type": "string" - }, - "cc": { - "description": "The C compiler", - "type": "string" - }, - "mkTemplate": { - "description": "Path to the mk template file", - "type": "string" - }, - "modelRoot": { - "description": "Path to the root for all model install files", - "type": "string" - }, - "RUNenv": { - "description": "Commands needed at the beginning of a RUN in dockerfile", - "type": ["array","string"] - }, - "container": { - "description": "True/False if using container to compile", - "type": "boolean" - }, - "containerBuild": { - "description": "Program used to build the container", - "type": "string" - }, - "containerRun": { - "description": "Program used to run the container", - "type": "string" - } - } - } - } -} diff --git a/fre/make/gfdlfremake/yamlfre.py b/fre/make/gfdlfremake/yamlfre.py index 6f638bbb..72458a8c 100644 --- a/fre/make/gfdlfremake/yamlfre.py +++ b/fre/make/gfdlfremake/yamlfre.py @@ -1,5 +1,6 @@ import os import json +from pathlib import Path import yaml from jsonschema import validate, ValidationError, SchemaError from . import platformfre @@ -176,8 +177,8 @@ def __init__(self,combinedyaml,v): #self.freyaml.update(self.platformsyaml) ## VALIDATION OF COMBINED YAML FOR COMPILATION - fremake_package_dir = os.path.dirname(os.path.abspath(__file__)) - schema_path = os.path.join(fremake_package_dir, 'schema.json') + fremake_package_dir = Path(__file__).resolve().parents[2] + schema_path = os.path.join(fremake_package_dir, 'gfdl_msd_schemas', 'FRE', 'fre_make.json') with open(schema_path, 'r') as f: s = f.read() schema = json.loads(s) From 91261eb099f872baf5bb482bd9ec75dfa802d6ca Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Fri, 1 Nov 2024 16:46:15 -0400 Subject: [PATCH 02/13] #187 Update docs and `platformfre.py` for gfdl_msd_schema submodule --- CONTRIBUTING.md | 2 +- docs/usage.rst | 2 +- fre/make/gfdlfremake/platformfre.py | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index a7bd7b6c..c37953c9 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -1,7 +1,7 @@ ## **For Developers** * Developers are free to use this repository's `README.md` to familiarize with the CLI and save time from having to install any dependencies, but development within a Conda environment is heavily recommended regardless -* Gain access to the repository with `git clone git@github.com:NOAA-GFDL/fre-cli.git` or your fork's link (recommended) and an SSH RSA key +* Gain access to the repository with `git clone --recursive git@github.com:NOAA-GFDL/fre-cli.git` or your fork's link (recommended) and an SSH RSA key - Once inside the repository, developers can test local changes by running a `pip install .` inside of the root directory to install the fre-cli package locally with the newest local changes on top of the installed Conda fre-cli dependencies - Test as a normal user would use the CLI * Create a GitHub issue to reflect your contribution's background and reference it with Git commits diff --git a/docs/usage.rst b/docs/usage.rst index 95afbe5a..b0fa001c 100644 --- a/docs/usage.rst +++ b/docs/usage.rst @@ -87,7 +87,7 @@ Usage (Developers) Developers are free to use the user guide above to familiarize with the CLI and save time from having to install any dependencies, but development within a Conda environment is heavily recommended regardless -Gain access to the repository with *git clone git@github.com:NOAA-GFDL/fre-cli.git* or your fork's link (recommended) and an SSH RSA key +Gain access to the repository with *git clone --recursive git@github.com:NOAA-GFDL/fre-cli.git* or your fork's link (recommended) and an SSH RSA key Once inside the repository, developers can test local changes by running a *pip install .* inside of the root directory to install the fre-cli package locally with the newest local changes diff --git a/fre/make/gfdlfremake/platformfre.py b/fre/make/gfdlfremake/platformfre.py index fe8924f9..4f8d0eed 100644 --- a/fre/make/gfdlfremake/platformfre.py +++ b/fre/make/gfdlfremake/platformfre.py @@ -52,7 +52,7 @@ def __init__(self,platforminfo): p["container"] except: p["container"] = False - p["RUNenv"] = "" + p["RUNenv"] = [""] p["containerBuild"] = "" p["containerRun"] = "" if p["container"]: From 8d59326ed5ba97b5889f7da35f2aab5a99d2c819 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 4 Nov 2024 16:33:45 -0500 Subject: [PATCH 03/13] #187 Include `git_url` to see how conda build works with submodules --- meta.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/meta.yaml b/meta.yaml index 07f76686..07e90bd2 100644 --- a/meta.yaml +++ b/meta.yaml @@ -5,10 +5,10 @@ package: version: '{{ environ.get("GIT_DESCRIBE_TAG", data.get("version")) }}' source: - path: . +# path: . # ideally we want this git_url path, but it messes with conda publish # where it builds only the sourced url and not local/branch changes -# git_url: https://github.com/NOAA-GFDL/fre-cli + git_url: https://github.com/NOAA-GFDL/fre-cli build: script: From 7f6fdac10e754c1b049c9461fbb61851a0d3e903 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Mon, 4 Nov 2024 17:12:55 -0500 Subject: [PATCH 04/13] #187 Update schema file path --- fre/yamltools/tests/test_combine_yamls.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/fre/yamltools/tests/test_combine_yamls.py b/fre/yamltools/tests/test_combine_yamls.py index f9e95fa2..dd17036b 100644 --- a/fre/yamltools/tests/test_combine_yamls.py +++ b/fre/yamltools/tests/test_combine_yamls.py @@ -87,8 +87,9 @@ def test_combined_compileyaml_validation(): Validate the combined compile yaml """ combined_yamlfile =f"{COMP_OUT_DIR}/combined-{COMP_EXPERIMENT}.yaml" - schema_file = os.path.join(f"{IN_DIR}","compile_yamls","schema.json") - +# schema_file = os.path.join(f"{IN_DIR}","compile_yamls","schema.json") + schema_file = os.path.join(Path(TEST_DIR).resolve().parents[1], "gfdl_msd_schemas", "FRE", "fre_make.json") + with open(combined_yamlfile,'r') as cf: yml = yaml.safe_load(cf) @@ -149,7 +150,9 @@ def test_combined_compileyaml_validatefail(): # Validate against schema; should fail wrong_combined = Path(f"{COMP_OUT_DIR}/combined-am5-wrong_datatype.yaml") - schema_file = os.path.join(f"{IN_DIR}","compile_yamls","schema.json") + #schema_file = os.path.join(f"{IN_DIR}","compile_yamls","schema.json") + schema_file = os.path.join(Path(TEST_DIR).resolve().parents[7], "gfdl_msd_schemas", "FRE", "fre_make.json") +# print(schema_file) # Open/load combined yaml file with open(wrong_combined,'r') as cf: From f937adeeadd60bfcf5872d28a2faf5e370373b24 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 12:16:43 -0500 Subject: [PATCH 05/13] #187 Fix fre-cli path --- meta.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/meta.yaml b/meta.yaml index 07e90bd2..a11f0151 100644 --- a/meta.yaml +++ b/meta.yaml @@ -8,7 +8,7 @@ source: # path: . # ideally we want this git_url path, but it messes with conda publish # where it builds only the sourced url and not local/branch changes - git_url: https://github.com/NOAA-GFDL/fre-cli + git_url: https://github.com/NOAA-GFDL/fre-cli.git build: script: From e9aba20784837933c3fb6989f281e21f779f8123 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 12:42:17 -0500 Subject: [PATCH 06/13] #187 Specify recursive submodule --- .github/workflows/build_conda.yml | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index d9ba9162..655ae5fd 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -11,6 +11,11 @@ jobs: steps: - name: Checkout Files uses: actions/checkout@v4 + with: + submodule: 'recursive' + - name: Update Submodules + run: | + git submodule update --init --recursive - name: Run Conda to Build run: | conda config --append channels conda-forge From 7b648ea5c584d90d0184ca2c76be3e77772c3404 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 12:52:37 -0500 Subject: [PATCH 07/13] #187 Was adding to wrong file --- .github/workflows/build_conda.yml | 5 ----- .github/workflows/create_test_conda_env.yml | 7 +++++-- 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 655ae5fd..d9ba9162 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -11,11 +11,6 @@ jobs: steps: - name: Checkout Files uses: actions/checkout@v4 - with: - submodule: 'recursive' - - name: Update Submodules - run: | - git submodule update --init --recursive - name: Run Conda to Build run: | conda config --append channels conda-forge diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index ce5de814..7ce15f37 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -7,16 +7,19 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + with: + submodule: 'recursive' +# - name: Update Submodules +# run: | +# git submodule update --init --recursive - name: Set up Python uses: actions/setup-python@v5 with: python-version: '>=3.9' - - name: Add conda to system path run: | # $CONDA is an env var pointing to root of miniconda dir echo $CONDA/bin >> $GITHUB_PATH - - name: Create fre-cli environment run: | # create environment containing all dependencies From 0bc9d54a7cc1f56b738e98e0920c062a20b65915 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 12:54:11 -0500 Subject: [PATCH 08/13] #187 Update create_test_conda_env.yml --- .github/workflows/create_test_conda_env.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index 7ce15f37..c26b8a62 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -8,7 +8,7 @@ jobs: steps: - uses: actions/checkout@v4 with: - submodule: 'recursive' + submodules: 'recursive' # - name: Update Submodules # run: | # git submodule update --init --recursive From b4ab3720efeddf330b51a8f414a86a7a4620b380 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 13:03:35 -0500 Subject: [PATCH 09/13] #187 Update schema path --- fre/yamltools/tests/test_combine_yamls.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fre/yamltools/tests/test_combine_yamls.py b/fre/yamltools/tests/test_combine_yamls.py index 58c402e3..abf98a50 100644 --- a/fre/yamltools/tests/test_combine_yamls.py +++ b/fre/yamltools/tests/test_combine_yamls.py @@ -136,8 +136,8 @@ def test_combined_compileyaml_validatefail(): # Validate against schema; should fail wrong_combined = Path(f"{COMP_OUT_DIR}/combined-am5-wrong_datatype.yaml") #schema_file = os.path.join(f"{IN_DIR}","compile_yamls","schema.json") - schema_file = os.path.join(Path(TEST_DIR).resolve().parents[7], "gfdl_msd_schemas", "FRE", "fre_make.json") -# print(schema_file) + schema_file = os.path.join(Path(TEST_DIR).resolve().parents[1], "gfdl_msd_schemas", "FRE", "fre_make.json") + print(schema_file) # Open/load combined yaml file with open(wrong_combined,'r') as cf: From 2281cc92523471f76a6910b335595e8ffff549e1 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 14:20:08 -0500 Subject: [PATCH 10/13] #187 Add recursive submodules in conda build and publish --- .github/workflows/build_conda.yml | 2 ++ .github/workflows/publish_conda.yml | 2 ++ 2 files changed, 4 insertions(+) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index d9ba9162..6c4f0621 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -11,6 +11,8 @@ jobs: steps: - name: Checkout Files uses: actions/checkout@v4 + with: + submodule: 'recursive' - name: Run Conda to Build run: | conda config --append channels conda-forge diff --git a/.github/workflows/publish_conda.yml b/.github/workflows/publish_conda.yml index e36a72ea..b7a20fd8 100644 --- a/.github/workflows/publish_conda.yml +++ b/.github/workflows/publish_conda.yml @@ -11,6 +11,8 @@ jobs: steps: - name: Checkout Files uses: actions/checkout@v4 + with: + submodules: 'recursive' - name: Run Conda to Build and Publish run: | conda config --append channels conda-forge From 19853a45f3fc6bec84c84b12db104e218bb07c89 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 14:23:49 -0500 Subject: [PATCH 11/13] #187 Change `submodule` to `submodules` --- .github/workflows/build_conda.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build_conda.yml b/.github/workflows/build_conda.yml index 6c4f0621..e11fb93d 100644 --- a/.github/workflows/build_conda.yml +++ b/.github/workflows/build_conda.yml @@ -12,7 +12,7 @@ jobs: - name: Checkout Files uses: actions/checkout@v4 with: - submodule: 'recursive' + submodules: 'recursive' - name: Run Conda to Build run: | conda config --append channels conda-forge From b49c4a713d5da891c6459e37594238633c1a314f Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Tue, 5 Nov 2024 14:38:37 -0500 Subject: [PATCH 12/13] #187 Remove comments - I don't think some of these are needed --- .github/workflows/create_test_conda_env.yml | 3 --- fre/yamltools/tests/test_combine_yamls.py | 3 --- 2 files changed, 6 deletions(-) diff --git a/.github/workflows/create_test_conda_env.yml b/.github/workflows/create_test_conda_env.yml index c26b8a62..24ecfdaf 100644 --- a/.github/workflows/create_test_conda_env.yml +++ b/.github/workflows/create_test_conda_env.yml @@ -9,9 +9,6 @@ jobs: - uses: actions/checkout@v4 with: submodules: 'recursive' -# - name: Update Submodules -# run: | -# git submodule update --init --recursive - name: Set up Python uses: actions/setup-python@v5 with: diff --git a/fre/yamltools/tests/test_combine_yamls.py b/fre/yamltools/tests/test_combine_yamls.py index abf98a50..d27aa0b3 100644 --- a/fre/yamltools/tests/test_combine_yamls.py +++ b/fre/yamltools/tests/test_combine_yamls.py @@ -81,7 +81,6 @@ def test_combined_compileyaml_validation(): Validate the combined compile yaml """ combined_yamlfile =f"{COMP_OUT_DIR}/combined-{COMP_EXPERIMENT}.yaml" -# schema_file = os.path.join(f"{IN_DIR}","compile_yamls","schema.json") schema_file = os.path.join(Path(TEST_DIR).resolve().parents[1], "gfdl_msd_schemas", "FRE", "fre_make.json") with open(combined_yamlfile,'r') as cf: @@ -135,9 +134,7 @@ def test_combined_compileyaml_validatefail(): # Validate against schema; should fail wrong_combined = Path(f"{COMP_OUT_DIR}/combined-am5-wrong_datatype.yaml") - #schema_file = os.path.join(f"{IN_DIR}","compile_yamls","schema.json") schema_file = os.path.join(Path(TEST_DIR).resolve().parents[1], "gfdl_msd_schemas", "FRE", "fre_make.json") - print(schema_file) # Open/load combined yaml file with open(wrong_combined,'r') as cf: From 071f46ac33e3a4b1989fd1006bae3fa8cf73ddb6 Mon Sep 17 00:00:00 2001 From: Dana Singh Date: Wed, 6 Nov 2024 10:41:10 -0500 Subject: [PATCH 13/13] #187 Update schema path --- fre/yamltools/tests/test_combine_yamls.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/fre/yamltools/tests/test_combine_yamls.py b/fre/yamltools/tests/test_combine_yamls.py index d27aa0b3..7df6eb36 100644 --- a/fre/yamltools/tests/test_combine_yamls.py +++ b/fre/yamltools/tests/test_combine_yamls.py @@ -16,6 +16,7 @@ #CWD = Path.cwd() TEST_DIR = Path("fre/yamltools/tests") IN_DIR = Path(f"{TEST_DIR}/AM5_example") +SCHEMA_DIR = Path("fre/gfdl_msd_schemas/FRE") # Create output directories COMP_OUT_DIR = Path(f"{TEST_DIR}/combine_yamls_out/compile") @@ -81,7 +82,7 @@ def test_combined_compileyaml_validation(): Validate the combined compile yaml """ combined_yamlfile =f"{COMP_OUT_DIR}/combined-{COMP_EXPERIMENT}.yaml" - schema_file = os.path.join(Path(TEST_DIR).resolve().parents[1], "gfdl_msd_schemas", "FRE", "fre_make.json") + schema_file = os.path.join(SCHEMA_DIR, "fre_make.json") with open(combined_yamlfile,'r') as cf: yml = yaml.safe_load(cf) @@ -134,7 +135,7 @@ def test_combined_compileyaml_validatefail(): # Validate against schema; should fail wrong_combined = Path(f"{COMP_OUT_DIR}/combined-am5-wrong_datatype.yaml") - schema_file = os.path.join(Path(TEST_DIR).resolve().parents[1], "gfdl_msd_schemas", "FRE", "fre_make.json") + schema_file = os.path.join(SCHEMA_DIR, "fre_make.json") # Open/load combined yaml file with open(wrong_combined,'r') as cf: