Merge branch 'v3.1-rc' into add_ICON_interpolator

.github/workflows/docs-latest.yml

@@ -8,18 +8,20 @@ on:
       - main
 
 jobs:
-  deploy-docs:
+  deploy-docs-latest:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+
       - name: Build docs
         uses: C2SM/sphinx-action@sphinx-latest
         with:
-          pre-build-command: "pip install sphinx_rtd_theme && pip install sphinx-copybutton"
           build-command: "sphinx-build -b html . _build"
           docs-folder: "docs/"
-      - uses: peaceiris/actions-gh-pages@v3
+
+      - name: Deploy on GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs/_build

.github/workflows/docs-pr-preview.yml

@@ -1,7 +1,9 @@
-name: Deploy PR previews
+name: Build and Deploy Documentation to PR Previews
 
 on:
   pull_request:
+    paths:
+      - 'docs/**'
     types:
       - opened
       - reopened
@@ -16,10 +18,10 @@ jobs:
     steps:
       - name: Checkout
         uses: actions/checkout@v3
+
       - name: Build docs
         uses: C2SM/sphinx-action@sphinx-latest
         with:
-          pre-build-command: "pip install sphinx_rtd_theme && pip install sphinx-copybutton"
           build-command: "sphinx-build -b html . _build"
           docs-folder: "docs/"
 

.github/workflows/docs-tag.yml

@@ -6,23 +6,26 @@ on:
       - '*'
 
 jobs:
-  deploy-docs:
+  deploy-docs-tag:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
         uses: actions/checkout@v3
+
       - name: Get release
         id: get_release
         uses: bruceadams/[email protected]
         env: 
           GITHUB_TOKEN: ${{ github.token }}
+
       - name: Build docs
         uses: C2SM/sphinx-action@sphinx-latest
         with:
-          pre-build-command: "pip install sphinx_rtd_theme && pip install sphinx-copybutton"
           build-command: "sphinx-build -b html . _build"
           docs-folder: "docs/"
-      - uses: peaceiris/actions-gh-pages@v3
+
+      - name: Deploy on GitHub Pages
+        uses: peaceiris/actions-gh-pages@v3
         with:
           github_token: ${{ secrets.GITHUB_TOKEN }}
           publish_dir: ./docs/_build

.gitignore

@@ -1,4 +1,5 @@
 docs/build/
+docs/_build/
 *__pycache__/
 *.pyc
 fieldextra.diagnostic
@@ -8,3 +9,4 @@ output/
 work/
 src/*/
 *.code-workspace
+.vscode/

README.md

@@ -1,4 +1,4 @@
-# Processing Chain for COSMO and ICON Simulations
+# <img src="https://polybox.ethz.ch/index.php/s/yc3zMmoXKyI2rJm/download" width="64" valign="middle" alt="Processing Chain"/> Processing Chain
 
 The Processing Chain is a python script that prepares necessary input
 data, submits compute-jobs to the queue on Piz Daint and does
@@ -9,71 +9,19 @@ e.g., by creating your own case or adding new jobs.
 
 ## Environment Setup
 
-The following steps allow you to create and use your own virtual
-environment to run the Processing Chain. We recommend to use a conda
-environment for the usage of the provided scripts. Please follow the
-instruction for the installation. The following steps only need to be 
-performed once.
-
-### 1\. Install Miniconda
-
-Install as user specific Miniconda, e.g. on your `$HOME` directory,
-which is the default location.
-
-> **Note**: Only conda itself should be installed in your `$HOME`.
-> All environments should be stored in your `$PROJECT` directory,
-> otherwise you risk filling up your `$HOME` directory. See below for instructions.
-
-To install the latest Miniconda, type:
-
-    wget https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh
-    bash Miniconda3-latest-Linux-x86_64.sh
-
-Further deails on Miniconda can be found on the [Miniconda documentation page](https://docs.conda.io/en/latest/miniconda.html).
-
-### 2\. Create the Conda Environment
-
-Create a conda environment `proc-chain` with and install requirements:
-
-    conda env create --prefix $PROJECT/envs/proc-chain -f env/environment.yml
-
-To be able to activate your conda environment by simply using `conda activate proc-chain` instead of the full path, add the following to your `.bashrc`:
-
-    export CONDA_ENVS_PATH=$PROJECT/envs
-
-Activate the environment (use "source activate" in case "conda activate"
-does not work):
-
-    conda activate proc-chain
-
-If you already have the environment but want to update it:
-
-    conda env update --file env/environment.yml --prune
-
-### 3\. Store user-specific data
-
-To register your email address and standard project account, store them into
-these files within your home directory:
-
-    echo <your_account_id> > ~/.acct
-    echo <your_email_address> > ~/.forward
-
-These settings are optional. The Processing Chain will first check the content
-of those files. If desired, the corresponding variables can be overridden by setting
-the `compute_account` and `user_mail` variables in the `config.yaml` file.
+To setup your conda environment for the Processing Chain, please refer
+to the part in the [official documentation](https://c2sm.github.io/processing-chain/latest/environment.html).
 
 ## Run the Chain
 
-Once everything has been set up correctly according to the above steps,
-you just need to execute the following command to activate your
-environment (if not done already):
+To activate your conda environment, type:
 
     conda activate proc-chain
 
 To test if your environment has been successfully set, use the command
 line help to display the available arguments for the main script:
 
-    python run_chain.py -h
+    ./run_chain.py -h
 
 To run the test cases with their standard jobs, please ensure
 that you clone the Processing Chain to `$SCRATCH`, as input and
@@ -89,9 +37,8 @@ For these pre-defined test cases, you can use the Jenkins script
 
     ./jenkins/scripts/jenkins.sh
 
-This script calls other scripts that are located in `jenkins/scripts/`. 
-They will
-- activate the conda environment (if not done already)
+This script calls other scripts that are located in `jenkins/scripts/` and will: 
+- activate the conda environment
 - setup spack-c2sm
 - download input data to `input/`
 - build `int2lm`, `cosmo-ghg`, `icon` and `icon-art`
@@ -104,16 +51,14 @@ They will
 
 To run the test cases manually, type:
 
-```bash
     # replace <casename> with one of the above tests
-    python run_chain.py <casename>
-```
+    ./run_chain.py <casename>
 
 ## Documentation
 
 For more information about the file structure, configuration options,
-namelist templates etc., please read the official
-[documentation](https://c2sm.github.io/processing-chain/).
+namelist templates etc., please read the [official
+documentation](https://c2sm.github.io/processing-chain/latest/).
 
 ## Contributing
 

cases/cosmo-ghg-spinup-test/config.yaml

@@ -4,7 +4,6 @@ model: cosmo-ghg
 constraint: gpu
 ntasks_per_node: 12
 restart_step: PT6H
-variant: spinup
 spinup: 3
 startdate: 2015-01-01T00:00:00Z
 enddate: 2015-01-01T18:00:00Z
@@ -47,7 +46,7 @@ online_vprm:
 int2lm:
     extpar_dir: ./input/cosmo-ghg/extpar
     extpar_filename: test_domain.nc
-    binary_file: ./src/int2lm/test/testsuite/int2lm
+    binary_file: ./ext/int2lm/test/testsuite/int2lm
     namelist_filename: int2lm_INPUT.cfg
     runjob_filename: int2lm_runjob.cfg
     compute_queue: normal
@@ -69,7 +68,7 @@ post_int2lm:
         - CO2_A2
 
 cosmo:
-    binary_file: ./src/cosmo-ghg/cosmo/ACC/cosmo_gpu
+    binary_file: ./ext/cosmo-ghg/cosmo/ACC/cosmo_gpu
     namelist_prefix: cosmo_INPUT_
     runjob_filename: cosmo_runjob.cfg
     compute_queue: normal

cases/cosmo-ghg-test/config.yaml

@@ -45,7 +45,7 @@ online_vprm:
 int2lm:
     extpar_dir: ./input/cosmo-ghg/extpar
     extpar_filename: test_domain.nc
-    binary_file: ./src/int2lm/test/testsuite/int2lm
+    binary_file: ./ext/int2lm/test/testsuite/int2lm
     namelist_filename: int2lm_INPUT.cfg
     runjob_filename: int2lm_runjob.cfg
     compute_queue: normal
@@ -67,7 +67,7 @@ post_int2lm:
         - CO2_A2
 
 cosmo:
-    binary_file: ./src/cosmo-ghg/cosmo/ACC/cosmo_gpu
+    binary_file: ./ext/cosmo-ghg/cosmo/ACC/cosmo_gpu
     namelist_prefix: cosmo_INPUT_
     runjob_filename: cosmo_runjob.cfg
     compute_queue: normal

cases/icon-art-global-test/config.yaml

@@ -59,7 +59,7 @@ input_files:
     pntSrc_xml_filename: ./input/icon-art-global/config/point-sources.xml
 
 icon:
-    binary_file: ./src/icon-art/bin/icon
+    binary_file: ./ext/icon-art/bin/icon
     runjob_filename: icon_runjob.cfg
     era5_inijob: icon_era5_inicond.sh
     era5_nudgingjob: icon_era5_nudging.sh

cases/icon-art-oem-test/config.yaml

@@ -60,7 +60,7 @@ input_files:
     oem_monthofyear_nc: ./input/icon-art-oem/OEM/monthofyear.nc
 
 icon:
-    binary_file: ./src/icon-art/bin/icon
+    binary_file: ./ext/icon-art/bin/icon
     runjob_filename: icon_runjob.cfg
     compute_queue: normal
     walltime: '00:10:00'

cases/icon-art-oem-test/icontools_remap_00_lbc_runjob.cfg

@@ -17,7 +17,7 @@ set -x
 
 export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions
 
-. {cfg.chain_src_dir}/src/spack-c2sm/setup-env.sh
+. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh
 spack load icontools
 
 #-----------------------------------------------------------------------------

cases/icon-art-oem-test/icontools_remap_ic_chem_runjob.cfg

@@ -17,7 +17,7 @@ set -e -x
 
 export ECCODES_DEFINITION_PATH={cfg.eccodes_dir}/definitions.edzw-2.12.5-2:{cfg.eccodes_dir}/definitions
 
-. {cfg.chain_src_dir}/src/spack-c2sm/setup-env.sh
+. {cfg.chain_src_dir}/ext/spack-c2sm/setup-env.sh
 spack load icontools
 
 #-----------------------------------------------------------------------------