diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 878f5dbe..4b09c0bd 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -4,7 +4,8 @@ This repository includes the python package `PyStemmusScope` for running the STE ## Configure the python package for development and testing -To contribute to development of the python package, we recommend installing the package in development mode. +To contribute to the development of the python package, we recommend installing +the package in development mode. ### Installation @@ -15,26 +16,20 @@ First, clone this repository: git clone https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing.git ``` -Then install the package: +Then install the package (On Windows, use `python` instead of `python3`): ```sh cd STEMMUS_SCOPE_Processing -pip install -e . -``` - -or - -```sh -python setup.py develop +python3 -m install -e .[dev] ``` ### Run tests The testing framework used here is [PyTest](https://pytest.org). You can run -tests as: +tests as (On Windows, use `python` instead of `python3`): ```sh -pytest +python3 -m pytest ``` ### Build documentation @@ -65,80 +60,4 @@ isort ## Development of STEMMUS_SCOPE model -To contribute to the STEMMUS_SCOPE model, you need access to the model source code that is stored in the repository [STEMMUS_SCOPE](https://github.com/EcoExtreML/STEMMUS_SCOPE). You also need a MATLAB license. - -### Development on Snellius using MATLAB - -[Snellius](https://servicedesk.surfsara.nl/wiki/display/WIKI/Snellius) is the -Dutch National supercomputer hosted at SURF. MATLAB `2021a` is installed on -Snellius, see the script -[`run_jupyter_lab_snellius_dev.sh`](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/run_jupyter_lab_snellius_dev.sh) -on how to load the module. - -The script `run_jupyter_lab_snellius_dev.sh` activates the conda environment `pystemmusscope` and creates a jupyter lab server on Snellius for running the notebook -interactively. Make sure that you create the `pystemmusscope` conda environment before submitting the the bash script. See **Create pystemmusscope environment** below. - -
- Create pystemmusscope environment - -Run the commands below in a terminal: - -```sh -# Download and install Mamba on linux -wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-pypy3-Linux-x86_64.sh -bash Mambaforge-pypy3-Linux-x86_64.sh -b -p ~/mamba - -# Update base environment -. ~/mamba/bin/activate -mamba update --name base mamba - -# Download environment file -wget https://raw.githubusercontent.com/EcoExtreML/STEMMUS_SCOPE_Processing/main/environment.yml - -# Create a conda environment called 'pystemmusscope' with all required dependencies -mamba env create -f environment.yml - -# The environment can be activated with -. ~/mamba/bin/activate pystemmusscope - -``` -
- - -### Development on CRIB using MATLAB - -[CRIB](https://crib.utwente.nl/) is the ITC Geospatial Computing Platform. - -MATLAB `2021a` is installed on CRIB, and supports Python `3.8`, see [Versions of Python Compatible with MATLAB Products](https://www.mathworks.com/content/dam/mathworks/mathworks-dot-com/support/sysreq/files/python-compatibility.pdf). - -1. Log in CRIB with your username and password and select a proper compute unit. -2. Install `PyStemmusScope` package. This step needs to be done once. -
- Install pystemmusscope with python 3.8 - - Run the commands below in a terminal: - - ```sh - # Download and install Mamba on linux - wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-pypy3-Linux-x86_64.sh - bash Mambaforge-pypy3-Linux-x86_64.sh -b -p ~/mamba - - # Update base environment - . ~/mamba/bin/activate - mamba update --name base mamba - - # Download environment file - wget https://raw.githubusercontent.com/EcoExtreML/STEMMUS_SCOPE_Processing/main/environment_3.8.yml - - # Create a conda environment called 'pystemmusscope' with all required dependencies - mamba env create -f environment_3.8.yml - ``` -
- -3. click on the `Remote Desktop` in the -Launcher. Click on the `Applications`. You will find the 'MATLAB' software under -the `Research`. -4. After clicking on 'MATLAB', it asks for your account information that is -connected to a MATLAB license. -5. Open the file `STEMMUS_SCOPE_run.m` and set the path of `config_file` to `../config_file_crib.txt` and change `WorkDir` and other configurations in `model.setup()`. -6. Then, run the main script `STEMMUS_SCOPE_run.m`. +To contribute to the STEMMUS_SCOPE model, you need access to the model source code that is stored in the repository [STEMMUS_SCOPE](https://github.com/EcoExtreML/STEMMUS_SCOPE). diff --git a/PyStemmusScope/stemmus_scope.py b/PyStemmusScope/stemmus_scope.py index 2231b860..36d168ea 100644 --- a/PyStemmusScope/stemmus_scope.py +++ b/PyStemmusScope/stemmus_scope.py @@ -2,7 +2,9 @@ import logging import os +import shlex import subprocess +from pathlib import Path from typing import Dict from . import config_io from . import forcing_io @@ -12,32 +14,99 @@ logger = logging.getLogger(__name__) +def _is_model_src_exe(model_src_path: Path): + """Check if input exists. Returns True if input is a file and False if it is + a directory. + + Args: + model_src_path(Path): path to Stemmus_Scope executable file or to a + directory containing model source codes. + """ + if model_src_path.is_file(): + msg = ("The model executable file can be used on a Unix system " + "where MCR is installed, see the " + "`documentaion`_.") + logger.info("%s", msg) + return True + if model_src_path.is_dir(): + return False + msg = ( + "Provide a valid path to an executable file or " + "to a directory containing model source codes, " + "see the `documentaion`_.") + raise ValueError(msg) + + +def _check_interpreter(interpreter: str): + if interpreter not in {"Octave" , "Matlab"}: + msg = ( + "Set `interpreter` as Octave or Matlab to run the model using source codes." + "Otherwise set `model_src_path` to the model executable file, " + "see the `documentaion`_.") + raise ValueError(msg) + + +def _run_sub_process(args: list, cwd): + # pylint: disable=consider-using-with + result = subprocess.Popen( + args, cwd=cwd, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, + ) + exit_code = result.wait() + stdout, stderr = result.communicate() + #TODO handle stderr properly + # when using octave, exit_code might be 139 + # see issue STEMMUS_SCOPE_Processing/issues/46 + if exit_code not in [0, 139]: + raise subprocess.CalledProcessError( + returncode=exit_code, cmd=args, stderr=stderr, output=stdout + ) + if exit_code == 139: + logger.warning(stderr) + + # TODO return log info line by line! + logger.info(stdout) + return stdout.decode('utf-8') + class StemmusScope(): """PyStemmusScope wrapper around Stemmus_Scope model. see https://gmd.copernicus.org/articles/14/1379/2021/ - It sets the model with a configuration file and executable file. - It also prepares forcing and soil data for model run. + Configures the model and prepares forcing and soil data for the model run. Args: config_file(str): path to Stemmus_Scope configuration file. An example config_file can be found in tests/test_data in `STEMMUS_SCOPE_Processing repository `_ - exe_file(str): path to Stemmus_Scope executable file. + model_src_path(str): path to Stemmus_Scope executable file or to a + directory containing model source codes. + interpreter(str, optional): use `Matlab` or `Octave`. Only required if + `model_src_path` is a path to model source codes. Example: See notebooks/run_model_in_notebook.ipynb in `STEMMUS_SCOPE_Processing repository `_ """ - def __init__(self, config_file: str, exe_file: str): + def __init__(self, config_file: str, model_src_path: str, interpreter: str = None): # make sure paths are abolute and path objects config_file = utils.to_absolute_path(config_file) - self.exe_file = utils.to_absolute_path(exe_file) + model_src_path = utils.to_absolute_path(model_src_path) + + # check the path to model source + self.exe_file = None + if _is_model_src_exe(model_src_path): + self.exe_file = model_src_path + else: + _check_interpreter(interpreter) + + self.model_src = model_src_path + self.interpreter = interpreter # read config template - self._configs = config_io.read_config(config_file) + self._config = config_io.read_config(config_file) def setup( self, @@ -61,30 +130,27 @@ def setup( """ # update config template if needed if WorkDir: - self._configs["WorkDir"] = WorkDir + self._config["WorkDir"] = WorkDir if ForcingFileName: - self._configs["ForcingFileName"] = ForcingFileName + self._config["ForcingFileName"] = ForcingFileName if NumberOfTimeSteps: - self._configs["NumberOfTimeSteps"] = NumberOfTimeSteps + self._config["NumberOfTimeSteps"] = NumberOfTimeSteps # create customized config file and input/output directories for model run _, _, self.cfg_file = config_io.create_io_dir( - self._configs["ForcingFileName"], self._configs + self._config["ForcingFileName"], self._config ) # read the run config file - self._configs = config_io.read_config(self.cfg_file) + self._config = config_io.read_config(self.cfg_file) # prepare forcing data - forcing_io.prepare_forcing(self._configs) + forcing_io.prepare_forcing(self._config) # prepare soil data - soil_io.prepare_soil_data(self._configs) - - # set matlab log dir - os.environ['MATLAB_LOG_DIR'] = str(self._configs["InputPath"]) + soil_io.prepare_soil_data(self._config) return str(self.cfg_file) @@ -94,23 +160,40 @@ def run(self) -> str: Args: Returns: - Tuple with stdout and stderr + string, the model log """ - - # run the model - args = [f"{self.exe_file} {self.cfg_file}"] - result = subprocess.run( - args, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True, - ) - stdout = result.stdout - - # TODO return log info line by line! - logger.info("%s", stdout) - - return stdout + if self.exe_file: + # run using MCR + args = [f"{self.exe_file} {self.cfg_file}"] + # set matlab log dir + os.environ['MATLAB_LOG_DIR'] = str(self._config["InputPath"]) + result = _run_sub_process(args, None) + if self.interpreter=="Matlab": + # set Matlab arguments + path_to_config = f"'{self.cfg_file}'" + eval_code= f'STEMMUS_SCOPE_exe({path_to_config});exit;' + args = ["matlab", "-r", eval_code, "-nodisplay", "-nosplash", "-nodesktop"] + # seperate args dont work on linux! + if utils.os_name() !="nt": + args = shlex.join(args) + result = _run_sub_process(args, self.model_src) + if self.interpreter=="Octave": + # set Octave arguments + # use subprocess instead of oct2py, + # see issue STEMMUS_SCOPE_Processing/issues/46 + path_to_config = f"'{self.cfg_file}'" + # fix for windows + path_to_config = path_to_config.replace("\\", "/") + eval_code = f'STEMMUS_SCOPE_exe({path_to_config});exit;' + args = ["octave", "--eval", eval_code, "--no-gui", "--silent"] + # seperate args dont work on linux! + if utils.os_name() !="nt": + args = shlex.join(args) + result = _run_sub_process(args, self.model_src) + return result @property def config(self) -> Dict: """Return the configurations for this model.""" - return self._configs + return self._config diff --git a/README.md b/README.md index 18095458..ee33e733 100644 --- a/README.md +++ b/README.md @@ -17,71 +17,97 @@ [![cffconvert](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/cffconvert.yml/badge.svg)](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/cffconvert.yml) [![markdown-link-check](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/markdown-link-check.yml/badge.svg)](https://github.com/EcoExtreML/stemmus_scope_processing/actions/workflows/markdown-link-check.yml) --> -This repository includes the python package `PyStemmusScope` for running the STEMMUS-SCOPE model. +This repository includes the python package `PyStemmusScope` for running the +STEMMUS-SCOPE model. + -The model source code is in MATLAB and available in the [STEMMUS_SCOPE repository](https://github.com/EcoExtreML/STEMMUS_SCOPE). See the relevant -instructions for `Users` or `Developers` on how to run the model. +The model source code, executable file and utility files are available in the [STEMMUS_SCOPE repository](https://github.com/EcoExtreML/STEMMUS_SCOPE). -## Users +The input datasets are available on Snellius and CRIB. First, make sure you have +right access to the repository and data. Then, see the notebook +[run_model_in_notebook.ipynb](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/notebooks/run_model_in_notebook.ipynb) +which provides different options to run the model, see [Run the +model](#run-the-model). -As a user, you don't need to have a MATLAB license to run the STEMMUS-SCOPE model. The workflow is executed using python and MATLAB Runtime on a Unix-like system. +## Run the model -As the STEMMUS-SCOPE executable only supports Unix-like systems, Windows users cannot run STEMMUS-SCOPE natively. -However, users of Windows 10 and newer can use WSL ([Windows Subsystem for Linux](https://docs.microsoft.com/en-us/windows/wsl/)) to run the model. +1. Using executable file: As a user, you don't need to have a MATLAB license to +run the STEMMUS-SCOPE model. If `PyStemmusScope` and `MATLAB Runtime` are +installed on a Unix-like system (e.g. your own machine, Snellius or WSL), you +can run STEMMUS_SCOPE using the executable file. +2. Using Matlab: If `PyStemmusScope` and `Matlab` are installed, you can run +STEMMUS_SCOPE from the source code, for example on Snellius or CRIB. +3. Using Octave: If `PyStemmusScope` and latest `Octave` including required +packages are installed, you can run STEMMUS_SCOPE from its source code, for +example on CRIB or your own machine. -
- WSL installation instructions. - Check the Microsoft Guide for a compatibility information and for general WSL instructions. +See section [Installations](#installations) for required packages. - If no installation exists, a Ubuntu distribution can be installed using the following commands: - ```sh - wsl --install - ``` +## Installations - After installation, you can start up the WSL instance and update the default software: +### On Snellius - ```sh - sudo apt update && sudo apt upgrade - ``` +[Snellius](https://servicedesk.surfsara.nl/wiki/display/WIKI/Snellius) is the +Dutch National supercomputer hosted at SURF. MATLAB and MATLAB Runtime are +installed on Snellius, see the script +[`run_jupyter_lab_snellius.sh`](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/run_jupyter_lab_snellius.sh) +on how to load the module. Also, use the same script to create a jupyter lab +server for running notebooks interactively. The script activates the conda +environment `pystemmusscope`. Make sure that you create the `pystemmusscope` +conda environment before submitting the the bash script. See +[Create pystemmusscope conda environment](#create-pystemmusscope-conda-environment). - You can now set up a python environment using either python's `venv`, or use Conda/Mamba. - Note that the command to run python and pip can be `python3` and `pip3` by default. +### On CRIB - For the rest of the installation instructions simply follow the steps below. - Note that it is possible to access files from the Windows filesystem from within WSL, by accessing, e.g., `/mnt/c/` instead of `C:\`. - This means that large input data files can be stored on your Windows installation instead of inside the WSL distro. -
+[CRIB](https://crib.utwente.nl/) is the ITC Geospatial Computing Platform. You +can run the model using `Matlab` or `Octave`. Currently, running the +exceutable file on CRIB is not supported because MATLAB Runtime can not be +installed there. See [Install PyStemmusScope](#install-pystemmusscope). -### Installations +### On your own machine -Follow the instructions below to install `PyStemmusScope`, `jupyterlab` and MATLAB Runtime. +Choose how do you want to run the model, see [Run the model](#run-the-model). -
- Install PyStemmusScope +### Install PyStemmusScope -Run the commands below in a terminal: +Run the commands below in a terminal (On Windows, use `python` instead of +`python3`): ```sh # will be replaced by `pip install pystemmusscope` -pip install git+https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing.git@main +python3 -m pip install git+https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing.git@main +``` + +or + +Open a jupyter notebook and run the code below in a cell: + +```python +!pip install git+https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing.git@main ``` -
+### Install jupyterlab -
- Install jupyterlab Jupyterlab is needed to run notebooks. Run the commands below in a terminal: ```sh -pip install jupyterlab +python3 -m pip install jupyterlab + +``` +Open a terminal, make sure the environment is activated. Then, run `jupyter lab`: + +```sh +jupyter lab ``` -
-
- Install MATLAB Runtime +JupyterLab will open automatically in your browser. Now, you can run the +notebook +[run_model_in_notebook.ipynb](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/notebooks/run_model_in_notebook.ipynb). -To run the STEMMUS_SCOPE, you need MATLAB Runtime version `2021a`. +### Install MATLAB Runtime + +To run the STEMMUS_SCOPE, you need MATLAB Runtime version `2021a` and a Unix-like system. In a terminal: @@ -101,40 +127,57 @@ For more information on how to download and install MATLAB Runtime, see the link - [download](https://nl.mathworks.com/products/compiler/matlab-runtime.html) - [installation](https://nl.mathworks.com/help/compiler/install-the-matlab-runtime.html) -
+### Install WSL -Open a terminal, make sure the environment is activated. Then, run `jupyter lab`: +As the STEMMUS-SCOPE executable only supports Unix-like systems, Windows users +cannot run STEMMUS-SCOPE natively. However, users of Windows 10 and newer can +use WSL ([Windows Subsystem for +Linux](https://docs.microsoft.com/en-us/windows/wsl/)) to run the model. +Check the Microsoft Guide +for a compatibility information and for general WSL instructions. +If no installation exists, a Ubuntu distribution can be installed using the following commands: +```sh +wsl --install +``` + +After installation, you can start up the WSL instance and update the default software: ```sh -jupyter lab +sudo apt update && sudo apt upgrade ``` -JupyterLab will open automatically in your browser. Now, you can run the notebook [run_model_in_notebook.ipynb](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/notebooks/run_model_in_notebook.ipynb). +You can now set up a python environment using either python's `venv`, or use Conda/Mamba. +Note that the command to run python and pip can be `python3` and `pip3` by default. -### On Snellius +For the rest of the installation instructions simply follow the steps below. +Note that it is possible to access files from the Windows filesystem from within +WSL, by accessing, e.g., `/mnt/c/` instead of `C:\`. This means that large input +data files can be stored on your Windows installation instead of inside the WSL +distro. However, WSL does not have write permission. Therefore, output data will +be stored within WSL. Make sure that `WorkDir` in the model config file is set +correctly. -[Snellius](https://servicedesk.surfsara.nl/wiki/display/WIKI/Snellius) is the -Dutch National supercomputer hosted at SURF. MATLAB Runtime is installed on -Snellius, see the script -[`run_jupyter_lab_snellius.sh`](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/run_jupyter_lab_snellius.sh) -on how to load the module. Also, use the script -[`run_jupyter_lab_snellius.sh`](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/run_jupyter_lab_snellius.sh) -to create a jupyter lab server on Snellius for running the notebook -[run_model_in_notebook.ipynb](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/notebooks/run_model_in_notebook.ipynb) -interactively. +### Create pystemmusscope conda environment -### On CRIB +If a conda environment is neeed, run the commands below in a terminal: -[CRIB](https://crib.utwente.nl/) is the ITC Geospatial Computing Platform. -Currently, running the model exceutable file and therefore the notebook -[run_model_in_notebook.ipynb](https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing/blob/main/notebooks/run_model_in_notebook.ipynb) -on CRIB is not supported because MATLAB Runtime can not be installed there. +```sh +# Download and install Mamba on linux +wget https://github.com/conda-forge/miniforge/releases/latest/download/Mambaforge-pypy3-Linux-x86_64.sh +bash Mambaforge-pypy3-Linux-x86_64.sh -b -p ~/mamba -## Developers +# Update base environment +. ~/mamba/bin/activate +mamba update --name base mamba -If you want to contribute to the development of PyStemmusScope, -have a look at the [contribution guidelines](https://pystemmusscope.readthedocs.io/en/latest/contributing_link.html). +# Download environment file +wget https://raw.githubusercontent.com/EcoExtreML/STEMMUS_SCOPE_Processing/main/environment.yml -## Credits +# Create a conda environment called 'pystemmusscope' with all required dependencies +mamba env create -f environment.yml -This package was created with [Cookiecutter](https://github.com/audreyr/cookiecutter) and the [NLeSC/python-template](https://github.com/NLeSC/python-template). +# The environment can be activated with +. ~/mamba/bin/activate pystemmusscope + +``` diff --git a/docs/conf.py b/docs/conf.py index 11a0b10a..ae6af940 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -48,6 +48,13 @@ "myst_parser", ] +source_suffix = { + '.rst': 'restructuredtext', + '.txt': 'markdown', + '.md': 'markdown', +} +myst_heading_anchors = 3 + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] diff --git a/docs/index.rst b/docs/index.rst index c2d9623c..f053c993 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -15,13 +15,20 @@ soil-plant-atmosphere continuum model based on the models `STEMMUS `_, and models canopy photosynthesis, fluorescence, and the transfer of energy, mass, and momentum. -For a guide on how to setup and run the model, see the :doc:`user guide `. +For a guide on how to setup and run the model, see the :doc:`User guide `. If you are interested in contributing to the PyStemmusScope code, see the -:doc:`contributing guide `. +:doc:`Contributing guide `. + +Credits +------- + +This package was created with +`Cookiecutter `_ and the +`NLeSC/python-template `_. .. toctree:: :maxdepth: 0 :hidden: User guide - Contributing guide \ No newline at end of file + Contributing guide diff --git a/environment_3.8.yml b/environment_3.8.yml deleted file mode 100644 index 36900b11..00000000 --- a/environment_3.8.yml +++ /dev/null @@ -1,10 +0,0 @@ ---- -name: pystemmusscope_3.8 -channels: - - conda-forge -dependencies: - - pip - - python=3.8 # supported by matlab R2021a, see issue #31 - - jupyterlab # needed to run notebooks - - pip: - - git+https://github.com/EcoExtreML/STEMMUS_SCOPE_Processing.git@main # will be replaced by `pip install pystemmusscope` diff --git a/notebooks/run_model_in_notebook.ipynb b/notebooks/run_model_in_notebook.ipynb index 514b9ca0..e02df5f0 100644 --- a/notebooks/run_model_in_notebook.ipynb +++ b/notebooks/run_model_in_notebook.ipynb @@ -4,13 +4,29 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## Run the STEMMUS_SCOPE model\n", - "Steps to run the STEMMUS_SCOPE model, including preprocessing and postprocessing, on Surf super computer Snellius or on a unix system." + "## STEMMUS_SCOPE model\n", + "This notebook shows steps to run the STEMMUS_SCOPE model, including preprocessing and postprocessing. STEMMUS_SCOPE files are located in a **private** repository https://github.com/EcoExtreML/STEMMUS_SCOPE. To clone the repository locally and access the files, make sure you have right access to the repository. Then specify the path to a config file e.g.`config_file_template.txt`, executable file `STEMMUS_SCOPE` or source code `STEMMUS_SCOPE/src` in the cells below. There are already config files for users on Snellius and CRIB, see `config_file_snellius.txt`, `config_file_crib.txt`. Choose how do you want to run the model:\n", + "\n", + "### 1. Using executable file:\n", + "If MATLAB Runtime is installed on a Unix-like system, you can run STEMMUS_SCOPE using the executable file, for example on Snellius or WSL.\n", + "\n", + "### 2. Using Matlab:\n", + "If Matlab is installed, you can run STEMMUS_SCOPE from the source code, for example on Snellius or CRIB.\n", + "\n", + "### 3. Using Octave:\n", + "If latest Octave including required packages is installed, you can run STEMMUS_SCOPE from the source code, for example on CRIB or your own machine." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1. Using executable file:" ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 16, "metadata": {}, "outputs": [], "source": [ @@ -20,107 +36,153 @@ ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 17, "metadata": {}, + "outputs": [], "source": [ - "#### Path to STEMMUS_SCOPE\n", - "\n", - "STEMMUS_SCOPE executable file is located in the **private** repository on GitHub https://github.com/EcoExtreML/STEMMUS_SCOPE. You need to clone the repository locally and specify the path to it in the cell below. Make sure you have right access to the repository. " + "# user must provide the correct path\n", + "path_to_config_file = \"~/my_config_template.txt\"\n", + "path_to_exe_file = \"~/STEMMUS_SCOPE/exe/STEMMUS_SCOPE\"" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "#### Update/set config files" + "If you run the model on your own machine, make sure `LD_LIBRARY_PATH` is set correctly. To do this, uncomment the cell below and run it:" ] }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "# user must provide the correct path\n", - "path_to_config_file = \"./config_file_template.txt\"\n", - "path_to_exe_file = \"./STEMMUS_SCOPE\"" + "# # Set LD_LIBRARY_PATH\n", + "# matlab_path = !whereis MATLAB\n", + "# matlab_path = matlab_path.s.split(\": \")[1]\n", + "# os.environ['LD_LIBRARY_PATH'] = (\n", + "# f\"{matlab_path}/MATLAB_Runtime/v910/runtime/glnxa64:\"\n", + "# f\"{matlab_path}/MATLAB_Runtime/v910/bin/glnxa64:\"\n", + "# f\"{matlab_path}/MATLAB_Runtime/v910/sys/os/glnxa64:\"\n", + "# f\"{matlab_path}/MATLAB_Runtime/v910/extern/bin/glnxa64:\"\n", + "# f\"{matlab_path}/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\")\n", + "# print(os.environ['LD_LIBRARY_PATH'])" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 19, "metadata": {}, + "outputs": [], "source": [ - "If you run the model locally, you need to first intsall MATLAB Runtime, see instructions in the [readme](https://github.com/EcoExtreML/processing#readme). Then set `LD_LIBRARY_PATH`. To do this, uncomment the cell below and run it:" + "# create an an instance of the model\n", + "model = StemmusScope(config_file=path_to_config_file, model_src_path=path_to_exe_file)" ] }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 20, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "/usr/local/MATLAB/MATLAB_Runtime/v910/runtime/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/bin/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/sys/os/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/extern/bin/glnxa64:/usr/local/MATLAB/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\n" + "New config file /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1642/ZA-Kru_2022-10-27-1642_config.txt\n", + "Model input dir /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1642/\n", + "Model output dir /gpfs/home2/alidoost/test_matlab/output/ZA-Kru_2022-10-27-1642/\n" ] } ], "source": [ - "# # Set LD_LIBRARY_PATH\n", - "# matlab_path = !whereis MATLAB\n", - "# matlab_path = matlab_path.s.split(\": \")[1]\n", - "# os.environ['LD_LIBRARY_PATH'] = (\n", - "# f\"{matlab_path}/MATLAB_Runtime/v910/runtime/glnxa64:\"\n", - "# f\"{matlab_path}/MATLAB_Runtime/v910/bin/glnxa64:\"\n", - "# f\"{matlab_path}/MATLAB_Runtime/v910/sys/os/glnxa64:\"\n", - "# f\"{matlab_path}/MATLAB_Runtime/v910/extern/bin/glnxa64:\"\n", - "# f\"{matlab_path}/MATLAB_Runtime/v910/sys/opengl/lib/glnxa64\")\n", - "# print(os.environ['LD_LIBRARY_PATH'])" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run the model for one forcing file" + "# setup the model\n", + "# here you can change the name of forcing file and number of time steps\n", + "config_path = model.setup(\n", + " ForcingFileName=\"ZA-Kru_2000-2002_FLUXNET2015_Met.nc\",\n", + " NumberOfTimeSteps=\"10\",\n", + ")\n", + "\n", + "# new config file genertaed to run the model \n", + "print(f\"New config file {config_path}\")\n", + "\n", + "# see input and output paths generated by the model\n", + "print(f'Model input dir {model.config[\"InputPath\"]}')\n", + "print(f'Model output dir {model.config[\"OutputPath\"]}')" ] }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 21, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n", - " 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n", - " 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n", - " 'ForcingFileName': 'FI-Hyy_1996-2014_FLUXNET2015_Met.nc',\n", - " 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n", - " 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n", - " 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n", - " 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n", - " 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n", - " 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n", - " 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n", - " 'NumberOfTimeSteps': '5',\n", - " 'InputPath': '',\n", - " 'OutputPath': ''}" + "b'Opening log file: /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1642//java.log.5036\\nReading config from /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1642/ZA-Kru_2022-10-27-1642_config.txt\\nThe calculations start now\\nThe calculations end now\\n'" ] }, - "execution_count": 5, + "execution_count": 21, "metadata": {}, "output_type": "execute_result" } ], + "source": [ + "# run the model\n", + "result = model.run()\n", + "result" + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "# save output in netcdf format\n", + "required_netcdf_variables = \"~/STEMMUS_SCOPE/utils/csv_to_nc/required_netcdf_variables.csv\"\n", + "nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", + "print(nc_file_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Using Matlab:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from PyStemmusScope import StemmusScope\n", + "from PyStemmusScope import save" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [], + "source": [ + "# user must provide the correct path\n", + "path_to_config_file = \"~/my_config_template.txt\"\n", + "path_to_model_src = \"~/STEMMUS_SCOPE/src\"" + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [], "source": [ "# create an an instance of the model\n", - "model = StemmusScope(config_file=path_to_config_file, exe_file=path_to_exe_file)\n", - "\n", - "# inspect model config template\n", - "model.config" + "model = StemmusScope(config_file=path_to_config_file, model_src_path=path_to_model_src, interpreter=\"Matlab\")" ] }, { @@ -132,17 +194,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "/gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1439/ZA-Kru_2022-08-08-1439_config.txt\n" + "New config file /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1651/ZA-Kru_2022-10-27-1651_config.txt\n", + "Model input dir /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1651/\n", + "Model output dir /gpfs/home2/alidoost/test_matlab/output/ZA-Kru_2022-10-27-1651/\n" ] } ], "source": [ "# setup the model\n", + "# here you can change the name of forcing file and number of time steps\n", "config_path = model.setup(\n", " ForcingFileName=\"ZA-Kru_2000-2002_FLUXNET2015_Met.nc\",\n", " NumberOfTimeSteps=\"10\",\n", ")\n", - "print(config_path)" + "\n", + "# new config file genertaed to run the model \n", + "print(f\"New config file {config_path}\")\n", + "\n", + "# see input and output paths generated by the model\n", + "print(f'Model input dir {model.config[\"InputPath\"]}')\n", + "print(f'Model output dir {model.config[\"OutputPath\"]}')" ] }, { @@ -153,20 +224,7 @@ { "data": { "text/plain": [ - "{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n", - " 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n", - " 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n", - " 'ForcingFileName': 'ZA-Kru_2000-2002_FLUXNET2015_Met.nc',\n", - " 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n", - " 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n", - " 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n", - " 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n", - " 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n", - " 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n", - " 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n", - " 'NumberOfTimeSteps': '10',\n", - " 'InputPath': '/gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1439/',\n", - " 'OutputPath': '/gpfs/scratch1/shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-08-08-1439/'}" + "b'MATLAB is selecting SOFTWARE OPENGL rendering.\\nOpening log file: /home/alidoost/java.log.37351\\n\\n < M A T L A B (R) >\\n Copyright 1984-2021 The MathWorks, Inc.\\n R2021a Update 3 (9.10.0.1684407) 64-bit (glnxa64)\\n May 27, 2021\\n\\n \\nTo get started, type doc.\\nFor product information, visit www.mathworks.com.\\n \\nReading config from /gpfs/home2/alidoost/test_matlab/input/ZA-Kru_2022-10-27-1651/ZA-Kru_2022-10-27-1651_config.txt\\nThe calculations start now\\nThe calculations end now\\n'" ] }, "execution_count": 7, @@ -175,103 +233,140 @@ } ], "source": [ - "# inspect model configs again\n", - "model.config" + "# run the model\n", + "result = model.run()\n", + "result" ] }, { "cell_type": "code", - "execution_count": 8, + "execution_count": null, "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "b'Opening log file: /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1439//java.log.51183\\nReading config from /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1439/ZA-Kru_2022-08-08-1439_config.txt\\nThe calculations start now\\nThe calculations end now\\n'" - ] - }, - "execution_count": 8, - "metadata": {}, - "output_type": "execute_result" - } - ], + "outputs": [], "source": [ - "# run the model\n", - "result = model.run()\n", - "result" + "# save output in netcdf format\n", + "required_netcdf_variables = \"~/STEMMUS_SCOPE/utils/csv_to_nc/required_netcdf_variables.csv\"\n", + "nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", + "print(nc_file_name)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. Using Octave:" + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "from PyStemmusScope import StemmusScope\n", + "from PyStemmusScope import save" + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# user must provide the correct path\n", + "path_to_config_file = \"~/my_config_template.txt\"\n", + "path_to_model_src = \"~/STEMMUS_SCOPE/src\"" ] }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# create an an instance of the model\n", + "model = StemmusScope(config_file=path_to_config_file, model_src_path=path_to_model_src, interpreter=\"Octave\")" + ] + }, + { + "cell_type": "code", + "execution_count": 4, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "/gpfs/scratch1/shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-08-08-1439/ZA-Kru_2022-08-08-1439_STEMMUS_SCOPE.nc\n" + "New config file /data/private/Sarah/test_run/input/ZA-Kru_2022-10-28-1159/ZA-Kru_2022-10-28-1159_config.txt\n", + "Model input dir /data/private/Sarah/test_run/input/ZA-Kru_2022-10-28-1159/\n", + "Model output dir /data/private/Sarah/test_run/output/ZA-Kru_2022-10-28-1159/\n" ] } ], "source": [ - "# save output in netcdf format\n", - "required_netcdf_variables = \"./required_netcdf_variables.csv\"\n", - "nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", - "print(nc_file_name)" + "# setup the model\n", + "# here you can change the name of forcing file and number of time steps\n", + "config_path = model.setup(\n", + " ForcingFileName=\"ZA-Kru_2000-2002_FLUXNET2015_Met.nc\",\n", + " NumberOfTimeSteps=\"10\",\n", + ")\n", + "\n", + "# new config file genertaed to run the model \n", + "print(f\"New config file {config_path}\")\n", + "\n", + "# see input and output paths generated by the model\n", + "print(f'Model input dir {model.config[\"InputPath\"]}')\n", + "print(f'Model output dir {model.config[\"OutputPath\"]}')" ] }, { - "cell_type": "markdown", + "cell_type": "code", + "execution_count": 5, "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Reading config from /data/private/Sarah/test_run/input/ZA-Kru_2022-10-28-1159/ZA-Kru_2022-10-28-1159_config.txt\n", + "The calculations start now\n", + "The calculations end now\n" + ] + } + ], "source": [ - "## Run the model for several forcing files" + "# run the model\n", + "result = model.run()\n", + "result" ] }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "b'Opening log file: /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1442//java.log.63885\\nReading config from /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1442/ZA-Kru_2022-08-08-1442_config.txt\\nThe calculations start now\\nThe calculations end now\\n'\n", - "/gpfs/scratch1/shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-08-08-1442/ZA-Kru_2022-08-08-1442_STEMMUS_SCOPE.nc\n", - "b'Opening log file: /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/AR-SLu_2022-08-08-1442//java.log.403\\nReading config from /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/AR-SLu_2022-08-08-1442/AR-SLu_2022-08-08-1442_config.txt\\nThe calculations start now\\nThe calculations end now\\n'\n", - "/gpfs/scratch1/shared/ecoextreml/stemmus_scope/output/AR-SLu_2022-08-08-1442/AR-SLu_2022-08-08-1442_STEMMUS_SCOPE.nc\n" + "/data/private/Sarah/test_run/output/ZA-Kru_2022-10-28-1159/ZA-Kru_2022-10-28-1159_STEMMUS_SCOPE.nc\n" ] } ], "source": [ - "from pathlib import Path\n", - "\n", - "forcing_filenames_list = [\n", - " \"ZA-Kru_2000-2002_FLUXNET2015_Met.nc\",\n", - " \"AR-SLu_2010-2010_FLUXNET2015_Met.nc\",\n", - "]\n", - "\n", - "full_run = False\n", - "if full_run:\n", - " forcing_filenames_list = [file.name for file in Path(model.config[\"ForcingPath\"]).iterdir()]\n", - "\n", - "for nc_file in forcing_filenames_list:\n", - " # setup the model\n", - " config_path = model.setup(\n", - " ForcingFileName=nc_file,\n", - " NumberOfTimeSteps=\"10\",\n", - " )\n", - "\n", - " # run the model\n", - " result = model.run()\n", - " print(result)\n", - " \n", - " # save results in a netcdf file\n", - " nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", - " print(nc_file_name)\n", - " " + "# save output in netcdf format\n", + "required_netcdf_variables = \"~/STEMMUS_SCOPE/utils/csv_to_nc/required_netcdf_variables.csv\"\n", + "nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", + "print(nc_file_name)" ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [] } ], "metadata": { diff --git a/notebooks/run_model_in_notebook_dev.ipynb b/notebooks/run_model_in_notebook_dev.ipynb deleted file mode 100644 index 952d89cc..00000000 --- a/notebooks/run_model_in_notebook_dev.ipynb +++ /dev/null @@ -1,343 +0,0 @@ -{ - "cells": [ - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Run the STEMMUS_SCOPE model\n", - "Steps to run the STEMMUS_SCOPE model, including preprocessing and postprocessing, on Surf super computer Snellius." - ] - }, - { - "cell_type": "code", - "execution_count": 9, - "metadata": {}, - "outputs": [], - "source": [ - "import os\n", - "from pathlib import Path\n", - "import subprocess\n", - "from PyStemmusScope import StemmusScope\n", - "from PyStemmusScope import save" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Path to STEMMUS_SCOPE\n", - "\n", - "STEMMUS_SCOPE source codes are located in the **private** repository on GitHub https://github.com/EcoExtreML/STEMMUS_SCOPE. You need to clone the repository locally and specify the path to it in the cell below. Make sure you have right access to the repository and you created a local branch for development purposes. " - ] - }, - { - "cell_type": "code", - "execution_count": 2, - "metadata": {}, - "outputs": [], - "source": [ - "# path to model repository\n", - "path_to_model = Path(\"path_to_STEMMUS_SCOPE_repository\")" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### set git branch and see status" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "dev_branch = \"python-compliant\"\n", - "!git -C {path_to_model} checkout {dev_branch}\n", - "!git -C {path_to_model} status" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Update/set config files" - ] - }, - { - "cell_type": "code", - "execution_count": 4, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n", - " 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n", - " 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n", - " 'ForcingFileName': 'FI-Hyy_1996-2014_FLUXNET2015_Met.nc',\n", - " 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n", - " 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n", - " 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n", - " 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n", - " 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n", - " 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n", - " 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n", - " 'NumberOfTimeSteps': '5',\n", - " 'InputPath': '',\n", - " 'OutputPath': ''}" - ] - }, - "execution_count": 4, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# the user must provide the correct path\n", - "# for example:\n", - "path_to_config_file = path_to_model / \"config_file_snellius.txt\"\n", - "path_to_exe_file = path_to_model / \"exe\" / \"STEMMUS_SCOPE\"\n", - "\n", - "# create an an instance of the model\n", - "model = StemmusScope(config_file=path_to_config_file, exe_file=path_to_exe_file)\n", - "\n", - "# inspect model config template\n", - "model.config" - ] - }, - { - "cell_type": "code", - "execution_count": 5, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1446/ZA-Kru_2022-08-08-1446_config.txt\n" - ] - } - ], - "source": [ - "# setup the model\n", - "config_path = model.setup(\n", - " ForcingFileName=\"ZA-Kru_2000-2002_FLUXNET2015_Met.nc\",\n", - " NumberOfTimeSteps=\"5\",\n", - ")\n", - "print(config_path)" - ] - }, - { - "cell_type": "code", - "execution_count": 6, - "metadata": {}, - "outputs": [ - { - "data": { - "text/plain": [ - "{'WorkDir': '/scratch-shared/ecoextreml/stemmus_scope/',\n", - " 'SoilPropertyPath': '/projects/0/einf2480/model_parameters/soil_property/',\n", - " 'ForcingPath': '/projects/0/einf2480/forcing/plumber2_data/',\n", - " 'ForcingFileName': 'ZA-Kru_2000-2002_FLUXNET2015_Met.nc',\n", - " 'directional': '/projects/0/einf2480/model_parameters/vegetation_property/directional/',\n", - " 'fluspect_parameters': '/projects/0/einf2480/model_parameters/vegetation_property/fluspect_parameters/',\n", - " 'leafangles': '/projects/0/einf2480/model_parameters/vegetation_property/leafangles/',\n", - " 'radiationdata': '/projects/0/einf2480/model_parameters/vegetation_property/radiationdata/',\n", - " 'soil_spectrum': '/projects/0/einf2480/model_parameters/vegetation_property/soil_spectrum/',\n", - " 'input_data': '/projects/0/einf2480/model_parameters/vegetation_property/input_data.xlsx',\n", - " 'InitialConditionPath': '/projects/0/einf2480/soil_initialcondition/',\n", - " 'NumberOfTimeSteps': '5',\n", - " 'InputPath': '/gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1446/',\n", - " 'OutputPath': '/gpfs/scratch1/shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-08-08-1446/'}" - ] - }, - "execution_count": 6, - "metadata": {}, - "output_type": "execute_result" - } - ], - "source": [ - "# inspect model config again\n", - "model.config" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Run the model\n", - "\n", - "`model.run()` uses executable file. Here, for development purposes, we call MATLAB on the main matlab script i.e. `STEMMUS_SCOPE_exe` instead of calling `model.run()`." - ] - }, - { - "cell_type": "code", - "execution_count": 7, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "MATLAB is selecting SOFTWARE OPENGL rendering.\n", - "Opening log file: /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1446//java.log.32841\n", - "\n", - " < M A T L A B (R) >\n", - " Copyright 1984-2021 The MathWorks, Inc.\n", - " R2021a Update 3 (9.10.0.1684407) 64-bit (glnxa64)\n", - " May 27, 2021\n", - "\n", - " \n", - "To get started, type doc.\n", - "For product information, visit www.mathworks.com.\n", - " \n", - "Reading config from /gpfs/scratch1/shared/ecoextreml/stemmus_scope/input/ZA-Kru_2022-08-08-1446/ZA-Kru_2022-08-08-1446_config.txt\n", - "The calculations start now\n", - "The calculations end now\n" - ] - } - ], - "source": [ - "# generate a src folder\n", - "path_to_code = path_to_model / \"src\"\n", - "\n", - "# set matlab arguments\n", - "path_to_config = f\"'{config_path}'\"\n", - "command_line = f'matlab -r \"STEMMUS_SCOPE_exe({path_to_config});exit;\"'\n", - "args = [command_line, \"-nodisplay\", \"-nosplash\", \"-nodesktop\"]\n", - "\n", - "# run the model\n", - "result = subprocess.run(args, cwd = path_to_code, shell=True)\n", - "result.check_returncode()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### save results in a netcdf file" - ] - }, - { - "cell_type": "code", - "execution_count": 10, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "/gpfs/scratch1/shared/ecoextreml/stemmus_scope/output/ZA-Kru_2022-08-08-1446/ZA-Kru_2022-08-08-1446_STEMMUS_SCOPE.nc\n" - ] - } - ], - "source": [ - "required_netcdf_variables = path_to_model / \"utils\" / \"csv_to_nc\" / \"required_netcdf_variables.csv\"\n", - "nc_file_name = save.to_netcdf(config_path, required_netcdf_variables)\n", - "print(nc_file_name)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### Generate execuable file" - ] - }, - { - "cell_type": "code", - "execution_count": 11, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Opening log file: ./java.log.52377\n" - ] - } - ], - "source": [ - "os.environ['MATLAB_LOG_DIR'] = \".\"\n", - "command_line = 'mcc -m ./src/STEMMUS_SCOPE_exe.m -a ./src -d ./exe -o STEMMUS_SCOPE -R nodisplay -R singleCompThread'\n", - "result = subprocess.run(command_line, cwd = path_to_model, shell=True)\n", - "result.check_returncode()" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "#### check changes by git" - ] - }, - { - "cell_type": "code", - "execution_count": 12, - "metadata": {}, - "outputs": [ - { - "name": "stdout", - "output_type": "stream", - "text": [ - "On branch python-compliant\n", - "Your branch is up to date with 'origin/python-compliant'.\n", - "\n", - "Changes not staged for commit:\n", - " (use \"git add ...\" to update what will be committed)\n", - " (use \"git restore ...\" to discard changes in working directory)\n", - "\t\u001b[31mmodified: exe/STEMMUS_SCOPE\u001b[m\n", - "\n", - "Untracked files:\n", - " (use \"git add ...\" to include in what will be committed)\n", - "\t\u001b[31m.ipynb_checkpoints/\u001b[m\n", - "\t\u001b[31mexe/mccExcludedFiles.log\u001b[m\n", - "\t\u001b[31mexe/readme.txt\u001b[m\n", - "\t\u001b[31mexe/requiredMCRProducts.txt\u001b[m\n", - "\t\u001b[31mexe/run_STEMMUS_SCOPE.sh\u001b[m\n", - "\t\u001b[31mexe/unresolvedSymbols.txt\u001b[m\n", - "\t\u001b[31mjava.log.52377\u001b[m\n", - "\t\u001b[31mtest_config.txt\u001b[m\n", - "\t\u001b[31mutils/csv_to_nc/__pycache__/\u001b[m\n", - "\n", - "no changes added to commit (use \"git add\" and/or \"git commit -a\")\n" - ] - } - ], - "source": [ - "!git -C {path_to_model} status" - ] - } - ], - "metadata": { - "interpreter": { - "hash": "28b136f154b3384fcb2854e5626613232692c304dbc5315064ef4c9363104a2c" - }, - "kernelspec": { - "display_name": "Python 3 (ipykernel)", - "language": "python", - "name": "python3" - }, - "language_info": { - "codemirror_mode": { - "name": "ipython", - "version": 3 - }, - "file_extension": ".py", - "mimetype": "text/x-python", - "name": "python", - "nbconvert_exporter": "python", - "pygments_lexer": "ipython3", - "version": "3.9.12" - }, - "metadata": { - "interpreter": { - "hash": "a8aa8210cb3d81d2b5c08b30572e78ad19bd30da92b4caf9890388bc07c5e3bf" - } - } - }, - "nbformat": 4, - "nbformat_minor": 4 -} diff --git a/run_jupyter_lab_snellius.sh b/run_jupyter_lab_snellius.sh index ca5ee3bf..cb3f5bd1 100644 --- a/run_jupyter_lab_snellius.sh +++ b/run_jupyter_lab_snellius.sh @@ -19,9 +19,12 @@ # Load module needed to run the model (no need for license) module load 2021 -### This is for Matlab +### This is for Matlab Runtime module load MCR/R2021a.3 +### This is for Matlab +module load MATLAB/2021a-upd3 + # Some security: stop script on error and undefined variables set -euo pipefail diff --git a/run_jupyter_lab_snellius_dev.sh b/run_jupyter_lab_snellius_dev.sh deleted file mode 100755 index e5b75783..00000000 --- a/run_jupyter_lab_snellius_dev.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# -# Serve a jupyter lab environment from a compute node on Snellius -# see https://servicedesk.surfsara.nl/wiki/pages/viewpage.action?pageId=30660252 -# usage: sbatch run_jupyter_lab_snellius_dev.sh - -# SLURM settings -#SBATCH -J jupyter_lab -#SBATCH -t 02:00:00 -#SBATCH -N 1 -#SBATCH --ntasks=1 -#SBATCH --cpus-per-task=32 -#SBATCH -p thin -#SBATCH --output=./slurm_%j.out -#SBATCH --error=./slurm_%j.out - -# Some security: stop script on error and undefined variables -set -euo pipefail - -# Use an appropriate conda environment -. ~/mamba/bin/activate pystemmusscope - -############ Use module MATLAB/2021a-upd3 to either run the source code or build the executable file ############ -############ This needs a matlab license, make sure your account is added to the license pool ############ -# Load matlab module -# On Snellius Try: "module spider MATLAB" to see how to load the module(s). -module load 2021 -module load MATLAB/2021a-upd3 - -# Choose random port and print instructions to connect -PORT=`shuf -i 5000-5999 -n 1` -LOGIN_HOST_EXT=int3-pub.snellius.surf.nl -LOGIN_HOST_INT=int3 - -echo "Selected port is: " $PORT -echo -echo "To connect to the notebook type the following command from your local terminal:" -echo "ssh -L ${PORT}:localhost:${PORT} ${USER}@${LOGIN_HOST_EXT}" - -ssh -o StrictHostKeyChecking=no -f -N -p 22 -R $PORT:localhost:$PORT $LOGIN_HOST_INT - -# Start the jupyter lab session -jupyter lab --no-browser --port $PORT diff --git a/setup.cfg b/setup.cfg index b855744f..c6f9a28b 100644 --- a/setup.cfg +++ b/setup.cfg @@ -33,7 +33,7 @@ include_package_data = True packages = find: install_requires = hdf5storage - netcdf4 + netcdf4<=1.5.8 numpy pandas xarray diff --git a/tests/test_stemmus_scope.py b/tests/test_stemmus_scope.py index 6270cfea..1e31b776 100644 --- a/tests/test_stemmus_scope.py +++ b/tests/test_stemmus_scope.py @@ -1,15 +1,50 @@ +import os +import shlex +import subprocess from pathlib import Path from unittest.mock import patch - import pytest - -import os -import subprocess from PyStemmusScope import StemmusScope from PyStemmusScope import config_io +from PyStemmusScope import utils from . import data_folder +class TestInit: + def test_model_without_exe(self, tmp_path): + config_file = str(data_folder / "config_file_test.txt") + exe_file = Path(tmp_path) / "STEMMUS_SCOPE" + if utils.os_name() == 'nt': + with pytest.raises(FileNotFoundError): + StemmusScope(config_file, model_src_path=exe_file) + else: + with pytest.raises(ValueError) as excinfo: + StemmusScope(config_file, model_src_path=exe_file) + assert "Provide a valid path to an executable file" in str(excinfo.value) + + def test_model_without_src(self): + config_file = str(data_folder / "config_file_test.txt") + if utils.os_name() == 'nt': + with pytest.raises(FileNotFoundError): + StemmusScope(config_file, model_src_path="src") + else: + with pytest.raises(ValueError) as excinfo: + StemmusScope(config_file, model_src_path="src") + assert "Provide a valid path to an executable file" in str(excinfo.value) + + def test_model_without_interpreter(self, tmp_path): + config_file = str(data_folder / "config_file_test.txt") + with pytest.raises(ValueError) as excinfo: + StemmusScope(config_file, model_src_path=tmp_path) + assert "Set `interpreter` as Octave or Matlab" in str(excinfo.value) + + def test_model_wrong_interpreter(self, tmp_path): + config_file = str(data_folder / "config_file_test.txt") + with pytest.raises(ValueError) as excinfo: + StemmusScope(config_file, model_src_path=tmp_path, interpreter="Nothing") + assert "Set `interpreter` as Octave or Matlab" in str(excinfo.value) + + class TestWithDefaults: @pytest.fixture def model(self, tmp_path): @@ -20,7 +55,7 @@ def model(self, tmp_path): with open(exe_file, "x", encoding="utf8") as dummy_file: dummy_file.close() - yield StemmusScope(config_file, exe_file) + yield StemmusScope(config_file, model_src_path=exe_file) @pytest.fixture def model_with_setup(self, model): @@ -41,30 +76,35 @@ def test_setup(self, model_with_setup): assert actual_output_dir == Path(model.config["OutputPath"]) assert actual_cfg_file == cfg_file - # matlab log dir - assert os.environ['MATLAB_LOG_DIR'] == str(model.config["InputPath"]) - - @patch("subprocess.run") - def test_run(self, mocked_run, model_with_setup): + @patch("subprocess.Popen") + def test_run_exe_file(self, mocked_popen, model_with_setup): - actual_cfg_file = data_folder / "directories" / "input" / "XX-dummy_2022-07-11-1200_config.txt" - output = ( + actual_cfg_file = data_folder / "directories" / "input" / "XX-dummy_2022-07-11-1200" / "XX-dummy_2022-07-11-1200_config.txt" + actual_log = ( f"b'Reading config from {actual_cfg_file}\n\n " "The calculations start now \r\n The calculations end now \r'" - ) - - mocked_run.return_value.stdout = output + ).encode() + mocked_popen.return_value.communicate.return_value = (actual_log, "error") + mocked_popen.return_value.wait.return_value = 0 model, cfg_file = model_with_setup result = model.run() expected = [f"{model.exe_file} {cfg_file}"] - mocked_run.assert_called_with( - expected, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True, + mocked_popen.assert_called_with( + expected, cwd=None, + stdout=subprocess.PIPE, + stderr=subprocess.PIPE, shell=True, ) # output of subprocess - assert result == output + expected_log = ( + f"b'Reading config from {cfg_file}\n\n " + "The calculations start now \r\n The calculations end now \r'" + ) + assert result == expected_log + # matlab log dir + assert os.environ['MATLAB_LOG_DIR'] == str(model.config["InputPath"]) class TestWithCustomSetup: @@ -76,7 +116,7 @@ def model(self, tmp_path): # create dummy exe file with open(exe_file, "x", encoding="utf8") as dummy_file: dummy_file.close() - yield StemmusScope(config_file, exe_file) + yield StemmusScope(config_file, model_src_path=exe_file) @pytest.fixture def model_with_setup(self, model, tmp_path): @@ -101,32 +141,142 @@ def test_setup(self, model_with_setup, tmp_path): assert actual_cfg_file == cfg_file assert model.config["NumberOfTimeSteps"] == "5" - # matlab log dir - assert os.environ['MATLAB_LOG_DIR'] == str(model.config["InputPath"]) - def test_config(self, model_with_setup): model, cfg_file = model_with_setup actual = config_io.read_config(cfg_file) assert actual == model.config - @patch("subprocess.run") - def test_run(self, mocked_run, model_with_setup, tmp_path): + @patch("subprocess.Popen") + def test_run_exe_file(self, mocked_popen, model_with_setup, tmp_path): - actual_cfg_file = tmp_path / "input" / "dummy_2022-07-11-1200_config.txt" - output = ( + actual_cfg_file = tmp_path / "input" / "dummy_2022-07-11-1200" / "dummy_2022-07-11-1200_config.txt" + actual_log = ( f"b'Reading config from {actual_cfg_file}\n\n " "The calculations start now \r\n The calculations end now \r'" - ) + ).encode() + mocked_popen.return_value.communicate.return_value = (actual_log, "error") + mocked_popen.return_value.wait.return_value = 0 - mocked_run.return_value.stdout = output model, cfg_file = model_with_setup result = model.run() expected = [f"{model.exe_file} {cfg_file}"] - mocked_run.assert_called_with( - expected, stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True, check=True, + mocked_popen.assert_called_with( + expected, cwd=None, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, ) # output of subprocess - assert result == output + expected_log = ( + f"b'Reading config from {cfg_file}\n\n " + "The calculations start now \r\n The calculations end now \r'" + ) + assert result == expected_log + # matlab log dir + assert os.environ['MATLAB_LOG_DIR'] == str(model.config["InputPath"]) + + +class TestWithMatlab: + @pytest.fixture + def model(self, tmp_path): + config_file = str(data_folder / "config_file_test.txt") + yield StemmusScope(config_file, model_src_path=tmp_path, interpreter="Matlab") + + @pytest.fixture + def model_with_setup(self, model): + with patch("time.strftime") as mocked_time: + mocked_time.return_value = "2022-07-11-1200" + + cfg_file = model.setup() + return model, cfg_file + + @patch("subprocess.Popen") + def test_run_matlab(self, mocked_popen, model_with_setup, tmp_path): + + actual_cfg_file = data_folder / "directories" / "input" / "XX-dummy_2022-07-11-1200" / "XX-dummy_2022-07-11-1200_config.txt" + actual_log = ( + "b'MATLAB is selecting SOFTWARE OPENGL rendering.\n..." + f"\nReading config from {actual_cfg_file}\n" + "The calculations start now\n The calculations end now\n'" + ).encode() + mocked_popen.return_value.communicate.return_value = (actual_log, "error") + mocked_popen.return_value.wait.return_value = 0 + + model, cfg_file = model_with_setup + result = model.run() + + path_to_config = f"'{actual_cfg_file}'" + eval_code= f'STEMMUS_SCOPE_exe({path_to_config});exit;' + args = ["matlab", "-r", eval_code, "-nodisplay", "-nosplash", "-nodesktop"] + # seperate args dont work on linux! + if utils.os_name() !="nt": + args = shlex.join(args) + + mocked_popen.assert_called_with( + args, cwd=tmp_path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, + ) + + # output of subprocess + expected_log = ( + "b'MATLAB is selecting SOFTWARE OPENGL rendering.\n..." + f"\nReading config from {cfg_file}\n" + "The calculations start now\n The calculations end now\n'" + ) + + assert result == expected_log + + +class TestWithOctave: + @pytest.fixture + def model(self, tmp_path): + config_file = str(data_folder / "config_file_test.txt") + yield StemmusScope(config_file, model_src_path=tmp_path, interpreter="Octave") + + @pytest.fixture + def model_with_setup(self, model): + with patch("time.strftime") as mocked_time: + mocked_time.return_value = "2022-07-11-1200" + + cfg_file = model.setup() + return model, cfg_file + + @patch("subprocess.Popen") + def test_run_matlab(self, mocked_popen, model_with_setup, tmp_path): + + actual_cfg_file = data_folder / "directories" / "input" / "XX-dummy_2022-07-11-1200" / "XX-dummy_2022-07-11-1200_config.txt" + actual_log = ( + f"b'Reading config from {actual_cfg_file}\n" + "The calculations start now\n The calculations end now \n'" + ).encode() + mocked_popen.return_value.communicate.return_value = (actual_log, "error") + mocked_popen.return_value.wait.return_value = 0 + + model, cfg_file = model_with_setup + result = model.run() + + path_to_config = f"'{actual_cfg_file}'" + # fix for windows + path_to_config = path_to_config.replace("\\", "/") + eval_code = f'STEMMUS_SCOPE_exe({path_to_config});exit;' + args = ["octave", "--eval", eval_code, "--no-gui", "--silent"] + # seperate args dont work on linux! + if utils.os_name() !="nt": + args = shlex.join(args) + + mocked_popen.assert_called_with( + args, cwd=tmp_path, + stdout=subprocess.PIPE, stderr=subprocess.PIPE, + shell=True, + ) + + # output of subprocess + expected_log = ( + f"b'Reading config from {cfg_file}\n" + "The calculations start now\n The calculations end now \n'" + ) + + assert result == expected_log