Merge branch 'devel' into wangc/github_actions

.gitignore

@@ -73,6 +73,7 @@ node_*[0-9]
 *.[oe][0-9][0-9][0-9][0-9][0-9][0-9]
 *.[oe][0-9][0-9][0-9][0-9][0-9][0-9][0-9]
 framework/states.txt
+slurm-*.out
 
 #OSX
 .DS_Store

dependencies.xml

@@ -38,20 +38,20 @@ Note all install methods after "main" take
 <dependencies>
   <main>
     <h5py/>
-    <numpy>1.24</numpy>
-    <scipy>1.9</scipy>
-    <scikit-learn>1.0</scikit-learn>
+    <numpy>1.26</numpy>
+    <scipy>1.12</scipy>
+    <scikit-learn>1.1</scikit-learn>
     <pandas/>
     <!-- Note most versions of xarray work, but some (such as 0.20) don't -->
-    <xarray>2023</xarray>
+    <xarray/>
     <netcdf4 source="pip">1.6</netcdf4>
-    <matplotlib>3.5</matplotlib>
+    <matplotlib>3.6</matplotlib>
     <statsmodels>0.13</statsmodels>
     <cloudpickle/>
-    <tensorflow source="pip">2.13</tensorflow>
+    <tensorflow source="pip">2.14</tensorflow>
     <grpcio source="pip" />
     <!-- conda is really slow on windows if the version is not specified.-->
-    <python skip_check='True' os='windows'>3.10</python>
+    <python skip_check='True' os='windows'>3.11</python>
     <python skip_check='True' os='mac,linux'>3</python>
     <hdf5 skip_check='True'/>
     <swig skip_check='True'/>
@@ -66,14 +66,14 @@ Note all install methods after "main" take
     <nomkl os='linux' skip_check='True'/>
     <cmake skip_check='True' optional='True'/>
     <dask source="pip" pip_extra="[complete]"/>
-    <ray source="pip" pip_extra="[default]">2.6</ray>
+    <ray source="pip" pip_extra="[default]">2.38</ray>
     <!-- redis is needed by ray, but on windows, this seems to need to be explicitly stated -->
     <redis source="pip" os='windows'/>
     <imageio source="pip">2.22</imageio>
     <line_profiler optional='True'/>
     <!-- <ete3 optional='True'/> -->
     <statsforecast/>
-    <pywavelets>1.2</pywavelets>
+    <pywavelets>1.4</pywavelets>
     <python-sensors source="pip"/>
     <numdifftools source="pip">0.9</numdifftools>
     <fmpy optional='True'/>
@@ -83,7 +83,7 @@ Note all install methods after "main" take
     <ipopt skip_check='True' optional='True'/>
     <cyipopt optional='True'/>
     <pyomo-extensions source="pyomo" skip_check='True' optional='True'/>
-    <setuptools>69</setuptools> <!-- ray 2.6 can't be installed with setuptools 70 -->
+    <setuptools />
     <!-- source="mamba" are the ones installed when mamba is installed -->
     <!-- mamba version 2.0.0 causes failures on mac: critical libmamba filesystem error: in permissions: Operation not permitted-->
     <mamba source="mamba" skip_check='True' os='mac'>1.5</mamba>

developer_tools/check_pip_packages.sh

@@ -1,11 +1,11 @@
 #!/bin/bash
 
-# This script tests that the 3.9 and 3.8 wheels built in directory dist
+# This script tests that the 3.10 and 3.11 wheels built in directory dist
 # are installable. It will not work if there is more than one
 # version of the wheel for a python version (including for other os's)
 # It is designed to be run after (or similar):
 # rm -Rf dist; ./developer_tools/make_pip_packages.sh
-# It will install raven_framework in the python37_pip and python38_pip
+# It will install raven_framework in the python310_pip and python311_pip
 # conda environments
 # It requires that .ravenrc has a working CONDA_DEFS statement.
 # To run from the raven directory:
@@ -28,21 +28,21 @@ ls -l dist
 #python -m pip install -f file://${RAVEN_DIR}/dist raven_framework || exit -1
 
 echo
-echo Checking Python 3.9
+echo Checking Python 3.10
 
-conda activate python39_pip
+conda activate python310_pip
 python -m pip uninstall -y raven_framework || echo not installed
-python -m pip install dist/raven_framework*cp39*.whl || exit -1
+python -m pip install dist/raven_framework*cp310*.whl || exit -1
 
 # Run some tests to check that the installed package is working. The user_guide
 # tests are all pretty simple, and there are only a few of them, so we'll use those.
-$RAVEN_DIR/run_tests --re="user_guide" --tester-command RavenFramework raven_framework --skip-load-env
+$RAVEN_DIR/run_tests --re="user_guide" --tester-command RavenFramework raven_framework --skip-load-env || exit -1
 
 echo
-echo Checking Python 3.10
+echo Checking Python 3.11
 
-conda activate python310_pip
+conda activate python311_pip
 python -m pip uninstall -y raven_framework || echo not installed
-python -m pip install dist/raven_framework*cp310*.whl || exit -1
+python -m pip install dist/raven_framework*cp311*.whl || exit -1
 
-$RAVEN_DIR/run_tests --re="user_guide" --tester-command RavenFramework raven_framework --skip-load-env
+$RAVEN_DIR/run_tests --re="user_guide" --tester-command RavenFramework raven_framework --skip-load-env || exit -1

developer_tools/make_pip_packages.sh

@@ -18,24 +18,24 @@ source $RAVEN_DIR/scripts/read_ravenrc.sh
 CONDA_DEFS=$(read_ravenrc "CONDA_DEFS")
 source ${CONDA_DEFS}
 
-conda env remove --name python39_pip
-conda create -y --name python39_pip python=3.9 swig
-
 conda env remove --name python310_pip
 conda create -y --name python310_pip python=3.10 swig
 
+conda env remove --name python311_pip
+conda create -y --name python311_pip python=3.11 swig
+
 cd $RAVEN_DIR
 
 rm -f setup.cfg
 python ./scripts/library_handler.py pip --action=setup.cfg > setup.cfg
 
-conda activate python39_pip
+conda activate python310_pip
 command -v python
 python -m ensurepip
 python -m pip install --upgrade build
 python -m build
 
-conda activate python310_pip
+conda activate python311_pip
 command -v python
 python -m ensurepip
 python -m pip install --upgrade build

developer_tools/rook.ini

@@ -1,4 +1,4 @@
 [rook]
-add_non_default_run_types = qsub
+add_non_default_run_types = qsub, slurm
 test_dir = tests, doc/workshop
 

doc/user_manual/raven_user_manual.bib

@@ -25,6 +25,12 @@ @misc{PBS
 	howpublished = {http://www.pbsworks.com},
 	Bdsk-Url-1 = {http://www.pbsworks.com}}
 
+@misc{SLURM,
+Title = {Slurm Workload Manager},
+howpublished = {https://slurm.schedmd.com/},
+}
+
+
 @misc{SciKitLearn,
 	Author = {David Cournapeau},
 	Date-Added = {2015-01-20 19:13:34 +0000},

doc/user_manual/runInfo.tex

@@ -320,52 +320,52 @@ \subsection{RunInfo: Input of Queue Modes}
 \item \xmlNode{mode}, \xmlDesc{string, optional field}, can specify which kind
 of protocol the parallel enviroment should use.
 %
-RAVEN currently supports one pre-defined ``mode'':
+RAVEN currently supports two pre-defined ``mode''s:
   \begin{itemize}
-    \item \textbf{mpi}: this ``mode'' uses \xmlNode{MPIExec}  command (default: \texttt{mpiexec})
+    \item \textbf{pbs}: this ``mode'' uses PBS and the \xmlNode{MPIExec}  command (default: \texttt{mpiexec})
       to distribute the running program; more information regarding this protocol can be found
-      in~\cite{MPI}.
+      in~\cite{MPI}.  (Note that this ``mode'' used to be called \textbf{mpi}, but was renamed to be more specific since the \textbf{slurm} ``mode'' also uses MPI.)
 
-      Mode ``MPI''  can either generate a \texttt{qsub} command or can execute
+      Mode ``PBS''  can either generate a \texttt{qsub} command or can execute
       on selected nodes.
       Parameters to be given to the mpi command can be specified with the \xmlNode{MPIParam} node.  These will be given after the \xmlNode{MPIExec}  command so that needed mpi parameters can be specified (such as \texttt{--nooversubscribe}).
 
-      In order to make the ``mpi'' mode generate a \texttt{qsub} command, an
+      In order to make the ``pbs'' mode generate a \texttt{qsub} command, an
       additional keyword (xml sub-node) needs to be specified:
          \begin{itemize}
          \item If RAVEN is executed in the HEAD node of an HPC system using
            \cite{PBS}, the user needs to input a sub-node, \xmlNode{runQSUB},
-           right after the specification of the mpi mode (i.e.,\\
-             \xmlNode{mode}\texttt{mpi}\xmlNode{runQSUB/}\xmlNode{/mode}).
+           right after the specification of the pbs mode (i.e.,\\
+             \xmlNode{mode}\texttt{pbs}\xmlNode{runQSUB/}\xmlNode{/mode}).
              If the keyword is provided, RAVEN generates a \texttt{qsub}
              command, instantiates itself, and submits itself to the queue
              system.
            \item If the user decides to execute RAVEN from an ``interactive
              node'' (a certain number of nodes that have been reserved in
-             interactive PBS mode), RAVEN, using the ``mpi'' system, is going to
+             interactive PBS mode), RAVEN, using the ``pbs'' system, is going to
              utilize the reserved resources (CPUs and nodes) to distribute the
              jobs, but, will not generate a \texttt{qsub} command.
          \end{itemize}
-     When the user decides to run in ``mpi'' mode without making RAVEN generate
+     When the user decides to run in ``pbs'' mode without making RAVEN generate
      a \texttt{qsub} command, different options are available:
       \begin{itemize}
            \item If the user decides to run on the local machine (either in
              local desktop/workstation or a remote machine), no additional
              keywords are needed (i.e.\\
-             \xmlNode{mode}\texttt{mpi}\xmlNode{/mode}).
+             \xmlNode{mode}\texttt{pbs}\xmlNode{/mode}).
            \item If the user is running on multiple nodes, the node ids have
              to be specified:
            \begin{itemize}
               \item the node ids can be specified in an external text file
                 (node ids separated by blank space).
                 This file needs to be provided in the XML node \xmlNode{mode},
                 introducing a sub-node named \xmlNode{nodefile} (e.g.\\
-\xmlNode{mode}\texttt{mpi}\xmlNode{nodefile}\texttt{/tmp/nodes}\xmlNode{/nodefile}\xmlNode{/mode}).
+\xmlNode{mode}\texttt{pbs}\xmlNode{nodefile}\texttt{/tmp/nodes}\xmlNode{/nodefile}\xmlNode{/mode}).
               \item the node ids can be contained in an enviromental variable
                 (node ids separated by blank space).
                 This variable needs to be provided in the \xmlNode{mode} XML
                 node, introducing a sub-node named \xmlNode{nodefileenv} (e.g.\\
-\xmlNode{mode}\texttt{mpi}\xmlNode{nodefileenv}\texttt{NODEFILE}\xmlNode{/nodefileenv}\xmlNode{/mode>}).
+\xmlNode{mode}\texttt{pbs}\xmlNode{nodefileenv}\texttt{NODEFILE}\xmlNode{/nodefileenv}\xmlNode{/mode>}).
                 \item If none of the above options are used, RAVEN will attempt
                   to find the nodes' information in the enviroment variable
                   \texttt{PBS\_NODEFILE}.
@@ -389,7 +389,68 @@ \subsection{RunInfo: Input of Queue Modes}
          \end{itemize}
          In addition, this flag activates the remote (PBS) execution of internal Models (e.g. ROMs,
          ExternalModels, PostProcessors, etc.). If this node is not present, the internal Models
-           are run using a multi-threading approach (i.e., master processor, multiple parallel threads)
+         are run using a multi-threading approach (i.e., master processor, multiple parallel threads)
+           \item \textbf{slurm}: this ``mode'' uses Slurm and the \xmlNode{MPIExec}  command (default: \texttt{mpiexec})
+      to distribute the running program; more information regarding this protocol can be found
+      in~\cite{SLURM}.
+
+      Mode ``Slurm''  can either generate a \texttt{sbatch} command or can execute
+      on selected nodes.
+      Parameters to be given to the mpi command can be specified with the \xmlNode{MPIParam} node.  These will be given after the \xmlNode{MPIExec}  command so that needed mpi parameters can be specified (such as \texttt{--nooversubscribe}).
+
+      In order to make the ``slurm'' mode generate a \texttt{sbatch} command, an
+      additional keyword (xml sub-node) needs to be specified:
+         \begin{itemize}
+         \item If RAVEN is executed in the HEAD node of an HPC system using
+           \cite{SLURM}, the user needs to input a sub-node, \xmlNode{runSbatch},
+           right after the specification of the slurm mode (i.e.,\\
+             \xmlNode{mode}\texttt{slurm}\xmlNode{runSbatch/}\xmlNode{/mode}).
+             If the keyword is provided, RAVEN generates a \texttt{sbatch}
+             command, instantiates itself, and submits itself to the queue
+             system.
+           \item If the user decides to execute RAVEN from an ``interactive
+             node'' (a certain number of nodes that have been reserved in
+             interactive Slurm mode), RAVEN, using the ``slurm'' system, is going to
+             utilize the reserved resources (CPUs and nodes) to distribute the
+             jobs, but, will not generate a \texttt{sbatch} command.
+         \end{itemize}
+     When the user decides to run in ``slurm'' mode without making RAVEN generate
+     a \texttt{sbatch} command, different options are available:
+      \begin{itemize}
+           \item If the user decides to run on the local machine (either in
+             local desktop/workstation or a remote machine), no additional
+             keywords are needed (i.e.\\
+             \xmlNode{mode}\texttt{slurm}\xmlNode{/mode}).
+           \item If the user is running on multiple nodes, the node ids have
+             to be specified:
+           \begin{itemize}
+              \item the node ids can be specified in an external text file
+                (node ids separated by blank space).
+                This file needs to be provided in the XML node \xmlNode{mode},
+                introducing a sub-node named \xmlNode{nodefile} (e.g.\\
+\xmlNode{mode}\texttt{slurm}\xmlNode{nodefile}\texttt{/tmp/nodes}\xmlNode{/nodefile}\xmlNode{/mode}).
+                \item If none of the above options are used, RAVEN will attempt
+                  to find the nodes' information by using: \texttt{srun --overlap -- hostname} 
+           \end{itemize}
+         \item The cores needed can be specified manually with the
+           \xmlNode{coresneeded}.  This is used in the
+           \texttt{sbatch} command \texttt{--ntasks} parameter.
+         \item The max memory needed can be specified with the
+           \xmlNode{memory} XML node.  This will be used in the
+           \texttt{sbatch} command \texttt{--mem} parameter.
+         \item If the automaticly generated mpi command is not needed,
+           \xmlNode{noPrecommand} can be used to completely disable
+           generating a precommand. This can be useful if calling a
+           script that will call \texttt{mpiexec} itself.  Note that
+           the machine host file is located at
+           \texttt{\%BASE\_WORKING\_DIR\%/node\_\%INDEX\%}
+         \item The placement can be specified with the \xmlNode{partition}
+           XML node.  This will be used in the \texttt{sbatch} partition
+           statement.
+         \end{itemize}
+         In addition, this flag activates the remote (Slurm) execution of \xmlNode{parallelMethod}  for internal Models (e.g. ROMs,
+         ExternalModels, PostProcessors, etc.). If this node is not present, the internal Models
+           are run on the same computer (for example: master processor, multiple parallel threads)
    \end{itemize}
 
 %Both methods can submit a qsub command or can be run from an already submitted interactive qsub command:
@@ -409,12 +470,14 @@ \subsection{RunInfo: Input of Queue Modes}
 Please refer to sub-section~\ref{subsec:runinfoadvanced} for advanced users.
 
 %%%%%% QUEUE SOFTWARE
-\item \xmlNode{queueingSoftware}, \xmlDesc{string, optional field}.
-RAVEN has support for the PBS queueing system. If the platform provides a
-different queueing system, the user can specify its name here (e.g., PBS
-PROFESSIONAL, etc.).
-%
-\default{PBS PROFESSIONAL}
+%%%%%% Note, this is not supported by any of the standard modes, so currently
+%%%%%% commented out.
+%\item \xmlNode{queueingSoftware}, \xmlDesc{string, optional field}.
+%RAVEN has support for the PBS queueing system. If the platform provides a
+%different queueing system, the user can specify its name here (e.g., PBS
+%PROFESSIONAL, etc.).
+%
+%\default{PBS PROFESSIONAL}
 
 %%%%%% EXPECTED TIME
 \item \xmlNode{expectedTime}, \xmlDesc{colum separated string, optional field

ravenframework/CustomModes/MPISimulationMode.py → ravenframework/CustomModes/PBSSimulationMode.py

@@ -24,13 +24,13 @@
 from ravenframework import Simulation
 
 #For the mode information
-modeName = "mpi"
-modeClassName = "MPISimulationMode"
+modeName = ["mpi","pbs"]
+modeClassName = "PBSSimulationMode"
 
-class MPISimulationMode(Simulation.SimulationMode):
+class PBSSimulationMode(Simulation.SimulationMode):
   """
-    MPISimulationMode is a specialized class of SimulationMode.
-    It is aimed to distribute the runs using the MPI protocol
+    PBSSimulationMode is a specialized class of SimulationMode.
+    It is aimed to distribute the runs using the MPI protocol on PBS
   """
   def __init__(self, *args):
     """