diff --git a/.github/pseudo-cluster/docker-compose.yml b/.github/pseudo-cluster/docker-compose.yml new file mode 100644 index 0000000000..a6adb767f4 --- /dev/null +++ b/.github/pseudo-cluster/docker-compose.yml @@ -0,0 +1,114 @@ +services: + munge-key-generator: + image: ghcr.io/reframe-hpc/munge-ubuntu:20.04 + hostname: munge-host + healthcheck: + test: ["CMD-SHELL", "test -f /scratch/munge.key"] + interval: 10s + timeout: 10s + retries: 5 + volumes: + - shared-scratch:/scratch + + frontend: + image: slurm-reframe + container_name: frontend + build: + dockerfile: .github/pseudo-cluster/reframe/Dockerfile + context: ../../ + hostname: login + user: admin + init: True + volumes: + - shared-home:/home/admin:rw + - shared-scratch:/scratch:rw + links: + - slurm-master + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + node0: + condition: service_started + node1: + condition: service_started + node2: + condition: service_started + environment: + - SLURM_CPUS_ON_NODE=1 + - BACKEND=${BACKEND:-squeue} + + slurm-master: + image: ghcr.io/reframe-hpc/slurm-master-ubuntu:20.04 + hostname: slurm-master + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + depends_on: + munge-key-generator: + condition: service_healthy + environment: + - SLURM_CPUS_ON_NODE=1 + + node0: + image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04 + hostname: nid00 + container_name: slurm-node0 + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + environment: + - SLURM_NODENAME=nid00 + - SLURM_CPUS_ON_NODE=1 + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + links: + - slurm-master + + node1: + image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04 + hostname: nid01 + container_name: slurm-node1 + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + environment: + - SLURM_NODENAME=nid01 + - SLURM_CPUS_ON_NODE=1 + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + links: + - slurm-master + + node2: + image: ghcr.io/reframe-hpc/slurm-node-ubuntu:20.04 + hostname: nid02 + container_name: slurm-node2 + user: admin + volumes: + - shared-home:/home/admin + - shared-scratch:/scratch:rw + environment: + - SLURM_NODENAME=nid02 + - SLURM_CPUS_ON_NODE=1 + depends_on: + munge-key-generator: + condition: service_healthy + slurm-master: + condition: service_started + links: + - slurm-master + +volumes: + shared-home: + shared-scratch: diff --git a/.github/pseudo-cluster/reframe/Dockerfile b/.github/pseudo-cluster/reframe/Dockerfile new file mode 100644 index 0000000000..afeac49f98 --- /dev/null +++ b/.github/pseudo-cluster/reframe/Dockerfile @@ -0,0 +1,42 @@ +FROM ubuntu:20.04 + +ARG DEBIAN_FRONTEND=noninteractive + +RUN apt update -y && \ + apt install -y \ + build-essential \ + clang jq libomp-dev tree vim \ + git \ + mariadb-client \ + munge \ + slurm-client \ + slurm-wlm-torque \ + sudo \ + python3 \ + python3-pip \ + wget \ + curl \ + mpich \ + libmpich-dev && \ + rm -rf /var/lib/apt/lists/* + +RUN useradd -m admin -s /usr/bin/bash -d /home/admin && \ + echo "admin:admin" | chpasswd && adduser admin sudo && \ + echo "admin ALL=(ALL) NOPASSWD:ALL" >> /etc/sudoers + +COPY .github/pseudo-cluster/reframe/slurm.conf /etc/slurm-llnl/ +COPY .github/pseudo-cluster/reframe/cgroup.conf /etc/slurm-llnl/ +COPY .github/pseudo-cluster/reframe/docker-entrypoint.sh /etc/slurm-llnl/ +COPY . /usr/local/share/reframe + +RUN mkdir /scratch && \ + chown -R admin:admin /scratch + +RUN chmod +rx /etc/slurm-llnl/docker-entrypoint.sh + +WORKDIR /home/admin + +ENV USER admin +ENV SHELL bash + +ENTRYPOINT ["/etc/slurm-llnl/docker-entrypoint.sh"] diff --git a/.github/pseudo-cluster/reframe/cgroup.conf b/.github/pseudo-cluster/reframe/cgroup.conf new file mode 120000 index 0000000000..c99f23956f --- /dev/null +++ b/.github/pseudo-cluster/reframe/cgroup.conf @@ -0,0 +1 @@ +../../../examples/tutorial/dockerfiles/slurm-cluster/reframe/cgroup.conf \ No newline at end of file diff --git a/.github/pseudo-cluster/reframe/docker-entrypoint.sh b/.github/pseudo-cluster/reframe/docker-entrypoint.sh new file mode 100755 index 0000000000..665f23ff8d --- /dev/null +++ b/.github/pseudo-cluster/reframe/docker-entrypoint.sh @@ -0,0 +1,24 @@ +#!/bin/bash + +trap exit 0 INT + +while [ ! -f /scratch/munge.key ] +do + sleep 1 +done + +sudo cp /scratch/munge.key /etc/munge/munge.key +sudo service munge start +sudo sed -i "s/REPLACE_IT/CPUs=${SLURM_CPUS_ON_NODE}/g" /etc/slurm-llnl/slurm.conf + +# Needs to be copied in the shared home directory +cp -r /usr/local/share/reframe . +cd reframe +./bootstrap.sh + +echo "Running unittests with backend scheduler: ${BACKEND}" + +tempdir=$(mktemp -d -p /scratch) +TMPDIR=$tempdir ./test_reframe.py -v \ + --rfm-user-config=ci-scripts/configs/ci-cluster.py \ + --rfm-user-system=pseudo-cluster:compute-${BACKEND:-squeue} diff --git a/.github/pseudo-cluster/reframe/slurm.conf b/.github/pseudo-cluster/reframe/slurm.conf new file mode 120000 index 0000000000..39eb5d7b6f --- /dev/null +++ b/.github/pseudo-cluster/reframe/slurm.conf @@ -0,0 +1 @@ +../../../examples/tutorial/dockerfiles/slurm-cluster/reframe/slurm.conf \ No newline at end of file diff --git a/.github/workflows/test-schedulers.yaml b/.github/workflows/test-schedulers.yaml new file mode 100644 index 0000000000..83285e0988 --- /dev/null +++ b/.github/workflows/test-schedulers.yaml @@ -0,0 +1,24 @@ +name: ReFrame CI / Scheduler backend tests +on: + pull_request: [] + +jobs: + scheduler-test: + runs-on: ubuntu-latest + strategy: + matrix: + scheduler: ['pbs', 'squeue', 'torque'] + steps: + - uses: actions/checkout@v4 + - name: Login to GitHub Container Registry + uses: docker/login-action@v2 + with: + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + - name: Build Images + run: | + docker compose -f .github/pseudo-cluster/docker-compose.yml build + - name: Run Unittests with ${{ matrix.scheduler }} sceduler + run: | + BACKEND=${{ matrix.scheduler }} docker compose -f .github/pseudo-cluster/docker-compose.yml up --abort-on-container-exit --exit-code-from frontend diff --git a/Jenkinsfile b/Jenkinsfile deleted file mode 100644 index 1553a9b1b1..0000000000 --- a/Jenkinsfile +++ /dev/null @@ -1,169 +0,0 @@ -#!/usr/bin/env groovy - -def dirPrefix = 'reframe-ci' -def loginBash = '#!/bin/bash -l' -def bashScript = 'ci-scripts/ci-runner.bash' -def machinesList = params.machines.split() -def machinesToRun = machinesList -def runTests = true -def uniqueID - -stage('Initialization') { - node('master') { - catchError(stageResult: 'FAILURE') { - uniqueID = "${env.ghprbActualCommit[0..6]}-${env.BUILD_ID}" - echo 'Environment Variables:' - echo sh(script: 'env|sort', returnStdout: true) - - def githubComment = env.ghprbCommentBody - if (githubComment == 'null' || !githubComment.trim().startsWith('@jenkins-cscs')) { - machinesToRun = machinesList - currentBuild.result = 'SUCCESS' - return - } - - def splittedComment = githubComment.split() - if (splittedComment.size() < 3) { - println 'No machines were found. Aborting...' - currentBuild.result = 'ABORTED' - return - } - if (splittedComment[1] != 'retry') { - println "Invalid command ${splittedComment[1]}. Aborting..." - currentBuild.result = 'ABORTED' - return - } - if (splittedComment[2] == 'all') { - machinesToRun = machinesList - currentBuild.result = 'SUCCESS' - return - } - else if (splittedComment[2] == 'none') { - runTests = false - currentBuild.result = 'SUCCESS' - return - } - - machinesRequested = [] - for (i = 2; i < splittedComment.size(); i++) { - machinesRequested.add(splittedComment[i]) - } - - machinesToRun = machinesRequested.findAll({it in machinesList}) - if (!machinesToRun) { - println 'No machines were found. Aborting...' - currentBuild.result = 'ABORTED' - return - } - currentBuild.result = 'SUCCESS' - } - } -} - -if (!runTests) { - println "Won't execute any test (${currentBuild.result}). Exiting..." - return -} - -if (currentBuild.result != 'SUCCESS') { - println "Initialization failed (${currentBuild.result}). Exiting..." - return -} - -def builds = [:] -stage('Unittest') { - for (mach in machinesToRun) { - def machineName = mach - builds[machineName] = { - node(machineName) { - def scratch = sh(returnStdout: true, - script: """${loginBash} - echo \$SCRATCH""").trim() - def reframeDir = "${scratch}/${dirPrefix}-${machineName}-${uniqueID}" - dir(reframeDir) { - checkout scm - sh("""${loginBash} - bash ${reframeDir}/${bashScript} -f ${reframeDir} -i ''""") - } - } - } - } - - catchError(stageResult: 'FAILURE') { - parallel builds - } -} - -builds = [:] -stage('Tutorial Check') { - if (currentBuild.result != 'SUCCESS') { - println 'Not executing "Tutorial Check" Stage' - return - } - else { - catchError(stageResult: 'FAILURE') { - if (!('daint' in machinesToRun)) { - return - } - node('daint') { - def scratch = sh(returnStdout: true, - script: """${loginBash} - echo \$SCRATCH""").trim() - def reframeDir = "${scratch}/${dirPrefix}-daint-${uniqueID}" - dir(reframeDir) { - sh("""${loginBash} - bash ${reframeDir}/${bashScript} -f ${reframeDir} -i '' -t""") - } - } - } - } -} - -builds = [:] -stage('Cleanup') { - if (currentBuild.result != 'SUCCESS') { - println 'Not executing "Cleanup" Stage' - return - } - else { - for (mach in machinesToRun) { - def machineName = mach - builds[machineName] = { - node(machineName) { - def scratch = sh(returnStdout: true, - script: """$loginBash - echo \$SCRATCH""").trim() - def reframeDir = "${scratch}/${dirPrefix}-${machineName}-${uniqueID}" - sh("""${loginBash} - rm -rf ${reframeDir} - date""") - - } - } - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - parallel builds - } - } -} - -def staleCleanupInterval = 3 -builds = [:] -stage('Cleanup Stale') { - for (mach in machinesToRun) { - def machineName = mach - builds[machineName] = { - node(machineName) { - def scratch = sh(returnStdout: true, - script: """${loginBash} - echo \$SCRATCH""").trim() - sh("""${loginBash} - find ${scratch} -maxdepth 1 -name 'reframe-ci*' -ctime +${staleCleanupInterval} -type d -exec printf 'Removing %s\\n' {} + - find ${scratch} -maxdepth 1 -name 'reframe-ci*' -ctime +${staleCleanupInterval} -type d -exec rm -rf {} +""") - } - } - } - catchError(buildResult: 'SUCCESS', stageResult: 'FAILURE') { - parallel builds - } -} diff --git a/README.md b/README.md index 1007efe4f4..4eb351f853 100644 --- a/README.md +++ b/README.md @@ -97,3 +97,37 @@ Please join the community's [Slack channel](https://join.slack.com/t/reframetalk ReFrame is an open-source project and we welcome and encourage contributions! Check out our Contribution Guide [here](https://github.com/reframe-hpc/reframe/wiki/contributing-to-reframe). + +## Citing ReFrame + +You can cite ReFrame in publications as follows: + +> Vasileios Karakasis et al. "Enabling Continuous Testing of HPC Systems Using ReFrame". In: *Tools and Techniques for High Performance Computing. HUST - Annual Workshop on HPC User Support Tools* (Denver, Colorado, USA, Nov. 17–18, 2019). Ed. by Guido Juckeland and Sunita Chandrasekaran. Vol. 1190. Communications in Computer and Information Science. Cham, Switzerland: Springer International Publishing, Mar. 2020, pp. 49–68. isbn: 978-3-030-44728-1. doi: 10.1007/978-3-030-44728-1_3. + +The corresponding BibTeX entry is the following: + +```bibtex +@InProceedings{karakasis20a, + author = {Karakasis, Vasileios and Manitaras, Theofilos and Rusu, Victor Holanda and + Sarmiento-P{\'e}rez, Rafael and Bignamini, Christopher and Kraushaar, Matthias and + Jocksch, Andreas and Omlin, Samuel and Peretti-Pezzi, Guilherme and + Augusto, Jo{\~a}o P. S. C. and Friesen, Brian and He, Yun and Gerhardt, Lisa and + Cook, Brandon and You, Zhi-Qiang and Khuvis, Samuel and Tomko, Karen}, + title = {Enabling Continuous Testing of {HPC} Systems Using {ReFrame}}, + booktitle = {Tools and Techniques for High Performance Computing}, + editor = {Juckeland, Guido and Chandrasekaran, Sunita}, + year = {2020}, + month = mar, + series = {Communications in Computer and Information Science}, + volume = {1190}, + pages = {49--68}, + address = {Cham, Switzerland}, + publisher = {Springer International Publishing}, + doi = {10.1007/978-3-030-44728-1_3}, + venue = {Denver, Colorado, USA}, + eventdate = {2019-11-17/2019-11-18}, + eventtitle = {{HUST} - Annual Workshop on {HPC} User Support Tools}, + isbn = {978-3-030-44728-1}, + issn = {1865-0937}, +} +``` diff --git a/README_minimal.md b/README_minimal.md index 027200d2dc..90044a6c97 100644 --- a/README_minimal.md +++ b/README_minimal.md @@ -39,3 +39,37 @@ Please join the community's [Slack channel](https://reframe-slack.herokuapp.com) ReFrame is an open-source project and we welcome and encourage contributions! Check out our Contribution Guide [here](https://github.com/reframe-hpc/reframe/wiki/contributing-to-reframe). + +## Citing ReFrame + +You can cite ReFrame in publications as follows: + +> Vasileios Karakasis et al. “Enabling Continuous Testing of HPC Systems Using ReFrame”. In: *Tools and Techniques for High Performance Computing. HUST - Annual Workshop on HPC User Support Tools* (Denver, Colorado, USA, Nov. 17–18, 2019). Ed. by Guido Juckeland and Sunita Chandrasekaran. Vol. 1190. Communications in Computer and Information Science. Cham, Switzerland: Springer International Publishing, Mar. 2020, pp. 49–68. isbn: 978-3-030-44728-1. doi: 10.1007/978-3-030-44728-1_3. + +The corresponding BibTeX entry is the following: + +```bibtex +@InProceedings{karakasis20a, + author = {Karakasis, Vasileios and Manitaras, Theofilos and Rusu, Victor Holanda and + Sarmiento-P{\'e}rez, Rafael and Bignamini, Christopher and Kraushaar, Matthias and + Jocksch, Andreas and Omlin, Samuel and Peretti-Pezzi, Guilherme and + Augusto, Jo{\~a}o P. S. C. and Friesen, Brian and He, Yun and Gerhardt, Lisa and + Cook, Brandon and You, Zhi-Qiang and Khuvis, Samuel and Tomko, Karen}, + title = {Enabling Continuous Testing of {HPC} Systems Using {ReFrame}}, + booktitle = {Tools and Techniques for High Performance Computing}, + editor = {Juckeland, Guido and Chandrasekaran, Sunita}, + year = {2020}, + month = mar, + series = {Communications in Computer and Information Science}, + volume = {1190}, + pages = {49--68}, + address = {Cham, Switzerland}, + publisher = {Springer International Publishing}, + doi = {10.1007/978-3-030-44728-1_3}, + venue = {Denver, Colorado, USA}, + eventdate = {2019-11-17/2019-11-18}, + eventtitle = {{HUST} - Annual Workshop on {HPC} User Support Tools}, + isbn = {978-3-030-44728-1}, + issn = {1865-0937}, +} +``` diff --git a/ci-scripts/ci-runner.bash b/ci-scripts/ci-runner.bash index 50130481b9..08e0062c9d 100644 --- a/ci-scripts/ci-runner.bash +++ b/ci-scripts/ci-runner.bash @@ -170,7 +170,7 @@ else for backend in slurm pbs torque; do echo "[INFO] Running unit tests with ${backend}" TMPDIR=$tempdir checked_exec ./test_reframe.py ${parallel_opts} \ - --rfm-user-config=config/cscs-ci.py \ + --rfm-user-config=ci-scripts/configs/cscs-ci.py \ -W=error::reframe.core.warnings.ReframeDeprecationWarning \ --rfm-user-system=dom:${backend} -ra done @@ -178,7 +178,7 @@ else else echo "[INFO] Running unit tests" TMPDIR=$tempdir checked_exec ./test_reframe.py ${parallel_opts} \ - --rfm-user-config=config/cscs-ci.py \ + --rfm-user-config=ci-scripts/configs/cscs-ci.py \ -W=error::reframe.core.warnings.ReframeDeprecationWarning -ra fi diff --git a/ci-scripts/configs/ci-cluster.py b/ci-scripts/configs/ci-cluster.py new file mode 100644 index 0000000000..3d30203fbd --- /dev/null +++ b/ci-scripts/configs/ci-cluster.py @@ -0,0 +1,73 @@ +# Copyright 2016-2024 Swiss National Supercomputing Centre (CSCS/ETH Zurich) +# ReFrame Project Developers. See the top-level LICENSE file for details. +# +# SPDX-License-Identifier: BSD-3-Clause + +site_configuration = { + 'systems': [ + { + 'name': 'pseudo-cluster', + 'descr': 'CI Slurm-based pseudo cluster', + 'hostnames': ['login'], + 'partitions': [ + { + 'name': 'login', + 'descr': 'Login nodes', + 'scheduler': 'local', + 'launcher': 'local', + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute-squeue', + 'descr': 'Squeue compute nodes', + 'scheduler': 'squeue', + 'launcher': 'srun', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute-torque', + 'descr': 'Torque compute nodes', + 'scheduler': 'torque', + 'launcher': 'mpiexec', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + }, + { + 'name': 'compute-pbs', + 'descr': 'PBS compute nodes', + 'scheduler': 'pbs', + 'launcher': 'mpiexec', + 'access': ['-p all'], + 'environs': ['gnu', 'clang'] + } + ] + }, + ], + 'environments': [ + { + 'name': 'baseline', + 'features': ['stream'] + }, + { + 'name': 'gnu', + 'cc': 'gcc', + 'cxx': 'g++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + }, + { + 'name': 'clang', + 'cc': 'clang', + 'cxx': 'clang++', + 'features': ['openmp'], + 'extras': {'omp_flag': '-fopenmp'} + } + ], + 'modes': [ + { + 'name': 'singlethread', + 'options': ['-E num_threads==1'] + } + ] +} diff --git a/config/cscs-ci.py b/ci-scripts/configs/cscs-ci.py similarity index 100% rename from config/cscs-ci.py rename to ci-scripts/configs/cscs-ci.py diff --git a/docs/config_reference.rst b/docs/config_reference.rst index c8715dbd36..8b84563be0 100644 --- a/docs/config_reference.rst +++ b/docs/config_reference.rst @@ -60,6 +60,13 @@ It consists of the following properties, which we also call conventionally *conf A list of `general configuration objects <#general-configuration>`__. +.. py:data:: storage + + :required: No + + A list of :ref:`storage configuration objects ` + + .. versionadded:: 4.7 .. py:data:: autodetect_methods @@ -432,6 +439,16 @@ System Partition Configuration No other test would be able to proceed. +.. py:attribute:: systems.partitions.sched_options.unqualified_hostnames + + :required: No + :default: ``false`` + + Use unqualified hostnames in the ``local`` scheduler backend. + + .. versionadded:: 4.7 + + .. py:attribute:: systems.partitions.sched_options.use_nodes_option :required: No @@ -1602,6 +1619,49 @@ The options of an execution mode will be passed to ReFrame as if they were speci For a detailed description of this property, have a look at the :attr:`~environments.target_systems` definition for environments. +.. _storage-configuration: + +Result storage configuration +============================ + +.. versionadded:: 4.7 + +.. py:attribute:: storage.backend + + :required: No + :default: ``"sqlite"`` + + The backend to use for storing the test results. + + Currently, only Sqlite can be used as a storage backend. + + +.. py:attribute:: storage.sqlite_conn_timeout + + :required: No + :default: ``60`` + + Timeout in seconds for SQLite database connections. + + +.. py:attribute:: storage.sqlite_db_file + + :required: No + :default: ``"${HOME}/.reframe/reports/results.db"`` + + The SQLite database file to use. + +.. py:attribute:: storage.sqlite_db_file_mode + + :required: No + :default: ``"644"`` + + The permissions of the SQLite database file in octal form. + + The mode will only taken into account upon creation of the DB file. + Permissions of an existing DB file have to be changed manually. + + General Configuration ===================== @@ -1859,6 +1919,20 @@ General Configuration For a detailed description of this property, have a look at the :attr:`~environments.target_systems` definition for environments. +.. py:attribute:: general.table_format + + :required: No + :default: ``"pretty"`` + + Set the formatting of tabular output. + + The acceptable values are the following: + + - ``csv``: Generate CSV output + - ``plain``: Generate a plain table without any lines + - ``pretty``: (default) Generate a pretty table + + .. py:attribute:: general.timestamp_dirs :required: No diff --git a/docs/hpctestlib.rst b/docs/hpctestlib.rst index 599aedab5e..a1db5c6ade 100644 --- a/docs/hpctestlib.rst +++ b/docs/hpctestlib.rst @@ -106,6 +106,10 @@ Scientific Applications :members: :show-inheritance: +.. automodule:: hpctestlib.sciapps.metalwalls.benchmarks + :members: + :show-inheritance: + System ======================= diff --git a/docs/index.rst b/docs/index.rst index 6d35d027a8..0d6811cc5c 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -18,6 +18,9 @@ Finally, ReFrame offers a powerful and efficient runtime for running and managin Publications ============ +Presentations & Talks +--------------------- + * Slides [`pdf `__][`talk `__] @ `9th EasyBuild User Meeting 2024 `__. * Slides [`part 1 `__][`part 2 `__][`talk `__] @ `8th EasyBuild User Meeting 2023 `__. * Slides [`pdf `__] @ `7th EasyBuild User Meeting 2022 `__. @@ -32,18 +35,26 @@ Publications * Slides [`pdf `__] @ `CSCS User Lab Day 2018 `__. * Slides [`pdf `__] @ `HPC Advisory Council 2018 `__. * Slides [`pdf `__] @ `SC17 `__. -* Slides [`pdf `__] @ `CUG 2017 `__. +* Slides [`pdf `__] @ `CUG 2017 `__. -Webinars and Tutorials -====================== +Webinars & Tutorials +-------------------- * "ReFrame – Efficient System and Application Performance Testing," CSCS Webinar, Aug. 29, 2022 [`slides `__] [`recording `__] [`demo run `__]. * Tutorial at 6th EasyBuild User Meeting 2021 [`YouTube `__] +Papers +------ + +- Vasileios Karakasis et al. "A regression framework for checking the health of large HPC systems". In: *Cray User Group 2017* (Redmond, Washington, USA, May 8--11, 2017). [`pdf `__] + +- Vasileios Karakasis et al. "Enabling Continuous Testing of HPC Systems Using ReFrame". In: *Tools and Techniques for High Performance Computing. HUST -- Annual Workshop on HPC User Support Tools* (Denver, Colorado, USA, Nov. 17--18, 2019). Ed. by Guido Juckeland and Sunita Chandrasekaran. Vol. 1190. Communications in Computer and Information Science. Cham, Switzerland: Springer International Publishing, Mar. 2020, pp. 49--68. isbn: 978-3-030-44728-1. doi: `10.1007/978-3-030-44728-1_3 `__. + .. toctree:: + :maxdepth: 2 :caption: Table of Contents started diff --git a/docs/manpage.rst b/docs/manpage.rst index 60b0f9d2d8..7c9227af8f 100644 --- a/docs/manpage.rst +++ b/docs/manpage.rst @@ -4,23 +4,234 @@ Command Line Reference Synopsis --------- +======== .. option:: reframe [OPTION]... ACTION Description ------------ +=========== ReFrame provides both a :doc:`programming interface ` for writing regression tests and a command-line interface for managing and running the tests, which is detailed here. The ``reframe`` command is part of ReFrame's frontend. This frontend is responsible for loading and running regression tests written in ReFrame. ReFrame executes tests by sending them down to a well defined pipeline. The implementation of the different stages of this pipeline is part of ReFrame's core architecture, but the frontend is responsible for driving this pipeline and executing tests through it. -There are three basic phases that the frontend goes through, which are described briefly in the following. +Usually, ReFrame processes tests in three phases: +1. It :ref:`discovers and loads tests ` from the filesystem. +2. It :ref:`filters ` the loaded tests based on the current system and any other criteria specified by the user. +3. It :ref:`acts ` upon the selected tests. + +There are also ReFrame commands that do not operate on a set of tests. + + +.. _commands: + +Commands +-------- + +ReFrame commands are mutually exclusive and one of them must always be specified. +There are commands that act upon the selected tests and others that have a helper function, such as querying the configuration, querying the results database etc. + +.. versionchanged:: 4.7 + + ReFrame commands are now mutually exclusive and only one can be specified every time. + + +Test commands +^^^^^^^^^^^^^ + +.. option:: --ci-generate=FILE + + Generate a Gitlab `child pipeline `__ specification in ``FILE`` that will run the selected tests. + + You can set up your Gitlab CI to use the generated file to run every test as a separate Gitlab job respecting test dependencies. + For more information, have a look in :ref:`generate-ci-pipeline`. + + .. note:: + This option will not work with the :ref:`test generation options `. + + + .. versionadded:: 3.4.1 + +.. option:: --describe + + Print a detailed description of the `selected tests <#test-filtering>`__ in JSON format and exit. + + .. note:: + The generated test description corresponds to its state after it has been initialized. + If any of its attributes are changed or set during its execution, their updated values will not be shown by this listing. + + .. versionadded:: 3.10.0 + + +.. option:: --dry-run + + Dry run the selected tests. + + The dry-run mode will try to execute as much of the test pipeline as possible. + More specifically, the tests will not be submitted and will not be run for real, + but their stage directory will be prepared and the corresponding job script will be emitted. + Similarly, the sanity and performance functions will not be evaluated but all the preparation will happen. + Tests run in dry-run mode will not fail unless there is a programming error in the test or if the test tries to use a resource that is not produced in dry run mode (e.g., access the standard output or a resource produced by a dependency outside any sanity or performance function). + In this case, users can call the :func:`~reframe.core.pipeline.RegressionTest.is_dry_run` method in their test and take a specific action if the test is run in dry-run mode. + + .. versionadded:: 4.1 + +.. option:: -L, --list-detailed[=T|C] + + List selected tests providing more details for each test. + + The unique id of each test (see also :attr:`~reframe.core.pipeline.RegressionTest.unique_name`) as well as the file where each test is defined are printed. + + This option accepts optionally a single argument denoting what type of listing is requested. + Please refer to :option:`-l` for an explanation of this argument. + + .. versionadded:: 3.10.0 + Support for different types of listing is added. + + .. versionchanged:: 4.0.5 + The variable names to which fixtures are bound are also listed. + See :ref:`test_naming_scheme` for more information. + +.. option:: -l, --list[=T|C] + + List selected tests and their dependencies. + + This option accepts optionally a single argument denoting what type of listing is requested. + There are two types of possible listings: + + - *Regular test listing* (``T``, the default): This type of listing lists the tests and their dependencies or fixtures using their :attr:`~reframe.core.pipeline.RegressionTest.display_name`. A test that is listed as a dependency of another test will not be listed separately. + - *Concretized test case listing* (``C``): This type of listing lists the exact test cases and their dependencies as they have been concretized for the current system and environment combinations. + This listing shows practically the exact test DAG that will be executed. + + .. versionadded:: 3.10.0 + Support for different types of listing is added. + + .. versionchanged:: 4.0.5 + The variable names to which fixtures are bound are also listed. + See :ref:`test_naming_scheme` for more information. + +.. option:: --list-tags + + List the unique tags of the selected tests. + + The tags are printed in alphabetical order. + + .. versionadded:: 3.6.0 + +.. option:: -r, --run + + Run the selected tests. + + +Result storage commands +^^^^^^^^^^^^^^^^^^^^^^^ + +.. option:: --delete-stored-session=UUID + + Delete the stored session with the specified UUID from the results database. + + .. versionadded:: 4.7 + +.. option:: --describe-stored-session=UUID + + Get detailed information of the session with the specified UUID. + The output is in JSON format. + + .. versionadded:: 4.7 + +.. option:: --describe-stored-testcases=SESSION_UUID|TIME_PERIOD + + Get detailed test case information of the session with the specified UUID or from the specified time period. + + If a session UUID is provided only information about the test cases of this session will be provided. + This option can be combined with :option:`--name` to restrict the listing to specific tests. + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section. + + .. versionadded:: 4.7 + +.. _--list-stored-sessions: + +.. option:: --list-stored-sessions[=TIME_PERIOD] + + List sessions stored in the results database. + + If ``TIME_PERIOD`` is ``all``, all stored sessions will be listed. + If not specified, only the sessions of last week will be listed. + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax`. + + .. versionadded:: 4.7 + +.. option:: --list-stored-testcases=SESSION_UUID|TIME_PERIOD + + List all test cases from the session with the specified UUID or from the specified time period. + + If a session UUID is provided only the test cases of this session will be listed. + This option can be combined with :option:`--name` to restrict the listing to specific tests. + For the exact syntax of ``TIME_PERIOD`` check the :ref:`time-period-syntax` section. + + .. versionadded:: 4.7 + +.. option:: --performance-compare=CMPSPEC + + Compare the performance of test cases that have run in the past. + + This option can be combined with :option:`--name` to restrict the comparison to specific tests. + Check the :ref:`performance-comparisons` section for the exact syntax of ``CMPSPEC``. + + .. versionadded:: 4.7 + +Other commands +^^^^^^^^^^^^^^ + +.. _--detect-host-topology: + +.. option:: --detect-host-topology[=FILE] + + Detect the local host processor topology, store it to ``FILE`` and exit. + + If no ``FILE`` is specified, the standard output will be used. + + .. versionadded:: 3.7.0 + +.. option:: --show-config [PARAM] + + Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit. + + The parameter value is printed in JSON format. + If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown. + Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter. + The ``/`` character acts as a selector of configuration object properties or an index in array objects. + The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property. + Here are some example queries: + + - Retrieve all the partitions of the current system: + + .. code:: bash + + reframe --show-config=systems/0/partitions + + - Retrieve the job scheduler of the partition named ``default``: + + .. code:: bash + + reframe --show-config=systems/0/partitions/@default/scheduler + + - Retrieve the check search path for system ``foo``: + + .. code:: bash + + reframe --system=foo --show-config=general/0/check_search_path + +.. option:: -V, --version + + Print version and exit. + + +.. _test-discovery: -------------------------------- Test discovery and test loading ------------------------------- @@ -56,7 +267,6 @@ This is something that test developers should bear in mind. .. _test-filtering: --------------- Test filtering -------------- @@ -234,107 +444,7 @@ This happens recursively so that if test ``T1`` depends on ``T2`` and ``T2`` dep The ``NAME`` pattern is matched anywhere in the test name and not at its beginning. If you want to match at the beginning of a test name, you should prepend ``^``. ------------- -Test actions ------------- - -ReFrame will finally act upon the selected tests. -There are currently two actions that can be performed on tests: (a) list the tests and (b) execute the tests. -An action must always be specified. - - -.. option:: --ci-generate=FILE - - Do not run the tests, but generate a Gitlab `child pipeline `__ specification in ``FILE``. - - You can set up your Gitlab CI to use the generated file to run every test as a separate Gitlab job respecting test dependencies. - For more information, have a look in :ref:`generate-ci-pipeline`. - - .. note:: - This option will not work with the :ref:`test generation options `. - - - .. versionadded:: 3.4.1 - -.. option:: --describe - - Print a detailed description of the `selected tests <#test-filtering>`__ in JSON format and exit. - - .. note:: - The generated test description corresponds to its state after it has been initialized. - If any of its attributes are changed or set during its execution, their updated values will not be shown by this listing. - - .. versionadded:: 3.10.0 - - -.. option:: --dry-run - - Dry run the selected tests. - - The dry-run mode will try to execute as much of the test pipeline as possible. - More specifically, the tests will not be submitted and will not be run for real, - but their stage directory will be prepared and the corresponding job script will be emitted. - Similarly, the sanity and performance functions will not be evaluated but all the preparation will happen. - Tests run in dry-run mode will not fail unless there is a programming error in the test or if the test tries to use a resource that is not produced in dry run mode (e.g., access the standard output or a resource produced by a dependency outside any sanity or performance function). - In this case, users can call the :func:`~reframe.core.pipeline.RegressionTest.is_dry_run` method in their test and take a specific action if the test is run in dry-run mode. - - .. versionadded:: 4.1 - - -.. option:: -L, --list-detailed[=T|C] - - List selected tests providing more details for each test. - - The unique id of each test (see also :attr:`~reframe.core.pipeline.RegressionTest.unique_name`) as well as the file where each test is defined are printed. - - This option accepts optionally a single argument denoting what type of listing is requested. - Please refer to :option:`-l` for an explanation of this argument. - - .. versionadded:: 3.10.0 - Support for different types of listing is added. - - .. versionchanged:: 4.0.5 - The variable names to which fixtures are bound are also listed. - See :ref:`test_naming_scheme` for more information. - -.. option:: -l, --list[=T|C] - - List selected tests and their dependencies. - - This option accepts optionally a single argument denoting what type of listing is requested. - There are two types of possible listings: - - - *Regular test listing* (``T``, the default): This type of listing lists the tests and their dependencies or fixtures using their :attr:`~reframe.core.pipeline.RegressionTest.display_name`. A test that is listed as a dependency of another test will not be listed separately. - - *Concretized test case listing* (``C``): This type of listing lists the exact test cases and their dependencies as they have been concretized for the current system and environment combinations. - This listing shows practically the exact test DAG that will be executed. - - .. versionadded:: 3.10.0 - Support for different types of listing is added. - - .. versionchanged:: 4.0.5 - The variable names to which fixtures are bound are also listed. - See :ref:`test_naming_scheme` for more information. - -.. option:: --list-tags - List the unique tags of the selected tests. - - The tags are printed in alphabetical order. - - .. versionadded:: 3.6.0 - -.. option:: -r, --run - - Execute the selected tests. - -If more than one action options are specified, the precedence order is the following: - - .. code-block:: console - - --describe > --list-detailed > --list > --list-tags > --ci-generate - - ----------------------------------- Options controlling ReFrame output ---------------------------------- @@ -455,7 +565,6 @@ Options controlling ReFrame output This option can also be set using the :envvar:`RFM_TIMESTAMP_DIRS` environment variable or the :attr:`~config.general.timestamp_dirs` general configuration parameter. -------------------------------------- Options controlling ReFrame execution ------------------------------------- @@ -590,6 +699,17 @@ Options controlling ReFrame execution .. versionchanged:: 3.6.1 Multiple report files are now accepted. + +.. option:: --retries-threshold=VALUE[%] + + Skip retries (see :option:`--max-retries`) if failures exceed the given threshold. + + Threshold can be specified either as an absolute value or as a percentage using the ``%`` character, e.g., ``--retries-threshold=30%``. + Note that in certain shells the ``%`` character may need to be escaped. + + .. versionadded:: 4.7 + + .. option:: -S, --setvar=[TEST.]VAR=VAL Set variable ``VAR`` in all tests or optionally only in test ``TEST`` to ``VAL``. @@ -696,7 +816,6 @@ Options controlling ReFrame execution Skip sanity checking phase. ----------------------------------- Options controlling job submission ---------------------------------- @@ -721,9 +840,9 @@ Options controlling job submission .. versionchanged:: 3.1 Use ``&`` to combine constraints. ------------------------- -Flexible node allocation ------------------------- + +Options controlling flexible node allocation +-------------------------------------------- ReFrame can automatically set the number of tasks of a test, if its :attr:`num_tasks ` attribute is set to a value less than or equal to zero. This scheme is conveniently called *flexible node allocation* and is valid only for the Slurm backend. @@ -764,7 +883,6 @@ If no node can be selected, the test will be marked as a failure with an appropr .. versionadded:: 4.7 ---------------------------------------- Options controlling ReFrame environment --------------------------------------- @@ -858,7 +976,6 @@ It does so by leveraging the selected system's environment modules system. .. _test-generators: ----------------------------------------- Options for generating tests dynamically ---------------------------------------- @@ -886,7 +1003,7 @@ The way the tests are generated and how they interact with the test filtering op - ``avail``: Tests will run on all the nodes of their respective valid partitions that are available for running jobs. Note that if a node is currently allocated to another job it is still considered as "available." - ``NODESTATE``: Tests will run on all the nodes of their respective valid partitions that are exclusively in state ``NODESTATE``. - aIf ``NODESTATE`` is not specified, ``idle`` is assumed. + If ``NODESTATE`` is not specified, ``idle`` is assumed. - ``NODESTATE*``: Tests will run on all the nodes of their respective valid partitions that are at least in state ``NODESTATE``. The state of the nodes will be determined once, before beginning the @@ -947,7 +1064,6 @@ The way the tests are generated and how they interact with the test filtering op .. versionadded:: 3.12.0 ---------------------- Miscellaneous options --------------------- @@ -971,16 +1087,6 @@ Miscellaneous options .. versionchanged:: 4.0.0 -.. _--detect-host-topology: - -.. option:: --detect-host-topology[=FILE] - - Detect the local host processor topology, store it to ``FILE`` and exit. - - If no ``FILE`` is specified, the standard output will be used. - - .. versionadded:: 3.7.0 - .. option:: --failure-stats Print failure statistics at the end of the run. @@ -995,11 +1101,21 @@ Miscellaneous options This option can also be set using the :envvar:`RFM_COLORIZE` environment variable or the :attr:`~config.general.colorize` general configuration parameter. -.. option:: --performance-report +.. _--performance-report: + +.. option:: --performance-report[=CMPSPEC] + + Print a report summarizing the performance of all performance tests that have run in the current session. - Print a performance report for all the performance tests that have been run. + For each test all of their performance variables are reported and optionally compared to past results based on the ``CMPSPEC`` specified. - The report shows the performance values retrieved for the different performance variables defined in the tests. + If not specified, the default ``CMPSPEC`` is ``now:now/last:/+job_nodelist+result``, meaning that the current performance will not be compared to any past run and, additionally, the ``job_nodelist`` and the test result (``pass`` or ``fail``) will be listed. + + For the exact syntax of ``CMPSPEC``, refer to :ref:`performance-comparisons`. + + .. versionchanged:: 4.7 + + The format of the performance report has changed and the optional ``CMPSPEC`` argument is now added. .. option:: -q, --quiet @@ -1014,34 +1130,14 @@ Miscellaneous options .. versionadded:: 3.9.3 -.. option:: --show-config [PARAM] +.. option:: --session-extras KV_DATA - Show the value of configuration parameter ``PARAM`` as this is defined for the currently selected system and exit. + Annotate the current session with custom key/value metadata. - The parameter value is printed in JSON format. - If ``PARAM`` is not specified or if it set to ``all``, the whole configuration for the currently selected system will be shown. - Configuration parameters are formatted as a path navigating from the top-level configuration object to the actual parameter. - The ``/`` character acts as a selector of configuration object properties or an index in array objects. - The ``@`` character acts as a selector by name for configuration objects that have a ``name`` property. - Here are some example queries: - - - Retrieve all the partitions of the current system: - - .. code:: bash + The key/value data is specified as a comma-separated list of `key=value` pairs. + When listing stored sessions with the :option:`--list-stored-sessions` option, any associated custom metadata will be presented by default. - reframe --show-config=systems/0/partitions - - - Retrieve the job scheduler of the partition named ``default``: - - .. code:: bash - - reframe --show-config=systems/0/partitions/@default/scheduler - - - Retrieve the check search path for system ``foo``: - - .. code:: bash - - reframe --system=foo --show-config=general/0/check_search_path + .. versionadded:: 4.7 .. option:: --system=NAME @@ -1056,16 +1152,30 @@ Miscellaneous options This option can also be set using the :envvar:`RFM_SYSTEM` environment variable. +.. option:: --table-format=csv|plain|pretty + + Set the formatting of tabular output printed by the options :option:`--performance-compare`, :option:`--performance-report` and the options controlling the stored sessions. + + The acceptable values are the following: + + - ``csv``: Generate CSV output + - ``plain``: Generate a plain table without any lines + - ``pretty``: (default) Generate a pretty table + + .. versionadded:: 4.7 + +.. option:: --table-hide-columns=COLUMNS + + Hide the specified comma-separated list of columns from the tabular output printed by the options :option:`--performance-compare`, :option:`--performance-report` and the options controlling the stored sessions. + + .. versionadded:: 4.7 + .. option:: --upgrade-config-file=OLD[:NEW] Convert the old-style configuration file ``OLD``, place it into the new file ``NEW`` and exit. If a new file is not given, a file in the system temporary directory will be created. -.. option:: -V, --version - - Print version and exit. - .. option:: -v, --verbose Increase verbosity level of output. @@ -1082,7 +1192,7 @@ Miscellaneous options .. _test_naming_scheme: Test Naming Scheme ------------------- +================== .. versionadded:: 3.10.0 @@ -1199,7 +1309,6 @@ Also users should not rely on how the variant numbers are assigned to a test, as A hash code is associated with each test. --------------------------------------- Differences from the old naming scheme -------------------------------------- @@ -1210,8 +1319,141 @@ Very large test names meant also very large path names which could also lead to Fixtures followed a similar naming pattern making them hard to debug. +Result storage +============== + +.. versionadded:: 4.7 + +ReFrame stores the results of every session that has executed at least one test into a database. +There is only one storage backend supported at the moment and this is SQLite. +The full session information as recorded in a run report file (see :option:`--report-file`) is stored in the database. +The test cases of the session are indexed by their run job completion time for quick retrieval of all the test cases that have run in a certain period of time. + +The database file is controlled by the :attr:`~config.storage.sqlite_db_file` configuration parameter and multiple ReFrame processes can access it safely simultaneously. + +There are several command-line options that allow users to query the results database, such as the :option:`--list-stored-sessions`, :option:`--list-stored-testcases`, :option:`--describe-stored-session` etc. +Other options that access the results database are the :option:`--performance-compare` and :option:`--performance-report` which compare the performance results of the same test cases in different periods of time or from different sessions. +Check the :ref:`commands` section for the complete list and details of each option related to the results database. + +Since the report file information is now kept in the results database, there is no need to keep the report files separately, although this remains the default behavior for backward compatibility. +You can disable the report generation by turning off the :attr:`~config.general.generate_file_reports` configuration parameter. +The file report of any session can be retrieved from the database with the :option:`--describe-stored-session` option. + + +.. _performance-comparisons: + +Performance comparisons +======================= + +.. versionadded:: 4.7 + +The :option:`--performance-compare` and :option:`--performance-report` options accept a ``CMPSPEC`` argument that specifies how to select and compare test cases. +The full syntax of ``CMPSPEC`` is the following: + +.. code-block:: console + + /// + +The ```` and ```` subspecs specify how the base and target test cases will be retrieved. +The base test cases will be compared against those from the target period. + +.. note:: + + The ```` subspec is ommitted from the ``CMPSPEC`` of the :option:`--performance-report` option as the base test cases are always the test cases from the current session. + +The test cases for comparison can either be retrieved from an existing past session or a past time period. +A past session is denoted with the ```` syntax and only the test cases of that particular session will be selected. +To view the UUIDs of all stored sessions, use the :option:`--list-stored-sessions` option. + +To retrieve results from a time period, check the :ref:`time period syntax ` below. + +The ```` subspec specifies how the performance of both the base and target cases should be grouped and aggregated. +The syntax is the following: + +.. code-block:: console + + :[+]* + +The ```` is a symbolic name for a function to aggregate the grouped test cases. +It can take one of the following values: + +- ``first``: retrieve the performance data of the first test case only +- ``last``: retrieve the performance data of the last test case only +- ``max``: retrieve the maximum of all test cases +- ``mean``: calculate the mean over all test cases +- ``median``: retrieve the median of all test cases +- ``min``: retrieve the minimum of all test cases + +The test cases are always grouped by the following attributes: + +- The test :attr:`~reframe.core.pipeline.RegressionTest.name` +- The system name +- The partition name +- The environment name +- The performance variable name (see :func:`@performance_function ` and :attr:`~reframe.core.pipeline.RegressionTest.perf_variables`) +- The performance variable unit + +The ``+`` subspec specifies additional attributes to group the test cases by. +Any loggable test attribute can be selected. + +.. note:: + + The loggable attributes of a test are the same as the ones list in the logging :attr:`~config.logging.handlers_perflog.format` option but without the ``check_`` prefix. + +Finally, the ```` subspec specifies additional test attributes to list as columns in the resulting comparison table. +The syntax is the following: + +.. code-block:: console + + [+]* + +``col`` refers to any loggable attribute of the test. +If these attributes have different values across the aggregated test cases, +the unique values will be joined using the ``|`` separator. + +Here are some examples of performance comparison specs: + +- Compare the test cases of the session ``7a70b2da-1544-4ac4-baf4-0fcddd30b672`` with the mean performance of the last 10 days: + + .. code-block:: console + + 7a70b2da-1544-4ac4-baf4-0fcddd30b672/now-10d:now/mean:/ + +- Compare the best performance of the test cases run on two specific days, group by the node list and report also the test result: + + .. code-block:: console + + 20240701:20240701+1d/20240705:20240705+1d/mean:+job_nodelist/+result + +.. _time-period-syntax: + +Time periods +============ + +A time period needs to be specified as part of the ``CMPSPEC`` of the :option:`--performance-compare` and :option:`--performance-report` options or as an argument to options that request past results from results database. + +The general syntax of time period subspec is the following: + +.. code-block:: console + + : + +```` and ```` are timestamp denoting the start and end of the requested period. +More specifically, the syntax of each timestamp is the following: + +.. code-block:: console + + [+|-w|d|h|m] + +The ```` is an absolute timestamp in one of the following ``strptime``-compatible formats or the special value ``now``: ``%Y%m%d``, ``%Y%m%dT%H%M``, ``%Y%m%dT%H%M%S``, ``%Y%m%dT%H%M%S%z``. + +Optionally, a shift argument can be appended with ``+`` or ``-`` signs, followed by an amount of weeks (``w``), days (``d``), hours (``h``) or minutes (``m``). + +For example, the period of the last 10 days can be specified as ``now-10d:now``. +Similarly, the period of the week starting on August 5, 2024 will be specified as ``20240805:20240805+1w``. + Environment ------------ +=========== Several aspects of ReFrame can be controlled through environment variables. Usually environment variables have counterparts in command line options or configuration parameters. @@ -1226,21 +1468,6 @@ Here is an alphabetical list of the environment variables recognized by ReFrame. Whenever an environment variable is associated with a configuration option, its default value is omitted as it is the same. -.. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT - - Pass access options in the submission command (relevant for LSF, OAR, PBS and Slurm). - - .. table:: - :align: left - - ================================== ================== - Associated command line option N/A - Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit` - ================================== ================== - -.. versionadded:: 4.7 - - .. envvar:: RFM_AUTODETECT_FQDN Use the fully qualified domain name as the hostname. @@ -1452,6 +1679,20 @@ Whenever an environment variable is associated with a configuration option, its .. versionadded:: 4.7 +.. envvar:: RFM_GENERATE_FILE_REPORTS + + Store session reports also in files. + + .. table:: + :align: left + + ================================== ================== + Associated command line option n/a + Associated configuration parameter :attr:`~config.general.generate_file_reports` + ================================== ================== + + .. versionadded:: 4.7 + .. envvar:: RFM_GIT_TIMEOUT Timeout value in seconds used when checking if a git repository exists. @@ -1600,6 +1841,21 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_PERF_REPORT_SPEC + + The default ``CMPSPEC`` of the :option:`--performance-report` option. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--performance-report` + Associated configuration parameter :attr:`~config.general.perf_report_spec` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_PERFLOG_DIR Directory prefix for logging performance data. @@ -1744,6 +2000,21 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_SCHED_ACCESS_IN_SUBMIT + + Pass access options in the submission command (relevant for LSF, OAR, PBS and Slurm). + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr::attr:`~config.systems.partitions.sched_options.sched_access_in_submit` + ================================== ================== + +.. versionadded:: 4.7 + + .. envvar:: RFM_STAGE_DIR Directory prefix for staging test resources. @@ -1757,6 +2028,51 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_SQLITE_CONN_TIMEOUT + + Timeout for SQLite database connections. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr:`~config.storage.sqlite_conn_timeout` + ================================== ================== + + .. versionadded:: 4.7 + + +.. envvar:: RFM_SQLITE_DB_FILE + + The SQLite database file for storing test results. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr:`~config.storage.sqlite_db_file` + ================================== ================== + + .. versionadded:: 4.7 + + +.. envvar:: RFM_SQLITE_DB_FILE_MODE + + The permissions of the SQLite database file in octal form. + + .. table:: + :align: left + + ================================== ================== + Associated command line option N/A + Associated configuration parameter :attr:`~config.storage.sqlite_db_file_mode` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_SYSLOG_ADDRESS The address of the Syslog server to send performance logs. @@ -1787,6 +2103,21 @@ Whenever an environment variable is associated with a configuration option, its ================================== ================== +.. envvar:: RFM_TABLE_FORMAT + + Set the format of the tables printed by various options accessing the results storage. + + .. table:: + :align: left + + ================================== ================== + Associated command line option :option:`--table-format` + Associated configuration parameter :attr:`~config.general.table_format` + ================================== ================== + + .. versionadded:: 4.7 + + .. envvar:: RFM_TIMESTAMP_DIRS Append a timestamp to the output and stage directory prefixes. @@ -1872,7 +2203,7 @@ Whenever an environment variable is associated with a configuration option, its Configuration File ------------------- +================== The configuration file of ReFrame defines the systems and environments to test as well as parameters controlling the framework's behavior. @@ -1885,12 +2216,12 @@ For a complete reference of the available configuration options, please refer to Reporting Bugs --------------- +============== For bugs, feature request, help, please open an issue on Github: See Also --------- +======== See full documentation online: diff --git a/docs/started.rst b/docs/started.rst index ccfc1f57ba..3ab07c0295 100644 --- a/docs/started.rst +++ b/docs/started.rst @@ -20,8 +20,8 @@ Requirements .. warning:: - Although ReFrame supports Python 3.6, you should note that Python 3.6 has reached its end-of-life and you are advised to use a newer Python version. - ReFrame installations on Python 3.6 may use out-of-date dependencies due to incompatibilities of their newer versions with Python 3.6. + Although ReFrame supports Python 3.6 and 3.7, you should note that these Python versions have reached end-of-life and you are strongly advised to use a newer version. + ReFrame installations on these Python versions may use out-of-date dependencies due to incompatibilities. Getting the Framework diff --git a/docs/tutorial.rst b/docs/tutorial.rst index 78b1d3a672..55c52bd1df 100644 --- a/docs/tutorial.rst +++ b/docs/tutorial.rst @@ -168,24 +168,28 @@ This can be suppressed by increasing the level at which this information is logg Run reports and performance logging ----------------------------------- -Once a test session finishes, ReFrame generates a detailed JSON report under ``$HOME/.reframe/reports``. -Every time ReFrame is run a new report will be generated automatically. -The latest one is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given. +Once a test session finishes, ReFrame stores the detailed session information in a database file located under ``$HOME/.reframe/reports``. +Past performance data can be retrieved from this database and compared with the current or another run. +We explain in detail the handling of the results database in section :ref:`inspecting-past-results`. + +By default, the session information is also saved in a JSON report file under ``$HOME/.reframe/reports``. +The latest report is always symlinked by the ``latest.json`` name, unless the :option:`--report-file` option is given. For performance tests, in particular, an additional CSV file is generated with all the relevant information. These files are located by default under ``perflogs///.log``. In our example, this translates to ``perflogs/generic/default/stream_test.log``. The information that is being logged is fully configurable and we will cover this in the :ref:`logging` section. -Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session. +Finally, you can use also the :option:`--performance-report` option, which will print a summary of the results of the performance tests that have run in the current session and compare them (by default) to their last obtained performance. .. code-block:: console - [stream_test /2e15a047 @generic:default:builtin] - num_tasks: 1 - performance: - - copy_bw: 22704.4 MB/s (r: 0 MB/s l: -inf% u: +inf%) - - triad_bw: 16040.9 MB/s (r: 0 MB/s l: -inf% u: +inf%) + ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━━━━━━━━┯━━━━━━━━━━┑ + │ name │ sysenv │ pvar │ pval │ punit │ pdiff │ job_nodelist │ result │ + ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━━━━━━━━┿━━━━━━━━━━┥ + │ stream_test │ generic:default+builtin │ copy_bw │ 40292.1 │ MB/s │ -0.04% │ myhost │ pass │ + │ stream_test │ generic:default+builtin │ triad_bw │ 30564.7 │ MB/s │ +0.12% │ myhost │ pass │ + ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━━━━━━━━┷━━━━━━━━━━┙ Inspecting the test artifacts @@ -1668,7 +1672,7 @@ By inspecting the generated script files, you will notice that ReFrame emits the .. code-block:: bash :caption: Run with the Docker compose setup. - cat /scratch/rfm-stage/output/pseudo-cluster/compute/gnu/stream_test_pseudo-cluster_compute_8de19aca/rfm_job.sh + cat /scratch/rfm-stage/output/pseudo-cluster/compute/gnu/stream_test_584fea5f/rfm_job.sh .. code-block:: bash @@ -1969,3 +1973,138 @@ The format function takes the raw log record, the extras and the keys to ignore Since we can't know the exact log record attributes, we iterate over its :attr:`__dict__` items and format the record keys as we go. Also note that we ignore all private field of the record starting with ``_``. Rerunning the previous example with ``CUSTOM_JSON=1`` will generated the modified JSON record. + + +.. _inspecting-past-results: + +Inspecting past results +======================= + +.. versionadded:: 4.7 + +For every session that has run at least one test case, ReFrame stores all its details, including the test cases, in a database. +Essentially, the stored information is the same as the one found in the :ref:`report file `. + +To list the stored sessions use the :option:`--list-stored-sessions` option: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe --list-stored-sessions + +This produces a table where the most important information about a session is listed: +its unique identifier, its start and end time and how many test cases have run: + +.. code-block:: console + + ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━┑ + │ UUID │ Start time │ End time │ Num runs │ Num cases │ + ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━┥ + │ fedb2cf8-6efa-43d8-a6dc-e72c868deba6 │ 20240823T104554+0000 │ 20240823T104557+0000 │ 1 │ 1 │ + │ 4253d6b3-3926-4c4c-a7e8-3f7dffe9bf23 │ 20240823T104608+0000 │ 20240823T104612+0000 │ 1 │ 1 │ + │ 453e64a2-f941-49e2-b628-bf50883a6387 │ 20240823T104721+0000 │ 20240823T104725+0000 │ 1 │ 1 │ + │ d923cca2-a72b-43ca-aca1-de741b65088b │ 20240823T104753+0000 │ 20240823T104757+0000 │ 1 │ 1 │ + │ 300b973b-84a6-4932-89eb-577a832fe357 │ 20240823T104814+0000 │ 20240823T104815+0000 │ 1 │ 2 │ + │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e │ 20240823T104834+0000 │ 20240823T104835+0000 │ 1 │ 4 │ + │ 2a00c55d-4492-498c-89f0-7cf821f308c1 │ 20240823T104843+0000 │ 20240823T104845+0000 │ 1 │ 4 │ + │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143 │ 20240823T104902+0000 │ 20240823T104903+0000 │ 1 │ 4 │ + │ 4bbc27bc-be50-4cca-9d1b-c5fb4988a5c0 │ 20240823T104922+0000 │ 20240823T104933+0000 │ 1 │ 26 │ + │ 200ea28f-6c3a-4973-a2b7-aa08408dbeec │ 20240823T104939+0000 │ 20240823T104943+0000 │ 1 │ 10 │ + │ b756755b-3181-4bb4-9eaa-cc8c3a9d7a43 │ 20240823T104955+0000 │ 20240823T104956+0000 │ 1 │ 10 │ + │ a8a99808-c22d-4b9c-83bc-164289fe6aa7 │ 20240823T105007+0000 │ 20240823T105007+0000 │ 1 │ 4 │ + │ f9b63cdc-7dda-44c5-ab85-1e9752047834 │ 20240823T105019+0000 │ 20240823T105020+0000 │ 1 │ 10 │ + │ 271fc2e7-b550-4325-b8bb-57bdf95f1d0d │ 20240823T105020+0000 │ 20240823T105020+0000 │ 1 │ 1 │ + │ 50cdb774-f231-4f61-8472-7daaa5199d57 │ 20240823T105031+0000 │ 20240823T105032+0000 │ 1 │ 5 │ + ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━┙ + +You can use the :option:`--list-stored-testcases` to list the test cases of a specific session or those that have run within a certain period of time: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe --list-stored-testcases=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e + +.. code-block:: console + + ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ + │ Name │ SysEnv │ Nodelist │ Completion Time │ Result │ UUID │ + ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ + │ build_stream ~tutorialsys:default+gnu │ tutorialsys:default+gnu │ │ n/a │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:0 │ + │ build_stream ~tutorialsys:default+clang │ tutorialsys:default+clang │ │ n/a │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:1 │ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:3 │ + ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ + + +The test case UUID comprises the UUID of the session where this test case belongs to, its run index (which run inside the session) and its test case index inside the run. +A session may have multiple runs if it has retried some failed test cases (see :option:`--max-retries`) or if it has run its tests repeatedly (see :option:`--reruns` and :option:`--duration`). + +You can also list the test cases that have run in a certain period of time use the :ref:`time period ` of :option:`--list-stored-testcases`: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe --list-stored-testcases=20240823T104835+0000:now + +.. code-block:: console + + ┍━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┑ + │ Name │ SysEnv │ Nodelist │ Completion Time │ Result │ UUID │ + ┝━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┥ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104835+0000 │ pass │ 1fb8488e-c361-4355-b7df-c0dcf3cdcc1e:0:3 │ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104844+0000 │ pass │ 2a00c55d-4492-498c-89f0-7cf821f308c1:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104845+0000 │ pass │ 2a00c55d-4492-498c-89f0-7cf821f308c1:0:3 │ + │ stream_test │ tutorialsys:default+gnu │ myhost │ 20240823T104903+0000 │ pass │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143:0:2 │ + │ stream_test │ tutorialsys:default+clang │ myhost │ 20240823T104903+0000 │ pass │ 98fe5a68-2582-49ca-9c3c-6bfd9b877143:0:3 │ + ... + │ T6 │ generic:default+builtin │ myhost │ 20240823T105020+0000 │ pass │ 271fc2e7-b550-4325-b8bb-57bdf95f1d0d:0:0 │ + │ T0 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:0 │ + │ T4 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:1 │ + │ T5 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:2 │ + │ T1 │ generic:default+builtin │ myhost │ 20240823T105031+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:3 │ + │ T6 │ generic:default+builtin │ myhost │ 20240823T105032+0000 │ pass │ 50cdb774-f231-4f61-8472-7daaa5199d57:0:4 │ + ┕━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┙ + +To get all the details of a session or a set of test cases you can use the :option:`--describe-stored-session` and :option:`--describe-stored-testcases` options which will return a JSON record with all the details. + +You can also combine the :option:`-n` option with the :option:`--list-stored-testcases` and :option:`--describe-stored-testcases` options in order to restrict the listing to specific tests only: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe --list-stored-testcases=20240823T104835+0000:now -n stream_test + + +Comparing performance of test cases +----------------------------------- + +ReFrame can be used to compare the performance of the same test cases run in different time periods using the :option:`--performance-compare` option. +The following will compare the performance of the test cases of the session ``1fb8488e-c361-4355-b7df-c0dcf3cdcc1e`` with any other same test case that has run the last 24h: + +.. code-block:: bash + :caption: Run in the single-node container. + + reframe --performance-compare=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e/now-1d:now/mean:/ + +.. code-block:: console + + ┍━━━━━━━━━━━━━┯━━━━━━━━━━━━━━━━━━━━━━━━━━━┯━━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┯━━━━━━━━━┑ + │ name │ sysenv │ pvar │ pval │ punit │ pdiff │ + ┝━━━━━━━━━━━━━┿━━━━━━━━━━━━━━━━━━━━━━━━━━━┿━━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┿━━━━━━━━━┥ + │ stream_test │ tutorialsys:default+gnu │ copy_bw │ 44139 │ MB/s │ +11.14% │ + │ stream_test │ tutorialsys:default+gnu │ triad_bw │ 39344.7 │ MB/s │ +20.77% │ + │ stream_test │ tutorialsys:default+clang │ copy_bw │ 44979.1 │ MB/s │ +10.81% │ + │ stream_test │ tutorialsys:default+clang │ triad_bw │ 39330.8 │ MB/s │ +8.28% │ + ┕━━━━━━━━━━━━━┷━━━━━━━━━━━━━━━━━━━━━━━━━━━┷━━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┷━━━━━━━━━┙ + +The :option:`-n` option can also be combined with :option:`--performance-compare` to restrict the test cases listed. +Similarly to the :option:`--performance-compare` option, the :option:`--performance-report` option can compare the performance of the current run with any arbitrary past session or past time period. + +Finally, you can delete completely a stored session using the :option:`--delete-stored-session` option: + +.. code-block:: bash + + reframe --delete-stored-session=1fb8488e-c361-4355-b7df-c0dcf3cdcc1e + +Deleting a session will also delete all its test cases from the database. diff --git a/examples/tutorial/dockerfiles/singlenode.Dockerfile b/examples/tutorial/dockerfiles/singlenode.Dockerfile index 647f26d231..2c8fed9c78 100644 --- a/examples/tutorial/dockerfiles/singlenode.Dockerfile +++ b/examples/tutorial/dockerfiles/singlenode.Dockerfile @@ -10,8 +10,16 @@ RUN apt-get -y update && \ ARG REFRAME_TAG=develop ARG REFRAME_REPO=reframe-hpc WORKDIR /usr/local/share -RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \ - cd reframe/ && ./bootstrap.sh + +# Clone reframe +# RUN git clone --depth 1 --branch $REFRAME_TAG https://github.com/$REFRAME_REPO/reframe.git && \ +# cd reframe/ && ./bootstrap.sh + +# Comment the above line and uncomment the following two for development + +COPY . /usr/local/share/reframe +RUN cd reframe && ./bootstrap.sh + ENV PATH=/usr/local/share/reframe/bin:$PATH # Install stream diff --git a/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml b/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml index 13ce87e9ff..d6ef75df9f 100644 --- a/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml +++ b/examples/tutorial/dockerfiles/slurm-cluster/docker-compose.yml @@ -12,7 +12,11 @@ services: frontend: image: slurm-reframe - build: examples/tutorial/dockerfiles/slurm-cluster/reframe + build: + context: examples/tutorial/dockerfiles/slurm-cluster/reframe + args: + REFRAME_REPO: reframe-hpc + REFRAME_TAG: develop container_name: frontend hostname: login user: admin diff --git a/hpctestlib/sciapps/metalwalls/benchmarks.py b/hpctestlib/sciapps/metalwalls/benchmarks.py new file mode 100644 index 0000000000..dda4f39edd --- /dev/null +++ b/hpctestlib/sciapps/metalwalls/benchmarks.py @@ -0,0 +1,301 @@ +"""ReFrame benchmark for MetalWalls""" +import re + +import reframe as rfm +import reframe.utility.sanity as sn +from reframe.core.builtins import (performance_function, run_after, run_before, + sanity_function) +from reframe.core.parameters import TestParam as parameter +from reframe.core.variables import TestVar as variable + +address_tpl = ( + 'https://gitlab.com/ampere2/metalwalls/-/raw/{version}/{bench}/{file}' + '?ref_type=tags&inline=false' +) + +extract_fields = [ + ####### + ('Ions->Atoms Coulomb potential', 'long range', 'I->A Cpot'), + ('Ions->Atoms Coulomb potential', 'k==0', 'I->A Cpot'), + ('Ions->Atoms Coulomb potential', 'short range', 'I->A Cpot'), + + ####### + ('Atoms->Atoms Coulomb potential', 'long range', 'A->A Cpot'), + ('Atoms->Atoms Coulomb potential', 'k==0', 'A->A Cpot'), + ('Atoms->Atoms Coulomb potential', 'short range', 'A->A Cpot'), + ('Atoms->Atoms Coulomb potential', 'self', 'A->A Cpot'), + + ####### + ('Atoms->Atoms Coulomb grad Q', 'long range', 'A->A gradQ'), + ('Atoms->Atoms Coulomb grad Q', 'k==0', 'A->A gradQ'), + ('Atoms->Atoms Coulomb grad Q', 'short range', 'A->A gradQ'), + ('Atoms->Atoms Coulomb grad Q', 'self', 'A->A gradQ'), + + ####### + ('Ions Coulomb forces', 'long range', 'Ions Cfrc'), + ('Ions Coulomb forces', 'k==0', 'Ions Cfrc'), + ('Ions Coulomb forces', 'short range', 'Ions Cfrc'), + ('Ions Coulomb forces', 'intramolecular', 'Ions Cfrc'), + + ####### + ('Ions Coulomb potential', 'long range', 'Ions Cpot'), + ('Ions Coulomb potential', 'k==0', 'Ions Cpot'), + ('Ions Coulomb potential', 'short range', 'Ions Cpot'), + ('Ions Coulomb potential', 'intramolecular', 'Ions Cpot'), + ('Ions Coulomb potential', 'self', 'Ions Cpot'), + + ####### + ( + 'Ions Coulomb electric field (due to charges)', + 'long range', + 'Ions Cfield(charges)' + ), + ( + 'Ions Coulomb electric field (due to charges)', + 'k==0', + 'Ions Cfield(charges)' + ), + ( + 'Ions Coulomb electric field (due to charges)', + 'short range', + 'Ions Cfield(charges)' + ), + ( + 'Ions Coulomb electric field (due to charges)', + 'intramolecular', + 'Ions Cfield(charges)' + ), + + ####### + ( + 'Ions Coulomb electric field (due to dipoles)', + 'long range', + 'Ions Cfield(dipoles)' + ), + ( + 'Ions Coulomb electric field (due to dipoles)', + 'k==0', + 'Ions Cfield(dipoles)' + ), + ( + 'Ions Coulomb electric field (due to dipoles)', + 'short range', + 'Ions Cfield(dipoles)' + ), + ( + 'Ions Coulomb electric field (due to dipoles)', + 'intramolecular', + 'Ions Cfield(dipoles)' + ), + ( + 'Ions Coulomb electric field (due to dipoles)', + 'self', + 'Ions Cfield(dipoles)' + ), + + ####### + ('Ions Coulomb electric field gradient', 'long range', 'Ions Cfield grad'), + ( + 'Ions Coulomb electric field gradient', + 'short range', + 'Ions Cfield grad' + ), + ('Ions Coulomb electric field gradient', 'self', 'Ions Cfield grad'), + + ####### + ('Ions Coulomb gradient mu', 'long range', 'Ions C mu_grad'), + ('Ions Coulomb gradient mu', 'k==0', 'Ions C mu_grad'), + ('Ions Coulomb gradient mu', 'short range', 'Ions C mu_grad'), + ('Ions Coulomb gradient mu', 'self', 'Ions C mu_grad'), + + ####### + ('Rattle', 'positions', 'Rattle'), + ('Rattle', 'velocities', 'Rattle'), + + ####### + ('van der Waals', 'vdW forces', 'van der Waals'), + ('van der Waals', 'vdW potential', 'van der Waals'), + + ####### + ('Intramolecular', 'Intramolecular forces', 'Intramolecular'), + ('Intramolecular', 'Intramolecular potential', 'Intramolecular'), + + ####### + ('Additional degrees', 'Electrode charge computation', 'Deg of freedom'), + ('Additional degrees', 'Inversion of the matrix', 'Deg of freedom'), + ('Additional degrees', 'One matrix-vector product', 'Deg of freedom'), + ('Additional degrees', 'Melt dipoles computation', 'Deg of freedom'), + ('Additional degrees', 'Inversion of the matrix', 'Deg of freedom'), + ('Additional degrees', 'One matrix-vector product', 'Deg of freedom'), + ('Additional degrees', 'AIM DOFs computation', 'Deg of freedom'), + + ####### + ('Diagnostics', 'diagnostics computations', 'Diagnostics'), + ('Diagnostics', 'IO', 'Diagnostics'), +] + + +@rfm.simple_test +class MetalWallsCheck(rfm.RunOnlyRegressionTest): + """MetalWalls benchmark test. + + `MetalWalls `__ is a molecular + dynamics code dedicated to the modelling of electrochemical systems. + Its main originality is the inclusion of a series of methods allowing to + apply a constant potential within the electrode materials. + + The benchmarks consist of a set of different inputs files that vary in the + number of atoms and the type of simulation performed. + They can be found in the following repository, which is also versioned: + https://gitlab.com/ampere2/metalwalls/. + + """ + + #: The name of the output files to keep. + #: + #: :type: :class:`List[str]` + #: :default: ``'run.out'`` + keep_files = ['run.out', ] + + #: The version of the benchmark suite to use. + #: + #: :type: :class:`str` + #: :default: ``'21.06.1'`` + benchmark_version = variable(str, value='21.06.1', loggable=True) + + executable = 'mw' + tags = {'sciapp', 'chemistry'} + descr = 'MetalWalls `mw` benchmark' + + #: Collect and report detailed performance metrics. + #: + #: :type: :class:`bool` + #: :default: ``False`` + debug_metrics = variable(bool, value=False, loggable=True) + + #: Parameter pack encoding the benchmark information. + #: + #: The first element of the tuple refers to the benchmark name, + #: the second is the final kinetic energy the third is the related + #: tolerance, the fourth is the absolute temperature and the fifth is + #: the related tolerance + #: + #: :type: `Tuple[str, float, float, float, float]` + #: :values: + benchmark_info = parameter([ + ('hackathonGPU/benchmark', 14.00, 0.05, 301.74, 0.5), + ('hackathonGPU/benchmark2', 14.00, 0.05, 301.74, 0.5), + ('hackathonGPU/benchmark3', 16.08, 0.05, 293.42, 0.5), + ('hackathonGPU/benchmark4', 16.08, 0.05, 293.42, 0.5), + ('hackathonGPU/benchmark5', 25.72, 0.05, 297.47, 0.5), + ('hackathonGPU/benchmark6', 25.72, 0.05, 297.47, 0.5), + ], fmt=lambda x: x[0], loggable=True) + + @run_after('init') + def prepare_test(self): + """Hook to the set the downloading of the pseudo-potentials""" + self.__bench, _, _, _, _ = self.benchmark_info + self.descr = f'MetalWalls {self.__bench} benchmark' + files_addresses = [ + address_tpl.format( + version=self.benchmark_version, + bench=self.__bench, + file=_ + ) for _ in ['data.inpt', 'runtime.inpt'] + ] + self.prerun_cmds += [ + f"curl -LJO '{_}'" for _ in files_addresses + ] + + @performance_function('s') + def total_elapsed_time(self): + """Extract the total elapsed time from the output file""" + return sn.extractsingle( + r'Total elapsed time:\s+(?P